mirror of
https://github.com/giongto35/cloud-game.git
synced 2026-07-17 16:50:31 +00:00
Use client-side stream vertical flip
Previously we used zero-cost x264 `X264_CSP_VFLIP` and vpx `vpx_img_flip` for OpenGL cores (which render their output buffers upside down). Now the flip is handled client-side during GPU compositing of the video stream. This will be needed later, when we have to use more codecs and Gstreamer.
This commit is contained in:
sergystepanov
parent
f55727a9c5
commit
e3b16d5feb
8 changed files with 168 additions and 162 deletions
|
|
@ -55,10 +55,11 @@ type (
|
|||
InitWebrtcStreamResponse string
|
||||
|
||||
AppVideoInfo struct {
|
||||
W int `json:"w"`
|
||||
H int `json:"h"`
|
||||
S int `json:"s"`
|
||||
A float32 `json:"a"`
|
||||
W int `json:"w"`
|
||||
H int `json:"h"`
|
||||
S int `json:"s"`
|
||||
A float32 `json:"a"`
|
||||
Flip bool `json:"flip,omitempty"`
|
||||
}
|
||||
|
||||
LibGameListInfo struct {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ type (
|
|||
Encode([]byte) []byte
|
||||
IntraRefresh()
|
||||
Info() string
|
||||
SetFlip(bool)
|
||||
Shutdown() error
|
||||
}
|
||||
)
|
||||
|
|
@ -119,14 +118,6 @@ func (v *Video) SetRot(a uint) {
|
|||
}
|
||||
}
|
||||
|
||||
// SetFlip tells the encoder to flip the frames vertically.
|
||||
func (v *Video) SetFlip(b bool) {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
v.codec.SetFlip(b)
|
||||
}
|
||||
|
||||
func (v *Video) Stop() {
|
||||
if v == nil {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -193,14 +193,6 @@ func (e *H264) IntraRefresh() {
|
|||
|
||||
func (e *H264) Info() string { return fmt.Sprintf("x264: v%v", Version()) }
|
||||
|
||||
func (e *H264) SetFlip(b bool) {
|
||||
if b {
|
||||
(*e.h).pic.img.i_csp |= C.X264_CSP_VFLIP
|
||||
} else {
|
||||
(*e.h).pic.img.i_csp &= ^C.X264_CSP_VFLIP
|
||||
}
|
||||
}
|
||||
|
||||
func (e *H264) Shutdown() error {
|
||||
if e.h != nil {
|
||||
C.h264_destroy(e.h)
|
||||
|
|
|
|||
|
|
@ -119,12 +119,9 @@ type Vpx struct {
|
|||
image C.vpx_image_t
|
||||
codecCtx C.vpx_codec_ctx_t
|
||||
kfi C.int
|
||||
flipped bool
|
||||
v int
|
||||
}
|
||||
|
||||
func (vpx *Vpx) SetFlip(b bool) { vpx.flipped = b }
|
||||
|
||||
func autoThreads(configured, cpus int) int {
|
||||
if configured != 0 {
|
||||
return configured
|
||||
|
|
@ -240,9 +237,6 @@ func NewEncoder(w, h int, th int, kfi int, version int, opts *Options) (*Vpx, er
|
|||
// see: https://chromium.googlesource.com/webm/libvpx/+/master/examples/simple_encoder.c
|
||||
func (vpx *Vpx) Encode(yuv []byte) []byte {
|
||||
C.vpx_img_read(&vpx.image, unsafe.Pointer(&yuv[0]))
|
||||
if vpx.flipped {
|
||||
C.vpx_img_flip(&vpx.image)
|
||||
}
|
||||
|
||||
var flags C.int
|
||||
if vpx.frameCount < 3 || (vpx.kfi > 0 && vpx.frameCount%vpx.kfi == 0) {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ type App interface {
|
|||
AudioSampleRate() int
|
||||
AspectRatio() float32
|
||||
AspectEnabled() bool
|
||||
Flipped() bool
|
||||
Init() error
|
||||
ViewportSize() (int, int)
|
||||
Scale() float64
|
||||
|
|
|
|||
|
|
@ -154,10 +154,11 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou
|
|||
data, err := api.Wrap(api.Out{
|
||||
T: uint8(api.AppVideoChange),
|
||||
Payload: api.AppVideoInfo{
|
||||
W: m.VideoW,
|
||||
H: m.VideoH,
|
||||
A: app.AspectRatio(),
|
||||
S: int(app.Scale()),
|
||||
W: m.VideoW,
|
||||
H: m.VideoH,
|
||||
A: app.AspectRatio(),
|
||||
S: int(app.Scale()),
|
||||
Flip: app.Flipped(),
|
||||
}})
|
||||
if err != nil {
|
||||
c.log.Error().Err(err).Msgf("wrap")
|
||||
|
|
@ -187,9 +188,6 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou
|
|||
return api.EmptyPacket
|
||||
}
|
||||
|
||||
if app.Flipped() {
|
||||
m.SetVideoFlip(true)
|
||||
}
|
||||
m.SetPixFmt(app.PixFormat())
|
||||
m.SetRot(app.Rotation())
|
||||
|
||||
|
|
@ -217,7 +215,13 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou
|
|||
}
|
||||
if r.App().AspectEnabled() {
|
||||
ww, hh := r.App().ViewportSize()
|
||||
response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio(), S: int(r.App().Scale())}
|
||||
response.AV = &api.AppVideoInfo{
|
||||
W: ww,
|
||||
H: hh,
|
||||
A: r.App().AspectRatio(),
|
||||
S: int(r.App().Scale()),
|
||||
Flip: r.App().Flipped(),
|
||||
}
|
||||
}
|
||||
|
||||
return api.Out{Payload: response}
|
||||
|
|
|
|||
|
|
@ -54,9 +54,8 @@ type WebrtcMediaPipe struct {
|
|||
initialized bool
|
||||
|
||||
// keep the old settings for reinit
|
||||
oldPf uint32
|
||||
oldRot uint
|
||||
oldFlip bool
|
||||
oldPf uint32
|
||||
oldRot uint
|
||||
}
|
||||
|
||||
func NewWebRtcMediaPipe(ac config.Audio, vc config.Video, log *logger.Logger) *WebrtcMediaPipe {
|
||||
|
|
@ -160,13 +159,11 @@ func (wmp *WebrtcMediaPipe) Reinit() error {
|
|||
// restore old
|
||||
wmp.SetPixFmt(wmp.oldPf)
|
||||
wmp.SetRot(wmp.oldRot)
|
||||
wmp.SetVideoFlip(wmp.oldFlip)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) IsInitialized() bool { return wmp.initialized }
|
||||
func (wmp *WebrtcMediaPipe) SetPixFmt(f uint32) { wmp.oldPf = f; wmp.v.SetPixFormat(f) }
|
||||
func (wmp *WebrtcMediaPipe) SetVideoFlip(b bool) { wmp.oldFlip = b; wmp.v.SetFlip(b) }
|
||||
func (wmp *WebrtcMediaPipe) SetRot(r uint) { wmp.oldRot = r; wmp.v.SetRot(r) }
|
||||
|
||||
func (wmp *WebrtcMediaPipe) Video() *encoder.Video {
|
||||
|
|
|
|||
270
web/js/stream.js
270
web/js/stream.js
|
|
@ -2,21 +2,21 @@ import {
|
|||
sub,
|
||||
APP_VIDEO_CHANGED,
|
||||
SETTINGS_CHANGED,
|
||||
TRANSFORM_CHANGE
|
||||
} from 'event';
|
||||
import {log} from 'log';
|
||||
import {opts, settings} from 'settings';
|
||||
TRANSFORM_CHANGE,
|
||||
} from "event";
|
||||
import { log } from "log";
|
||||
import { opts, settings } from "settings";
|
||||
|
||||
const videoEl = document.getElementById('stream')
|
||||
const mirrorEl = document.getElementById('mirror-stream')
|
||||
const playEl = document.getElementById('play-stream')
|
||||
const videoEl = document.getElementById("stream");
|
||||
const mirrorEl = document.getElementById("mirror-stream");
|
||||
const playEl = document.getElementById("play-stream");
|
||||
|
||||
const options = {
|
||||
volume: 0.5,
|
||||
poster: '/img/screen_loading.gif',
|
||||
poster: "/img/screen_loading.gif",
|
||||
mirrorMode: null,
|
||||
mirrorUpdateRate: 1 / 60,
|
||||
}
|
||||
};
|
||||
|
||||
const state = {
|
||||
screen: videoEl,
|
||||
|
|
@ -24,185 +24,211 @@ const state = {
|
|||
w: 0,
|
||||
h: 0,
|
||||
aspect: 4 / 3,
|
||||
fit: 'contain',
|
||||
fit: "contain",
|
||||
flip: false,
|
||||
ready: false,
|
||||
autoplayWait: false
|
||||
}
|
||||
autoplayWait: false,
|
||||
};
|
||||
|
||||
const mute = (mute) => (videoEl.muted = mute)
|
||||
const flip = (flip) => (videoEl.style.transform = flip ? "scaleY(-1)" : "");
|
||||
const mute = (mute) => (videoEl.muted = mute);
|
||||
|
||||
const onPlay = () => {
|
||||
state.ready = true
|
||||
videoEl.poster = ''
|
||||
resize(state.w, state.h, state.aspect, state.fit)
|
||||
useCustomScreen(options.mirrorMode === 'mirror')
|
||||
}
|
||||
state.ready = true;
|
||||
videoEl.poster = "";
|
||||
flip(state.flip);
|
||||
resize(state.w, state.h, state.aspect, state.fit);
|
||||
useCustomScreen(options.mirrorMode === "mirror");
|
||||
};
|
||||
|
||||
const play = () => {
|
||||
const promise = videoEl.play()
|
||||
const promise = videoEl.play();
|
||||
|
||||
if (promise === undefined) {
|
||||
log.error('oh no, the video is not a promise!')
|
||||
return
|
||||
log.error("oh no, the video is not a promise!");
|
||||
return;
|
||||
}
|
||||
|
||||
promise
|
||||
.then(onPlay)
|
||||
.catch(error => {
|
||||
if (error.name === 'NotAllowedError') {
|
||||
showPlayButton()
|
||||
} else {
|
||||
log.error('Playback fail', error)
|
||||
}
|
||||
})
|
||||
}
|
||||
promise.then(onPlay).catch((error) => {
|
||||
if (error.name === "NotAllowedError") {
|
||||
showPlayButton();
|
||||
} else {
|
||||
log.error("Playback fail", error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const toggle = (show) => state.screen.toggleAttribute('hidden', show === undefined ? show : !show)
|
||||
const toggle = (show) =>
|
||||
state.screen.toggleAttribute("hidden", show === undefined ? show : !show);
|
||||
|
||||
const resize = (w, h, aspect, fit) => {
|
||||
if (!state.ready) return;
|
||||
|
||||
state.screen.setAttribute('width', '' + w)
|
||||
state.screen.setAttribute('height', '' + h)
|
||||
aspect !== undefined && (state.screen.style.aspectRatio = '' + aspect)
|
||||
fit !== undefined && (state.screen.style['object-fit'] = fit)
|
||||
}
|
||||
state.screen.setAttribute("width", "" + w);
|
||||
state.screen.setAttribute("height", "" + h);
|
||||
if (aspect !== undefined) {
|
||||
state.screen.style.aspectRatio = "" + aspect;
|
||||
}
|
||||
if (fit !== undefined) {
|
||||
state.screen.style["object-fit"] = fit;
|
||||
}
|
||||
};
|
||||
|
||||
const showPlayButton = () => {
|
||||
state.autoplayWait = true
|
||||
toggle()
|
||||
playEl.removeAttribute('hidden')
|
||||
}
|
||||
state.autoplayWait = true;
|
||||
toggle();
|
||||
playEl.removeAttribute("hidden");
|
||||
};
|
||||
|
||||
playEl.addEventListener('click', () => {
|
||||
playEl.setAttribute('hidden', "")
|
||||
state.autoplayWait = false
|
||||
play()
|
||||
toggle()
|
||||
})
|
||||
playEl.addEventListener("click", () => {
|
||||
playEl.setAttribute("hidden", "");
|
||||
state.autoplayWait = false;
|
||||
play();
|
||||
toggle();
|
||||
});
|
||||
|
||||
// Track resize even when the underlying media stream changes its video size
|
||||
videoEl.addEventListener('resize', () => {
|
||||
recalculateSize()
|
||||
if (state.screen === videoEl) return
|
||||
resize(videoEl.videoWidth, videoEl.videoHeight)
|
||||
})
|
||||
videoEl.addEventListener("resize", () => {
|
||||
recalculateSize();
|
||||
if (state.screen === videoEl) return;
|
||||
resize(videoEl.videoWidth, videoEl.videoHeight);
|
||||
});
|
||||
|
||||
videoEl.addEventListener('loadstart', () => {
|
||||
videoEl.volume = options.volume / 100
|
||||
videoEl.poster = options.poster
|
||||
})
|
||||
videoEl.addEventListener("loadstart", () => {
|
||||
videoEl.volume = options.volume / 100;
|
||||
videoEl.poster = options.poster;
|
||||
});
|
||||
|
||||
videoEl.onfocus = () => videoEl.blur()
|
||||
videoEl.onerror = (e) => log.error('Playback error', e)
|
||||
videoEl.onfocus = () => videoEl.blur();
|
||||
videoEl.onerror = (e) => log.error("Playback error", e);
|
||||
|
||||
const onFullscreen = (fullscreen) => {
|
||||
const el = document.fullscreenElement
|
||||
const el = document.fullscreenElement;
|
||||
|
||||
if (fullscreen) {
|
||||
// timeout is due to a chrome bug
|
||||
setTimeout(() => {
|
||||
// aspect ratio calc
|
||||
const w = window.screen.width ?? window.innerWidth
|
||||
const hh = el.innerHeight || el.clientHeight || 0
|
||||
const dw = (w - hh * state.aspect) / 2
|
||||
state.screen.style.padding = `0 ${dw}px`
|
||||
state.screen.classList.toggle('with-footer')
|
||||
}, 1)
|
||||
const w = window.screen.width ?? window.innerWidth;
|
||||
const hh = el.innerHeight || el.clientHeight || 0;
|
||||
const dw = (w - hh * state.aspect) / 2;
|
||||
state.screen.style.padding = `0 ${dw}px`;
|
||||
state.screen.classList.toggle("with-footer");
|
||||
}, 1);
|
||||
} else {
|
||||
state.screen.style.padding = '0'
|
||||
state.screen.classList.toggle('with-footer')
|
||||
state.screen.style.padding = "0";
|
||||
state.screen.classList.toggle("with-footer");
|
||||
}
|
||||
|
||||
if (el === videoEl) {
|
||||
videoEl.classList.toggle('no-media-controls', !fullscreen)
|
||||
videoEl.blur()
|
||||
videoEl.classList.toggle("no-media-controls", !fullscreen);
|
||||
videoEl.blur();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const vs = {w: 1, h: 1}
|
||||
const vs = { w: 1, h: 1 };
|
||||
|
||||
const recalculateSize = () => {
|
||||
const fullscreen = document.fullscreenElement !== null
|
||||
const {aspect, screen} = state
|
||||
const fullscreen = document.fullscreenElement !== null;
|
||||
const { aspect, screen } = state;
|
||||
|
||||
let width, height
|
||||
let width, height;
|
||||
if (fullscreen) {
|
||||
// we can't get the real <video> size on the screen without the black bars,
|
||||
// so we're using one side (without the bar) for the length calc of another
|
||||
const horizontal = videoEl.videoWidth > videoEl.videoHeight
|
||||
width = horizontal ? aspect * screen.offsetHeight : screen.offsetWidth
|
||||
height = horizontal ? screen.offsetHeight : aspect * screen.offsetWidth
|
||||
const horizontal = videoEl.videoWidth > videoEl.videoHeight;
|
||||
width = horizontal ? aspect * screen.offsetHeight : screen.offsetWidth;
|
||||
height = horizontal ? screen.offsetHeight : aspect * screen.offsetWidth;
|
||||
} else {
|
||||
({width, height} = screen.getBoundingClientRect())
|
||||
({ width, height } = screen.getBoundingClientRect());
|
||||
}
|
||||
|
||||
vs.w = width
|
||||
vs.h = height
|
||||
}
|
||||
vs.w = width;
|
||||
vs.h = height;
|
||||
};
|
||||
|
||||
const useCustomScreen = (use) => {
|
||||
if (use) {
|
||||
if (videoEl.paused || videoEl.ended) return
|
||||
if (videoEl.paused || videoEl.ended) return;
|
||||
|
||||
if (state.screen === mirrorEl) return
|
||||
if (state.screen === mirrorEl) return;
|
||||
|
||||
toggle(false)
|
||||
state.screen = mirrorEl
|
||||
resize(videoEl.videoWidth, videoEl.videoHeight)
|
||||
toggle(false);
|
||||
state.screen = mirrorEl;
|
||||
resize(videoEl.videoWidth, videoEl.videoHeight);
|
||||
|
||||
// stretch depending on the video orientation
|
||||
const isPortrait = videoEl.videoWidth < videoEl.videoHeight
|
||||
state.screen.style.width = isPortrait ? 'auto' : videoEl.videoWidth
|
||||
const isPortrait = videoEl.videoWidth < videoEl.videoHeight;
|
||||
state.screen.style.width = isPortrait ? "auto" : videoEl.videoWidth;
|
||||
|
||||
let surface = state.screen.getContext('2d')
|
||||
state.ready && toggle(true)
|
||||
let surface = state.screen.getContext("2d");
|
||||
if (state.ready) {
|
||||
toggle(true);
|
||||
}
|
||||
state.timerId = setInterval(function () {
|
||||
if (videoEl.paused || videoEl.ended || !surface) return
|
||||
surface.drawImage(videoEl, 0, 0)
|
||||
}, options.mirrorUpdateRate)
|
||||
if (videoEl.paused || videoEl.ended || !surface) return;
|
||||
if (state.flip) {
|
||||
surface.setTransform(1, 0, 0, -1, 0, mirrorEl.height);
|
||||
surface.drawImage(videoEl, 0, 0);
|
||||
surface.setTransform(1, 0, 0, 1, 0, 0);
|
||||
} else {
|
||||
surface.drawImage(videoEl, 0, 0);
|
||||
}
|
||||
}, options.mirrorUpdateRate);
|
||||
} else {
|
||||
clearInterval(state.timerId)
|
||||
toggle(false)
|
||||
state.screen = videoEl
|
||||
state.ready && toggle(true)
|
||||
clearInterval(state.timerId);
|
||||
toggle(false);
|
||||
state.screen = videoEl;
|
||||
if (state.ready) {
|
||||
toggle(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const init = () => {
|
||||
options.mirrorMode = settings.loadOr(opts.MIRROR_SCREEN, 'none')
|
||||
options.volume = settings.loadOr(opts.VOLUME, 50)
|
||||
options.mirrorMode = settings.loadOr(opts.MIRROR_SCREEN, "none");
|
||||
options.volume = settings.loadOr(opts.VOLUME, 50);
|
||||
sub(SETTINGS_CHANGED, () => {
|
||||
if (settings.changed(opts.MIRROR_SCREEN, options, 'mirrorMode')) {
|
||||
useCustomScreen(options.mirrorMode === 'mirror')
|
||||
if (settings.changed(opts.MIRROR_SCREEN, options, "mirrorMode")) {
|
||||
useCustomScreen(options.mirrorMode === "mirror");
|
||||
}
|
||||
if (settings.changed(opts.VOLUME, options, 'volume')) {
|
||||
videoEl.volume = options.volume / 100
|
||||
if (settings.changed(opts.VOLUME, options, "volume")) {
|
||||
videoEl.volume = options.volume / 100;
|
||||
}
|
||||
})
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
sub(APP_VIDEO_CHANGED, (payload) => {
|
||||
const {w, h, a, s} = payload
|
||||
const { w, h, a, s, flip } = payload;
|
||||
|
||||
const scale = !s ? 1 : s
|
||||
const ww = w * scale
|
||||
const hh = h * scale
|
||||
if (flip !== undefined) {
|
||||
state.flip = flip;
|
||||
if (state.ready) flip();
|
||||
}
|
||||
|
||||
state.aspect = a
|
||||
const scale = !s ? 1 : s;
|
||||
const ww = w * scale;
|
||||
const hh = h * scale;
|
||||
|
||||
const a2 = (ww / hh).toFixed(6)
|
||||
state.aspect = a;
|
||||
|
||||
state.h = hh
|
||||
state.w = Math.floor(hh * a)
|
||||
resize(ww, hh, state.aspect, a > 1 && a.toFixed(6) !== a2 ? 'fill' : 'contain')
|
||||
recalculateSize()
|
||||
})
|
||||
const a2 = (ww / hh).toFixed(6);
|
||||
|
||||
state.h = hh;
|
||||
state.w = Math.floor(hh * a);
|
||||
resize(
|
||||
ww,
|
||||
hh,
|
||||
state.aspect,
|
||||
a > 1 && a.toFixed(6) !== a2 ? "fill" : "contain",
|
||||
);
|
||||
recalculateSize();
|
||||
});
|
||||
|
||||
sub(TRANSFORM_CHANGE, () => {
|
||||
// cache stream element size when the interface is transformed
|
||||
recalculateSize()
|
||||
})
|
||||
recalculateSize();
|
||||
});
|
||||
|
||||
/**
|
||||
* Game streaming module.
|
||||
|
|
@ -211,11 +237,11 @@ sub(TRANSFORM_CHANGE, () => {
|
|||
* @version 1
|
||||
*/
|
||||
export const stream = {
|
||||
audio: {mute},
|
||||
audio: { mute },
|
||||
video: {
|
||||
el: videoEl,
|
||||
get size() {
|
||||
return vs
|
||||
return vs;
|
||||
},
|
||||
},
|
||||
play,
|
||||
|
|
@ -224,4 +250,4 @@ export const stream = {
|
|||
hasDisplay: true,
|
||||
init,
|
||||
onFullscreen,
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue