From b7f47f5519661abd68e8353b496ae8a93c4589b2 Mon Sep 17 00:00:00 2001 From: giongto35 Date: Sun, 20 Oct 2019 13:00:52 +0800 Subject: [PATCH] MISC fix, fix aspect ratio setting + Only play audio when has user interaction (#117) * Update encoding flow * unmute when play * only play audio when action * Update version * Update more comments * Remove indirect * Fix comments --- pkg/config/worker/config.go | 6 ++-- pkg/emulator/libretro/nanoarch/nanoarch.go | 11 ++----- pkg/worker/room/media.go | 2 +- pkg/worker/room/room.go | 37 +++++++++++++--------- web/game.html | 20 ++++++------ web/js/gesture_keyboard.js | 2 ++ web/js/gesture_touch.js | 7 ++-- web/js/ws.js | 1 - 8 files changed, 44 insertions(+), 42 deletions(-) diff --git a/pkg/config/worker/config.go b/pkg/config/worker/config.go index 8101c4f0..b80bf3b0 100644 --- a/pkg/config/worker/config.go +++ b/pkg/config/worker/config.go @@ -11,7 +11,7 @@ type Config struct { // video Scale int - DisableCustomSize bool + EnableAspectRatio bool Width int Height int @@ -23,7 +23,7 @@ func NewDefaultConfig() Config { Port: 8800, OverlordAddress: "ws://localhost:8000/wso", Scale: 1, - DisableCustomSize: false, + EnableAspectRatio: false, Width: 320, Height: 240, MonitoringConfig: monitoring.ServerMonitoringConfig{ @@ -39,7 +39,7 @@ func (c *Config) AddFlags(fs *pflag.FlagSet) *Config { fs.StringVarP(&c.OverlordAddress, "overlordhost", "", c.OverlordAddress, "OverWorker URL to connect") fs.IntVarP(&c.Scale, "scale", "s", c.Scale, "Set output viewport scale factor") - fs.BoolVarP(&c.DisableCustomSize, "disable-custom-size", "", c.DisableCustomSize, "Disable custom size") + fs.BoolVarP(&c.EnableAspectRatio, "ar", "", c.EnableAspectRatio, "Enable Aspect Ratio") fs.IntVarP(&c.Width, "width", "w", c.Width, "Set custom viewport width") fs.IntVarP(&c.Height, "height", "h", c.Height, "Set custom viewport height") diff --git a/pkg/emulator/libretro/nanoarch/nanoarch.go b/pkg/emulator/libretro/nanoarch/nanoarch.go index 1e43e311..7f9d7176 100644 --- a/pkg/emulator/libretro/nanoarch/nanoarch.go +++ b/pkg/emulator/libretro/nanoarch/nanoarch.go @@ -104,20 +104,13 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned, bytes := int(height) * packedWidth * int(video.bpp) data_ := (*[1 << 30]byte)(data)[:bytes:bytes] - // !to move it on the other side of the channel - image.DrawRgbaImage(int(video.pixFmt), image.ScaleBilinear, int(width), int(height), + // image is resized here and push to channel. On the other side, images will be fan out + image.DrawRgbaImage(int(video.pixFmt), image.ScaleNearestNeighbour, int(width), int(height), packedWidth, ewidth, eheight, int(video.bpp), data_, outputImg) NAEmulator.imageChannel <- outputImg } -//export coreInputPoll -func coreInputPoll() { - //for i := range NAEmulator.keys { - //joy[i] = NAEmulator.keys[i] - //} -} - //export coreInputState func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.unsigned) C.int16_t { if port > 0 || index > 0 || device != C.RETRO_DEVICE_JOYPAD { diff --git a/pkg/worker/room/media.go b/pkg/worker/room/media.go index 971b9901..55cf0a2e 100644 --- a/pkg/worker/room/media.go +++ b/pkg/worker/room/media.go @@ -60,7 +60,6 @@ func (r *Room) startAudio(sampleRate int) { dstBufferSize := config.AUDIO_FRAME srcBufferSize := dstBufferSize * srcSampleRate / config.AUDIO_RATE - fmt.Println("src BufferSize", srcBufferSize) pcm := make([]int16, srcBufferSize) // 640 * 1000 / 16000 == 40 ms idx := 0 @@ -107,6 +106,7 @@ func (r *Room) startAudio(sampleRate int) { } } +// startVideo listen from imageChannel and push to Encoder. The output of encoder will be pushed to webRTC func (r *Room) startVideo(width, height int, videoEncoderType string) { var encoder encoder.Encoder var err error diff --git a/pkg/worker/room/room.go b/pkg/worker/room/room.go index 6e03ae31..8d620c1f 100644 --- a/pkg/worker/room/room.go +++ b/pkg/worker/room/room.go @@ -54,6 +54,9 @@ type Room struct { const separator = "___" +// TODO: Remove after fully migrate +const oldSeparator = "|" + // NewRoom creates a new room func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStorage *storage.Client, cfg worker.Config) *Room { // If no roomID is given, generate it from gameName @@ -89,34 +92,31 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor go func(game gamelist.GameInfo, roomID string) { // Check room is on local or fetch from server savepath := util.GetSavePath(roomID) - log.Println("Check ", savepath, " on local : ", room.isGameOnLocal(savepath)) - if !room.isGameOnLocal(savepath) { - // Fetch room from GCP to server - log.Println("Load room from online storage", savepath) - if err := room.saveOnlineRoomToLocal(roomID, savepath); err != nil { - log.Printf("Warn: Room %s is not in online storage, error %s", roomID, err) - } + log.Println("Check ", savepath, " on online storage : ", room.isGameOnLocal(savepath)) + if err := room.saveOnlineRoomToLocal(roomID, savepath); err != nil { + log.Printf("Warn: Room %s is not in online storage, error %s", roomID, err) } + // If not then load room or create room from local. log.Printf("Room %s started. GamePath: %s, GameName: %s", roomID, game.Path, game.Name) // Spawn new emulator based on gameName and plug-in all channels emuName, _ := config.FileTypeToEmulator[game.Type] - room.director = getEmulator(emuName, roomID, imageChannel, audioChannel, inputChannel) gameMeta := room.director.LoadMeta(game.Path) + // nwidth, nheight are the webRTC output size. + // There are currently two approach var nwidth, nheight int - if !cfg.DisableCustomSize { + if cfg.EnableAspectRatio { baseAspectRatio := float64(gameMeta.BaseWidth) / float64(gameMeta.Height) nwidth, nheight = resizeToAspect(baseAspectRatio, cfg.Width, cfg.Height) log.Printf("Viewport size will be changed from %dx%d (%f) -> %dx%d", cfg.Width, cfg.Height, baseAspectRatio, nwidth, nheight) } else { - log.Println("Viewport custom size is disabled, base size will be used instead") nwidth, nheight = gameMeta.BaseWidth, gameMeta.BaseHeight + log.Printf("Viewport custom size is disabled, base size will be used instead %dx%d", nwidth, nheight) } - if cfg.Scale > 1 { nwidth, nheight = nwidth*cfg.Scale, nheight*cfg.Scale log.Printf("Viewport size has scaled to %dx%d", nwidth, nheight) @@ -126,6 +126,7 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor log.Println("meta: ", gameMeta) + // Spawn video and audio encoding for webRTC go room.startVideo(nwidth, nheight, videoEncoderType) go room.startAudio(gameMeta.AudioSampleRate) room.director.Start() @@ -150,7 +151,7 @@ func resizeToAspect(ratio float64, sw int, sh int) (dw int, dh int) { return } -// create director +// getEmulator creates new emulator and run it func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- []int16, inputChannel <-chan int) emulator.CloudEmulator { nanoarch.Init(emuName, roomID, imageChannel, audioChannel, inputChannel) @@ -160,10 +161,15 @@ func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA, // getGameNameFromRoomID parse roomID to get roomID and gameName func getGameNameFromRoomID(roomID string) string { parts := strings.Split(roomID, separator) - if len(parts) <= 1 { - return "" + if len(parts) > 1 { + return parts[1] } - return parts[1] + // TODO: Remove when fully migrate + parts = strings.Split(roomID, oldSeparator) + if len(parts) > 1 { + return parts[1] + } + return "" } // generateRoomID generate a unique room ID containing 16 digits @@ -259,6 +265,7 @@ func (r *Room) Close() { //close(r.audioChannel) } +// SaveGame will save game to local and trigger a callback to store game on onlineStorage, so the game can be accessed later func (r *Room) SaveGame() error { onlineSaveFunc := func() error { // Try to save the game to gCloud diff --git a/web/game.html b/web/game.html index 696500f9..c6e8abf9 100644 --- a/web/game.html +++ b/web/game.html @@ -33,7 +33,7 @@ There is still audio because current audio flow is not from media but it is manually encoded (technical webRTC challenge). Later, when we can integrate audio to media, we can face the issue with mute again . https://developers.google.com/web/updates/2017/09/autoplay-policy-changes --> - +