MISC fix, fix aspect ratio setting + Only play audio when has user interaction (#117)

* Update encoding flow

* unmute when play

* only play audio when action

* Update version

* Update more comments

* Remove indirect

* Fix comments

pkg/config/worker/config.go

@@ -11,7 +11,7 @@ type Config struct {
 
 	// video
 	Scale             int
-	DisableCustomSize bool
+	EnableAspectRatio bool
 	Width             int
 	Height            int
 
@@ -23,7 +23,7 @@ func NewDefaultConfig() Config {
 		Port:              8800,
 		OverlordAddress:   "ws://localhost:8000/wso",
 		Scale:             1,
-		DisableCustomSize: false,
+		EnableAspectRatio: false,
 		Width:             320,
 		Height:            240,
 		MonitoringConfig: monitoring.ServerMonitoringConfig{
@@ -39,7 +39,7 @@ func (c *Config) AddFlags(fs *pflag.FlagSet) *Config {
 	fs.StringVarP(&c.OverlordAddress, "overlordhost", "", c.OverlordAddress, "OverWorker URL to connect")
 
 	fs.IntVarP(&c.Scale, "scale", "s", c.Scale, "Set output viewport scale factor")
-	fs.BoolVarP(&c.DisableCustomSize, "disable-custom-size", "", c.DisableCustomSize, "Disable custom size")
+	fs.BoolVarP(&c.EnableAspectRatio, "ar", "", c.EnableAspectRatio, "Enable Aspect Ratio")
 	fs.IntVarP(&c.Width, "width", "w", c.Width, "Set custom viewport width")
 	fs.IntVarP(&c.Height, "height", "h", c.Height, "Set custom viewport height")
 

pkg/emulator/libretro/nanoarch/nanoarch.go

@@ -104,20 +104,13 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned,
 	bytes := int(height) * packedWidth * int(video.bpp)
 	data_ := (*[1 << 30]byte)(data)[:bytes:bytes]
 
-	// !to move it on the other side of the channel
-	image.DrawRgbaImage(int(video.pixFmt), image.ScaleBilinear, int(width), int(height),
+	// image is resized here and push to channel. On the other side, images will be fan out
+	image.DrawRgbaImage(int(video.pixFmt), image.ScaleNearestNeighbour, int(width), int(height),
 		packedWidth, ewidth, eheight, int(video.bpp), data_, outputImg)
 
 	NAEmulator.imageChannel <- outputImg
 }
 
-//export coreInputPoll
-func coreInputPoll() {
-	//for i := range NAEmulator.keys {
-	//joy[i] = NAEmulator.keys[i]
-	//}
-}
-
 //export coreInputState
 func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.unsigned) C.int16_t {
 	if port > 0 || index > 0 || device != C.RETRO_DEVICE_JOYPAD {

pkg/worker/room/media.go

@@ -60,7 +60,6 @@ func (r *Room) startAudio(sampleRate int) {
 
 	dstBufferSize := config.AUDIO_FRAME
 	srcBufferSize := dstBufferSize * srcSampleRate / config.AUDIO_RATE
-	fmt.Println("src BufferSize", srcBufferSize)
 	pcm := make([]int16, srcBufferSize) // 640 * 1000 / 16000 == 40 ms
 	idx := 0
 
@@ -107,6 +106,7 @@ func (r *Room) startAudio(sampleRate int) {
 	}
 }
 
+// startVideo listen from imageChannel and push to Encoder. The output of encoder will be pushed to webRTC
 func (r *Room) startVideo(width, height int, videoEncoderType string) {
 	var encoder encoder.Encoder
 	var err error

pkg/worker/room/room.go

@@ -54,6 +54,9 @@ type Room struct {
 
 const separator = "___"
 
+// TODO: Remove after fully migrate
+const oldSeparator = "|"
+
 // NewRoom creates a new room
 func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStorage *storage.Client, cfg worker.Config) *Room {
 	// If no roomID is given, generate it from gameName
@@ -89,34 +92,31 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor
 	go func(game gamelist.GameInfo, roomID string) {
 		// Check room is on local or fetch from server
 		savepath := util.GetSavePath(roomID)
-		log.Println("Check ", savepath, " on local : ", room.isGameOnLocal(savepath))
-		if !room.isGameOnLocal(savepath) {
-			// Fetch room from GCP to server
-			log.Println("Load room from online storage", savepath)
-			if err := room.saveOnlineRoomToLocal(roomID, savepath); err != nil {
-				log.Printf("Warn: Room %s is not in online storage, error %s", roomID, err)
-			}
+		log.Println("Check ", savepath, " on online storage : ", room.isGameOnLocal(savepath))
+		if err := room.saveOnlineRoomToLocal(roomID, savepath); err != nil {
+			log.Printf("Warn: Room %s is not in online storage, error %s", roomID, err)
 		}
 
+		// If not then load room or create room from local.
 		log.Printf("Room %s started. GamePath: %s, GameName: %s", roomID, game.Path, game.Name)
 
 		// Spawn new emulator based on gameName and plug-in all channels
 		emuName, _ := config.FileTypeToEmulator[game.Type]
-
 		room.director = getEmulator(emuName, roomID, imageChannel, audioChannel, inputChannel)
 		gameMeta := room.director.LoadMeta(game.Path)
 
+		// nwidth, nheight are the webRTC output size.
+		// There are currently two approach
 		var nwidth, nheight int
-		if !cfg.DisableCustomSize {
+		if cfg.EnableAspectRatio {
 			baseAspectRatio := float64(gameMeta.BaseWidth) / float64(gameMeta.Height)
 			nwidth, nheight = resizeToAspect(baseAspectRatio, cfg.Width, cfg.Height)
 			log.Printf("Viewport size will be changed from %dx%d (%f) -> %dx%d", cfg.Width, cfg.Height,
 				baseAspectRatio, nwidth, nheight)
 		} else {
-			log.Println("Viewport custom size is disabled, base size will be used instead")
 			nwidth, nheight = gameMeta.BaseWidth, gameMeta.BaseHeight
+			log.Printf("Viewport custom size is disabled, base size will be used instead %dx%d", nwidth, nheight)
 		}
-
 		if cfg.Scale > 1 {
 			nwidth, nheight = nwidth*cfg.Scale, nheight*cfg.Scale
 			log.Printf("Viewport size has scaled to %dx%d", nwidth, nheight)
@@ -126,6 +126,7 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor
 
 		log.Println("meta: ", gameMeta)
 
+		// Spawn video and audio encoding for webRTC
 		go room.startVideo(nwidth, nheight, videoEncoderType)
 		go room.startAudio(gameMeta.AudioSampleRate)
 		room.director.Start()
@@ -150,7 +151,7 @@ func resizeToAspect(ratio float64, sw int, sh int) (dw int, dh int) {
 	return
 }
 
-// create director
+// getEmulator creates new emulator and run it
 func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- []int16, inputChannel <-chan int) emulator.CloudEmulator {
 	nanoarch.Init(emuName, roomID, imageChannel, audioChannel, inputChannel)
 
@@ -160,10 +161,15 @@ func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA,
 // getGameNameFromRoomID parse roomID to get roomID and gameName
 func getGameNameFromRoomID(roomID string) string {
 	parts := strings.Split(roomID, separator)
-	if len(parts) <= 1 {
-		return ""
+	if len(parts) > 1 {
+		return parts[1]
 	}
-	return parts[1]
+	// TODO: Remove when fully migrate
+	parts = strings.Split(roomID, oldSeparator)
+	if len(parts) > 1 {
+		return parts[1]
+	}
+	return ""
 }
 
 // generateRoomID generate a unique room ID containing 16 digits
@@ -259,6 +265,7 @@ func (r *Room) Close() {
 	//close(r.audioChannel)
 }
 
+// SaveGame will save game to local and trigger a callback to store game on onlineStorage, so the game can be accessed later
 func (r *Room) SaveGame() error {
 	onlineSaveFunc := func() error {
 		// Try to save the game to gCloud

web/game.html

@@ -33,7 +33,7 @@
                 There is still audio because current audio flow is not from media but it is manually encoded (technical webRTC challenge). Later, when we can integrate audio to media, we can face the issue with mute again .
                 https://developers.google.com/web/updates/2017/09/autoplay-policy-changes
             -->
-            <video id="game-screen" playinfullscreen="false" playsinline onloadstart="this.volume=0.5" poster="/static/img/screen_loading.gif"></video>
+            <video id="game-screen" muted playinfullscreen="false" playsinline onloadstart="this.volume=0.5" poster="/static/img/screen_loading.gif"></video>
             <!--<video id="game-screen" autoplay=true poster="/static/img/screen_loading.gif"></video>-->
 
             <div id="menu-screen">
@@ -85,18 +85,18 @@
         STUNTURN = {{.STUNTURN}};
     </script>
 
-    <script src="/static/js/jquery-3.3.1.min.js?1"></script>
-    <script src="/static/js/utils.js?1"></script>
+    <script src="/static/js/jquery-3.3.1.min.js?3"></script>
+    <script src="/static/js/utils.js?3"></script>
 
     <!-- https://rawgit.com/Rillke/opus.js-sample/master/index.xhtml -->
-    <script src="/static/js/libopus.js?1"></script>
-    <script src="/static/js/opus.js?1"></script>
+    <script src="/static/js/libopus.js?3"></script>
+    <script src="/static/js/opus.js?3"></script>
 
-    <script src="/static/js/global.js?2"></script>
-    <script src="/static/js/controller.js?2"></script>
-    <script src="/static/js/gesture_keyboard.js?2"></script>
-    <script src="/static/js/gesture_touch.js?2"></script>
-    <script src="/static/js/gesture_joystick.js?2"></script>
+    <script src="/static/js/global.js?3"></script>
+    <script src="/static/js/controller.js?3"></script>
+    <script src="/static/js/gesture_keyboard.js?3"></script>
+    <script src="/static/js/gesture_touch.js?3"></script>
+    <script src="/static/js/gesture_joystick.js?3"></script>
     <script src="/static/js/ws.js?6"></script>
 
     <script src="/static/js/init.js?2"></script>

web/js/gesture_keyboard.js

@@ -27,6 +27,8 @@ const KEYBOARD_MAP = {
 $("body").on("keyup", function (event) {
     if (event.keyCode in KEYBOARD_MAP) {
         doButtonUp(KEYBOARD_MAP[event.keyCode]);
+        // unmute when there is user interaction
+        document.getElementById("game-screen").muted = false;
     }
 });
 

web/js/gesture_touch.js

@@ -272,14 +272,15 @@ function handleWindowMove(event) {
 }
 
 function handleWindowUp(event) {
+    // unmute when there is user interaction
+    document.getElementById("game-screen").muted = false;
+
     handleVpadJoystickUp(event);
     handleMenuUp(event);
     $(".btn").trigger("touchend");
 }
 
-
-
 // Bind events for window
 $(window).on("mousemove", handleWindowMove);
 window.addEventListener("touchmove", handleWindowMove, {passive: false});
-$(window).on("mouseup", handleWindowUp);
+$(window).on("mouseup", handleWindowUp);

web/js/ws.js

@@ -262,7 +262,6 @@ function startGame() {
         return false;
     }
 
-    document.getElementById("game-screen").muted = false
     var promise = document.getElementById("game-screen").play();
     if (promise !== undefined) {
         promise.then(_ => {