From 4e57f8ea280d269a19cfbb1de7a8dacc6c2cde1a Mon Sep 17 00:00:00 2001 From: hasagi Date: Sat, 13 Apr 2019 00:40:08 +0700 Subject: [PATCH] WIP audio --- main.go | 95 ++++++++++++++++++++++++++++++++++++++---------- static/js/ws.js | 18 +++++++-- ui/audio.go | 58 +++++++++++++++++++++++++++++ ui/director.go | 21 ++++++++--- ui/gameview.go | 15 +++++--- webrtc/webrtc.go | 27 ++++++++++++-- 6 files changed, 197 insertions(+), 37 deletions(-) create mode 100644 ui/audio.go diff --git a/main.go b/main.go index 376a2ab7..aa630c51 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,8 @@ import ( "github.com/giongto35/cloud-game/webrtc" "github.com/gorilla/websocket" pionRTC "github.com/pion/webrtc" + + "github.com/hraban/opus" ) const ( @@ -48,6 +50,7 @@ type WSPacket struct { // A room stores all the channel for interaction between all webRTCs session and emulator type Room struct { imageChannel chan *image.RGBA + audioChanel chan float32 inputChannel chan int // closedChannel is to fire exit event when there is no webRTC session running closedChannel chan bool @@ -96,21 +99,24 @@ func initRoom(roomID, gameName string) string { roomID = generateRoomID() } imageChannel := make(chan *image.RGBA, 100) + audioChannel := make(chan float32, 48000) inputChannel := make(chan int, 100) closedChannel := make(chan bool) // create director - director := ui.NewDirector(roomID, imageChannel, inputChannel, closedChannel) + director := ui.NewDirector(roomID, imageChannel, audioChannel, inputChannel, closedChannel) rooms[roomID] = &Room{ imageChannel: imageChannel, + audioChanel: audioChannel, inputChannel: inputChannel, closedChannel: closedChannel, rtcSessions: []*webrtc.WebRTC{}, - director: director, + director: director, } go fanoutScreen(imageChannel, roomID) + go fanoutAudio(audioChannel, roomID) go director.Start([]string{"games/" + gameName}) return roomID @@ -144,7 +150,7 @@ func startSession(webRTC *webrtc.WebRTC, gameName string, roomID string, playerI // TODO: Might have race condition rooms[roomID].rtcSessions = append(rooms[roomID].rtcSessions, webRTC) - faninInput(rooms[roomID].inputChannel, webRTC, playerIndex) + go faninInput(rooms[roomID].inputChannel, webRTC, playerIndex) return roomID } @@ -298,30 +304,79 @@ func fanoutScreen(imageChannel chan *image.RGBA, roomID string) { } if isRoomRunning == false { - log.Println("Closed room", roomID) + log.Println("Closed room from screen routine", roomID) rooms[roomID].closedChannel <- true } } } -// faninInput fan-in of the same room to inputChannel -func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) { - go func() { - for { - // Client stopped - if webRTC.IsClosed() { - return +// fanoutAudio fanout outputs to all webrtc in the same room +func fanoutAudio(audioChanel chan float32, roomID string) { + enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP) + if err != nil { + log.Println("[!] Cannot create audio encoder") + return + } + pcm := make([]float32, 120) + c := 0 + + for audio := range audioChanel { + if c >= cap(pcm) { + data := make([]byte, 1000) + n, err := enc.EncodeFloat32(pcm, data) + if err != nil { + log.Println("[!] Failed to decode") + continue + } + data = data[:n] + + isRoomRunning := false + for _, webRTC := range rooms[roomID].rtcSessions { + // Client stopped + if webRTC.IsClosed() { + continue + } + + // encode frame + // fanout imageChannel + if webRTC.IsConnected() { + // NOTE: can block here + webRTC.AudioChannel <- data + } + isRoomRunning = true } - // encode frame - if webRTC.IsConnected() { - input := <-webRTC.InputChannel - // the first 8 bits belong to player 1 - // the next 8 belongs to player 2 ... - // We standardize and put it to inputChannel (16 bits) - input = input << ((uint(playerIndex) - 1) * ui.NumKeys) - inputChannel <- input + if isRoomRunning == false { + log.Println("Closed room from audio routine", roomID) + rooms[roomID].closedChannel <- true } + + c = 0 + } else { + pcm[c] = audio + c++ } - }() + + + } +} + +// faninInput fan-in of the same room to inputChannel +func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) { + for { + // Client stopped + if webRTC.IsClosed() { + return + } + + // encode frame + if webRTC.IsConnected() { + input := <-webRTC.InputChannel + // the first 8 bits belong to player 1 + // the next 8 belongs to player 2 ... + // We standardize and put it to inputChannel (16 bits) + input = input << ((uint(playerIndex) - 1) * ui.NumKeys) + inputChannel <- input + } + } } diff --git a/static/js/ws.js b/static/js/ws.js index eedbb3b0..0d803384 100644 --- a/static/js/ws.js +++ b/static/js/ws.js @@ -86,9 +86,18 @@ function startGame() { // stream channel pc.ontrack = function (event) { - log("New stream, yay!"); - document.getElementById("game-screen").srcObject = event.streams[0]; - $("#game-screen").show(); + console.log(event.streams); + var el = document.createElement(event.track.kind); + el.srcObject = event.streams[0]; + el.autoplay = true; + el.width = 800; + el.height = 600; + el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif"); + document.getElementById('remoteVideos').appendChild(el) + + // log("New stream, yay!"); + // document.getElementById("game-screen").srcObject = event.streams[0]; + // $("#game-screen").show(); } @@ -108,9 +117,10 @@ function startGame() { function startWebRTC() { // receiver only tracks pc.addTransceiver('video', {'direction': 'recvonly'}); + pc.addTransceiver('audio', {'direction': 'recvonly'}); // create SDP - pc.createOffer({offerToReceiveVideo: true, offerToReceiveAudio: false}).then(d => { + pc.createOffer({offerToReceiveVideo: true, offerToReceiveAudio: true}).then(d => { pc.setLocalDescription(d).catch(log); }) } diff --git a/ui/audio.go b/ui/audio.go new file mode 100644 index 00000000..1d576d4a --- /dev/null +++ b/ui/audio.go @@ -0,0 +1,58 @@ +package ui + +import ( + "github.com/gordonklaus/portaudio" +) + + +type Audio struct { + stream *portaudio.Stream + sampleRate float64 + outputChannels int + channel chan float32 +} + +func NewAudio() *Audio { + a := Audio{} + a.channel = make(chan float32, 16000) + return &a +} + +func (a *Audio) Start() error { + host, err := portaudio.DefaultHostApi() + if err != nil { + return err + } + + parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice) + stream, err := portaudio.OpenStream(parameters, a.Callback) + if err != nil { + return err + } + if err := stream.Start(); err != nil { + return err + } + a.stream = stream + a.sampleRate = parameters.SampleRate + a.outputChannels = parameters.Output.Channels + return nil +} + +func (a *Audio) Stop() error { + return a.stream.Close() +} + +func (a *Audio) Callback(out []float32) { + var output float32 + for i := range out { + if i%a.outputChannels == 0 { + select { + case sample := <-a.channel: + output = sample + default: + output = 0 + } + } + out[i] = output + } +} \ No newline at end of file diff --git a/ui/director.go b/ui/director.go index a8d73bc9..6b2d398f 100644 --- a/ui/director.go +++ b/ui/director.go @@ -6,13 +6,15 @@ import ( "time" "github.com/giongto35/cloud-game/nes" + // "github.com/gordonklaus/portaudio" ) type Director struct { - // audio *Audio + // audio *Audio view *GameView timestamp float64 imageChannel chan *image.RGBA + audioChanel chan float32 inputChannel chan int closedChannel chan bool roomID string @@ -21,10 +23,11 @@ type Director struct { const FPS = 60 -func NewDirector(roomID string, imageChannel chan *image.RGBA, inputChannel chan int, closedChannel chan bool) *Director { +func NewDirector(roomID string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int, closedChannel chan bool) *Director { director := Director{} // director.audio = audio director.imageChannel = imageChannel + director.audioChanel = audioChanel director.inputChannel = inputChannel director.closedChannel = closedChannel director.roomID = roomID @@ -53,6 +56,13 @@ func (d *Director) Step() { } func (d *Director) Start(paths []string) { + // portaudio.Initialize() + // defer portaudio.Terminate() + + // audio := NewAudio() + // audio.Start() + // d.audio = audio + if len(paths) == 1 { d.PlayGame(paths[0]) } @@ -60,9 +70,10 @@ func (d *Director) Start(paths []string) { } func (d *Director) Run() { - c := time.Tick(time.Second / FPS) + // c := time.Tick(time.Second / FPS) L: - for range c { + // for range c { + for { // quit game select { @@ -89,7 +100,7 @@ func (d *Director) PlayGame(path string) { log.Fatalln(err) } // Set GameView as current view - d.SetView(NewGameView(console, path, hash, d.imageChannel, d.inputChannel)) + d.SetView(NewGameView(console, path, hash, d.imageChannel, d.audioChanel, d.inputChannel)) } func (d *Director) SaveGame() error { diff --git a/ui/gameview.go b/ui/gameview.go index 4380379c..5de50641 100644 --- a/ui/gameview.go +++ b/ui/gameview.go @@ -37,16 +37,19 @@ type GameView struct { keyPressed [NumKeys * 2]bool imageChannel chan *image.RGBA + audioChanel chan float32 inputChannel chan int + } -func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, inputChannel chan int) *GameView { +func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int) *GameView { gameview := &GameView{ console: console, title: title, hash: hash, keyPressed: [NumKeys * 2]bool{false}, imageChannel: imageChannel, + audioChanel: audioChanel, inputChannel: inputChannel, } @@ -70,8 +73,10 @@ func (view *GameView) ListenToInputChannel() { func (view *GameView) Enter() { // Always reset game // Legacy Audio code. TODO: Add it back to support audio - // view.console.SetAudioChannel(view.director.audio.channel) - // view.console.SetAudioSampleRate(view.director.audio.sampleRate) + // view.console.SetAudioChannel(view.audio.channel) + // view.console.SetAudioSampleRate(view.audio.sampleRate) + view.console.SetAudioSampleRate(48000) + view.console.SetAudioChannel(view.audioChanel) // load state if err := view.console.LoadState(savePath(view.hash)); err == nil { @@ -92,8 +97,8 @@ func (view *GameView) Enter() { // Exit ... func (view *GameView) Exit() { - // view.console.SetAudioChannel(nil) - // view.console.SetAudioSampleRate(0) + view.console.SetAudioChannel(nil) + view.console.SetAudioSampleRate(0) // save sram cartridge := view.console.Cartridge if cartridge.Battery != 0 { diff --git a/webrtc/webrtc.go b/webrtc/webrtc.go index 03e279d7..505b6d4f 100644 --- a/webrtc/webrtc.go +++ b/webrtc/webrtc.go @@ -93,6 +93,7 @@ func Decode(in string, obj interface{}) { func NewWebRTC() *WebRTC { w := &WebRTC{ ImageChannel: make(chan []byte, 2), + AudioChannel: make(chan []byte, 2), InputChannel: make(chan int, 2), } return w @@ -106,6 +107,7 @@ type WebRTC struct { isClosed bool // for yuvI420 image ImageChannel chan []byte + AudioChannel chan []byte InputChannel chan int } @@ -137,7 +139,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e return "", err } - vp8Track, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeVP8, rand.Uint32(), "video", "pion2") + vp8Track, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeVP8, rand.Uint32(), "video", "pion2a") if err != nil { return "", err } @@ -146,6 +148,16 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e return "", err } + + opusTrack, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeOpus, rand.Uint32(), "audio", "pion2b") + if err != nil { + return "", err + } + _, err = w.connection.AddTrack(opusTrack) + if err != nil { + return "", err + } + // WebRTC state callback w.connection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { log.Printf("ICE Connection State has changed: %s\n", connectionState.String()) @@ -153,7 +165,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e go func() { w.isConnected = true log.Println("ConnectionStateConnected") - w.startStreaming(vp8Track) + w.startStreaming(vp8Track, opusTrack) }() } @@ -235,7 +247,7 @@ func (w *WebRTC) IsClosed() bool { return w.isClosed } -func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) { +func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track) { log.Println("Start streaming") // send screenshot go func() { @@ -254,4 +266,13 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) { vp8Track.WriteSample(media.Sample{Data: bs, Samples: 1}) } }() + + // send audio + go func() { + for w.isConnected { + data := <-w.AudioChannel + opusTrack.WriteSample(media.Sample{Data: data, Samples: 1}) + } + }() + }