diff --git a/main.go b/main.go index aa630c51..b69e75a8 100644 --- a/main.go +++ b/main.go @@ -19,9 +19,76 @@ import ( "github.com/gorilla/websocket" pionRTC "github.com/pion/webrtc" - "github.com/hraban/opus" + // "github.com/hraban/opus" + "gopkg.in/hraban/opus.v2" + // "github.com/xlab/opus-go/opus" ) + +// fanoutAudio fanout outputs to all webrtc in the same room +func fanoutAudio(audioChannel chan float32, roomID string) { + var output float32 + pcm := make([]float32, 240) + + enc, err := opus.NewEncoder(48000, 1, opus.AppAudio) + if err != nil { + log.Println("[!] Cannot create audio encoder") + return + } + + // var err2 int32 + // var err error + // enc := opus.EncoderCreate(48000, 1, opus.ApplicationAudio, &err2) + + c := time.Tick(time.Millisecond * 5) + + for range c { + for i := 0; i < len(pcm); i++ { + select { + case sample := <- audioChannel: + output = sample + default: + output = 0 + + } + pcm[i] = output + } + + + data := make([]byte, 1000) + n, err := enc.EncodeFloat32(pcm, data) + // n := opus.EncodeFloat(enc, pcm, int32(c), data, 1000) + + if err != nil { + log.Println("[!] Failed to decode") + continue + } + data = data[:n] + + isRoomRunning := false + for _, webRTC := range rooms[roomID].rtcSessions { + // Client stopped + if webRTC.IsClosed() { + continue + } + + // encode frame + // fanout imageChannel + if webRTC.IsConnected() { + // NOTE: can block here + webRTC.AudioChannel <- data + } + isRoomRunning = true + } + + if isRoomRunning == false { + log.Println("Closed room from audio routine", roomID) + rooms[roomID].closedChannel <- true + } + + } +} + const ( width = 256 height = 240 @@ -310,57 +377,6 @@ func fanoutScreen(imageChannel chan *image.RGBA, roomID string) { } } -// fanoutAudio fanout outputs to all webrtc in the same room -func fanoutAudio(audioChanel chan float32, roomID string) { - enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP) - if err != nil { - log.Println("[!] Cannot create audio encoder") - return - } - pcm := make([]float32, 120) - c := 0 - - for audio := range audioChanel { - if c >= cap(pcm) { - data := make([]byte, 1000) - n, err := enc.EncodeFloat32(pcm, data) - if err != nil { - log.Println("[!] Failed to decode") - continue - } - data = data[:n] - - isRoomRunning := false - for _, webRTC := range rooms[roomID].rtcSessions { - // Client stopped - if webRTC.IsClosed() { - continue - } - - // encode frame - // fanout imageChannel - if webRTC.IsConnected() { - // NOTE: can block here - webRTC.AudioChannel <- data - } - isRoomRunning = true - } - - if isRoomRunning == false { - log.Println("Closed room from audio routine", roomID) - rooms[roomID].closedChannel <- true - } - - c = 0 - } else { - pcm[c] = audio - c++ - } - - - } -} - // faninInput fan-in of the same room to inputChannel func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) { for { diff --git a/static/index_ws.html b/static/index_ws.html index ed39156a..ebd54ff5 100644 --- a/static/index_ws.html +++ b/static/index_ws.html @@ -23,7 +23,7 @@ Play as player(1,2): $("#gameOp").on("change", function() { gameIdx = gameOp.selectedIndex; }); + + gameIdx = 7; diff --git a/static/js/ws.js b/static/js/ws.js index 0d803384..6e5c3b99 100644 --- a/static/js/ws.js +++ b/static/js/ws.js @@ -84,18 +84,21 @@ function startGame() { } } + var stream = new MediaStream(); + document.getElementById("game-screen2").srcObject = stream; // stream channel pc.ontrack = function (event) { - console.log(event.streams); - var el = document.createElement(event.track.kind); - el.srcObject = event.streams[0]; - el.autoplay = true; - el.width = 800; - el.height = 600; - el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif"); - document.getElementById('remoteVideos').appendChild(el) + console.log(event); + stream.addTrack(event.track); + // var el = document.createElement(event.track.kind); + // el.srcObject = event.streams[0]; + // el.autoplay = true; + // el.width = 800; + // el.height = 600; + // el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif"); + // document.getElementById('remoteVideos').appendChild(el) - // log("New stream, yay!"); + log("New stream, yay!"); // document.getElementById("game-screen").srcObject = event.streams[0]; // $("#game-screen").show(); } @@ -109,6 +112,7 @@ function startGame() { localSessionDescription = session; log("Send SDP to remote peer"); conn.send(JSON.stringify({"id": "sdp", "data": session})); + console.log(session); } else { console.log(JSON.stringify(event.candidate)); } diff --git a/ui/audio.go b/ui/audio.go index 1d576d4a..ac20e64a 100644 --- a/ui/audio.go +++ b/ui/audio.go @@ -2,6 +2,10 @@ package ui import ( "github.com/gordonklaus/portaudio" + "log" + + "gopkg.in/hraban/opus.v2" + // "github.com/xlab/opus-go/opus" ) @@ -19,13 +23,17 @@ func NewAudio() *Audio { } func (a *Audio) Start() error { - host, err := portaudio.DefaultHostApi() - if err != nil { - return err - } - - parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice) - stream, err := portaudio.OpenStream(parameters, a.Callback) + parameters := portaudio.StreamParameters{} + parameters.SampleRate = 44100 + + // host, err := portaudio.DefaultHostApi() + // if err != nil { + // return err + // } + + // parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice) + stream, err := portaudio.OpenDefaultStream(0, 1, 48000, 0, a.Callback) + // stream, err := portaudio.OpenStream(parameters, a.Callback) if err != nil { return err } @@ -34,7 +42,11 @@ func (a *Audio) Start() error { } a.stream = stream a.sampleRate = parameters.SampleRate - a.outputChannels = parameters.Output.Channels + // a.outputChannels = parameters.Output.Channels + a.outputChannels = 1 + + log.Println(a.sampleRate, a.outputChannels, parameters.FramesPerBuffer) + return nil } @@ -44,8 +56,9 @@ func (a *Audio) Stop() error { func (a *Audio) Callback(out []float32) { var output float32 + log.Println(len(out)) for i := range out { - if i%a.outputChannels == 0 { + if i%a.outputChannels == 1 { select { case sample := <-a.channel: output = sample @@ -55,4 +68,18 @@ func (a *Audio) Callback(out []float32) { } out[i] = output } + + enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP) + if err != nil { + log.Println("[!] Cannot create audio encoder", err) + return + } + data := make([]byte, 1000) + n, err := enc.EncodeFloat32(out, data) + + if err != nil { + log.Println("[!] Failed to decode") + return + } + data = data[:n] } \ No newline at end of file diff --git a/ui/director.go b/ui/director.go index 6b2d398f..0dd8a39a 100644 --- a/ui/director.go +++ b/ui/director.go @@ -10,7 +10,6 @@ import ( ) type Director struct { - // audio *Audio view *GameView timestamp float64 imageChannel chan *image.RGBA @@ -25,7 +24,6 @@ const FPS = 60 func NewDirector(roomID string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int, closedChannel chan bool) *Director { director := Director{} - // director.audio = audio director.imageChannel = imageChannel director.audioChanel = audioChanel director.inputChannel = inputChannel diff --git a/ui/gameview.go b/ui/gameview.go index 5de50641..535e5967 100644 --- a/ui/gameview.go +++ b/ui/gameview.go @@ -40,6 +40,7 @@ type GameView struct { audioChanel chan float32 inputChannel chan int + } func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int) *GameView { @@ -71,10 +72,9 @@ func (view *GameView) ListenToInputChannel() { // Enter enter the game view. func (view *GameView) Enter() { - // Always reset game - // Legacy Audio code. TODO: Add it back to support audio // view.console.SetAudioChannel(view.audio.channel) // view.console.SetAudioSampleRate(view.audio.sampleRate) + view.console.SetAudioSampleRate(48000) view.console.SetAudioChannel(view.audioChanel) diff --git a/webrtc/webrtc.go b/webrtc/webrtc.go index 505b6d4f..56bc04e1 100644 --- a/webrtc/webrtc.go +++ b/webrtc/webrtc.go @@ -93,7 +93,7 @@ func Decode(in string, obj interface{}) { func NewWebRTC() *WebRTC { w := &WebRTC{ ImageChannel: make(chan []byte, 2), - AudioChannel: make(chan []byte, 2), + AudioChannel: make(chan []byte, 1000), InputChannel: make(chan int, 2), } return w @@ -135,6 +135,12 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e log.Println("=== StartClient ===") w.connection, err = webrtc.NewPeerConnection(config) + // m := webrtc.MediaEngine{} + // m.RegisterCodec(webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, 48000)) + // m.RegisterCodec(webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, 1)) + // api := webrtc.NewAPI(webrtc.WithMediaEngine(m)) + // w.connection, err = api.NewPeerConnection(config) + if err != nil { return "", err } @@ -271,7 +277,8 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track) go func() { for w.isConnected { data := <-w.AudioChannel - opusTrack.WriteSample(media.Sample{Data: data, Samples: 1}) + // opusTrack.Write(data) + opusTrack.WriteSample(media.Sample{Data: data, Samples: 1000}) } }()