Mirrors/cloud-game
1
0
Fork 0
mirror of https://github.com/giongto35/cloud-game.git synced 2026-07-21 00:59:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

WIP audio

Browse source
This commit is contained in:
hasagi 2019-04-13 00:40:08 +07:00
parent 2f89612072
commit 55e7f47e76
6 changed files with 197 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

95
main.go
View file

@ -18,6 +18,8 @@ import (
"github.com/giongto35/cloud-game/webrtc"
"github.com/gorilla/websocket"
pionRTC "github.com/pion/webrtc"
"github.com/hraban/opus"
)
const (
@ -48,6 +50,7 @@ type WSPacket struct {
// A room stores all the channel for interaction between all webRTCs session and emulator
type Room struct {
imageChannel chan *image.RGBA
audioChanel chan float32
inputChannel chan int
// closedChannel is to fire exit event when there is no webRTC session running
closedChannel chan bool
@ -96,21 +99,24 @@ func initRoom(roomID, gameName string) string {
roomID = generateRoomID()
}
imageChannel := make(chan *image.RGBA, 100)
audioChannel := make(chan float32, 48000)
inputChannel := make(chan int, 100)
closedChannel := make(chan bool)
// create director
director := ui.NewDirector(roomID, imageChannel, inputChannel, closedChannel)
director := ui.NewDirector(roomID, imageChannel, audioChannel, inputChannel, closedChannel)
rooms[roomID] = &Room{
imageChannel: imageChannel,
audioChanel: audioChannel,
inputChannel: inputChannel,
closedChannel: closedChannel,
rtcSessions: []*webrtc.WebRTC{},
director: director,
director: director,
}
go fanoutScreen(imageChannel, roomID)
go fanoutAudio(audioChannel, roomID)
go director.Start([]string{"games/" + gameName})
return roomID
@ -144,7 +150,7 @@ func startSession(webRTC *webrtc.WebRTC, gameName string, roomID string, playerI
// TODO: Might have race condition
rooms[roomID].rtcSessions = append(rooms[roomID].rtcSessions, webRTC)
faninInput(rooms[roomID].inputChannel, webRTC, playerIndex)
go faninInput(rooms[roomID].inputChannel, webRTC, playerIndex)
return roomID
}
@ -298,30 +304,79 @@ func fanoutScreen(imageChannel chan *image.RGBA, roomID string) {
}
if isRoomRunning == false {
log.Println("Closed room", roomID)
log.Println("Closed room from screen routine", roomID)
rooms[roomID].closedChannel <- true
}
}
}
// faninInput fan-in of the same room to inputChannel
func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) {
go func() {
for {
// Client stopped
if webRTC.IsClosed() {
return
// fanoutAudio fanout outputs to all webrtc in the same room
func fanoutAudio(audioChanel chan float32, roomID string) {
enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP)
if err != nil {
log.Println("[!] Cannot create audio encoder")
return
}
pcm := make([]float32, 120)
c := 0
for audio := range audioChanel {
if c >= cap(pcm) {
data := make([]byte, 1000)
n, err := enc.EncodeFloat32(pcm, data)
if err != nil {
log.Println("[!] Failed to decode")
continue
}
data = data[:n]
isRoomRunning := false
for _, webRTC := range rooms[roomID].rtcSessions {
// Client stopped
if webRTC.IsClosed() {
continue
}
// encode frame
// fanout imageChannel
if webRTC.IsConnected() {
// NOTE: can block here
webRTC.AudioChannel <- data
}
isRoomRunning = true
}
// encode frame
if webRTC.IsConnected() {
input := <-webRTC.InputChannel
// the first 8 bits belong to player 1
// the next 8 belongs to player 2 ...
// We standardize and put it to inputChannel (16 bits)
input = input << ((uint(playerIndex) - 1) * ui.NumKeys)
inputChannel <- input
if isRoomRunning == false {
log.Println("Closed room from audio routine", roomID)
rooms[roomID].closedChannel <- true
}
c = 0
} else {
pcm[c] = audio
c++
}
}()
}
}
// faninInput fan-in of the same room to inputChannel
func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) {
for {
// Client stopped
if webRTC.IsClosed() {
return
}
// encode frame
if webRTC.IsConnected() {
input := <-webRTC.InputChannel
// the first 8 bits belong to player 1
// the next 8 belongs to player 2 ...
// We standardize and put it to inputChannel (16 bits)
input = input << ((uint(playerIndex) - 1) * ui.NumKeys)
inputChannel <- input
}
}
}

18
static/js/ws.js
View file

@ -86,9 +86,18 @@ function startGame() {
// stream channel
pc.ontrack = function (event) {
log("New stream, yay!");
document.getElementById("game-screen").srcObject = event.streams[0];
$("#game-screen").show();
console.log(event.streams);
var el = document.createElement(event.track.kind);
el.srcObject = event.streams[0];
el.autoplay = true;
el.width = 800;
el.height = 600;
el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif");
document.getElementById('remoteVideos').appendChild(el)
// log("New stream, yay!");
// document.getElementById("game-screen").srcObject = event.streams[0];
// $("#game-screen").show();
}
@ -108,9 +117,10 @@ function startGame() {
function startWebRTC() {
// receiver only tracks
pc.addTransceiver('video', {'direction': 'recvonly'});
pc.addTransceiver('audio', {'direction': 'recvonly'});
// create SDP
pc.createOffer({offerToReceiveVideo: true, offerToReceiveAudio: false}).then(d => {
pc.createOffer({offerToReceiveVideo: true, offerToReceiveAudio: true}).then(d => {
pc.setLocalDescription(d).catch(log);
})
}

58
ui/audio.go Normal file
View file

@ -0,0 +1,58 @@
package ui
import (
"github.com/gordonklaus/portaudio"
)
type Audio struct {
stream *portaudio.Stream
sampleRate float64
outputChannels int
channel chan float32
}
func NewAudio() *Audio {
a := Audio{}
a.channel = make(chan float32, 16000)
return &a
}
func (a *Audio) Start() error {
host, err := portaudio.DefaultHostApi()
if err != nil {
return err
}
parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice)
stream, err := portaudio.OpenStream(parameters, a.Callback)
if err != nil {
return err
}
if err := stream.Start(); err != nil {
return err
}
a.stream = stream
a.sampleRate = parameters.SampleRate
a.outputChannels = parameters.Output.Channels
return nil
}
func (a *Audio) Stop() error {
return a.stream.Close()
}
func (a *Audio) Callback(out []float32) {
var output float32
for i := range out {
if i%a.outputChannels == 0 {
select {
case sample := <-a.channel:
output = sample
default:
output = 0
}
}
out[i] = output
}
}

21
ui/director.go
View file

@ -6,13 +6,15 @@ import (
"time"
"github.com/giongto35/cloud-game/nes"
// "github.com/gordonklaus/portaudio"
)
type Director struct {
// audio *Audio
// audio *Audio
view *GameView
timestamp float64
imageChannel chan *image.RGBA
audioChanel chan float32
inputChannel chan int
closedChannel chan bool
roomID string
@ -21,10 +23,11 @@ type Director struct {
const FPS = 60
func NewDirector(roomID string, imageChannel chan *image.RGBA, inputChannel chan int, closedChannel chan bool) *Director {
func NewDirector(roomID string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int, closedChannel chan bool) *Director {
director := Director{}
// director.audio = audio
director.imageChannel = imageChannel
director.audioChanel = audioChanel
director.inputChannel = inputChannel
director.closedChannel = closedChannel
director.roomID = roomID
@ -53,6 +56,13 @@ func (d *Director) Step() {
}
func (d *Director) Start(paths []string) {
// portaudio.Initialize()
// defer portaudio.Terminate()
// audio := NewAudio()
// audio.Start()
// d.audio = audio
if len(paths) == 1 {
d.PlayGame(paths[0])
}
@ -60,9 +70,10 @@ func (d *Director) Start(paths []string) {
}
func (d *Director) Run() {
c := time.Tick(time.Second / FPS)
// c := time.Tick(time.Second / FPS)
L:
for range c {
// for range c {
for {
// quit game
select {
@ -89,7 +100,7 @@ func (d *Director) PlayGame(path string) {
log.Fatalln(err)
}
// Set GameView as current view
d.SetView(NewGameView(console, path, hash, d.imageChannel, d.inputChannel))
d.SetView(NewGameView(console, path, hash, d.imageChannel, d.audioChanel, d.inputChannel))
}
func (d *Director) SaveGame() error {

15
ui/gameview.go
View file

@ -37,16 +37,19 @@ type GameView struct {
keyPressed [NumKeys * 2]bool
imageChannel chan *image.RGBA
audioChanel chan float32
inputChannel chan int
}
func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, inputChannel chan int) *GameView {
func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int) *GameView {
gameview := &GameView{
console: console,
title: title,
hash: hash,
keyPressed: [NumKeys * 2]bool{false},
imageChannel: imageChannel,
audioChanel: audioChanel,
inputChannel: inputChannel,
}
@ -70,8 +73,10 @@ func (view *GameView) ListenToInputChannel() {
func (view *GameView) Enter() {
// Always reset game
// Legacy Audio code. TODO: Add it back to support audio
// view.console.SetAudioChannel(view.director.audio.channel)
// view.console.SetAudioSampleRate(view.director.audio.sampleRate)
// view.console.SetAudioChannel(view.audio.channel)
// view.console.SetAudioSampleRate(view.audio.sampleRate)
view.console.SetAudioSampleRate(48000)
view.console.SetAudioChannel(view.audioChanel)
// load state
if err := view.console.LoadState(savePath(view.hash)); err == nil {
@ -92,8 +97,8 @@ func (view *GameView) Enter() {
// Exit ...
func (view *GameView) Exit() {
// view.console.SetAudioChannel(nil)
// view.console.SetAudioSampleRate(0)
view.console.SetAudioChannel(nil)
view.console.SetAudioSampleRate(0)
// save sram
cartridge := view.console.Cartridge
if cartridge.Battery != 0 {

27
webrtc/webrtc.go
View file

@ -93,6 +93,7 @@ func Decode(in string, obj interface{}) {
func NewWebRTC() *WebRTC {
w := &WebRTC{
ImageChannel: make(chan []byte, 2),
AudioChannel: make(chan []byte, 2),
InputChannel: make(chan int, 2),
}
return w
@ -106,6 +107,7 @@ type WebRTC struct {
isClosed bool
// for yuvI420 image
ImageChannel chan []byte
AudioChannel chan []byte
InputChannel chan int
}
@ -137,7 +139,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
return "", err
}
vp8Track, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeVP8, rand.Uint32(), "video", "pion2")
vp8Track, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeVP8, rand.Uint32(), "video", "pion2a")
if err != nil {
return "", err
}
@ -146,6 +148,16 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
return "", err
}
opusTrack, err := w.connection.NewTrack(webrtc.DefaultPayloadTypeOpus, rand.Uint32(), "audio", "pion2b")
if err != nil {
return "", err
}
_, err = w.connection.AddTrack(opusTrack)
if err != nil {
return "", err
}
// WebRTC state callback
w.connection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
log.Printf("ICE Connection State has changed: %s\n", connectionState.String())
@ -153,7 +165,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
go func() {
w.isConnected = true
log.Println("ConnectionStateConnected")
w.startStreaming(vp8Track)
w.startStreaming(vp8Track, opusTrack)
}()
}
@ -235,7 +247,7 @@ func (w *WebRTC) IsClosed() bool {
return w.isClosed
}
func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) {
func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track) {
log.Println("Start streaming")
// send screenshot
go func() {
@ -254,4 +266,13 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) {
vp8Track.WriteSample(media.Sample{Data: bs, Samples: 1})
}
}()
// send audio
go func() {
for w.isConnected {
data := <-w.AudioChannel
opusTrack.WriteSample(media.Sample{Data: data, Samples: 1})
}
}()
}