mirror of
https://github.com/giongto35/cloud-game.git
synced 2026-07-26 11:34:09 +00:00
Add audio streaming support with WebRTC and Opus (#102)
* Update README.md with additional info about Windows builds * Add initial Opus stream audio support
This commit is contained in:
sergystepanov
giongto35
parent
937bbb2857
commit
4e06d6a27e
8 changed files with 39 additions and 38 deletions
2
go.mod
vendored
2
go.mod
vendored
|
|
@ -9,8 +9,6 @@ require (
|
|||
github.com/gofrs/uuid v3.2.0+incompatible
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
|
||||
github.com/gorilla/websocket v1.4.0
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/pion/webrtc/v2 v2.1.2
|
||||
github.com/prometheus/client_golang v1.1.0
|
||||
github.com/spf13/pflag v1.0.3
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ const DefaultSTUNTURN = `[{"urls":"stun:stun-turn.webgame2d.com:3478"},{"urls":"
|
|||
const CODEC_VP8 = "VP8"
|
||||
const CODEC_H264 = "H264"
|
||||
|
||||
const AUDIO_RATE = 48000
|
||||
const AUDIO_CHANNELS = 2
|
||||
const AUDIO_MS = 10
|
||||
const AUDIO_FRAME = AUDIO_RATE * AUDIO_MS / 1000 * AUDIO_CHANNELS
|
||||
|
||||
var Port = flag.String("port", "8000", "Port of the game")
|
||||
var FrontendSTUNTURN = flag.String("stunturn", DefaultSTUNTURN, "Frontend STUN TURN servers")
|
||||
var Mode = flag.String("mode", "dev", "Environment")
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ import "C"
|
|||
// naEmulator implements CloudEmulator
|
||||
type naEmulator struct {
|
||||
imageChannel chan<- *image.RGBA
|
||||
audioChannel chan<- float32
|
||||
audioChannel chan<- int16
|
||||
inputChannel <-chan int
|
||||
|
||||
meta config.EmulatorMeta
|
||||
|
|
@ -65,7 +65,7 @@ var NAEmulator *naEmulator
|
|||
var outputImg *image.RGBA
|
||||
|
||||
// NAEmulator implements CloudEmulator interface based on NanoArch(golang RetroArch)
|
||||
func NewNAEmulator(etype string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- float32, inputChannel <-chan int) *naEmulator {
|
||||
func NewNAEmulator(etype string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- int16, inputChannel <-chan int) *naEmulator {
|
||||
meta := config.EmulatorConfig[etype]
|
||||
ewidth = meta.Width
|
||||
eheight = meta.Height
|
||||
|
|
@ -84,7 +84,7 @@ func NewNAEmulator(etype string, roomID string, imageChannel chan<- *image.RGBA,
|
|||
}
|
||||
|
||||
// Init initialize new RetroArch cloud emulator
|
||||
func Init(etype string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- float32, inputChannel <-chan int) {
|
||||
func Init(etype string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- int16, inputChannel <-chan int) {
|
||||
NAEmulator = NewNAEmulator(etype, roomID, imageChannel, audioChannel, inputChannel)
|
||||
go NAEmulator.listenInput()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -234,18 +234,17 @@ func min(a, b C.size_t) C.size_t {
|
|||
}
|
||||
|
||||
func audioWrite2(buf unsafe.Pointer, frames C.size_t) C.size_t {
|
||||
numFrames := int(frames) * 2
|
||||
pcm := (*[1 << 30]int16)(unsafe.Pointer(buf))[:numFrames:numFrames]
|
||||
// !to make it mono/stereo independent
|
||||
samples := int(frames) * 2
|
||||
pcm := (*[1 << 30]int16)(buf)[:samples:samples]
|
||||
|
||||
for i := 0; i < numFrames; i += 1 {
|
||||
s := float32(pcm[i])
|
||||
select {
|
||||
case NAEmulator.audioChannel <- s:
|
||||
default:
|
||||
}
|
||||
// !to rewrite this stuff
|
||||
// (channels are not that fast to put bytes one by one)
|
||||
for i := 0; i < samples; i += 1 {
|
||||
NAEmulator.audioChannel <- pcm[i]
|
||||
}
|
||||
|
||||
return 2 * frames
|
||||
return frames
|
||||
}
|
||||
|
||||
//export coreAudioSample
|
||||
|
|
|
|||
|
|
@ -291,7 +291,10 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track)
|
|||
if !w.isConnected {
|
||||
return
|
||||
}
|
||||
err := opusTrack.WriteSample(media.Sample{Data: data, Samples: uint32(len(data))})
|
||||
err := opusTrack.WriteSample(media.Sample{
|
||||
Data: data,
|
||||
Samples: uint32(config.AUDIO_FRAME / config.AUDIO_CHANNELS),
|
||||
})
|
||||
if err != nil {
|
||||
log.Println("Warn: Err write sample: ", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,10 +11,10 @@ import (
|
|||
"gopkg.in/hraban/opus.v2"
|
||||
)
|
||||
|
||||
func resample(pcm []float32, targetSize int, srcSampleRate int, dstSampleRate int) []float32 {
|
||||
newPCML := make([]float32, targetSize/2)
|
||||
newPCMR := make([]float32, targetSize/2)
|
||||
newPCM := make([]float32, targetSize)
|
||||
func resample(pcm []int16, targetSize int, srcSampleRate int, dstSampleRate int) []int16 {
|
||||
newPCML := make([]int16, targetSize/2)
|
||||
newPCMR := make([]int16, targetSize/2)
|
||||
newPCM := make([]int16, targetSize)
|
||||
for i := 0; i+1 < len(pcm); i += 2 {
|
||||
newPCML[(i/2)*dstSampleRate/srcSampleRate] = pcm[i]
|
||||
newPCMR[(i/2)*dstSampleRate/srcSampleRate] = pcm[i+1]
|
||||
|
|
@ -39,29 +39,25 @@ func resample(pcm []float32, targetSize int, srcSampleRate int, dstSampleRate in
|
|||
|
||||
func (r *Room) startAudio(sampleRate int) {
|
||||
log.Println("Enter fan audio")
|
||||
// srcSampleRate := 32768
|
||||
srcSampleRate := sampleRate
|
||||
dstSampleRate := 48000
|
||||
|
||||
enc, err := opus.NewEncoder(dstSampleRate, 2, opus.AppVoIP)
|
||||
enc, err := opus.NewEncoder(config.AUDIO_RATE, 2, opus.AppAudio)
|
||||
if err != nil {
|
||||
log.Println("[!] Cannot create audio encoder", err)
|
||||
}
|
||||
|
||||
enc.SetMaxBandwidth(opus.Fullband)
|
||||
enc.SetBitrateToAuto()
|
||||
enc.SetComplexity(10)
|
||||
|
||||
dstBufferSize := 240
|
||||
srcBufferSize := dstBufferSize * srcSampleRate / dstSampleRate
|
||||
dstBufferSize := config.AUDIO_FRAME
|
||||
srcBufferSize := dstBufferSize * srcSampleRate / config.AUDIO_RATE
|
||||
fmt.Println("src BufferSize", srcBufferSize)
|
||||
pcm := make([]float32, srcBufferSize) // 640 * 1000 / 16000 == 40 ms
|
||||
pcm := make([]int16, srcBufferSize) // 640 * 1000 / 16000 == 40 ms
|
||||
idx := 0
|
||||
|
||||
if err != nil {
|
||||
log.Println("[!] Cannot create audio encoder", err)
|
||||
return
|
||||
}
|
||||
|
||||
// fanout Audio
|
||||
fmt.Println("listening audiochanel", r.IsRunning)
|
||||
fmt.Println("listening audio channel", r.IsRunning)
|
||||
for sample := range r.audioChannel {
|
||||
if !r.IsRunning {
|
||||
log.Println("Room ", r.ID, " audio channel closed")
|
||||
|
|
@ -72,9 +68,9 @@ func (r *Room) startAudio(sampleRate int) {
|
|||
pcm[idx] = sample
|
||||
idx++
|
||||
if idx == len(pcm) {
|
||||
data := make([]byte, dstBufferSize)
|
||||
dstpcm := resample(pcm, dstBufferSize, srcSampleRate, dstSampleRate)
|
||||
n, err := enc.EncodeFloat32(dstpcm, data)
|
||||
data := make([]byte, 1024*2)
|
||||
dstpcm := resample(pcm, dstBufferSize, srcSampleRate, config.AUDIO_RATE)
|
||||
n, err := enc.Encode(dstpcm, data)
|
||||
|
||||
if err != nil {
|
||||
log.Println("[!] Failed to decode", err)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ type Room struct {
|
|||
// imageChannel is image stream received from director
|
||||
imageChannel <-chan *image.RGBA
|
||||
// audioChannel is audio stream received from director
|
||||
audioChannel <-chan float32
|
||||
audioChannel <-chan int16
|
||||
// inputChannel is input stream from websocket send to room
|
||||
inputChannel chan<- int
|
||||
// State of room
|
||||
|
|
@ -63,7 +63,7 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor
|
|||
|
||||
log.Println("Init new room: ", roomID, gameName, gameInfo)
|
||||
imageChannel := make(chan *image.RGBA, 30)
|
||||
audioChannel := make(chan float32, 30)
|
||||
audioChannel := make(chan int16, 30)
|
||||
inputChannel := make(chan int, 100)
|
||||
|
||||
room := &Room{
|
||||
|
|
@ -115,7 +115,7 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor
|
|||
}
|
||||
|
||||
// create director
|
||||
func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- float32, inputChannel <-chan int) emulator.CloudEmulator {
|
||||
func getEmulator(emuName string, roomID string, imageChannel chan<- *image.RGBA, audioChannel chan<- int16, inputChannel <-chan int) emulator.CloudEmulator {
|
||||
nanoarch.Init(emuName, roomID, imageChannel, audioChannel, inputChannel)
|
||||
|
||||
return nanoarch.NAEmulator
|
||||
|
|
|
|||
2
web/game.html
vendored
2
web/game.html
vendored
|
|
@ -33,7 +33,7 @@
|
|||
There is still audio because current audio flow is not from media but it is manually encoded (technical webRTC challenge). Later, when we can integrate audio to media, we can face the issue with mute again .
|
||||
https://developers.google.com/web/updates/2017/09/autoplay-policy-changes
|
||||
-->
|
||||
<video id="game-screen" muted playinfullscreen="false" playsinline poster="/static/img/screen_loading.gif"></video>
|
||||
<video id="game-screen" playinfullscreen="false" playsinline onloadstart="this.volume=0.5" poster="/static/img/screen_loading.gif"></video>
|
||||
<!--<video id="game-screen" autoplay=true poster="/static/img/screen_loading.gif"></video>-->
|
||||
|
||||
<div id="menu-screen">
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue