mirror of
https://github.com/giongto35/cloud-game.git
synced 2026-01-23 18:46:11 +00:00
Use dynamic Opus frames with config
This commit is contained in:
Sergey Stepanov
parent
eae8c71bb1
commit
123ef4c3bc
9 changed files with 182 additions and 225 deletions
|
|
@ -168,7 +168,7 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
|
|||
}
|
||||
|
||||
m.AudioSrcHz = app.AudioSampleRate()
|
||||
m.AudioFrame = w.conf.Encoder.Audio.Frame
|
||||
m.AudioFrames = w.conf.Encoder.Audio.Frames
|
||||
m.VideoW, m.VideoH = app.ViewportSize()
|
||||
m.VideoScale = app.Scale()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,107 +1,119 @@
|
|||
package media
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type buffer2 struct {
|
||||
s samples
|
||||
wi int
|
||||
dst int
|
||||
// buffer is a simple non-concurrent safe buffer for audio samples.
|
||||
type buffer struct {
|
||||
stretch bool
|
||||
frameHz []int
|
||||
|
||||
dstHz2 int
|
||||
|
||||
raw samples
|
||||
buckets []Bucket
|
||||
cur *Bucket
|
||||
|
||||
sym bool
|
||||
}
|
||||
|
||||
type Bucket struct {
|
||||
mem samples
|
||||
vol int
|
||||
ms float32
|
||||
lv int
|
||||
dst int
|
||||
}
|
||||
|
||||
func NewBucket(level int, size int) Bucket {
|
||||
return Bucket{
|
||||
mem: make(samples, size),
|
||||
vol: level,
|
||||
func newBuffer(frames []float32, hz int) (*buffer, error) {
|
||||
if hz < 2000 {
|
||||
return nil, errors.New("hz should be > than 2000")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bucket) Reset() {
|
||||
b.lv = 0
|
||||
}
|
||||
buf := buffer{}
|
||||
|
||||
func (b *Bucket) IsEmpty() bool {
|
||||
return b.lv == 0
|
||||
}
|
||||
|
||||
var frames = [...]int{10, 5}
|
||||
|
||||
func newOpusBuffer(hz int) buffer2 {
|
||||
buf := buffer2{}
|
||||
|
||||
var fz = make([]int, 3)
|
||||
sum := 0
|
||||
for i, f := range frames {
|
||||
sum += f
|
||||
fz[i] = frame(hz, float32(f))
|
||||
buf.buckets = append(buf.buckets, NewBucket(f, fz[i]))
|
||||
// preallocate continuous array
|
||||
s := 0
|
||||
for _, f := range frames {
|
||||
s += frame(hz, f)
|
||||
}
|
||||
buf.cur = &buf.buckets[0]
|
||||
buf.raw = make(samples, s)
|
||||
|
||||
//buf.enableStretch(frame(hz, float32(buf.cur.vol)))
|
||||
|
||||
return buf
|
||||
next := 0
|
||||
for _, f := range frames {
|
||||
s := frame(hz, f)
|
||||
buf.buckets = append(buf.buckets, Bucket{
|
||||
mem: buf.raw[next : next+s],
|
||||
ms: f,
|
||||
})
|
||||
next += s
|
||||
}
|
||||
buf.cur = &buf.buckets[len(buf.buckets)-1]
|
||||
return &buf, nil
|
||||
}
|
||||
|
||||
func (b *buffer2) chooseBucket(l int) {
|
||||
//b.cur = &b.buckets[0]
|
||||
func (b *buffer) choose(l int) {
|
||||
for _, bb := range b.buckets {
|
||||
if l >= len(bb.mem) {
|
||||
b.cur = &bb
|
||||
b.sym = false
|
||||
if b.stretch {
|
||||
b.enableStretch(frame(b.dstHz2, float32(b.cur.vol)))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// enableStretch adds a simple stretching of buffer to a desired size before
|
||||
// the onFull callback call.
|
||||
func (b *buffer2) enableStretch(l int) { b.stretch = true; b.dst = frame(b.dstHz2, float32(b.cur.vol)) }
|
||||
|
||||
func (b *buffer2) dstHz(hz int) {
|
||||
b.dstHz2 = hz
|
||||
b.enableStretch(frame(hz, float32(b.cur.vol)))
|
||||
func (b *buffer) resample(hz int) {
|
||||
b.stretch = true
|
||||
for i := range b.buckets {
|
||||
b.buckets[i].dst = frame(hz, float32(b.buckets[i].ms))
|
||||
}
|
||||
}
|
||||
|
||||
func (b *buffer2) write(s samples, onFull func(samples, int)) (r int) {
|
||||
// select bucket
|
||||
//b.chooseBucket(len(s))
|
||||
// write fills the buffer until it's full and then passes the gathered data into a callback.
|
||||
//
|
||||
// There are two cases to consider:
|
||||
// 1. Underflow, when the length of the written data is less than the buffer's available space.
|
||||
// 2. Overflow, when the length exceeds the current available buffer space.
|
||||
//
|
||||
// We overwrite any previous values in the buffer and move the internal write pointer
|
||||
// by the length of the written data.
|
||||
// In the first case, we won't call the callback, but it will be called every time
|
||||
// when the internal buffer overflows until all samples are read.
|
||||
func (b *buffer) write(s samples, onFull func(samples, float32)) (r int) {
|
||||
for r < len(s) {
|
||||
buf := b.cur
|
||||
|
||||
w := copy(buf.mem[buf.lv:], s[r:])
|
||||
r += w
|
||||
buf.lv += w
|
||||
if buf.lv == len(buf.mem) {
|
||||
b.sym = true
|
||||
if b.stretch {
|
||||
onFull(buf.mem.stretch(b.dst), buf.vol)
|
||||
onFull(buf.mem.stretch(buf.dst), buf.ms)
|
||||
} else {
|
||||
onFull(buf.mem, buf.vol)
|
||||
onFull(buf.mem, buf.ms)
|
||||
}
|
||||
if !b.sym {
|
||||
fmt.Printf(">>>>>>>>>")
|
||||
}
|
||||
b.chooseBucket(len(s) - r)
|
||||
b.cur.Reset()
|
||||
b.choose(len(s) - r)
|
||||
b.cur.lv = 0
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// frame calculates an audio stereo frame size, i.e. 48k*frame/1000*2
|
||||
// with round(x / 2) * 2 for the closest even number
|
||||
func frame(hz int, frame float32) int {
|
||||
return int(math.Round(float64(hz)*float64(frame)/1000/2) * 2 * 2)
|
||||
}
|
||||
|
||||
// stretch does a simple stretching of audio samples.
|
||||
// something like: [1,2,3,4,5,6] -> [1,2,x,x,3,4,x,x,5,6,x,x] -> [1,2,1,2,3,4,3,4,5,6,5,6]
|
||||
func (s samples) stretch(size int) []int16 {
|
||||
out := buf[:size]
|
||||
n := len(s)
|
||||
ratio := float32(size) / float32(n)
|
||||
sPtr := unsafe.Pointer(&s[0])
|
||||
for i, l, r := 0, 0, 0; i < n; i += 2 {
|
||||
l, r = r, int(float32((i+2)>>1)*ratio)<<1 // index in src * ratio -> approximated index in dst *2 due to int16
|
||||
for j := l; j < r; j += 2 {
|
||||
*(*int32)(unsafe.Pointer(&out[j])) = *(*int32)(sPtr) // out[j] = s[i]; out[j+1] = s[i+1]
|
||||
}
|
||||
sPtr = unsafe.Add(sPtr, uintptr(4))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,77 @@
|
|||
package media
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_buffer2_write2(t *testing.T) {
|
||||
|
||||
buf := newOpusBuffer(1000)
|
||||
|
||||
t.Logf("%+v", buf)
|
||||
type bufWrite struct {
|
||||
sample int16
|
||||
len int
|
||||
}
|
||||
|
||||
func TestBufferWrite(t *testing.T) {
|
||||
tests := []struct {
|
||||
bufLen int
|
||||
writes []bufWrite
|
||||
expect samples
|
||||
}{
|
||||
{
|
||||
bufLen: 2000,
|
||||
writes: []bufWrite{
|
||||
{sample: 1, len: 10},
|
||||
{sample: 2, len: 20},
|
||||
{sample: 3, len: 30},
|
||||
},
|
||||
expect: samples{3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
|
||||
},
|
||||
{
|
||||
bufLen: 2000,
|
||||
writes: []bufWrite{
|
||||
{sample: 1, len: 3},
|
||||
{sample: 2, len: 18},
|
||||
{sample: 3, len: 2},
|
||||
},
|
||||
expect: samples{2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
var lastResult samples
|
||||
buf, err := newBuffer([]float32{10, 5}, test.bufLen)
|
||||
if err != nil {
|
||||
t.Fatalf("oof, %v", err)
|
||||
}
|
||||
for _, w := range test.writes {
|
||||
buf.write(samplesOf(w.sample, w.len),
|
||||
func(s samples, ms float32) { lastResult = s },
|
||||
)
|
||||
}
|
||||
if !reflect.DeepEqual(test.expect, lastResult) {
|
||||
t.Errorf("not expted buffer, %v != %v, %v", lastResult, test.expect, len(buf.cur.mem))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBufferWrite(b *testing.B) {
|
||||
fn := func(_ samples, _ float32) {}
|
||||
l := 2000
|
||||
buf, err := newBuffer([]float32{10}, l)
|
||||
if err != nil {
|
||||
b.Fatalf("oof: %v", err)
|
||||
}
|
||||
samples1 := samplesOf(1, l/2)
|
||||
samples2 := samplesOf(2, l*2)
|
||||
for i := 0; i < b.N; i++ {
|
||||
buf.write(samples1, fn)
|
||||
buf.write(samples2, fn)
|
||||
}
|
||||
}
|
||||
|
||||
func samplesOf(v int16, len int) (s samples) {
|
||||
s = make(samples, len)
|
||||
for i := range s {
|
||||
s[i] = v
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,16 +2,13 @@ package media
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/giongto35/cloud-game/v3/pkg/config"
|
||||
"github.com/giongto35/cloud-game/v3/pkg/encoder"
|
||||
"github.com/giongto35/cloud-game/v3/pkg/encoder/opus"
|
||||
"github.com/giongto35/cloud-game/v3/pkg/logger"
|
||||
"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -19,16 +16,7 @@ const (
|
|||
sampleBufLen = 1024 * 4
|
||||
)
|
||||
|
||||
// buffer is a simple non-concurrent safe ring buffer for audio samples.
|
||||
type (
|
||||
buffer struct {
|
||||
s samples
|
||||
wi int
|
||||
dst int
|
||||
stretch bool
|
||||
}
|
||||
samples []int16
|
||||
)
|
||||
type samples []int16
|
||||
|
||||
var (
|
||||
encoderOnce = sync.Once{}
|
||||
|
|
@ -36,39 +24,6 @@ var (
|
|||
buf = make([]int16, sampleBufLen)
|
||||
)
|
||||
|
||||
func newBuffer(srcLen int) buffer { return buffer{s: make(samples, srcLen)} }
|
||||
|
||||
// enableStretch adds a simple stretching of buffer to a desired size before
|
||||
// the onFull callback call.
|
||||
func (b *buffer) enableStretch(l int) { b.stretch = true; b.dst = l }
|
||||
|
||||
// write fills the buffer until it's full and then passes the gathered data into a callback.
|
||||
//
|
||||
// There are two cases to consider:
|
||||
// 1. Underflow, when the length of the written data is less than the buffer's available space.
|
||||
// 2. Overflow, when the length exceeds the current available buffer space.
|
||||
//
|
||||
// We overwrite any previous values in the buffer and move the internal write pointer
|
||||
// by the length of the written data.
|
||||
// In the first case, we won't call the callback, but it will be called every time
|
||||
// when the internal buffer overflows until all samples are read.
|
||||
func (b *buffer) write(s samples, onFull func(samples)) (r int) {
|
||||
for r < len(s) {
|
||||
w := copy(b.s[b.wi:], s[r:])
|
||||
r += w
|
||||
b.wi += w
|
||||
if b.wi == len(b.s) {
|
||||
b.wi = 0
|
||||
if b.stretch {
|
||||
onFull(b.s.stretch(b.dst))
|
||||
} else {
|
||||
onFull(b.s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func DefaultOpus() (*opus.Encoder, error) {
|
||||
var err error
|
||||
encoderOnce.Do(func() { opusCoder, err = opus.NewEncoder(audioHz) })
|
||||
|
|
@ -81,34 +36,11 @@ func DefaultOpus() (*opus.Encoder, error) {
|
|||
return opusCoder, nil
|
||||
}
|
||||
|
||||
// frame calculates an audio stereo frame size, i.e. 48k*frame/1000*2
|
||||
// with round(x / 2) * 2 for the closest even number
|
||||
func frame(hz int, frame float32) int {
|
||||
return int(math.Round(float64(hz)*float64(frame)/1000/2) * 2 * 2)
|
||||
}
|
||||
|
||||
// stretch does a simple stretching of audio samples.
|
||||
// something like: [1,2,3,4,5,6] -> [1,2,x,x,3,4,x,x,5,6,x,x] -> [1,2,1,2,3,4,3,4,5,6,5,6]
|
||||
func (s samples) stretch(size int) []int16 {
|
||||
out := buf[:size]
|
||||
n := len(s)
|
||||
ratio := float32(size) / float32(n)
|
||||
sPtr := unsafe.Pointer(&s[0])
|
||||
for i, l, r := 0, 0, 0; i < n; i += 2 {
|
||||
l, r = r, int(float32((i+2)>>1)*ratio)<<1 // index in src * ratio -> approximated index in dst *2 due to int16
|
||||
for j := l; j < r; j += 2 {
|
||||
*(*int32)(unsafe.Pointer(&out[j])) = *(*int32)(sPtr) // out[j] = s[i]; out[j+1] = s[i+1]
|
||||
}
|
||||
sPtr = unsafe.Add(sPtr, uintptr(4))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type WebrtcMediaPipe struct {
|
||||
a *opus.Encoder
|
||||
v *encoder.Video
|
||||
onAudio func([]byte, int)
|
||||
audioBuf buffer2
|
||||
onAudio func([]byte, float32)
|
||||
audioBuf *buffer
|
||||
log *logger.Logger
|
||||
|
||||
mua sync.RWMutex
|
||||
|
|
@ -118,7 +50,7 @@ type WebrtcMediaPipe struct {
|
|||
vConf config.Video
|
||||
|
||||
AudioSrcHz int
|
||||
AudioFrame float32
|
||||
AudioFrames []float32
|
||||
VideoW, VideoH int
|
||||
VideoScale float64
|
||||
|
||||
|
|
@ -135,11 +67,8 @@ func NewWebRtcMediaPipe(ac config.Audio, vc config.Video, log *logger.Logger) *W
|
|||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) SetAudioCb(cb func([]byte, int32)) {
|
||||
//fr := int32(time.Duration(wmp.AudioFrame) * time.Millisecond)
|
||||
wmp.onAudio = func(bytes []byte, l int) {
|
||||
fr := int32(time.Duration(l) * time.Millisecond)
|
||||
//wmp.log.Info().Msgf(">>> %v", fr)
|
||||
cb(bytes, fr)
|
||||
wmp.onAudio = func(bytes []byte, ms float32) {
|
||||
cb(bytes, int32(time.Duration(ms)*time.Millisecond))
|
||||
}
|
||||
}
|
||||
func (wmp *WebrtcMediaPipe) Destroy() {
|
||||
|
|
@ -153,7 +82,7 @@ func (wmp *WebrtcMediaPipe) PushAudio(audio []int16) {
|
|||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) Init() error {
|
||||
if err := wmp.initAudio(wmp.AudioSrcHz, wmp.AudioFrame); err != nil {
|
||||
if err := wmp.initAudio(wmp.AudioSrcHz, wmp.AudioFrames); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil {
|
||||
|
|
@ -172,31 +101,34 @@ func (wmp *WebrtcMediaPipe) Init() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) initAudio(srcHz int, frameSize float32) error {
|
||||
func (wmp *WebrtcMediaPipe) initAudio(srcHz int, frameSizes []float32) error {
|
||||
au, err := DefaultOpus()
|
||||
if err != nil {
|
||||
return fmt.Errorf("opus fail: %w", err)
|
||||
}
|
||||
wmp.log.Debug().Msgf("Opus: %v", au.GetInfo())
|
||||
wmp.SetAudio(au)
|
||||
buf := newOpusBuffer(srcHz) //newBuffer(frame(srcHz, frameSize))
|
||||
buf, err := newBuffer(frameSizes, srcHz)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wmp.log.Debug().Msgf("Opus frames (ms): %v", frameSizes)
|
||||
dstHz, _ := au.SampleRate()
|
||||
if srcHz != dstHz {
|
||||
buf.dstHz(dstHz)
|
||||
buf.enableStretch(frame(dstHz, frameSize))
|
||||
buf.resample(dstHz)
|
||||
wmp.log.Debug().Msgf("Resample %vHz -> %vHz", srcHz, dstHz)
|
||||
}
|
||||
wmp.audioBuf = buf
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) encodeAudio(pcm samples, l int) {
|
||||
func (wmp *WebrtcMediaPipe) encodeAudio(pcm samples, ms float32) {
|
||||
data, err := wmp.Audio().Encode(pcm)
|
||||
if err != nil {
|
||||
wmp.log.Error().Err(err).Msgf("opus encode fail")
|
||||
return
|
||||
}
|
||||
wmp.onAudio(data, l)
|
||||
wmp.onAudio(data, ms)
|
||||
}
|
||||
|
||||
func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) (err error) {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package media
|
|||
import (
|
||||
"image"
|
||||
"math/rand/v2"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/giongto35/cloud-game/v3/pkg/config"
|
||||
|
|
@ -154,69 +153,6 @@ func gen(l int) []int16 {
|
|||
return nums
|
||||
}
|
||||
|
||||
type bufWrite struct {
|
||||
sample int16
|
||||
len int
|
||||
}
|
||||
|
||||
func TestBufferWrite(t *testing.T) {
|
||||
tests := []struct {
|
||||
bufLen int
|
||||
writes []bufWrite
|
||||
expect samples
|
||||
}{
|
||||
{
|
||||
bufLen: 20,
|
||||
writes: []bufWrite{
|
||||
{sample: 1, len: 10},
|
||||
{sample: 2, len: 20},
|
||||
{sample: 3, len: 30},
|
||||
},
|
||||
expect: samples{3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
|
||||
},
|
||||
{
|
||||
bufLen: 11,
|
||||
writes: []bufWrite{
|
||||
{sample: 1, len: 3},
|
||||
{sample: 2, len: 18},
|
||||
{sample: 3, len: 2},
|
||||
},
|
||||
expect: samples{3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
var lastResult samples
|
||||
buf := newBuffer(test.bufLen)
|
||||
for _, w := range test.writes {
|
||||
buf.write(samplesOf(w.sample, w.len), func(s samples) { lastResult = s })
|
||||
}
|
||||
if !reflect.DeepEqual(test.expect, lastResult) {
|
||||
t.Errorf("not expted buffer, %v != %v, %v", lastResult, test.expect, buf.s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBufferWrite(b *testing.B) {
|
||||
fn := func(_ samples) {}
|
||||
l := 1920
|
||||
buf := newBuffer(l)
|
||||
samples1 := samplesOf(1, l/2)
|
||||
samples2 := samplesOf(2, l*2)
|
||||
for i := 0; i < b.N; i++ {
|
||||
buf.write(samples1, fn)
|
||||
buf.write(samples2, fn)
|
||||
}
|
||||
}
|
||||
|
||||
func samplesOf(v int16, len int) (s samples) {
|
||||
s = make(samples, len)
|
||||
for i := range s {
|
||||
s[i] = v
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestFrame(t *testing.T) {
|
||||
type args struct {
|
||||
hz int
|
||||
|
|
|
|||
|
|
@ -229,7 +229,7 @@ func room(cfg conf) testRoom {
|
|||
|
||||
m := media.NewWebRtcMediaPipe(conf.Encoder.Audio, conf.Encoder.Video, l)
|
||||
m.AudioSrcHz = emu.AudioSampleRate()
|
||||
m.AudioFrame = conf.Encoder.Audio.Frame
|
||||
m.AudioFrames = conf.Encoder.Audio.Frames
|
||||
m.VideoW, m.VideoH = emu.ViewportSize()
|
||||
m.VideoScale = emu.Scale()
|
||||
if err := m.Init(); err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue