From 76b7cafb715cfdb2f25f6467e80ffcc859b42e2e Mon Sep 17 00:00:00 2001 From: sergystepanov Date: Mon, 18 Jan 2021 23:20:50 +0300 Subject: [PATCH] Refactor OPUS encoder for quality and speed (#262) * Refactor OPUS encoder for quality and speed. Use OPUS low delay mode (https://www.opus-codec.org/docs/opus_api-1.1.3/group__opus__ctlvalues.html#ga592232fb39db60c1369989c5c5d19a07) * Slightly tweak linear resample function and disable it on 48kHz input * Rewrite OPUS encoder as a struct with some internal buffer * Use OPUS 192Kbps preset by default * Add encoder callback function on a full buffer * Reuse OPUS output buffer * Add write limiter for samples --- go.mod | 19 +++-- go.sum | 48 +++++++---- pkg/encoder/opus/buffer.go | 20 +++++ pkg/encoder/opus/encoder.go | 137 +++++++++++++++++++++++++++++++ pkg/encoder/opus/encoder_test.go | 27 ++++++ pkg/util/common.go | 8 -- pkg/worker/room/media.go | 102 ++++++----------------- 7 files changed, 253 insertions(+), 108 deletions(-) create mode 100644 pkg/encoder/opus/buffer.go create mode 100644 pkg/encoder/opus/encoder.go create mode 100644 pkg/encoder/opus/encoder_test.go delete mode 100644 pkg/util/common.go diff --git a/go.mod b/go.mod index 73b24edd..069e1be9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/giongto35/cloud-game/v2 go 1.13 require ( - cloud.google.com/go v0.74.0 // indirect + cloud.google.com/go v0.75.0 // indirect cloud.google.com/go/storage v1.12.0 github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec github.com/faiface/mainthread v0.0.0-20171120011319-8b78f0a41ae3 @@ -13,24 +13,29 @@ require ( github.com/gofrs/flock v0.8.0 github.com/gofrs/uuid v4.0.0+incompatible github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/google/uuid v1.1.3 // indirect github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/kkyr/fig v0.2.0 - github.com/mitchellh/mapstructure v1.4.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/pelletier/go-toml v1.8.1 // indirect + github.com/pion/ice/v2 v2.0.15 // indirect github.com/pion/interceptor v0.0.9 github.com/pion/rtp v1.6.2 + github.com/pion/sdp/v3 v3.0.4 // indirect github.com/pion/webrtc/v3 v3.0.3 github.com/prometheus/client_golang v1.9.0 + github.com/prometheus/procfs v0.3.0 // indirect github.com/spf13/pflag v1.0.5 github.com/veandco/go-sdl2 v0.4.5 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/image v0.0.0-20201208152932-35266b937fa6 - golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect - golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 // indirect - golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee // indirect - google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect + golang.org/x/mod v0.4.1 // indirect + golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 // indirect + golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 // indirect + golang.org/x/text v0.3.5 // indirect + golang.org/x/tools v0.0.0-20210115202250-e0d201561e39 // indirect + google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect + google.golang.org/grpc v1.35.0 // indirect gopkg.in/hraban/opus.v2 v2.0.0-20201025103112-d779bb1cc5a2 gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 747ed0a3..e8b40891 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.66.0 h1:DZeAkuQGQqnm9Xv36SbMJEU8aFBz4wL04UpMWPWwjzg= cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0 h1:kpgPA77kSSbjSs+fWHkPTxQ6J5Z2Qkruo5jfXEkHxNQ= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -80,6 +80,7 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -101,6 +102,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/faiface/mainthread v0.0.0-20171120011319-8b78f0a41ae3 h1:baVdMKlASEHrj19iqjARrPbaRisD7EuZEVJj6ZMLl1Q= github.com/faiface/mainthread v0.0.0-20171120011319-8b78f0a41ae3/go.mod h1:VEPNJUlxl5KdWjDvz6Q1l+rJlxF2i6xqDeGuGAxa87M= @@ -199,13 +201,13 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.3 h1:twObb+9XcuH5B9V1TBCvvvZoO6iEdILi2a76PYn5rJI= -github.com/google/uuid v1.1.3/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -293,8 +295,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks= -github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -349,6 +351,8 @@ github.com/pion/dtls/v2 v2.0.4 h1:WuUcqi6oYMu/noNTz92QrF1DaFj4eXbhQ6dzaaAwOiI= github.com/pion/dtls/v2 v2.0.4/go.mod h1:qAkFscX0ZHoI1E07RfYPoRw3manThveu+mlTDdOxoGI= github.com/pion/ice/v2 v2.0.14 h1:FxXxauyykf89SWAtkQCfnHkno6G8+bhRkNguSh9zU+4= github.com/pion/ice/v2 v2.0.14/go.mod h1:wqaUbOq5ObDNU5ox1hRsEst0rWfsKuH1zXjQFEWiZwM= +github.com/pion/ice/v2 v2.0.15 h1:KZrwa2ciL9od8+TUVJiYTNsCW9J5lktBjGwW1MacEnQ= +github.com/pion/ice/v2 v2.0.15/go.mod h1:ZIiVGevpgAxF/cXiIVmuIUtCb3Xs4gCzCbXB6+nFkSI= github.com/pion/interceptor v0.0.9 h1:fk5hTdyLO3KURQsf/+RjMpEm4NE3yeTY9Kh97b5BvwA= github.com/pion/interceptor v0.0.9/go.mod h1:dHgEP5dtxOTf21MObuBAjJeAayPxLUAZjerGH8Xr07c= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -367,6 +371,8 @@ github.com/pion/sctp v1.7.11 h1:UCnj7MsobLKLuP/Hh+JMiI/6W5Bs/VF45lWKgHFjSIE= github.com/pion/sctp v1.7.11/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0= github.com/pion/sdp/v3 v3.0.3 h1:gJK9hk+JFD2NGIM1nXmqNCq1DkVaIZ9dlA3u3otnkaw= github.com/pion/sdp/v3 v3.0.3/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk= +github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8= +github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk= github.com/pion/srtp/v2 v2.0.1 h1:kgfh65ob3EcnFYA4kUBvU/menCp9u7qaJLXwWgpobzs= github.com/pion/srtp/v2 v2.0.1/go.mod h1:c8NWHhhkFf/drmHTAblkdu8++lsISEBBdAuiyxgqIsE= github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg= @@ -377,6 +383,7 @@ github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJ github.com/pion/transport v0.10.1 h1:2W+yJT+0mOQ160ThZYUx5Zp2skzshiNgxrNE9GUfhJM= github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= github.com/pion/transport v0.12.0/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= +github.com/pion/transport v0.12.1/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= github.com/pion/transport v0.12.2 h1:WYEjhloRHt1R86LhUKjC5y+P52Y11/QqEUalvtzVoys= github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA= @@ -424,6 +431,8 @@ github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFB github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -536,6 +545,8 @@ golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -582,7 +593,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 h1:42cLlJJdEh+ySyeUUbEQ5bsTi golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 h1:3uJsdck53FDIpWwLeAXlia9p4C8j0BO2xZrqzKpL0D8= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -596,6 +606,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 h1:Lm4OryKCca1vehdsWogr9N4t7NfZxLbJoc/H0w4K4S4= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 h1:BaN3BAqnopnKjvl+15DYP6LLrbBHfbfmlFYzmFj/Q9Q= +golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -655,8 +667,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg= -golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -668,6 +680,8 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -726,9 +740,9 @@ golang.org/x/tools v0.0.0-20200918232735-d647fc253266 h1:k7tVuG0g1JwmD3Jh8oAl1vQ golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee h1:5xKxdl/RhlelmSPaxyVeq5PYSmJ4H14yeQT58qP1F6o= -golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210115202250-e0d201561e39 h1:BTs2GMGSMWpgtCpv1CE7vkJTv7XcHdcLLnAMu7UbgTY= +golang.org/x/tools v0.0.0-20210115202250-e0d201561e39/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -806,9 +820,9 @@ google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f h1:izedQ6yVIc5mZsRuXzmSreCOlzI0lCU1HpG8yEdMiKw= +google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -833,6 +847,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/encoder/opus/buffer.go b/pkg/encoder/opus/buffer.go new file mode 100644 index 00000000..1f91ffbe --- /dev/null +++ b/pkg/encoder/opus/buffer.go @@ -0,0 +1,20 @@ +package opus + +type Buffer struct { + Data []int16 + idx int +} + +func (b *Buffer) Write(samples []int16) (written int) { + w := copy(b.Data[b.idx:], samples) + b.idx += w + return w +} + +func (b *Buffer) Full() bool { + full := b.idx == len(b.Data) + if full { + b.idx = 0 + } + return full +} diff --git a/pkg/encoder/opus/encoder.go b/pkg/encoder/opus/encoder.go new file mode 100644 index 00000000..d41fef1e --- /dev/null +++ b/pkg/encoder/opus/encoder.go @@ -0,0 +1,137 @@ +package opus + +import ( + "fmt" + "log" + + "gopkg.in/hraban/opus.v2" +) + +type Encoder struct { + *opus.Encoder + + channels int + inFrequency int + outFrequency int + // OPUS output buffer, 1K should be enough + outBuffer []byte + + buffer Buffer + onFullBuffer func(data []byte) + resampleBufSize int +} + +func NewEncoder(inputSampleRate, outputSampleRate, channels int, options ...func(*Encoder) error) (Encoder, error) { + encoder, err := opus.NewEncoder( + outputSampleRate, + channels, + // be aware that low delay option is not optimized for voice + opus.AppRestrictedLowdelay, + ) + if err != nil { + return Encoder{}, err + } + enc := &Encoder{ + Encoder: encoder, + buffer: Buffer{Data: make([]int16, inputSampleRate*20/1000*channels)}, + channels: channels, + inFrequency: inputSampleRate, + outFrequency: outputSampleRate, + outBuffer: make([]byte, 1024), + onFullBuffer: func(data []byte) {}, + } + + _ = enc.SetMaxBandwidth(opus.Fullband) + _ = enc.SetBitrate(192000) + _ = enc.SetComplexity(10) + + for _, option := range options { + err := option(enc) + if err != nil { + return Encoder{}, err + } + } + return *enc, nil +} + +func SampleBuffer(ms int, resampling bool) func(*Encoder) error { + return func(e *Encoder) (err error) { + e.buffer = Buffer{Data: make([]int16, e.inFrequency*ms/1000*e.channels)} + if resampling { + e.resampleBufSize = e.outFrequency * ms / 1000 * e.channels + } + return + } +} + +func CallbackOnFullBuffer(fn func(_ []byte)) func(*Encoder) error { + return func(e *Encoder) (err error) { + e.onFullBuffer = fn + return + } +} + +func (e *Encoder) BufferWrite(samples []int16) (written int) { + n := len(samples) + for k := n / len(e.buffer.Data); written < n || k >= 0; k-- { + written += e.buffer.Write(samples[written:]) + if e.buffer.Full() { + data, err := e.Encode(e.buffer.Data) + if err != nil { + log.Println("[!] Failed to encode", err) + continue + } + e.onFullBuffer(data) + } + } + return +} + +func (e *Encoder) Encode(pcm []int16) ([]byte, error) { + if e.resampleBufSize > 0 { + pcm = resampleFn(pcm, e.resampleBufSize) + } + n, err := e.Encoder.Encode(pcm, e.outBuffer) + if err != nil { + return nil, err + } + return e.outBuffer[:n], nil +} + +func (e *Encoder) GetInfo() string { + bitrate, _ := e.Encoder.Bitrate() + complexity, _ := e.Encoder.Complexity() + dtx, _ := e.Encoder.DTX() + fec, _ := e.Encoder.InBandFEC() + maxBandwidth, _ := e.Encoder.MaxBandwidth() + lossPercent, _ := e.Encoder.PacketLossPerc() + sampleRate, _ := e.Encoder.SampleRate() + return fmt.Sprintf( + "Bitrate: %v bps, Complexity: %v, DTX: %v, FEC: %v, Max bandwidth: *%v, Loss%%: %v, Rate: %v Hz", + bitrate, complexity, dtx, fec, maxBandwidth, lossPercent, sampleRate, + ) +} + +// resampleFn does a simple linear interpolation of audio samples. +func resampleFn(pcm []int16, size int) []int16 { + r, l, audio := make([]int16, size/2), make([]int16, size/2), make([]int16, size) + // ratio is basically the destination sample rate + // divided by the origin sample rate (i.e. 48000/44100) + ratio := float32(size) / float32(len(pcm)) + for i, n := 0, len(pcm)-1; i < n; i += 2 { + idx := int(float32(i/2) * ratio) + r[idx], l[idx] = pcm[i], pcm[i+1] + } + for i, n := 1, len(r); i < n; i++ { + if r[i] == 0 { + r[i] = r[i-1] + } + if l[i] == 0 { + l[i] = l[i-1] + } + } + for i := 0; i < size-1; i += 2 { + audio[i], audio[i+1] = r[i/2], l[i/2] + } + return audio +} diff --git a/pkg/encoder/opus/encoder_test.go b/pkg/encoder/opus/encoder_test.go new file mode 100644 index 00000000..a61ee9c7 --- /dev/null +++ b/pkg/encoder/opus/encoder_test.go @@ -0,0 +1,27 @@ +package opus + +import ( + "math/rand" + "testing" + "time" +) + +var resampleData []int16 + +func init() { + rand.Seed(time.Now().Unix()) + l := rand.Perm(getBufferLength(44000)) + for _, n := range l { + resampleData = append(resampleData, int16(n)) + } +} + +func BenchmarkResample(b *testing.B) { + sr := 48000 + bs := getBufferLength(sr) + for i := 0; i < b.N; i++ { + resampleFn(resampleData, bs) + } +} + +func getBufferLength(sampleRate int) int { return sampleRate * 20 / 1000 * 2 } diff --git a/pkg/util/common.go b/pkg/util/common.go deleted file mode 100644 index 31933ec1..00000000 --- a/pkg/util/common.go +++ /dev/null @@ -1,8 +0,0 @@ -package util - -func MinInt(x int, y int) int { - if x < y { - return x - } - return y -} diff --git a/pkg/worker/room/media.go b/pkg/worker/room/media.go index 0ede36ab..6fa363f7 100644 --- a/pkg/worker/room/media.go +++ b/pkg/worker/room/media.go @@ -7,38 +7,11 @@ import ( encoderConfig "github.com/giongto35/cloud-game/v2/pkg/config/encoder" "github.com/giongto35/cloud-game/v2/pkg/encoder" "github.com/giongto35/cloud-game/v2/pkg/encoder/h264encoder" + "github.com/giongto35/cloud-game/v2/pkg/encoder/opus" vpxencoder "github.com/giongto35/cloud-game/v2/pkg/encoder/vpx-encoder" - "github.com/giongto35/cloud-game/v2/pkg/util" "github.com/giongto35/cloud-game/v2/pkg/webrtc" - "gopkg.in/hraban/opus.v2" ) -func resample(pcm []int16, targetSize int, srcSampleRate int, dstSampleRate int) []int16 { - newPCML := make([]int16, targetSize/2) - newPCMR := make([]int16, targetSize/2) - newPCM := make([]int16, targetSize) - for i := 0; i+1 < len(pcm); i += 2 { - newPCML[(i/2)*dstSampleRate/srcSampleRate] = pcm[i] - newPCMR[(i/2)*dstSampleRate/srcSampleRate] = pcm[i+1] - } - for i := 1; i < len(newPCML); i++ { - if newPCML[i] == 0 { - newPCML[i] = newPCML[i-1] - } - } - for i := 1; i < len(newPCMR); i++ { - if newPCMR[i] == 0 { - newPCMR[i] = newPCMR[i-1] - } - } - for i := 0; i+1 < targetSize; i += 2 { - newPCM[i] = newPCML[i/2] - newPCM[i+1] = newPCMR[i/2] - } - - return newPCM -} - func (r *Room) startVoice() { // broadcast voice go func() { @@ -64,61 +37,36 @@ func (r *Room) startVoice() { } func (r *Room) startAudio(sampleRate int, audio encoderConfig.Audio) { - log.Println("Enter fan audio") - - srcSampleRate := sampleRate - enc, err := opus.NewEncoder(audio.Frequency, audio.Channels, opus.AppAudio) + sound, err := opus.NewEncoder( + sampleRate, + audio.Frequency, + audio.Channels, + opus.SampleBuffer(audio.Frame, sampleRate != audio.Frequency), + // we use callback on full buffer in order to + // send data to all the clients ASAP + opus.CallbackOnFullBuffer(r.broadcastAudio), + ) if err != nil { - log.Println("[!] Cannot create audio encoder", err) + log.Fatalf("error: cannot create audio encoder, %v", err) + } + log.Printf("OPUS: %v", sound.GetInfo()) + + for samples := range r.audioChannel { + sound.BufferWrite(samples) } - enc.SetMaxBandwidth(opus.Fullband) - enc.SetBitrateToAuto() - enc.SetComplexity(10) - - dstBufferSize := audio.GetFrameDuration() - srcBufferSize := dstBufferSize * srcSampleRate / audio.Frequency - pcm := make([]int16, srcBufferSize) // 640 * 1000 / 16000 == 40 ms - idx := 0 - - // fanout Audio - for sample := range r.audioChannel { - for i := 0; i < len(sample); { - rem := util.MinInt(len(sample)-i, len(pcm)-idx) - copy(pcm[idx:idx+rem], sample[i:i+rem]) - i += rem - idx += rem - - if idx == len(pcm) { - data := make([]byte, 1024*2) - dstpcm := resample(pcm, dstBufferSize, srcSampleRate, audio.Frequency) - n, err := enc.Encode(dstpcm, data) - - if err != nil { - log.Println("[!] Failed to decode", err) - - idx = 0 - continue - } - data = data[:n] - - // TODO: r.rtcSessions is rarely updated. Lock will hold down perf - //r.sessionsLock.Lock() - for _, webRTC := range r.rtcSessions { - if webRTC.IsConnected() { - // NOTE: can block here - webRTC.AudioChannel <- data - } - } - - idx = 0 - } - } - - } log.Println("Room ", r.ID, " audio channel closed") } +func (r *Room) broadcastAudio(audio []byte) { + for _, webRTC := range r.rtcSessions { + if webRTC.IsConnected() { + // NOTE: can block here + webRTC.AudioChannel <- audio + } + } +} + // startVideo processes imageChannel images with an encoder (codec) then pushes the result to WebRTC. func (r *Room) startVideo(width, height int, videoCodec encoder.VideoCodec) { var enc encoder.Encoder