diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 40fed385..c6cd144f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,85 +4,82 @@ name: build on: - push: - branches: - - master - tags-ignore: - - 'v*' - pull_request: - branches: - - master + push: + branches: + - master + tags-ignore: + - "v*" + pull_request: + branches: + - master jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 - build: - strategy: - matrix: - os: [ ubuntu-latest, windows-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "stable" - - uses: actions/setup-go@v5 - with: - go-version: 'stable' + - name: Linux + if: matrix.os == 'ubuntu-latest' + env: + MESA_GL_VERSION_OVERRIDE: 3.3COMPAT + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y \ + make pkg-config \ + libsdl2-dev libgl1 libglx-mesa0 \ + libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \ + gstreamer1.0-plugins-good - - name: Linux - if: matrix.os == 'ubuntu-latest' - env: - MESA_GL_VERSION_OVERRIDE: 3.3COMPAT - run: | - sudo apt-get -qq update - sudo apt-get -qq install -y \ - make pkg-config \ - libvpx-dev libx264-dev libopus-dev libyuv-dev libjpeg-turbo8-dev \ - libsdl2-dev libgl1 libglx-mesa0 libspeexdsp-dev - - make build - xvfb-run --auto-servernum make test verify-cores + make build + xvfb-run --auto-servernum make test verify-cores - - name: macOS - if: matrix.os == 'macos-12' - run: | - brew install libvpx x264 sdl2 speexdsp - make build test verify-cores + - name: macOS + if: matrix.os == 'macos-12' + run: | + brew install sdl2 gstreamer gst-plugins-base gst-plugins-good + make build test verify-cores - - uses: msys2/setup-msys2@v2 - if: matrix.os == 'windows-latest' - with: - msystem: ucrt64 - path-type: inherit - release: false - install: > - mingw-w64-ucrt-x86_64-gcc - mingw-w64-ucrt-x86_64-pkgconf - mingw-w64-ucrt-x86_64-dlfcn - mingw-w64-ucrt-x86_64-libvpx - mingw-w64-ucrt-x86_64-opus - mingw-w64-ucrt-x86_64-libx264 - mingw-w64-ucrt-x86_64-SDL2 - mingw-w64-ucrt-x86_64-libyuv - mingw-w64-ucrt-x86_64-libjpeg-turbo - mingw-w64-ucrt-x86_64-speexdsp + - uses: msys2/setup-msys2@v2 + if: matrix.os == 'windows-latest' + with: + msystem: ucrt64 + path-type: inherit + release: false + install: > + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-dlfcn + mingw-w64-ucrt-x86_64-SDL2 + mingw-w64-ucrt-x86_64-gstreamer + mingw-w64-ucrt-x86_64-gst-plugins-base + mingw-w64-ucrt-x86_64-gst-plugins-good - - name: Windows - if: matrix.os == 'windows-latest' - env: - MESA_VERSION: '24.0.7' - MESA_GL_VERSION_OVERRIDE: 3.3COMPAT - shell: msys2 {0} - run: | - set MSYSTEM=UCRT64 - - wget -q https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z - "/c/Program Files/7-Zip/7z.exe" x mesa3d-$MESA_VERSION-release-msvc.7z -omesa - echo -e " 1\r\n 9\r\n " >> commands - ./mesa/systemwidedeploy.cmd < ./commands - - make build test verify-cores + - name: Windows + if: matrix.os == 'windows-latest' + env: + MESA_VERSION: "24.0.7" + MESA_GL_VERSION_OVERRIDE: 3.3COMPAT + shell: msys2 {0} + run: | + set MSYSTEM=UCRT64 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: emulator-test-frames-${{ matrix.os }} - path: _rendered/*.png + wget -q https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z + "/c/Program Files/7-Zip/7z.exe" x mesa3d-$MESA_VERSION-release-msvc.7z -omesa + echo -e " 1\r\n 9\r\n " >> commands + ./mesa/systemwidedeploy.cmd < ./commands + + make build test verify-cores + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: emulator-test-frames-${{ matrix.os }} + path: _rendered/*.png diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index e13c330c..7ba231eb 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -1,80 +1,48 @@ # ---------------------------------------------------------------------------------- -# Publish Docker images from the current master branch or v* into Github repository +# Publish Docker image from the current master branch or v* into Github repository # ---------------------------------------------------------------------------------- name: docker-publish on: - push: - branches: - - master - tags: - - "v*" - workflow_dispatch: + push: + branches: + - master + tags: + - 'v*' jobs: - docker-publish: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v3 - - run: echo "V=$(./scripts/version.sh)" >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta (full) - id: meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/${{ github.repository }}/cloud-game - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - - uses: docker/build-push-action@v4 - with: - build-args: VERSION=${{ env.V }} - context: . - push: true - provenance: false - sbom: false - tags: | - ${{ steps.meta.outputs.tags }} - labels: | - ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Docker meta (worker no-pgo) - id: meta-nopgo - uses: docker/metadata-action@v4 - with: - images: ghcr.io/${{ github.repository }}/cloud-game-worker - tags: | - type=ref,event=branch,suffix=-nopgo - type=semver,pattern={{version}},suffix=-nopgo - type=semver,pattern={{major}}.{{minor}},suffix=-nopgo - type=semver,pattern={{major}},suffix=-nopgo - - - uses: docker/build-push-action@v4 - with: - file: Dockerfile.worker - build-args: VERSION=${{ env.V }} - context: . - push: true - provenance: false - sbom: false - tags: | - ${{ steps.meta-nopgo.outputs.tags }} - labels: | - ${{ steps.meta-nopgo.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + docker-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: echo "V=$(./scripts/version.sh)" >> $GITHUB_ENV + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }}/cloud-game + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + - uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v4 + with: + build-args: VERSION=${{ env.V }} + context: . + push: true + provenance: false + sbom: false + tags: | + ${{ steps.meta.outputs.tags }} + labels: | + ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index efa3b4ce..03e0ba29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,79 @@ ARG BUILD_PATH=/tmp/cloud-game ARG VERSION=master +# gstreamer minimal build +FROM ubuntu:resolute AS gst-builder +ARG GST_VERSION=1.28.4 +ENV DEBIAN_FRONTEND=noninteractive + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get -q update && apt-get -q install --no-install-recommends -y \ + build-essential ca-certificates curl flex bison git \ + meson ninja-build pkg-config python3 \ + libglib2.0-dev libffi-dev zlib1g-dev libssl-dev \ + libopus-dev libvpx-dev liborc-0.4-dev + +WORKDIR /gst + +RUN git clone --single-branch --depth 1 --branch ${GST_VERSION} \ + https://gitlab.freedesktop.org/gstreamer/gstreamer.git . + +# fix meson build error +RUN sed -i '1i if get_option('"'"'doc'"'"').disabled()\n subdir_done()\nendif' \ + subprojects/gst-plugins-good/docs/meson.build + +RUN meson setup builddir \ + --prefix=/usr/local \ + --buildtype=release \ + -Doptimization=3 \ + -Db_lto=true \ + -Dauto_features=disabled \ + -Dtools=disabled \ + -Dgstreamer:tools=disabled \ + -Dbase=enabled \ + -Dgood=enabled \ + -Dbad=disabled \ + -Dugly=disabled \ + -Dlibav=disabled \ + -Dges=disabled \ + -Drtsp_server=disabled \ + -Drs=disabled \ + -Dpython=disabled \ + -Dsharp=disabled \ + -Dintrospection=disabled \ + -Ddoc=disabled \ + -Dtests=disabled \ + -Dexamples=disabled \ + -Dnls=disabled \ + -Dorc=enabled \ + -Dgpl=disabled \ + -Dlibnice=disabled \ + -Dtls=disabled \ + -Dgst-plugins-base:app=enabled \ + -Dgst-plugins-base:videoconvertscale=enabled \ + -Dgst-plugins-base:audioresample=enabled \ + -Dgst-plugins-base:opus=enabled \ + -Dgst-plugins-good:vpx=enabled && \ + meson compile -C builddir && \ + meson install -C builddir && \ + find /usr/local/lib -name "*.so*" -exec strip {} \; + # base build stage FROM ubuntu:resolute AS build0 -ARG GO=1.26.3 +ENV DEBIAN_FRONTEND=noninteractive +ARG GO=1.26.4 ARG GO_DIST=go${GO}.linux-amd64.tar.gz ADD https://go.dev/dl/$GO_DIST ./ -RUN tar -C /usr/local -xzf $GO_DIST && rm $GO_DIST && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + tar -C /usr/local -xzf $GO_DIST && rm $GO_DIST && \ apt-get -q update && apt-get -q install --no-install-recommends -y \ ca-certificates \ make \ - upx \ - && rm -rf /var/lib/apt/lists/* + upx && \ + rm -rf /var/lib/apt/lists/* ENV PATH="${PATH}:/usr/local/go/bin" # coordinator build stage @@ -22,9 +83,11 @@ ARG VERSION ENV GIT_VERSION=${VERSION} WORKDIR ${BUILD_PATH} +COPY go.mod go.sum ./ +RUN go mod download COPY . ./ RUN --mount=type=cache,target=/root/.cache/go-build make build.coordinator && \ - find ./bin/* | xargs upx --best --lzma + find ./bin/* | xargs upx -9 WORKDIR /usr/local/share/cloud-game RUN mv ${BUILD_PATH}/bin/* ./ && \ @@ -39,22 +102,25 @@ ARG BUILD_PATH ARG VERSION ENV GIT_VERSION=${VERSION} -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ +COPY --from=gst-builder /usr/local /usr/local + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get -q update && apt-get -q install --no-install-recommends -y \ build-essential \ - libopus-dev \ libsdl2-dev \ - libvpx-dev \ - libyuv-dev \ - libjpeg-turbo8-dev \ - libx264-dev \ - libspeexdsp-dev \ pkg-config \ - && rm -rf /var/lib/apt/lists/* + libglib2.0-dev && \ + rm -rf /var/lib/apt/lists/* + +ENV PKG_CONFIG_PATH=/usr/local/lib/x86_64-linux-gnu/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig WORKDIR ${BUILD_PATH} +COPY go.mod go.sum ./ +RUN go mod download COPY . ./ -RUN --mount=type=cache,target=/root/.cache/go-build make GO_TAGS=static,st,x264s build.worker && \ - find ./bin/* | xargs upx --best --lzma +RUN --mount=type=cache,target=/root/.cache/go-build make GO_TAGS=static build.worker && \ + find ./bin/* | xargs upx -9 WORKDIR /usr/local/share/cloud-game RUN mv ${BUILD_PATH}/bin/* ./ && \ @@ -65,27 +131,41 @@ RUN mv ${BUILD_PATH}/bin/* ./ && \ FROM scratch AS coordinator COPY --from=build_coordinator /usr/local/share/cloud-game /cloud-game -# autocertbot (SSL) requires these on the first run COPY --from=build_coordinator /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ # worker runtime FROM ubuntu:resolute AS worker +ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get -q update && apt-get -q install --no-install-recommends -y \ curl \ + libglib2.0-0 \ + libopus0 \ + libvpx12 \ + liborc-0.4-0 \ libx11-6 \ - libxext6 \ - libx264-165 \ - && apt-get autoremove \ - && rm -rf /var/lib/apt/lists/* /var/log/* /usr/share/bug /usr/share/doc /usr/share/doc-base \ + libxext6 && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /var/log/* /usr/share/bug /usr/share/doc /usr/share/doc-base \ /usr/share/X11/locale/* +COPY --from=gst-builder /usr/local/lib/x86_64-linux-gnu/libgst* /usr/local/lib/x86_64-linux-gnu/ +COPY --from=gst-builder /usr/local/lib/x86_64-linux-gnu/gstreamer-1.0/ /usr/local/lib/x86_64-linux-gnu/gstreamer-1.0/ + +ENV LD_LIBRARY_PATH=/usr/local/lib/x86_64-linux-gnu +ENV GST_PLUGIN_PATH=/usr/local/lib/x86_64-linux-gnu/gstreamer-1.0 + COPY --from=build_worker /usr/local/share/cloud-game /cloud-game COPY --from=build_worker /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ ADD https://github.com/sergystepanov/mesa-llvmpipe/releases/download/v1.0.0/libGL.so.1.5.0 \ /usr/lib/x86_64-linux-gnu/ -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libGL.so.1.5.0 libGL.so.1 && ln -s libGL.so.1 libGL.so +RUN cd /usr/lib/x86_64-linux-gnu && \ + rm -f libGL.so.1 libGL.so && \ + ln -s libGL.so.1.5.0 libGL.so.1 && \ + ln -s libGL.so.1 libGL.so # final image FROM worker AS cloud-game diff --git a/Dockerfile.worker b/Dockerfile.worker deleted file mode 100644 index eb51d9ae..00000000 --- a/Dockerfile.worker +++ /dev/null @@ -1,66 +0,0 @@ -ARG BUILD_PATH=/tmp/cloud-game -ARG VERSION=master - -# base build stage -FROM ubuntu:resolute AS build0 -ARG GO=1.26.3 -ARG GO_DIST=go${GO}.linux-amd64.tar.gz - -ADD https://go.dev/dl/$GO_DIST ./ -RUN tar -C /usr/local -xzf $GO_DIST && rm $GO_DIST && \ - apt-get -q update && apt-get -q install --no-install-recommends -y \ - ca-certificates \ - make \ - upx \ - && rm -rf /var/lib/apt/lists/* -ENV PATH="${PATH}:/usr/local/go/bin" - -# worker build stage (no PGO) -FROM build0 AS build_worker -ARG BUILD_PATH -ARG VERSION -ENV GIT_VERSION=${VERSION} - -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - build-essential \ - libopus-dev \ - libsdl2-dev \ - libvpx-dev \ - libyuv-dev \ - libjpeg-turbo8-dev \ - libx264-dev \ - libspeexdsp-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR ${BUILD_PATH} -COPY . ./ -RUN --mount=type=cache,target=/root/.cache/go-build make GO_TAGS=static,st,x264s build.worker-nopgo && \ - find ./bin/* | xargs upx --best --lzma - -WORKDIR /usr/local/share/cloud-game -RUN mv ${BUILD_PATH}/bin/* ./ && \ - mv ${BUILD_PATH}/LICENSE ./ && \ - ${BUILD_PATH}/scripts/mkdirs.sh worker - -# worker runtime -FROM ubuntu:resolute AS worker - -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - curl \ - libx11-6 \ - libxext6 \ - libx264-165 \ - && apt-get autoremove \ - && rm -rf /var/lib/apt/lists/* /var/log/* /usr/share/bug /usr/share/doc /usr/share/doc-base \ - /usr/share/X11/locale/* - -COPY --from=build_worker /usr/local/share/cloud-game /usr/local/share/cloud-game -COPY --from=build_worker /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - -ADD https://github.com/sergystepanov/mesa-llvmpipe/releases/download/v1.0.0/libGL.so.1.5.0 \ - /usr/lib/x86_64-linux-gnu/ -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libGL.so.1.5.0 libGL.so.1 && ln -s libGL.so.1 libGL.so - -WORKDIR /usr/local/share/cloud-game -CMD ["./worker"] diff --git a/LICENSE b/LICENSE index f1d7177f..091d6558 100644 --- a/LICENSE +++ b/LICENSE @@ -199,7 +199,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + ------------------------------------------------------------------------------------ This product bundles various third-party components under other open source licenses. @@ -209,5 +209,24 @@ MIT License https://github.com/pion/webrtc/ https://github.com/fogleman/nes https://github.com/poi5305/go-yuv2webRTC -https://github.com/bchanx/animated-gameboy-in-css +https://github.com/bchanx/animated-gameboy-in-css +LGPL-2.1+ +--------- + +GStreamer core and plugins (dynamically linked at runtime): +https://gstreamer.freedesktop.org/ +- gstreamer (core) +- gst-plugins-base (videoconvert, videoscale, opusenc, audioresample, appsrc/sink) +- gst-plugins-good (vp8enc, vp9enc) +- go-gst bindings: https://github.com/go-gst/go-gst + +SDL2 (Zlib) +----------- + +https://github.com/libsdl-org/SDL + +libretro cores (various licenses) +--------------------------------- +Individual emulator cores are loaded as shared libraries at runtime. +See each core's respective license. diff --git a/Makefile b/Makefile index a5037e4e..24986cd3 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,6 @@ build.worker: -ldflags "-w -s -X 'main.Version=$(GIT_VERSION)'" $(EXT_WFLAGS) \ -o bin/ ./cmd/worker -build.worker-nopgo: - mkdir -p bin/ - CGO_CFLAGS=${CGO_CFLAGS} CGO_LDFLAGS=${CGO_LDFLAGS} \ - go build $(TRIMPATH) -buildmode=exe $(if $(GO_TAGS),-tags $(GO_TAGS),) \ - -ldflags "-w -s -X 'main.Version=$(GIT_VERSION)'" $(EXT_WFLAGS) \ - -o bin/ ./cmd/worker - build: build.coordinator build.worker test: @@ -57,11 +50,18 @@ dev.build-local: dev.run: dev.build-local ifeq ($(OS),Windows_NT) - ./bin/coordinator.exe & ./bin/worker.exe + ./bin/coordinator.exe & ./bin/worker.exe else ./bin/coordinator & ./bin/worker endif +dev.run.trace: dev.build-local +ifeq ($(OS),Windows_NT) + ./bin/coordinator.exe & GST_DEBUG_COLOR_MODE=off GST_TRACERS="latency(flags=pipeline+element)" GST_DEBUG=GST_TRACER:7 GST_DEBUG_FILE=traces.log ./bin/worker.exe +else + GST_DEBUG_COLOR_MODE=off GST_TRACERS="latency(flags=pipeline+element)" GST_DEBUG=GST_TRACER:7 GST_DEBUG_FILE=traces.log ./bin/worker & ./bin/coordinator +endif + dev.run.debug: go build -race -o bin/ ./cmd/coordinator CGO_CFLAGS=${CGO_CFLAGS} CGO_LDFLAGS=${CGO_LDFLAGS} \ diff --git a/README.md b/README.md index 1054d2ab..25391f2c 100644 --- a/README.md +++ b/README.md @@ -54,26 +54,19 @@ a better sense of performance. ## Development environment * Install [Go](https://golang.org/doc/install) -* Install [libvpx](https://www.webmproject.org/code/), [libx264](https://www.videolan.org/developers/x264.html) - , [libopus](http://opus-codec.org/), [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) - , [sdl2](https://wiki.libsdl.org/Installation), [libyuv](https://chromium.googlesource.com/libyuv/libyuv/)+[libjpeg-turbo](https://github.com/libjpeg-turbo/libjpeg-turbo) +* Install GStreamer and SDL2: ``` # Ubuntu / Windows (WSL2) -apt-get install -y make gcc pkg-config libvpx-dev libx264-dev libopus-dev libsdl2-dev libyuv-dev libjpeg-turbo8-dev libspeexdsp-dev +apt-get install -y make gcc pkg-config libsdl2-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-plugins-good -# MacOS -brew install pkg-config libvpx x264 opus sdl2 jpeg-turbo speexdsp +# macOS +brew install pkg-config sdl2 gstreamer gst-plugins-base gst-plugins-good # Windows (MSYS2) -pacman -Sy --noconfirm --needed git make mingw-w64-ucrt-x86_64-{gcc,pkgconf,dlfcn,libvpx,opus,libx264,SDL2,libyuv,libjpeg-turbo,speexdsp} +pacman -Sy --noconfirm --needed git make mingw-w64-ucrt-x86_64-{gcc,pkgconf,dlfcn,SDL2,gstreamer,gst-plugins-base,gst-plugins-good} ``` -(You don't need to download libyuv on macOS) - -(If you need to use the app on an older version of Ubuntu that does not have libyuv (when it says: unable to locate package libyuv-dev), you can add a custom apt repository: -`add sudo add-apt-repository ppa:savoury1/graphics`) - Because the coordinator and workers need to run simultaneously. Workers connect to the coordinator. 1. Script @@ -97,7 +90,7 @@ It will spawn a docker environment and you can access the service on `localhost: ## Configuration -The default configuration file is stored in the [`pkg/configs/config.yaml`](pkg/config/config.yaml) file. +The default configuration file is stored in the [`pkg/config/config.yaml`](pkg/config/config.yaml) file. This configuration file will be embedded into the applications and loaded automatically during startup. In order to change the default parameters you can specify environment variables with the `CLOUD_GAME_` prefix, or place a custom `config.yaml` file into one of these places: just near the application, `.cr` folder in user's home, or diff --git a/cmd/worker/main.go b/cmd/worker/main.go index a20cd819..a1b2db4d 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -1,8 +1,6 @@ package main import ( - "time" - "github.com/giongto35/cloud-game/v3/pkg/config" "github.com/giongto35/cloud-game/v3/pkg/logger" "github.com/giongto35/cloud-game/v3/pkg/os" @@ -31,7 +29,6 @@ func run() { } w.Start(done) <-done - time.Sleep(100 * time.Millisecond) // hack if err := w.Stop(); err != nil { log.Error().Err(err).Msg("shutdown fail") } diff --git a/docker-compose.yml b/docker-compose.yml index dff2c59a..ab9bdc63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,33 +1,39 @@ services: + cloud-game: + build: . + image: cloud-game-local + container_name: cloud-game-local + depends_on: + - xvfb + environment: + - DISPLAY=:99 + - MESA_GL_VERSION_OVERRIDE=4.5 + - CLOUD_GAME_WEBRTC_SINGLEPORT=8443 + # - CLOUD_GAME_WEBRTC_ICEIPMAP=127.0.0.1 + - CLOUD_GAME_COORDINATOR_DEBUG=true + - CLOUD_GAME_WORKER_DEBUG=true + - CLOUD_GAME_ENCODER_VIDEO_CODEC=vp8 + # - PION_LOG_TRACE=all + ports: + - "8000:8000" + - "9000:9000" + - "8443:8443/udp" + command: > + bash -c "./coordinator & while [ ! -S /tmp/.X11-unix/X99 ]; do sleep 0.1; done; ./worker" + volumes: + - ./assets/cores:/usr/local/share/cloud-game/assets/cores + - ./assets/games:/usr/local/share/cloud-game/assets/games + - x11:/tmp/.X11-unix - cloud-game: - build: . - image: cloud-game-local - container_name: cloud-game-local - environment: - - DISPLAY=:99 - - MESA_GL_VERSION_OVERRIDE=4.5 - - CLOUD_GAME_WEBRTC_SINGLEPORT=8443 - # - CLOUD_GAME_WEBRTC_ICEIPMAP=127.0.0.1 - - CLOUD_GAME_COORDINATOR_DEBUG=true - - CLOUD_GAME_WORKER_DEBUG=true - # - PION_LOG_TRACE=all - ports: - - "8000:8000" - - "9000:9000" - - "8443:8443/udp" - command: > - bash -c "./coordinator & ./worker" - volumes: - - ./assets/cores:/usr/local/share/cloud-game/assets/cores - - ./assets/games:/usr/local/share/cloud-game/assets/games - - x11:/tmp/.X11-unix - - xvfb: - image: kcollins/xvfb:latest - volumes: - - x11:/tmp/.X11-unix - command: [ ":99", "-screen", "0", "320x240x16" ] + xvfb: + image: kcollins/xvfb:latest + entrypoint: ["sh", "-c"] + command: + [ + "rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 && exec Xvfb :99 -screen 0 320x240x16", + ] + volumes: + - x11:/tmp/.X11-unix volumes: - x11: + x11: diff --git a/go.mod b/go.mod index cfd74eb7..94bcb74e 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,13 @@ module github.com/giongto35/cloud-game/v3 go 1.26 require ( - github.com/VictoriaMetrics/metrics v1.43.2 + github.com/VictoriaMetrics/metrics v1.44.0 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/fsnotify/fsnotify v1.10.1 - github.com/goccy/go-json v0.10.6 + github.com/go-gst/go-gst v1.4.0 github.com/gofrs/flock v0.13.0 github.com/gorilla/websocket v1.5.3 + github.com/klauspost/cpuid/v2 v2.3.0 github.com/knadh/koanf/maps v0.1.2 github.com/knadh/koanf/v2 v2.3.5 github.com/minio/minio-go/v7 v7.2.0 @@ -27,28 +28,29 @@ require ( require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-gst/go-glib v1.4.0 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.18.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pion/datachannel v1.6.0 // indirect + github.com/pion/datachannel v1.6.2 // indirect github.com/pion/dtls/v3 v3.1.4 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.16 // indirect github.com/pion/rtp v1.10.2 // indirect - github.com/pion/sctp v1.10.0 // indirect - github.com/pion/sdp/v3 v3.0.18 // indirect - github.com/pion/srtp/v3 v3.0.11 // indirect - github.com/pion/stun/v3 v3.1.5 // indirect + github.com/pion/sctp v1.10.1 // indirect + github.com/pion/sdp/v3 v3.0.19 // indirect + github.com/pion/srtp/v3 v3.0.12 // indirect + github.com/pion/stun/v3 v3.1.6 // indirect github.com/pion/transport/v4 v4.0.2 // indirect github.com/pion/turn/v5 v5.0.10 // indirect github.com/tinylib/msgp v1.6.4 // indirect @@ -57,6 +59,7 @@ require ( github.com/wlynxg/anet v0.0.5 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect diff --git a/go.sum b/go.sum index b5b240c0..516ed7ad 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/VictoriaMetrics/metrics v1.43.2 h1:+8pIQEGwchKS5CYFyvv3LKvNXGi7baZ9hmIV4RHqibY= -github.com/VictoriaMetrics/metrics v1.43.2/go.mod h1:xDM82ULLYCYdFRgQ2JBxi8Uf1+8En1So9YUwlGTOqTc= +github.com/VictoriaMetrics/metrics v1.44.0 h1:Fr8yqQSV+ZfYaDD/anqk1E8e9YPgfleSleJmAI0M0Tw= +github.com/VictoriaMetrics/metrics v1.44.0/go.mod h1:xDM82ULLYCYdFRgQ2JBxi8Uf1+8En1So9YUwlGTOqTc= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -11,10 +11,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-gst/go-glib v1.4.0 h1:FB2uVfB0uqz7/M6EaDdWWlBZRQpvFAbWfL7drdw8lAE= +github.com/go-gst/go-glib v1.4.0/go.mod h1:GUIpWmkxQ1/eL+FYSjKpLDyTZx6Vgd9nNXt8dA31d5M= +github.com/go-gst/go-gst v1.4.0 h1:EikB43u4c3wc8d2RzlFRSfIGIXYzDy6Zls2vJqrG2BU= +github.com/go-gst/go-gst v1.4.0/go.mod h1:p8TLGtOxJLcrp6PCkTPdnanwWBxPZvYiHDbuSuwgO3c= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= -github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -40,6 +42,8 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -52,8 +56,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0= -github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk= +github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= +github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= github.com/pion/dtls/v3 v3.1.4 h1:QhvtMflMfu9Kf0RcDC5BJBle4caPskByrKQR6uuYqpY= github.com/pion/dtls/v3 v3.1.4/go.mod h1:cr/qotLISUw/9C1m83ZPNZtj9WnXkYLpfCptPqbkInc= github.com/pion/ice/v4 v4.2.7 h1:zDEbC6MiEdhQpF8TxBOTws+NU6ZgGpveHrQq4Lc1kao= @@ -70,14 +74,14 @@ github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= github.com/pion/rtp v1.10.2 h1:l+f6tTDcAH6xwepaAoW791ddhuYsJlqRATOzirO04Mo= github.com/pion/rtp v1.10.2/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= -github.com/pion/sctp v1.10.0 h1:qeoD6swF/2M5bYRcAGayqSbTKX3m4AW29CiQxG1+Pfg= -github.com/pion/sctp v1.10.0/go.mod h1:N20Dq6LY+JvJDAh9VVh1JELngb2rQ8dPgds5yBWiPgw= -github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= -github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= -github.com/pion/srtp/v3 v3.0.11 h1:GiESUr54/K4UuPigfq/CvWUed80JenQAHXn0C2MQQIQ= -github.com/pion/srtp/v3 v3.0.11/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= -github.com/pion/stun/v3 v3.1.5 h1:Y1FHlhaI6+4UoC5i/zQf4F7JvdZtB24/05oyy/GF1x8= -github.com/pion/stun/v3 v3.1.5/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= +github.com/pion/sctp v1.10.1 h1:IDwqRiteu1gqPnqYysDLxVTwIOMFR0zvwOS6cjhzX7k= +github.com/pion/sctp v1.10.1/go.mod h1:N20Dq6LY+JvJDAh9VVh1JELngb2rQ8dPgds5yBWiPgw= +github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= +github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= +github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc= +github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8= +github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= @@ -121,6 +125,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= diff --git a/pkg/api/worker.go b/pkg/api/worker.go index a071fa17..d8c73246 100644 --- a/pkg/api/worker.go +++ b/pkg/api/worker.go @@ -57,9 +57,11 @@ type ( AppVideoInfo struct { W int `json:"w"` H int `json:"h"` - S int `json:"s"` A float32 `json:"a"` - Flip bool `json:"flip,omitempty"` + S int `json:"s"` + SM string `json:"sm,omitempty"` + Flip bool `json:"flip"` + Rot uint `json:"rot"` } LibGameListInfo struct { diff --git a/pkg/com/net.go b/pkg/com/net.go index 722ce9b5..5b2e0f2a 100644 --- a/pkg/com/net.go +++ b/pkg/com/net.go @@ -1,6 +1,7 @@ package com import ( + "encoding/json" "errors" "fmt" "net/http" @@ -8,7 +9,6 @@ import ( "time" "github.com/giongto35/cloud-game/v3/pkg/network/websocket" - "github.com/goccy/go-json" "github.com/rs/xid" ) diff --git a/pkg/config/config.yaml b/pkg/config/config.yaml index bf9feda2..61ee209d 100644 --- a/pkg/config/config.yaml +++ b/pkg/config/config.yaml @@ -216,7 +216,20 @@ emulator: # - altRepo (bool) prioritize secondary repo as the download source # - lib (string) # - roms ([]string) - # - scale (int) scales the output video frames by this factor. + # - scale (int) [0 - none] + # scales the output video frames by this factor. + # - scaleMethod (string) [bilinear2] -- Gstreamer method of scaling + # nearest-neighbour (0) – Nearest Neighbour + # bilinear (1) – Bilinear (2-tap) + # 4-tap (2) – 4-tap Sinc + # lanczos (3) – Lanczos + # bilinear2 (4) – Bilinear (multi-tap) + # sinc (5) – Sinc (multi-tap) + # hermite (6) – Hermite (multi-tap) + # spline (7) – Spline (multi-tap) + # catrom (8) – Catmull-Rom (multi-tap) + # mitchell (9) – Mitchell (multi-tap) + # - maxThreads (int) [0] -- cap the number of threads used by the encoder # - folder (string) # By default emulator selection is based on the folder named as cores # in the list (i.e. nes, snes) but if you specify folder param, @@ -259,9 +272,11 @@ emulator: gba: lib: mgba_libretro roms: ["gba", "gbc"] + maxThreads: 4 options: - mgba_audio_low_pass_filter: disabled - mgba_audio_low_pass_range: 50 + mgba_audio_low_pass_filter: enabled + mgba_audio_low_pass_range: 60 + # mgba_frameskip: auto pcsx: lib: pcsx_rearmed_libretro roms: ["cue", "chd"] @@ -269,20 +284,25 @@ emulator: folder: psx # see: https://github.com/libretro/pcsx_rearmed/blob/master/frontend/libretro_core_options.h options: - "pcsx_rearmed_show_bios_bootlogo": enabled - "pcsx_rearmed_drc": enabled - "pcsx_rearmed_display_internal_fps": disabled + pcsx_rearmed_show_bios_bootlogo: enabled + pcsx_rearmed_drc: enabled + pcsx_rearmed_frameskip_type: auto + pcsx_rearmed_spu_interpolation: gaussian + pcsx_rearmed_rgb32_output: enabled # MAME core requires additional manual setup, please read: # https://docs.libretro.com/library/fbneo/ mame: lib: fbneo_libretro folder: mame roms: ["zip"] + scaleMethod: bilinear2 nes: lib: nestopia_libretro roms: ["nes"] options: - nestopia_aspect: "uncorrected" + nestopia_aspect: "4:3" + nestopia_overscan_v_top: 0 + nestopia_overscan_v_bottom: 0 snes: lib: snes9x_libretro roms: ["smc", "sfc", "swc", "fig", "bs"] @@ -328,57 +348,45 @@ emulator: 3: [257, 513] options: "dosbox_pure_conf": "outside" - "dosbox_pure_force60fps": "true" + # "dosbox_pure_force60fps": "true" encoder: audio: - # audio frame duration needed for WebRTC (Opus) - # most of the emulators have ~1400 samples per a video frame, - # so we keep the frame buffer roughly half of that size or 2 RTC packets per frame - # (deprecated) due to frames - frame: 10 - # dynamic frames for Opus encoder - frames: - - 10 - # speex (2), linear (1) or nearest neighbour (0) audio resampler - # linear should sound slightly better than 0 + codec: opus + # resampler qualiy: 0 - low, 1 - medium, 2 - best resampler: 2 video: - # h264, vpx (vp8) or vp9 - codec: h264 + # vp8 | vp9 + codec: vp8 # Threaded encoder if supported, 0 - auto, 1 - nope, >1 - multi-threaded threads: 0 - # see: https://trac.ffmpeg.org/wiki/Encode/H.264 - h264: - # crf, cbr - mode: crf - # Constant Rate Factor (CRF) 0-51 (default: 23) - crf: 23 - # Rate control options - # set the maximum bitrate - maxRate: 0 - # set the expected client buffer size - bufSize: 0 - # ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo - preset: superfast - # baseline, main, high, high10, high422, high444 - profile: baseline - # film, animation, grain, stillimage, psnr, ssim, fastdecode, zerolatency - tune: zerolatency - # 0-3 - logLevel: 0 - # see: https://www.webmproject.org/docs/encoder-parameters - vpx: - # target bitrate (KBit/s) - bitrate: 1200 - # force keyframe interval - keyframeInterval: 120 - # speed/quality tradeoff (0=auto, 1=slow, 8=fast) - cpuUsed: 0 - # log2 of tile columns (0=auto, 0-6). VP9 only. - tileColumns: 0 - # content tuning: "screen" for games. VP9 only. - tune: screen + # max interval between keyframes (0 = encoder default) + keyframeInterval: 300 + # colorimetry caps: bt601 | bt709 | smpte240m + colorimetry: bt709,chroma-site=jpeg + list: + opus: + encoder: opusenc + params: audio-type=restricted-lowdelay complexity=8 bitrate=96000 frame-size=5 + # h264 (requires gstreamer1.0-plugins-ugly + x264 [GPL]): + # encoder: x264enc + # params: speed-preset=superfast tune=zerolatency pass=quant quantizer=20 + # caps: video/x-h264,stream-format=byte-stream,alignment=au + vp8: + encoder: vp8enc + params: >- + keyframe-mode=disabled tuning=ssim cpu-used=6 target-bitrate=3200000 deadline=1 sharpness=1 end-usage=cbr + static-threshold=100 + error-resilient=default+partitions overshoot=20 undershoot=100 + caps: video/x-vp8 + vp9: + encoder: vp9enc + params: >- + keyframe-mode=disabled tuning=ssim cpu-used=6 target-bitrate=3200000 deadline=1 sharpness=1 end-usage=cbr + static-threshold=100 + error-resilient=default+partitions overshoot=20 undershoot=100 + tile-columns=6 lag-in-frames=0 row-mt=true + caps: video/x-vp9 # game recording # (experimental) diff --git a/pkg/config/emulator.go b/pkg/config/emulator.go index 6a0ad9bb..baf701e9 100644 --- a/pkg/config/emulator.go +++ b/pkg/config/emulator.go @@ -86,6 +86,8 @@ type LibretroCoreConfig struct { Roms []string SaveStateFs string Scale float64 + ScaleMethod string + MaxThreads int UniqueSaveDir bool UsesLibCo bool VFR bool diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 08e17dd3..33772293 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -9,10 +9,8 @@ import ( func TestConfigEnv(t *testing.T) { var out WorkerConfig - _ = os.Setenv("CLOUD_GAME_ENCODER_AUDIO_FRAMES[0]", "10") - _ = os.Setenv("CLOUD_GAME_ENCODER_AUDIO_FRAMES[1]", "5") - defer func() { _ = os.Unsetenv("CLOUD_GAME_ENCODER_AUDIO_FRAMES[0]") }() - defer func() { _ = os.Unsetenv("CLOUD_GAME_ENCODER_AUDIO_FRAMES[1]") }() + _ = os.Setenv("CLOUD_GAME_ENCODER_AUDIO_FRAME", "10") + defer func() { _ = os.Unsetenv("CLOUD_GAME_ENCODER_AUDIO_FRAME") }() _ = os.Setenv("CLOUD_GAME_EMULATOR_LIBRETRO_CORES_LIST_PCSX_OPTIONS__PCSX_REARMED_DRC", "x") defer func() { @@ -24,11 +22,9 @@ func TestConfigEnv(t *testing.T) { t.Fatal(err) } - for i, x := range []float32{10, 5} { - if out.Encoder.Audio.Frames[i] != x { - t.Errorf("%v is not [10, 5]", out.Encoder.Audio.Frames) - t.Failed() - } + if out.Encoder.Audio.Frame != 10 { + t.Errorf("%v is not [10]", out.Encoder.Audio.Frame) + t.Failed() } v := out.Emulator.Libretro.Cores.List["pcsx"].Options["pcsx_rearmed_drc"] diff --git a/pkg/config/worker.go b/pkg/config/worker.go index 9a156430..be5ac424 100644 --- a/pkg/config/worker.go +++ b/pkg/config/worker.go @@ -48,10 +48,28 @@ type Worker struct { type Encoder struct { Audio Audio Video Video + List map[string]CodecSettings +} + +func (e Encoder) AudioSettings() *CodecSettings { + codec := strings.ToLower(e.Audio.Codec) + if cs, ok := e.List[codec]; ok { + return &cs + } + return nil +} + +func (e Encoder) VideoSettings() (*CodecSettings, string) { + codec := strings.ToLower(e.Video.Codec) + if cs, ok := e.List[codec]; ok { + return &cs, codec + } + return nil, "" } type Audio struct { - Frames []float32 + Codec string + Frame int Resampler int } @@ -59,22 +77,13 @@ type Video struct { Codec string KeyframeInterval int Threads int - H264 struct { - Mode string - Crf uint8 - MaxRate int - BufSize int - LogLevel int32 - Preset string - Profile string - Tune string - } - Vpx struct { - Bitrate uint - CpuUsed int - TileColumns int - Tune string - } + Colorimetry string +} + +type CodecSettings struct { + Encoder string + Params string + Caps string } // allows custom config path @@ -113,7 +122,7 @@ func (c *WorkerConfig) expandSpecialTags() { if err != nil { panic(fmt.Sprintf("couldn't read user home directory, %v", err)) } - *dir = strings.Replace(*dir, tag, userHomeDir, -1) + *dir = strings.ReplaceAll(*dir, tag, userHomeDir) *dir = filepath.FromSlash(*dir) } } diff --git a/pkg/encoder/color/bgra/bgra.go b/pkg/encoder/color/bgra/bgra.go deleted file mode 100644 index 39a50c22..00000000 --- a/pkg/encoder/color/bgra/bgra.go +++ /dev/null @@ -1,56 +0,0 @@ -package bgra - -import ( - "image" - "image/color" -) - -type BGRA struct { - image.RGBA -} - -var BGRAModel = color.ModelFunc(func(c color.Color) color.Color { - if _, ok := c.(BGRAColor); ok { - return c - } - r, g, b, a := c.RGBA() - return BGRAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} -}) - -// BGRAColor represents a BGRA color. -type BGRAColor struct { - R, G, B, A uint8 -} - -func (c BGRAColor) RGBA() (r, g, b, a uint32) { - r = uint32(c.B) - r |= r << 8 - g = uint32(c.G) - g |= g << 8 - b = uint32(c.R) - b |= b << 8 - a = uint32(255) //uint32(c.A) - a |= a << 8 - return -} - -func NewBGRA(r image.Rectangle) *BGRA { - return &BGRA{*image.NewRGBA(r)} -} - -func (p *BGRA) ColorModel() color.Model { return BGRAModel } -func (p *BGRA) At(x, y int) color.Color { - i := p.PixOffset(x, y) - s := p.Pix[i : i+4 : i+4] - return BGRAColor{s[0], s[1], s[2], s[3]} -} - -func (p *BGRA) Set(x, y int, c color.Color) { - i := p.PixOffset(x, y) - c1 := BGRAModel.Convert(c).(BGRAColor) - s := p.Pix[i : i+4 : i+4] - s[0] = c1.R - s[1] = c1.G - s[2] = c1.B - s[3] = 255 -} diff --git a/pkg/encoder/color/rgb565/rgb565.go b/pkg/encoder/color/rgb565/rgb565.go deleted file mode 100644 index 11c66c8b..00000000 --- a/pkg/encoder/color/rgb565/rgb565.go +++ /dev/null @@ -1,62 +0,0 @@ -package rgb565 - -import ( - "encoding/binary" - "image" - "image/color" - "math" -) - -// RGB565 is an in-memory image whose At method returns RGB565 values. -type RGB565 struct { - // Pix holds the image's pixels, as RGB565 values in big-endian format. The pixel at - // (x, y) starts at Pix[(y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2]. - Pix []uint8 - // Stride is the Pix stride (in bytes) between vertically adjacent pixels. - Stride int - // Rect is the image's bounds. - Rect image.Rectangle -} - -// Model is the model for RGB565 colors. -var Model = color.ModelFunc(func(c color.Color) color.Color { - //if _, ok := c.(Color); ok { - // return c - //} - r, g, b, _ := c.RGBA() - return Color(uint16((r<<8)&rMask | (g<<3)&gMask | (b>>3)&bMask)) -}) - -const ( - rMask = 0b1111100000000000 - gMask = 0b0000011111100000 - bMask = 0b0000000000011111 -) - -// Color represents an RGB565 color. -type Color uint16 - -func (c Color) RGBA() (r, g, b, a uint32) { - return uint32(math.Round(float64(c&rMask>>11)*255.0/31.0)) << 8, - uint32(math.Round(float64(c&gMask>>5)*255.0/63.0)) << 8, - uint32(math.Round(float64(c&bMask)*255.0/31.0)) << 8, - 0xffff -} - -func NewRGB565(r image.Rectangle) *RGB565 { - return &RGB565{Pix: make([]uint8, r.Dx()*r.Dy()<<1), Stride: r.Dx() << 1, Rect: r} -} - -func (p *RGB565) Bounds() image.Rectangle { return p.Rect } -func (p *RGB565) ColorModel() color.Model { return Model } -func (p *RGB565) PixOffset(x, y int) int { return (x-p.Rect.Min.X)<<1 + (y-p.Rect.Min.Y)*p.Stride } - -func (p *RGB565) At(x, y int) color.Color { - i := p.PixOffset(x, y) - return Color(binary.LittleEndian.Uint16(p.Pix[i : i+2])) -} - -func (p *RGB565) Set(x, y int, c color.Color) { - i := p.PixOffset(x, y) - binary.LittleEndian.PutUint16(p.Pix[i:i+2], uint16(Model.Convert(c).(Color))) -} diff --git a/pkg/encoder/color/rgba/rgba.go b/pkg/encoder/color/rgba/rgba.go deleted file mode 100644 index 5bb2e9bc..00000000 --- a/pkg/encoder/color/rgba/rgba.go +++ /dev/null @@ -1,24 +0,0 @@ -package rgba - -import ( - "image" - "image/color" -) - -func ToRGBA(img image.Image, flipped bool) *image.RGBA { - bounds := img.Bounds() - sw, sh := bounds.Dx(), bounds.Dy() - dst := image.NewRGBA(image.Rect(0, 0, sw, sh)) - for y := range sh { - yy := y - if flipped { - yy = sh - y - } - for x := range sw { - px := img.At(x, y) - rgba := color.RGBAModel.Convert(px).(color.RGBA) - dst.Set(x, yy, rgba) - } - } - return dst -} diff --git a/pkg/encoder/encoder.go b/pkg/encoder/encoder.go deleted file mode 100644 index 2252f5e0..00000000 --- a/pkg/encoder/encoder.go +++ /dev/null @@ -1,137 +0,0 @@ -package encoder - -import ( - "fmt" - "sync/atomic" - - "github.com/giongto35/cloud-game/v3/pkg/config" - "github.com/giongto35/cloud-game/v3/pkg/encoder/h264" - "github.com/giongto35/cloud-game/v3/pkg/encoder/vpx" - "github.com/giongto35/cloud-game/v3/pkg/encoder/yuv" - "github.com/giongto35/cloud-game/v3/pkg/logger" -) - -type ( - InFrame yuv.RawFrame - OutFrame []byte - Encoder interface { - Encode([]byte) []byte - IntraRefresh() - Info() string - Shutdown() error - } -) - -type Video struct { - codec Encoder - log *logger.Logger - stopped atomic.Bool - y yuv.Conv - pf yuv.PixFmt - rot uint -} - -type VideoCodec string - -const ( - H264 VideoCodec = "h264" - VP8 VideoCodec = "vp8" - VP9 VideoCodec = "vp9" - VPX VideoCodec = "vpx" -) - -// NewVideoEncoder returns new video encoder. -// By default, it waits for RGBA images on the input channel, -// converts them into YUV I420 format, -// encodes with provided video encoder, and -// puts the result into the output channel. -func NewVideoEncoder(w, h, dw, dh int, scale float64, conf config.Video, log *logger.Logger) (*Video, error) { - var enc Encoder - var err error - codec := VideoCodec(conf.Codec) - switch codec { - case H264: - opts := h264.Options(conf.H264) - enc, err = h264.NewEncoder(dw, dh, conf.Threads, conf.KeyframeInterval, &opts) - case VP8, VP9, VPX: - opts := vpx.Options(conf.Vpx) - v := 8 - if codec == VP9 { - v = 9 - } - enc, err = vpx.NewEncoder(dw, dh, conf.Threads, conf.KeyframeInterval, v, &opts) - default: - err = fmt.Errorf("unsupported codec: %v", conf.Codec) - } - if err != nil { - return nil, err - } - if enc == nil { - return nil, fmt.Errorf("no encoder") - } - - return &Video{codec: enc, y: yuv.NewYuvConv(w, h, scale), log: log}, nil -} - -func (v *Video) Encode(frame InFrame) OutFrame { - if v.stopped.Load() { - return nil - } - - yCbCr := v.y.Process(yuv.RawFrame(frame), v.rot, v.pf) - //defer v.y.Put(&yCbCr) - if bytes := v.codec.Encode(yCbCr); len(bytes) > 0 { - return bytes - } - return nil -} - -func (v *Video) Info() string { - return fmt.Sprintf("%v, libyuv: %v", v.codec.Info(), v.y.Version()) -} - -func (v *Video) SetPixFormat(f uint32) { - if v == nil { - return - } - - switch f { - case 0: - v.pf = yuv.PixFmt(yuv.FourccRgb0) - case 1: - v.pf = yuv.PixFmt(yuv.FourccArgb) - case 2: - v.pf = yuv.PixFmt(yuv.FourccRgbp) - default: - v.pf = yuv.PixFmt(yuv.FourccAbgr) - } -} - -// SetRot sets the de-rotation angle of the frames. -func (v *Video) SetRot(a uint) { - if v == nil { - return - } - - if a > 0 { - v.rot = (a + 180) % 360 - } -} - -func (v *Video) Stop() { - if v == nil { - return - } - - if v.stopped.Swap(true) { - return - } - v.rot = 0 - - defer func() { v.codec = nil }() - if err := v.codec.Shutdown(); err != nil { - if v.log != nil { - v.log.Error().Err(err).Msg("failed to close the encoder") - } - } -} diff --git a/pkg/encoder/h264/x264.go b/pkg/encoder/h264/x264.go deleted file mode 100644 index 9429b926..00000000 --- a/pkg/encoder/h264/x264.go +++ /dev/null @@ -1,203 +0,0 @@ -package h264 - -/* -// See: [x264](https://www.videolan.org/developers/x264.html) -#cgo !st pkg-config: x264 -#cgo x264s pkg-config: x264 -#cgo st,!x264s LDFLAGS: -l:libx264.a - -#include "stdint.h" -#include "x264.h" -#include - -typedef struct -{ - x264_t *h; - x264_nal_t *nal; // array of NALs - int i_nal; // number of NALs - int y; // Y size - int uv; // U or V size - x264_picture_t pic; - x264_picture_t pic_out; -} h264; - -h264 *h264_new(x264_param_t *param) -{ - h264 tmp; - x264_picture_t pic; - - tmp.h = x264_encoder_open(param); - if (!tmp.h) - return NULL; - - x264_picture_init(&pic); - pic.img.i_csp = param->i_csp; - pic.img.i_plane = 3; - pic.img.i_stride[0] = param->i_width; - pic.img.i_stride[1] = param->i_width >> 1; - pic.img.i_stride[2] = param->i_width >> 1; - tmp.pic = pic; - - // crashes during x264_picture_clean :/ - //if (x264_picture_alloc(&pic, param->i_csp, param->i_width, param->i_height) < 0) - // return NULL; - - tmp.y = param->i_width * param->i_height; - tmp.uv = tmp.y >> 2; - - h264 *h = malloc(sizeof(h264)); - *h = tmp; - return h; -} - -int h264_encode(h264 *h, uint8_t *yuv) -{ - h->pic.img.plane[0] = yuv; - h->pic.img.plane[1] = h->pic.img.plane[0] + h->y; - h->pic.img.plane[2] = h->pic.img.plane[1] + h->uv; - h->pic.i_pts += 1; - return x264_encoder_encode(h->h, &h->nal, &h->i_nal, &h->pic, &h->pic_out); -} - -void h264_destroy(h264 *h) -{ - if (h == NULL) return; - x264_encoder_close(h->h); - free(h); -} -*/ -import "C" - -import ( - "fmt" - "strings" - "unsafe" -) - -type H264 struct { - h *C.h264 -} - -type Options struct { - Mode string - // Constant Rate Factor (CRF) - // This method allows the encoder to attempt to achieve a certain output quality for the whole file - // when output file size is of less importance. - // The range of the CRF scale is 0–51, where 0 is lossless, 23 is the default, and 51 is the worst quality possible. - Crf uint8 - // vbv-maxrate - MaxRate int - // vbv-bufsize - BufSize int - LogLevel int32 - // ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo. - Preset string - // baseline, main, high, high10, high422, high444. - Profile string - // film, animation, grain, stillimage, psnr, ssim, fastdecode, zerolatency. - Tune string -} - -func NewEncoder(w, h int, th int, kfi int, opts *Options) (encoder *H264, err error) { - ver := Version() - - if ver < 150 { - return nil, fmt.Errorf("x264: the library version should be newer than v150, you have got version %v", ver) - } - - if opts == nil { - opts = &Options{ - Mode: "crf", - Crf: 23, - Tune: "zerolatency", - Preset: "superfast", - Profile: "baseline", - } - } - - param := C.x264_param_t{} - - if opts.Preset != "" && opts.Tune != "" { - preset := C.CString(opts.Preset) - tune := C.CString(opts.Tune) - defer C.free(unsafe.Pointer(preset)) - defer C.free(unsafe.Pointer(tune)) - if C.x264_param_default_preset(¶m, preset, tune) < 0 { - return nil, fmt.Errorf("x264: invalid preset/tune name") - } - } else { - C.x264_param_default(¶m) - } - - if opts.Profile != "" { - profile := C.CString(opts.Profile) - defer C.free(unsafe.Pointer(profile)) - if C.x264_param_apply_profile(¶m, profile) < 0 { - return nil, fmt.Errorf("x264: invalid profile name") - } - } - - param.i_bitdepth = 8 - if ver > 155 { - param.i_csp = C.X264_CSP_I420 - } else { - param.i_csp = 1 - } - param.i_width = C.int(w) - param.i_height = C.int(h) - param.i_log_level = C.int(opts.LogLevel) - if kfi > 0 { - param.i_keyint_max = C.int(kfi) - } else { - param.i_keyint_max = 120 - } - param.i_sync_lookahead = 0 - param.i_threads = C.int(th) - if th != 1 { - param.b_sliced_threads = 1 - } - - param.rc.i_rc_method = C.X264_RC_CRF - param.rc.f_rf_constant = C.float(opts.Crf) - - if strings.ToLower(opts.Mode) == "cbr" { - param.rc.i_rc_method = C.X264_RC_ABR - param.i_nal_hrd = C.X264_NAL_HRD_CBR - } - - if opts.MaxRate > 0 { - param.rc.i_bitrate = C.int(opts.MaxRate) - param.rc.i_vbv_max_bitrate = C.int(opts.MaxRate) - } - if opts.BufSize > 0 { - param.rc.i_vbv_buffer_size = C.int(opts.BufSize) - } - - h264 := C.h264_new(¶m) - if h264 == nil { - return nil, fmt.Errorf("x264: cannot open the encoder") - } - return &H264{h264}, nil -} - -func (e *H264) Encode(yuv []byte) []byte { - bytes := C.h264_encode(e.h, (*C.uchar)(unsafe.SliceData(yuv))) - // we merge multiple NALs stored in **nal into a single byte stream - // ret contains the total size of NALs in bytes, i.e. each e.nal[...].p_payload * i_payload - return unsafe.Slice((*byte)(e.h.nal.p_payload), bytes) -} - -func (e *H264) IntraRefresh() { - // !to implement -} - -func (e *H264) Info() string { return fmt.Sprintf("x264: v%v", Version()) } - -func (e *H264) Shutdown() error { - if e.h != nil { - C.h264_destroy(e.h) - } - return nil -} - -func Version() int { return int(C.X264_BUILD) } diff --git a/pkg/encoder/h264/x264_test.go b/pkg/encoder/h264/x264_test.go deleted file mode 100644 index 2a98bdb5..00000000 --- a/pkg/encoder/h264/x264_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package h264 - -import "testing" - -func TestH264Encode(t *testing.T) { - h264, err := NewEncoder(120, 120, 0, 120, nil) - if err != nil { - t.Error(err) - return - } - data := make([]byte, 120*120*1.5) - h264.Encode(data) - if err := h264.Shutdown(); err != nil { - t.Error(err) - } -} - -func Benchmark(b *testing.B) { - w, h := 1920, 1080 - h264, err := NewEncoder(w, h, 0, 120, nil) - if err != nil { - b.Error(err) - return - } - data := make([]byte, int(float64(w)*float64(h)*1.5)) - for b.Loop() { - h264.Encode(data) - } -} diff --git a/pkg/encoder/opus/opus.go b/pkg/encoder/opus/opus.go deleted file mode 100644 index 95b111f1..00000000 --- a/pkg/encoder/opus/opus.go +++ /dev/null @@ -1,199 +0,0 @@ -package opus - -/* -#cgo pkg-config: opus -#cgo st LDFLAGS: -l:libopus.a - -#include - -int get_bitrate(OpusEncoder *st, opus_int32 *bitrate) { return opus_encoder_ctl(st, OPUS_GET_BITRATE(bitrate)); } -int get_complexity(OpusEncoder *st, opus_int32 *complexity) { return opus_encoder_ctl(st, OPUS_GET_COMPLEXITY(complexity)); } -int get_dtx(OpusEncoder *st, opus_int32 *dtx) { return opus_encoder_ctl(st, OPUS_GET_DTX(dtx)); } -int get_inband_fec(OpusEncoder *st, opus_int32 *fec) { return opus_encoder_ctl(st, OPUS_GET_INBAND_FEC(fec)); } -int get_max_bandwidth(OpusEncoder *st, opus_int32 *max_bw) { return opus_encoder_ctl(st, OPUS_GET_MAX_BANDWIDTH(max_bw)); } -int get_packet_loss_perc(OpusEncoder *st, opus_int32 *loss_perc) { return opus_encoder_ctl(st, OPUS_GET_PACKET_LOSS_PERC(loss_perc)); } -int get_sample_rate(OpusEncoder *st, opus_int32 *sample_rate) { return opus_encoder_ctl(st, OPUS_GET_SAMPLE_RATE(sample_rate)); } -int set_bitrate(OpusEncoder *st, opus_int32 bitrate) { return opus_encoder_ctl(st, OPUS_SET_BITRATE(bitrate)); } -int set_complexity(OpusEncoder *st, opus_int32 complexity) { return opus_encoder_ctl(st, OPUS_SET_COMPLEXITY(complexity)); } -int set_dtx(OpusEncoder *st, opus_int32 use_dtx) { return opus_encoder_ctl(st, OPUS_SET_DTX(use_dtx)); } -int set_inband_fec(OpusEncoder *st, opus_int32 fec) { return opus_encoder_ctl(st, OPUS_SET_INBAND_FEC(fec)); } -int set_max_bandwidth(OpusEncoder *st, opus_int32 max_bw) { return opus_encoder_ctl(st, OPUS_SET_MAX_BANDWIDTH(max_bw)); } -int set_packet_loss_perc(OpusEncoder *st, opus_int32 loss_perc) { return opus_encoder_ctl(st, OPUS_SET_PACKET_LOSS_PERC(loss_perc)); } -int reset_state(OpusEncoder *st) { return opus_encoder_ctl(st, OPUS_RESET_STATE); } -*/ -import "C" -import ( - "fmt" - "unsafe" -) - -type ( - Application int - Bandwidth int - Bitrate int - Error int -) - -const ( - // AppRestrictedLowDelay optimizes encoding for low latency applications - AppRestrictedLowDelay = Application(C.OPUS_APPLICATION_RESTRICTED_LOWDELAY) - // FullBand is 20 kHz bandpass - FullBand = Bandwidth(C.OPUS_BANDWIDTH_FULLBAND) -) -const stereo = C.int(2) - -type Encoder struct { - mem []byte - out []byte - st *C.struct_OpusEncoder -} - -func NewEncoder(outFq int) (*Encoder, error) { - mem := make([]byte, C.opus_encoder_get_size(stereo)) - out := make([]byte, 1024) - enc := Encoder{ - mem: mem, - st: (*C.OpusEncoder)(unsafe.Pointer(&mem[0])), - out: out, - } - err := unwrap(C.opus_encoder_init(enc.st, C.opus_int32(outFq), stereo, C.int(AppRestrictedLowDelay))) - if err != nil { - return nil, fmt.Errorf("opus: initializatoin error (%v)", err) - } - _ = enc.SetMaxBandwidth(FullBand) - _ = enc.SetBitrate(96000) - _ = enc.SetComplexity(5) - - return &enc, nil -} - -func (e *Encoder) Reset() error { return e.ResetState() } - -func (e *Encoder) Encode(pcm []int16) ([]byte, error) { - if len(pcm) == 0 { - return nil, nil - } - n := C.opus_encode(e.st, (*C.opus_int16)(&pcm[0]), C.int(len(pcm)>>1), (*C.uchar)(&e.out[0]), C.opus_int32(cap(pcm))) - err := unwrap(n) - // n = 1 is DTX - if err != nil || n == 1 { - return []byte{}, err - } - return e.out[:int(n)], nil -} - -func (e *Encoder) GetInfo() string { - bitrate, _ := e.Bitrate() - complexity, _ := e.Complexity() - dtx, _ := e.DTX() - fec, _ := e.FEC() - maxBandwidth, _ := e.MaxBandwidth() - lossPercent, _ := e.PacketLossPerc() - sampleRate, _ := e.SampleRate() - return fmt.Sprintf( - "%v, Bitrate: %v bps, Complexity: %v, DTX: %v, FEC: %v, Max bandwidth: *%v, Loss%%: %v, Rate: %v Hz", - CodecVersion(), bitrate, complexity, dtx, fec, maxBandwidth, lossPercent, sampleRate, - ) -} - -// SampleRate returns the sample rate of the encoder. -func (e *Encoder) SampleRate() (int, error) { - var sampleRate C.opus_int32 - res := C.get_sample_rate(e.st, &sampleRate) - return int(sampleRate), unwrap(res) -} - -// Bitrate returns the bitrate of the encoder. -func (e *Encoder) Bitrate() (int, error) { - var bitrate C.opus_int32 - res := C.get_bitrate(e.st, &bitrate) - return int(bitrate), unwrap(res) -} - -// SetBitrate sets the bitrate of the encoder. -// BitrateMax / BitrateAuto can be used here. -func (e *Encoder) SetBitrate(b Bitrate) error { - return unwrap(C.set_bitrate(e.st, C.opus_int32(b))) -} - -// Complexity returns the value of the complexity. -func (e *Encoder) Complexity() (int, error) { - var complexity C.opus_int32 - res := C.get_complexity(e.st, &complexity) - return int(complexity), unwrap(res) -} - -// SetComplexity sets the complexity factor for the encoder. -// Complexity is a value from 1 to 10, where 1 is the lowest complexity and 10 is the highest. -func (e *Encoder) SetComplexity(complexity int) error { - return unwrap(C.set_complexity(e.st, C.opus_int32(complexity))) -} - -// DTX says if discontinuous transmission is enabled. -func (e *Encoder) DTX() (bool, error) { - var dtx C.opus_int32 - res := C.get_dtx(e.st, &dtx) - return dtx > 0, unwrap(res) -} - -// SetDTX switches discontinuous transmission. -func (e *Encoder) SetDTX(dtx bool) error { - var i int - if dtx { - i = 1 - } - return unwrap(C.set_dtx(e.st, C.opus_int32(i))) -} - -// MaxBandwidth returns the maximum allowed bandpass value. -func (e *Encoder) MaxBandwidth() (Bandwidth, error) { - var b C.opus_int32 - res := C.get_max_bandwidth(e.st, &b) - return Bandwidth(b), unwrap(res) -} - -// SetMaxBandwidth sets the upper limit of the bandpass. -func (e *Encoder) SetMaxBandwidth(b Bandwidth) error { - return unwrap(C.set_max_bandwidth(e.st, C.opus_int32(b))) -} - -// FEC says if forward error correction (FEC) is enabled. -func (e *Encoder) FEC() (bool, error) { - var fec C.opus_int32 - res := C.get_inband_fec(e.st, &fec) - return fec > 0, unwrap(res) -} - -// SetFEC switches the forward error correction (FEC). -func (e *Encoder) SetFEC(fec bool) error { - var i int - if fec { - i = 1 - } - return unwrap(C.set_inband_fec(e.st, C.opus_int32(i))) -} - -// PacketLossPerc returns configured packet loss percentage. -func (e *Encoder) PacketLossPerc() (int, error) { - var lossPerc C.opus_int32 - res := C.get_packet_loss_perc(e.st, &lossPerc) - return int(lossPerc), unwrap(res) -} - -// SetPacketLossPerc sets expected packet loss percentage. -func (e *Encoder) SetPacketLossPerc(lossPerc int) error { - return unwrap(C.set_packet_loss_perc(e.st, C.opus_int32(lossPerc))) -} - -func (e *Encoder) ResetState() error { return unwrap(C.reset_state(e.st)) } - -func (e Error) Error() string { return fmt.Sprintf("opus: %v", C.GoString(C.opus_strerror(C.int(e)))) } - -func unwrap(error C.int) (err error) { - if error < C.OPUS_OK { - err = Error(int(error)) - } - return -} - -func CodecVersion() string { return C.GoString(C.opus_get_version_string()) } diff --git a/pkg/encoder/vpx/libvpx.go b/pkg/encoder/vpx/libvpx.go deleted file mode 100644 index bc3a9f70..00000000 --- a/pkg/encoder/vpx/libvpx.go +++ /dev/null @@ -1,270 +0,0 @@ -package vpx - -/* -#cgo pkg-config: vpx -#cgo st LDFLAGS: -l:libvpx.a - -#include "vpx/vpx_encoder.h" -#include "vpx/vpx_image.h" -#include "vpx/vp8cx.h" - -#include -#include - -#define VP8_FOURCC 0x30385056 -#define VP9_FOURCC 0x30395056 - -#ifndef VP9E_TUNE_SCREEN -#define VP9E_TUNE_SCREEN 1 -#endif - -typedef struct VpxInterface { - const char *const name; - const uint32_t fourcc; - vpx_codec_iface_t *(*const codec_interface)(); -} VpxInterface; - -typedef struct FrameBuffer { - void *ptr; - int size; -} FrameBuffer; - -vpx_codec_err_t call_vpx_codec_enc_config_default(const VpxInterface *encoder, vpx_codec_enc_cfg_t *cfg) { - return vpx_codec_enc_config_default(encoder->codec_interface(), cfg, 0); -} -vpx_codec_err_t call_vpx_codec_enc_init(vpx_codec_ctx_t *codec, const VpxInterface *encoder, vpx_codec_enc_cfg_t *cfg) { - return vpx_codec_enc_init(codec, encoder->codec_interface(), cfg, 0); -} - -// Drains all encoded packets, returning the first frame found. -FrameBuffer get_frame_buffer(vpx_codec_ctx_t *codec, vpx_codec_iter_t *iter) { - FrameBuffer fb = {NULL, 0}; - const vpx_codec_cx_pkt_t *pkt; - while ((pkt = vpx_codec_get_cx_data(codec, iter)) != NULL) { - if (pkt->kind == VPX_CODEC_CX_FRAME_PKT && fb.ptr == NULL) { - fb.ptr = pkt->data.frame.buf; - fb.size = pkt->data.frame.sz; - } - } - return fb; -} - -vpx_codec_err_t set_vpx_cpu_used(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP8E_SET_CPUUSED, value); -} - -vpx_codec_err_t set_vp9_tile_columns(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP9E_SET_TILE_COLUMNS, value); -} - -vpx_codec_err_t set_vp9_tune_content(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP9E_SET_TUNE_CONTENT, value); -} - -vpx_codec_err_t set_vp9_row_mt(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP9E_SET_ROW_MT, value); -} - -vpx_codec_err_t set_vp8_auto_alt_ref(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP8E_SET_ENABLEAUTOALTREF, value); -} - -vpx_codec_err_t set_vp8_noise_sensitivity(vpx_codec_ctx_t *ctx, int value) { - return vpx_codec_control_(ctx, VP8E_SET_NOISE_SENSITIVITY, value); -} - -const VpxInterface vpx_encoders[] = { - { "vp8", VP8_FOURCC, &vpx_codec_vp8_cx }, - { "vp9", VP9_FOURCC, &vpx_codec_vp9_cx }, -}; - -int vpx_img_plane_width(const vpx_image_t *img, int plane) { - if (plane > 0 && img->x_chroma_shift > 0) - return (img->d_w + 1) >> img->x_chroma_shift; - else - return img->d_w; -} - -int vpx_img_plane_height(const vpx_image_t *img, int plane) { - if (plane > 0 && img->y_chroma_shift > 0) - return (img->d_h + 1) >> img->y_chroma_shift; - else - return img->d_h; -} - -void vpx_img_read(vpx_image_t *dst, void *src) { - for (int plane = 0; plane < 3; ++plane) { - unsigned char *buf = dst->planes[plane]; - const int stride = dst->stride[plane]; - const int w = vpx_img_plane_width(dst, plane); - const int h = vpx_img_plane_height(dst, plane); - - for (int y = 0; y < h; ++y) { - memcpy(buf, src, w); - buf += stride; - src += w; - } - } -} -*/ -import "C" -import ( - "fmt" - "runtime" - "unsafe" -) - -type Vpx struct { - frameCount C.int - image C.vpx_image_t - codecCtx C.vpx_codec_ctx_t - kfi C.int - v int -} - -func autoThreads(configured, cpus int) int { - if configured != 0 { - return configured - } - reserve := max(cpus/4, 1) - if t := cpus - reserve; t > 0 { - return t - } - return 1 -} - -type Options struct { - // Target bandwidth to use for this stream, in kilobits per second. - Bitrate uint - // Speed/quality tradeoff (0=auto, 1–8 manual). Defaults to 0. - // Applies to both VP8 and VP9. - CpuUsed int - // Log2 of tile columns for VP9 frame-level parallelism (0–6). VP9 only. - TileColumns int - // Content tuning: "screen" for game/screen content. VP9 only. - Tune string -} - -func NewEncoder(w, h int, th int, kfi int, version int, opts *Options) (*Vpx, error) { - if opts == nil { - opts = &Options{ - Bitrate: 1200, - TileColumns: 2, - Tune: "screen", - } - } - if kfi == 0 { - kfi = 120 - } - - vpx := Vpx{ - frameCount: C.int(0), - kfi: C.int(kfi), - v: version, - } - - if C.vpx_img_alloc(&vpx.image, C.VPX_IMG_FMT_I420, C.uint(w), C.uint(h), 1) == nil { - return nil, fmt.Errorf("vpx_img_alloc failed") - } - - idx := 0 - if version == 9 { - idx = 1 - } - encoder := &C.vpx_encoders[idx] - - var cfg C.vpx_codec_enc_cfg_t - if C.call_vpx_codec_enc_config_default(encoder, &cfg) != 0 { - return nil, fmt.Errorf("failed to get default codec config") - } - - cfg.g_w = C.uint(w) - cfg.g_h = C.uint(h) - cfg.g_threads = C.uint(autoThreads(th, runtime.NumCPU())) - cfg.g_lag_in_frames = 0 - cfg.rc_target_bitrate = C.uint(opts.Bitrate) - cfg.g_error_resilient = 0 - - if C.call_vpx_codec_enc_init(&vpx.codecCtx, encoder, &cfg) != 0 { - return nil, fmt.Errorf("failed to initialize encoder") - } - - // Speed/quality tradeoff: higher = faster, lower = better quality. - if opts.CpuUsed == 0 { - thr := autoThreads(th, runtime.NumCPU()) - switch { - case thr <= 2: - opts.CpuUsed = 8 - case thr <= 4: - opts.CpuUsed = 7 - default: - opts.CpuUsed = 6 - } - } - C.set_vpx_cpu_used(&vpx.codecCtx, C.int(opts.CpuUsed)) - - // VP9-specific tuning. - if version == 9 { - if opts.TileColumns == 0 { - thr := autoThreads(th, runtime.NumCPU()) - if thr >= 4 { - opts.TileColumns = 2 - } else if thr >= 2 { - opts.TileColumns = 1 - } - } - if opts.TileColumns > 0 { - C.set_vp9_tile_columns(&vpx.codecCtx, C.int(opts.TileColumns)) - } - if opts.Tune == "screen" { - C.set_vp9_tune_content(&vpx.codecCtx, C.VP9E_TUNE_SCREEN) - } - C.set_vp9_row_mt(&vpx.codecCtx, 1) - } - - // VP8 real-time tuning: disable features that add latency. - if version == 8 { - // No alt-ref frames for zero-latency. - C.set_vp8_auto_alt_ref(&vpx.codecCtx, 0) - // Reduce noise sensitivity — game content is low-noise. - C.set_vp8_noise_sensitivity(&vpx.codecCtx, 0) - } - - return &vpx, nil -} - -// Encode encodes yuv image with the VPX8 encoder. -// see: https://chromium.googlesource.com/webm/libvpx/+/master/examples/simple_encoder.c -func (vpx *Vpx) Encode(yuv []byte) []byte { - C.vpx_img_read(&vpx.image, unsafe.Pointer(&yuv[0])) - - var flags C.int - if vpx.frameCount < 3 || (vpx.kfi > 0 && vpx.frameCount%vpx.kfi == 0) { - flags |= C.VPX_EFLAG_FORCE_KF - } - C.vpx_codec_encode(&vpx.codecCtx, &vpx.image, C.vpx_codec_pts_t(vpx.frameCount), 1, C.vpx_enc_frame_flags_t(flags), C.VPX_DL_REALTIME) - vpx.frameCount++ - - var iter C.vpx_codec_iter_t - fb := C.get_frame_buffer(&vpx.codecCtx, &iter) - if fb.ptr == nil { - return nil - } - - // zero-copy slice view into the encoder's internal buffer - return unsafe.Slice((*byte)(fb.ptr), fb.size) -} - -func (vpx *Vpx) Info() string { - return fmt.Sprintf("vpx (%v): %v", vpx.v, C.GoString(C.vpx_codec_version_str())) -} - -func (vpx *Vpx) IntraRefresh() { - // !to implement -} - -func (vpx *Vpx) Shutdown() error { - C.vpx_img_free(&vpx.image) - C.vpx_codec_destroy(&vpx.codecCtx) - return nil -} diff --git a/pkg/encoder/yuv/libyuv/libyuv.go b/pkg/encoder/yuv/libyuv/libyuv.go deleted file mode 100644 index 0848c095..00000000 --- a/pkg/encoder/yuv/libyuv/libyuv.go +++ /dev/null @@ -1,274 +0,0 @@ -// Package libyuv contains the wrapper for: https://chromium.googlesource.com/libyuv/libyuv. -// MacOS libs are from: https://packages.macports.org/libyuv/. -package libyuv - -/* -#cgo !darwin,!st LDFLAGS: -lyuv -#cgo !darwin,st LDFLAGS: -l:libyuv.a -l:libjpeg.a -l:libstdc++.a -static-libgcc - -#cgo darwin CFLAGS: -DINCLUDE_LIBYUV_VERSION_H_ -#cgo darwin LDFLAGS: -L${SRCDIR} -lstdc++ -#cgo darwin,amd64 LDFLAGS: -lyuv_darwin_x86_64 -ljpeg -lstdc++ -#cgo darwin,arm64 LDFLAGS: -lyuv_darwin_arm64 -ljpeg -lstdc++ - -#include // for uintptr_t and C99 types -#include - -#if !defined(LIBYUV_API) -#define LIBYUV_API -#endif // LIBYUV_API - -#ifndef INCLUDE_LIBYUV_VERSION_H_ -#include "libyuv/version.h" -#else -#define LIBYUV_VERSION 1874 // darwin static libs version -#endif // INCLUDE_LIBYUV_VERSION_H_ - -// Supported rotation. -typedef enum RotationMode { - kRotate0 = 0, // No rotation. - kRotate90 = 90, // Rotate 90 degrees clockwise. - kRotate180 = 180, // Rotate 180 degrees. - kRotate270 = 270, // Rotate 270 degrees clockwise. -} RotationModeEnum; - -// RGB16 (RGBP fourcc) little endian to I420. -LIBYUV_API -int RGB565ToI420(const uint8_t* src_rgb565, int src_stride_rgb565, uint8_t* dst_y, int dst_stride_y, - uint8_t* dst_u, int dst_stride_u, uint8_t* dst_v, int dst_stride_v, int width, int height); - -// Rotate I420 frame. -LIBYUV_API -int I420Rotate(const uint8_t* src_y, int src_stride_y, const uint8_t* src_u, int src_stride_u, - const uint8_t* src_v, int src_stride_v, uint8_t* dst_y, int dst_stride_y, uint8_t* dst_u, - int dst_stride_u, uint8_t* dst_v, int dst_stride_v, int width, int height, enum RotationMode mode); - -// RGB15 (RGBO fourcc) little endian to I420. -LIBYUV_API -int ARGB1555ToI420(const uint8_t* src_argb1555, int src_stride_argb1555, uint8_t* dst_y, int dst_stride_y, - uint8_t* dst_u, int dst_stride_u, uint8_t* dst_v, int dst_stride_v, int width, int height); - -// ABGR little endian (rgba in memory) to I420. -LIBYUV_API -int ABGRToI420(const uint8_t* src_abgr, int src_stride_abgr, uint8_t* dst_y, int dst_stride_y, uint8_t* dst_u, - int dst_stride_u, uint8_t* dst_v, int dst_stride_v, int width, int height); - -// ARGB little endian (bgra in memory) to I420. -LIBYUV_API -int ARGBToI420(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_y, int dst_stride_y, uint8_t* dst_u, - int dst_stride_u, uint8_t* dst_v, int dst_stride_v, int width, int height); - - -void ConvertToI420Custom(const uint8_t* sample, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_u, - int dst_stride_u, - uint8_t* dst_v, - int dst_stride_v, - int src_width, - int src_height, - int crop_width, - int crop_height, - uint32_t fourcc); - -#ifdef __cplusplus -namespace libyuv { -extern "C" { -#endif - -#define FOURCC(a, b, c, d) \ - (((uint32_t)(a)) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24)) - -enum FourCC { - FOURCC_I420 = FOURCC('I', '4', '2', '0'), - FOURCC_ARGB = FOURCC('A', 'R', 'G', 'B'), - FOURCC_ABGR = FOURCC('A', 'B', 'G', 'R'), - FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), - FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. - FOURCC_ANY = -1, -}; - -inline void ConvertToI420Custom(const uint8_t* sample, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_u, - int dst_stride_u, - uint8_t* dst_v, - int dst_stride_v, - int src_width, - int src_height, - int crop_width, - int crop_height, - uint32_t fourcc) { - const int stride = src_width << 1; - - switch (fourcc) { - case FOURCC_RGBP: - RGB565ToI420(sample, stride, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, crop_width, crop_height); - break; - case FOURCC_RGBO: - ARGB1555ToI420(sample, stride, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, crop_width, crop_height); - break; - case FOURCC_ARGB: - ARGBToI420(sample, stride << 1, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, crop_width, crop_height); - break; - case FOURCC_ABGR: - ABGRToI420(sample, stride << 1, dst_y, dst_stride_y, dst_u, - dst_stride_u, dst_v, dst_stride_v, crop_width, crop_height); - break; - } -} - -void rotateI420(const uint8_t* sample, - uint8_t* dst_y, - int dst_stride_y, - uint8_t* dst_u, - int dst_stride_u, - uint8_t* dst_v, - int dst_stride_v, - int src_width, - int src_height, - int crop_width, - int crop_height, - enum RotationMode rotation, - uint32_t fourcc) { - - uint8_t* tmp_y = dst_y; - uint8_t* tmp_u = dst_u; - uint8_t* tmp_v = dst_v; - int tmp_y_stride = dst_stride_y; - int tmp_u_stride = dst_stride_u; - int tmp_v_stride = dst_stride_v; - - uint8_t* rotate_buffer = NULL; - - int y_size = crop_width * crop_height; - int uv_size = y_size >> 1; - rotate_buffer = (uint8_t*)malloc(y_size + y_size); - if (!rotate_buffer) { - return; - } - dst_y = rotate_buffer; - dst_u = dst_y + y_size; - dst_v = dst_u + uv_size; - dst_stride_y = crop_width; - dst_stride_u = dst_stride_v = crop_width >> 1; - ConvertToI420Custom(sample, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, - src_width, src_height, crop_width, crop_height, fourcc); - I420Rotate(dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v, - dst_stride_v, tmp_y, tmp_y_stride, tmp_u, tmp_u_stride, - tmp_v, tmp_v_stride, crop_width, crop_height, rotation); - free(rotate_buffer); -} - -// Supported filtering. -typedef enum FilterMode { - kFilterNone = 0, // Point sample; Fastest. - kFilterLinear = 1, // Filter horizontally only. - kFilterBilinear = 2, // Faster than box, but lower quality scaling down. - kFilterBox = 3 // Highest quality. -} FilterModeEnum; - -LIBYUV_API -int I420Scale(const uint8_t *src_y, int src_stride_y, const uint8_t *src_u, int src_stride_u, - const uint8_t *src_v, int src_stride_v, int src_width, int src_height, uint8_t *dst_y, - int dst_stride_y, uint8_t *dst_u, int dst_stride_u, uint8_t *dst_v, int dst_stride_v, - int dst_width, int dst_height, enum FilterMode filtering); - -#ifdef __cplusplus -} // extern "C" -} // namespace libyuv -#endif -*/ -import "C" -import "fmt" - -const FourccRgbp uint32 = C.FOURCC_RGBP -const FourccArgb uint32 = C.FOURCC_ARGB -const FourccAbgr uint32 = C.FOURCC_ABGR -const FourccRgb0 uint32 = C.FOURCC_RGBO - -func Y420(src []byte, dst []byte, _, h, stride int, dw, dh int, rot uint, pix uint32, cx, cy int) { - cw := (dw + 1) / 2 - ch := (dh + 1) / 2 - i0 := dw * dh - i1 := i0 + cw*ch - yStride := dw - cStride := cw - - if rot == 0 { - C.ConvertToI420Custom( - (*C.uchar)(&src[0]), - (*C.uchar)(&dst[0]), - C.int(yStride), - (*C.uchar)(&dst[i0]), - C.int(cStride), - (*C.uchar)(&dst[i1]), - C.int(cStride), - C.int(stride), - C.int(h), - C.int(cx), - C.int(cy), - C.uint32_t(pix)) - } else { - C.rotateI420( - (*C.uchar)(&src[0]), - (*C.uchar)(&dst[0]), - C.int(yStride), - (*C.uchar)(&dst[i0]), - C.int(cStride), - (*C.uchar)(&dst[i1]), - C.int(cStride), - C.int(stride), - C.int(h), - C.int(cx), - C.int(cy), - C.enum_RotationMode(rot), - C.uint32_t(pix)) - } -} - -func Y420Scale(src []byte, dst []byte, w, h int, dw, dh int) { - srcWidthUV, dstWidthUV := (w+1)>>1, (dw+1)>>1 - srcHeightUV, dstHeightUV := (h+1)>>1, (dh+1)>>1 - - srcYPlaneSize, dstYPlaneSize := w*h, dw*dh - srcUVPlaneSize, dstUVPlaneSize := srcWidthUV*srcHeightUV, dstWidthUV*dstHeightUV - - srcStrideY, dstStrideY := w, dw - srcStrideU, dstStrideU := srcWidthUV, dstWidthUV - srcStrideV, dstStrideV := srcWidthUV, dstWidthUV - - srcY := (*C.uchar)(&src[0]) - srcU := (*C.uchar)(&src[srcYPlaneSize]) - srcV := (*C.uchar)(&src[srcYPlaneSize+srcUVPlaneSize]) - - dstY := (*C.uchar)(&dst[0]) - dstU := (*C.uchar)(&dst[dstYPlaneSize]) - dstV := (*C.uchar)(&dst[dstYPlaneSize+dstUVPlaneSize]) - - C.I420Scale( - srcY, - C.int(srcStrideY), - srcU, - C.int(srcStrideU), - srcV, - C.int(srcStrideV), - C.int(w), - C.int(h), - dstY, - C.int(dstStrideY), - dstU, - C.int(dstStrideU), - dstV, - C.int(dstStrideV), - C.int(dw), - C.int(dh), - C.enum_FilterMode(C.kFilterNone)) -} - -func Version() string { return fmt.Sprintf("%v", int(C.LIBYUV_VERSION)) } diff --git a/pkg/encoder/yuv/libyuv/libyuv_darwin_arm64.a b/pkg/encoder/yuv/libyuv/libyuv_darwin_arm64.a deleted file mode 100644 index f399a41c..00000000 Binary files a/pkg/encoder/yuv/libyuv/libyuv_darwin_arm64.a and /dev/null differ diff --git a/pkg/encoder/yuv/libyuv/libyuv_darwin_x86_64.a b/pkg/encoder/yuv/libyuv/libyuv_darwin_x86_64.a deleted file mode 100644 index 63cd5c74..00000000 Binary files a/pkg/encoder/yuv/libyuv/libyuv_darwin_x86_64.a and /dev/null differ diff --git a/pkg/encoder/yuv/yuv.go b/pkg/encoder/yuv/yuv.go deleted file mode 100644 index 4718c7c1..00000000 --- a/pkg/encoder/yuv/yuv.go +++ /dev/null @@ -1,92 +0,0 @@ -package yuv - -import ( - "image" - - "github.com/giongto35/cloud-game/v3/pkg/encoder/yuv/libyuv" -) - -type Conv struct { - w, h int - sw, sh int - scale float64 - frame []byte - frameSc []byte -} - -type RawFrame struct { - Data []byte - Stride int - W, H int -} - -type PixFmt uint32 - -const FourccRgbp = libyuv.FourccRgbp -const FourccArgb = libyuv.FourccArgb -const FourccAbgr = libyuv.FourccAbgr -const FourccRgb0 = libyuv.FourccRgb0 - -func NewYuvConv(w, h int, scale float64) Conv { - if scale < 1 { - scale = 1 - } - - sw, sh := round(w, scale), round(h, scale) - conv := Conv{w: w, h: h, sw: sw, sh: sh, scale: scale} - bufSize := int(float64(w) * float64(h) * 1.5) - - if scale == 1 { - conv.frame = make([]byte, bufSize) - } else { - bufSizeSc := int(float64(sw) * float64(sh) * 1.5) - // [original frame][scaled frame ] - frames := make([]byte, bufSize+bufSizeSc) - conv.frame = frames[:bufSize] - conv.frameSc = frames[bufSize:] - } - - return conv -} - -// Process converts an image to YUV I420 format inside the internal buffer. -func (c *Conv) Process(frame RawFrame, rot uint, pf PixFmt) []byte { - cx, cy := c.w, c.h // crop - if rot == 90 || rot == 270 { - cx, cy = cy, cx - } - - var stride int - switch pf { - case PixFmt(libyuv.FourccRgbp), PixFmt(libyuv.FourccRgb0): - stride = frame.Stride >> 1 - default: - stride = frame.Stride >> 2 - } - - libyuv.Y420(frame.Data, c.frame, frame.W, frame.H, stride, c.w, c.h, rot, uint32(pf), cx, cy) - - if c.scale > 1 { - libyuv.Y420Scale(c.frame, c.frameSc, c.w, c.h, c.sw, c.sh) - return c.frameSc - } - - return c.frame -} - -func (c *Conv) Version() string { return libyuv.Version() } -func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) & ^1 } - -func ToYCbCr(bytes []byte, w, h int) *image.YCbCr { - cw, ch := (w+1)/2, (h+1)/2 - - i0 := w*h + 0*cw*ch - i1 := w*h + 1*cw*ch - i2 := w*h + 2*cw*ch - - yuv := image.NewYCbCr(image.Rect(0, 0, w, h), image.YCbCrSubsampleRatio420) - yuv.Y = bytes[:i0:i0] - yuv.Cb = bytes[i0:i1:i1] - yuv.Cr = bytes[i1:i2:i2] - return yuv -} diff --git a/pkg/encoder/yuv/yuv_test.go b/pkg/encoder/yuv/yuv_test.go deleted file mode 100644 index 4e0ebbf7..00000000 --- a/pkg/encoder/yuv/yuv_test.go +++ /dev/null @@ -1,284 +0,0 @@ -package yuv - -import ( - "archive/zip" - "fmt" - "image" - "image/color" - "image/png" - "io" - "math" - "math/rand/v2" - "os" - "path/filepath" - "testing" - - "github.com/giongto35/cloud-game/v3/pkg/encoder/yuv/libyuv" - _ "github.com/giongto35/cloud-game/v3/test" -) - -func TestYuvPredefined(t *testing.T) { - t.Skip("Skipped because on Windows some colors are different") - im := []uint8{101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 101, 0, 106, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255, 18, 226, 78, 255} - should := []byte{ - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 52, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 52, 142, 142, 142, 142, 142, 142, 142, - 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, - 110, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, - 110, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 110, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, - 94, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, - 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, - 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, - 110, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, - 110, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, 110, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94, 110, 94, 94, 94, - 94, 94, 94, 94, 110, 94, 94, 94, 110, 94, 94, 94, 94, 94, 94, 94, 76, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 76, - 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 76, 47, - 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, - 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 76, - 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, - 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, - 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 76, 47, 47, 47, 47, 47, 47, 47, 76, 47, 47, 47, 76, 47, 47, 47, 47, 47, 47, 47, - } - - pc := NewYuvConv(32, 32, 1) - frame := RawFrame{Data: im, Stride: 32, W: 32, H: 32} - a := pc.Process(frame, 0, PixFmt(libyuv.FourccAbgr)) - - v := libyuv.Version() - t.Logf("%v", v) - - if len(a) != len(should) { - t.Fatalf("different size a: %v, o: %v", len(a), len(should)) - } - - for i := range a { - if a[i] != should[i] { - t.Fatalf("diff in %vth, %v != %v \n%v\n%v", i, a[i], should[i], im, should) - } - } -} - -func TestYuvScale(t *testing.T) { - name := "001_alsa_ABGR_256_240_1024.raw" - path := filepath.Join("./test/testdata/raw/", name) - - data, err := ReadZip(path + ".zip") - if err != nil { - t.Error(err) - } - - pf, w, h, stride := PixFmt(libyuv.FourccArgb), 256, 240, 1024 - scale := 2 - - conv := NewYuvConv(w, h, float64(scale)) - frame := RawFrame{Data: data, Stride: stride, W: w, H: h} - out := conv.Process(frame, 0, pf) - - d := float64(len(out)) / float64(len(data)) - if d != 1.5 { - t.Errorf("Scaled not by factor %v, %v", scale, d) - } - - // save as RGBA - //sw, sh := w*scale, h*scale - //yuv := ToYCbCr(out, sw, sh) - //if f, err := os.Create(filepath.Join("./", name+".png")); err == nil { - // if err = png.Encode(f, yuv); err != nil { - // t.Logf("Couldn't encode the image, %v", err) - // } - // _ = f.Close() - //} -} - -func BenchmarkYuv(b *testing.B) { - tests := []struct { - w int - h int - }{ - {w: 1920, h: 1080}, - {w: 320, h: 240}, - } - r1 := rand.Float32() - - for _, test := range tests { - w, h := test.w, test.h - frame := genFrame(w, h, r1) - b.Run(fmt.Sprintf("%vx%v YUV", w, h), func(b *testing.B) { - pc := NewYuvConv(w, h, 1) - for i := 0; i < b.N; i++ { - pc.Process(frame, 0, PixFmt(libyuv.FourccAbgr)) - b.SetBytes(int64(len(frame.Data))) - } - b.ReportAllocs() - }) - } -} - -func genFrame(w, h int, seed float32) RawFrame { - img := image.NewRGBA(image.Rectangle{Max: image.Point{X: w, Y: h}}) - for x := range w { - for y := range h { - col := color.RGBA{R: uint8(seed * 255), G: uint8(seed * 255), B: uint8(seed * 255), A: 0xff} - img.Set(x, y, col) - } - } - return RawFrame{ - Data: img.Pix, - Stride: img.Stride, - W: img.Bounds().Dx(), - H: img.Bounds().Dy(), - } -} - -func TestGen24bitFull(t *testing.T) { - t.Skip() - const tau = 2 * math.Pi - const deg = 3 * math.Pi / 2 - //const depth = 1 << 24 - var wh = 255 //int(math.Sqrt(depth)) - - img := image.NewRGBA(image.Rectangle{Max: image.Point{X: wh, Y: wh}}) - - centerX, centerY := wh/2, wh/2 - radius := centerX - //if centerY < radius { - // radius = centerY - //} - - for y := range wh { - dy := float64(y - centerY) - for x := range wh { - dx := float64(x - centerX) - dist := math.Sqrt(dx*dx + dy*dy) - if dist <= float64(radius) { - hue := (math.Atan2(dx, dy) + deg) / tau - r, g, b := hsb2rgb(hue, linear(0, float64(centerX), dist), 1) - img.Set(x, y, color.RGBA{R: uint8(r), G: uint8(g), B: uint8(b), A: 255}) - } - } - } - - f, _ := os.Create("out_image.png") - defer func() { _ = f.Close() }() - - // Encode to `PNG` with `DefaultCompression` level - // then save to file - _ = png.Encode(f, img) -} - -func linear(a, b, x float64) float64 { return (x - a) / (b - a) } - -func hsb2rgb(hue, s, bri float64) (r, g, b int) { - u := int(bri*255 + 0.5) - if s == 0 { - return u, u, u - } - h := (hue - math.Floor(hue)) * 6 - f := h - math.Floor(h) - p := int(bri*(1-s)*255 + 0.5) - q := int(bri*(1-s*f)*255 + 0.5) - t := int(bri*(1-s*(1-f))*255 + 0.5) - switch int(h) { - case 0: - r, g, b = u, t, p - case 1: - r, g, b = q, u, p - case 2: - r, g, b = p, u, t - case 3: - r, g, b = p, q, u - case 4: - r, g, b = t, p, u - case 5: - r, g, b = u, p, q - } - return -} - -func ReadZip(path string) ([]byte, error) { - zf, err := zip.OpenReader(path) - if err != nil { - return nil, err - } - defer func() { _ = zf.Close() }() - - f, err := zf.File[0].Open() - if err != nil { - return nil, err - } - defer func() { _ = f.Close() }() - - return io.ReadAll(f) -} diff --git a/pkg/network/webrtc/webrtc.go b/pkg/network/webrtc/webrtc.go index 53b933c7..b342b6b5 100644 --- a/pkg/network/webrtc/webrtc.go +++ b/pkg/network/webrtc/webrtc.go @@ -220,27 +220,27 @@ func (p *Peer) Channel(label string, conf *webrtc.DataChannelInit, onMessage fun return ch, nil } -func (p *Peer) SendAudio(dat []byte, dur int32) { - if err := p.send(dat, int64(dur), p.a.WriteSample); err != nil { +func (p *Peer) SendAudio(dat []byte, dur time.Duration) { + if err := p.send(dat, dur, p.a.WriteSample); err != nil { p.log.Error().Err(err).Send() } } -func (p *Peer) SendVideo(data []byte, dur int32) { - if err := p.send(data, int64(dur), p.v.WriteSample); err != nil { +func (p *Peer) SendVideo(data []byte, dur time.Duration) { + if err := p.send(data, dur, p.v.WriteSample); err != nil { p.log.Error().Err(err).Send() } } func (p *Peer) SendData(data []byte) { _ = p.d.Send(data) } -func (p *Peer) send(data []byte, duration int64, fn func(media.Sample) error) error { +func (p *Peer) send(data []byte, duration time.Duration, fn func(media.Sample) error) error { sample, _ := samplePool.Get().(*media.Sample) if sample == nil { sample = new(media.Sample) } sample.Data = data - sample.Duration = time.Duration(duration) + sample.Duration = duration err := fn(*sample) if err != nil { return err diff --git a/pkg/resampler/simple.go b/pkg/resampler/simple.go deleted file mode 100644 index 39e509c0..00000000 --- a/pkg/resampler/simple.go +++ /dev/null @@ -1,62 +0,0 @@ -package resampler - -func Linear(dst, src []int16) { - nSrc, nDst := len(src), len(dst) - if nSrc < 2 || nDst < 2 { - return - } - - srcPairs, dstPairs := nSrc>>1, nDst>>1 - - // replicate single pair input or output - if srcPairs == 1 || dstPairs == 1 { - for i := range dstPairs { - dst[i*2], dst[i*2+1] = src[0], src[1] - } - return - } - - ratio := ((srcPairs - 1) << 16) / (dstPairs - 1) - lastSrc := nSrc - 2 - - // interpolate all pairs except the last - for i, pos := 0, 0; i < dstPairs-1; i, pos = i+1, pos+ratio { - idx := (pos >> 16) << 1 - di := i << 1 - frac := int32(pos & 0xFFFF) - l0, r0 := int32(src[idx]), int32(src[idx+1]) - - // L = L0 + (L1-L0)*frac - dst[di] = int16(l0 + ((int32(src[idx+2])-l0)*frac)>>16) - // R = R0 + (R1-R0)*frac - dst[di+1] = int16(r0 + ((int32(src[idx+3])-r0)*frac)>>16) - } - - // last output pair = last input pair (avoids precision loss at the edge) - lastDst := (dstPairs - 1) << 1 - dst[lastDst], dst[lastDst+1] = src[lastSrc], src[lastSrc+1] -} - -func Nearest(dst, src []int16) { - nSrc, nDst := len(src), len(dst) - if nSrc < 2 || nDst < 2 { - return - } - - srcPairs, dstPairs := nSrc>>1, nDst>>1 - - if srcPairs == 1 || dstPairs == 1 { - for i := range dstPairs { - dst[i*2], dst[i*2+1] = src[0], src[1] - } - return - } - - ratio := (srcPairs << 16) / dstPairs - - for i, pos := 0, 0; i < dstPairs; i, pos = i+1, pos+ratio { - si := (pos >> 16) << 1 - di := i << 1 - dst[di], dst[di+1] = src[si], src[si+1] - } -} diff --git a/pkg/resampler/speex.go b/pkg/resampler/speex.go deleted file mode 100644 index b62d2be1..00000000 --- a/pkg/resampler/speex.go +++ /dev/null @@ -1,106 +0,0 @@ -package resampler - -/* - #cgo pkg-config: speexdsp - #cgo st LDFLAGS: -l:libspeexdsp.a - - #include - #include "speex_resampler.h" -*/ -import "C" - -import ( - "errors" - "unsafe" -) - -// Quality -const ( - QualityMax = 10 - QualityMin = 0 - QualityDefault = 4 - QualityDesktop = 5 - QualityVoid = 3 -) - -// Errors -const ( - ErrorSuccess = iota - ErrorAllocFailed - ErrorBadState - ErrorInvalidArg - ErrorPtrOverlap - ErrorMaxError -) - -type Resampler struct { - resampler *C.SpeexResamplerState - channels int - inRate int - outRate int -} - -func Init(channels, inRate, outRate, quality int) (*Resampler, error) { - var err C.int - r := &Resampler{ - channels: channels, - inRate: inRate, - outRate: outRate, - } - - r.resampler = C.speex_resampler_init( - C.spx_uint32_t(channels), - C.spx_uint32_t(inRate), - C.spx_uint32_t(outRate), - C.int(quality), - &err, - ) - - if r.resampler == nil { - return nil, StrError(int(err)) - } - - C.speex_resampler_skip_zeros(r.resampler) - - return r, nil -} - -func (r *Resampler) Destroy() { - if r.resampler != nil { - C.speex_resampler_destroy(r.resampler) - r.resampler = nil - } -} - -// Process performs resampling. -// Returns written samples count and error if any. -func (r *Resampler) Process(out, in []int16) (int, error) { - if len(in) == 0 || len(out) == 0 { - return 0, nil - } - - inLen := C.spx_uint32_t(len(in) / r.channels) - outLen := C.spx_uint32_t(len(out) / r.channels) - - res := C.speex_resampler_process_interleaved_int( - r.resampler, - (*C.spx_int16_t)(unsafe.Pointer(&in[0])), - &inLen, - (*C.spx_int16_t)(unsafe.Pointer(&out[0])), - &outLen, - ) - - if res != ErrorSuccess { - return 0, StrError(int(res)) - } - - return int(outLen) * r.channels, nil -} - -func StrError(errorCode int) error { - cS := C.speex_resampler_strerror(C.int(errorCode)) - if cS == nil { - return nil - } - return errors.New(C.GoString(cS)) -} diff --git a/pkg/resampler/speex_resampler.h b/pkg/resampler/speex_resampler.h deleted file mode 100644 index 9e046ed7..00000000 --- a/pkg/resampler/speex_resampler.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef SPEEX_RESAMPLER_H -#define SPEEX_RESAMPLER_H - -#define spx_int16_t short -#define spx_int32_t int -#define spx_uint16_t unsigned short -#define spx_uint32_t unsigned int - -#define SPEEX_RESAMPLER_QUALITY_MAX 10 -#define SPEEX_RESAMPLER_QUALITY_MIN 0 -#define SPEEX_RESAMPLER_QUALITY_DEFAULT 4 -#define SPEEX_RESAMPLER_QUALITY_VOIP 3 -#define SPEEX_RESAMPLER_QUALITY_DESKTOP 5 -enum { - RESAMPLER_ERR_SUCCESS = 0, - RESAMPLER_ERR_ALLOC_FAILED = 1, - RESAMPLER_ERR_BAD_STATE = 2, - RESAMPLER_ERR_INVALID_ARG = 3, - RESAMPLER_ERR_PTR_OVERLAP = 4, - - RESAMPLER_ERR_MAX_ERROR -}; -struct SpeexResamplerState_; -typedef struct SpeexResamplerState_ SpeexResamplerState; -/** Create a new resampler with integer input and output rates. - * @param nb_channels Number of channels to be processed - * @param in_rate Input sampling rate (integer number of Hz). - * @param out_rate Output sampling rate (integer number of Hz). - * @param quality Resampling quality between 0 and 10, where 0 has poor quality - * and 10 has very high quality. - * @return Newly created resampler state - * @retval NULL Error: not enough memory - */ -SpeexResamplerState *speex_resampler_init(spx_uint32_t nb_channels, - spx_uint32_t in_rate, - spx_uint32_t out_rate, - int quality, - int *err); -/** Destroy a resampler state. - * @param st Resampler state - */ -void speex_resampler_destroy(SpeexResamplerState *st); - - -/** Make sure that the first samples to go out of the resamplers don't have - * leading zeros. This is only useful before starting to use a newly created - * resampler. It is recommended to use that when resampling an audio file, as - * it will generate a file with the same length. For real-time processing, - * it is probably easier not to use this call (so that the output duration - * is the same for the first frame). - * @param st Resampler state - */ -int speex_resampler_skip_zeros(SpeexResamplerState *st); - -/** Resample an interleaved int array. The input and output buffers must *not* overlap. - * @param st Resampler state - * @param in Input buffer - * @param in_len Number of input samples in the input buffer. Returns the number - * of samples processed. This is all per-channel. - * @param out Output buffer - * @param out_len Size of the output buffer. Returns the number of samples written. - * This is all per-channel. - */ -int speex_resampler_process_interleaved_int(SpeexResamplerState *st, - const spx_int16_t *in, - spx_uint32_t *in_len, - spx_int16_t *out, - spx_uint32_t *out_len); -const char *speex_resampler_strerror(int err); -#endif \ No newline at end of file diff --git a/pkg/worker/caged/app/app.go b/pkg/worker/caged/app/app.go index db398a61..e6cbdd47 100644 --- a/pkg/worker/caged/app/app.go +++ b/pkg/worker/caged/app/app.go @@ -1,5 +1,7 @@ package app +import "time" + type App interface { AudioSampleRate() int AspectRatio() float32 @@ -7,7 +9,9 @@ type App interface { Flipped() bool Init() error ViewportSize() (int, int) - Scale() float64 + Scale() (float64, string) + Rotation() uint + PixFormat() uint32 Start() Close() @@ -19,13 +23,13 @@ type App interface { } type Audio struct { - Data []int16 - Duration int32 // up to 6y nanosecond-wise + Data []byte + Duration time.Duration } type Video struct { Frame RawFrame - Duration int32 + Duration time.Duration } type RawFrame struct { diff --git a/pkg/worker/caged/libretro/caged.go b/pkg/worker/caged/libretro/caged.go index 3d21db11..f3553b1a 100644 --- a/pkg/worker/caged/libretro/caged.go +++ b/pkg/worker/caged/libretro/caged.go @@ -92,7 +92,7 @@ func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat( func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } func (c *Caged) ViewportSize() (int, int) { return c.base.ViewportSize() } -func (c *Caged) Scale() float64 { return c.Emulator.Scale() } +func (c *Caged) Scale() (float64, string) { return c.Emulator.Scale() } func (c *Caged) Input(p int, d byte, data []byte) { c.base.Input(p, d, data) } func (c *Caged) KbMouseSupport() bool { return c.base.KbMouseSupport() } func (c *Caged) Start() { go c.Emulator.Start() } diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go index c0857e24..b1b16378 100644 --- a/pkg/worker/caged/libretro/frontend.go +++ b/pkg/worker/caged/libretro/frontend.go @@ -7,6 +7,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" "unsafe" @@ -47,14 +48,16 @@ type Emulator interface { Close() // Input passes input to the emulator Input(player int, device byte, data []byte) - // Scale returns set video scale factor - Scale() float64 + // Scale returns set video scale factor and method + Scale() (float64, string) Reset() } type Frontend struct { conf config.Emulator done chan struct{} + doneEvt chan struct{} // closed when Start() fully exits + closed atomic.Bool log *logger.Logger nano *nanoarch.Nanoarch onAudio func(app.Audio) @@ -62,12 +65,17 @@ type Frontend struct { onVideo func(app.Video) storage Storage scale float64 + scaleM string isGL bool th int // draw threads vw, vh int // out frame size // directives + // SpinBatch batches nanotime checks in the spin loop. + // Higher values = check every N iterations, reducing syscall overhead. + SpinBatch int + // skipVideo used when new frame was too late skipVideo bool @@ -128,14 +136,15 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) { // set global link to the Libretro f := &Frontend{ - conf: conf, - done: make(chan struct{}), - log: log, - onAudio: noAudio, - onData: noData, - onVideo: noVideo, - storage: store, - th: conf.Threads, + conf: conf, + done: make(chan struct{}), + log: log, + onAudio: noAudio, + onData: noData, + onVideo: noVideo, + storage: store, + th: conf.Threads, + SpinBatch: 2048, // ~ 3GHz 3M / 2048 calls } f.linkNano(nano) @@ -187,6 +196,7 @@ func (f *Frontend) LoadCore(emu string) { } f.storage.SetNonBlocking(conf.NonBlockingSave) f.scale = scale + f.scaleM = conf.ScaleMethod f.isGL = conf.IsGlAllowed f.nano.CoreLoad(meta) f.mu.Unlock() @@ -198,14 +208,14 @@ func (f *Frontend) handleAudio(audio unsafe.Pointer, samples int) { fr = new(app.Audio) } // !to look if we need a copy - fr.Data = unsafe.Slice((*int16)(audio), samples) + fr.Data = unsafe.Slice((*byte)(audio), samples<<1) // due to audio buffering for opus fixed frames and const duration up in the hierarchy, // we skip Duration here f.onAudio(*fr) audioPool.Put(fr) } -func (f *Frontend) handleVideo(data []byte, delta int32, fi nanoarch.FrameInfo) { +func (f *Frontend) handleVideo(data []byte, delta time.Duration, fi nanoarch.FrameInfo) { if f.conf.SkipLateFrames && f.skipVideo { return } @@ -278,7 +288,8 @@ func (f *Frontend) Start() { f.mui.Lock() f.done = make(chan struct{}) - f.nano.LastFrameTime = time.Now().UnixNano() + f.doneEvt = make(chan struct{}) + f.nano.LastFrameTime = time.Now() defer func() { // Save game on quit if it was saved before (shared or click-saved). @@ -290,6 +301,7 @@ func (f *Frontend) Start() { } f.mui.Unlock() f.Shutdown() + close(f.doneEvt) }() if f.HasSave() { @@ -308,7 +320,6 @@ func (f *Frontend) Start() { // The main loop of Libretro - // calculate the exact duration required for a frame (e.g., 16.666ms = 60 FPS) targetFrameTime := time.Duration(float64(time.Second) / f.nano.VideoFramerate()) // stop sleeping and start spinning in the remaining 1ms @@ -317,6 +328,8 @@ func (f *Frontend) Start() { // how many frames will be considered not normal const lateFramesThreshold = 3 + batch := max(f.SpinBatch, 1) + lastFrameStart := time.Now() for { @@ -324,11 +337,11 @@ func (f *Frontend) Start() { case <-f.done: return default: - // run one tick of the emulation f.Tick() elapsed := time.Since(lastFrameStart) sleepTime := targetFrameTime - elapsed + deadline := lastFrameStart.Add(targetFrameTime) if sleepTime > 0 { // SLEEP @@ -340,9 +353,15 @@ func (f *Frontend) Start() { // SPIN // if we are close to the target, - // burn CPU and check the clock with ns resolution - for time.Since(lastFrameStart) < targetFrameTime { - // CPU burn! + // burn CPU and check the clock with ns resolution. + // SpinBatch batches the time check to reduce + // syscall overhead while keeping sub-us precision. + for { + for range batch - 1 { + } + if !time.Now().Before(deadline) { + break + } } f.skipVideo = false } else { @@ -394,7 +413,7 @@ func (f *Frontend) Rotation() uint { return f.nano.Rot } func (f *Frontend) SRAMPath() string { return f.storage.GetSRAMPath() } func (f *Frontend) SaveGameState() error { return f.Save() } func (f *Frontend) SaveStateName() string { return filepath.Base(f.HashPath()) } -func (f *Frontend) Scale() float64 { return f.scale } +func (f *Frontend) Scale() (float64, string) { return f.scale, f.scaleM } func (f *Frontend) SetAudioCb(cb func(app.Audio)) { f.onAudio = cb } func (f *Frontend) SetSessionId(name string) { f.storage.SetMainSaveName(name) } func (f *Frontend) SetDataCb(cb func([]byte)) { f.onData = cb } @@ -429,11 +448,20 @@ func (f *Frontend) ViewportCalc() (nw int, nh int) { func (f *Frontend) Close() { f.log.Debug().Msgf("frontend close") + if f.closed.Swap(true) { + return + } close(f.done) f.mui.Lock() f.nano.Close() + f.mui.Unlock() + if f.doneEvt != nil { + <-f.doneEvt + } + + f.mui.Lock() if f.UniqueSaveDir && !f.HasSave() { if err := f.nano.DeleteSaveDir(); err != nil { f.log.Error().Msgf("couldn't delete save dir: %v", err) diff --git a/pkg/worker/caged/libretro/frontend_test.go b/pkg/worker/caged/libretro/frontend_test.go index 2cacd5a4..4cb76be1 100644 --- a/pkg/worker/caged/libretro/frontend_test.go +++ b/pkg/worker/caged/libretro/frontend_test.go @@ -226,7 +226,7 @@ func TestSavePersistence(t *testing.T) { for range 10 { v, _ := front.save() - if v == nil || len(v) == 0 { + if len(v) == 0 { t.Errorf("couldn't persist the state") t.Fail() } diff --git a/pkg/worker/caged/libretro/nanoarch/input.go b/pkg/worker/caged/libretro/nanoarch/input.go index 2ff7379e..ed31bf2e 100644 --- a/pkg/worker/caged/libretro/nanoarch/input.go +++ b/pkg/worker/caged/libretro/nanoarch/input.go @@ -85,7 +85,7 @@ func (s *InputState) SetInput(port int, data []byte) { // SyncToCache syncs input state to C-side cache before Run(). func (s *InputState) SyncToCache() { - for p := uint(0); p < maxPort; p++ { + for p := range uint(maxPort) { keys := atomic.LoadUint32(&s[p].keys) axes := atomic.LoadInt64(&s[p].axes) triggers := atomic.LoadInt32(&s[p].triggers) diff --git a/pkg/worker/caged/libretro/nanoarch/input_test.go b/pkg/worker/caged/libretro/nanoarch/input_test.go index 1df81da7..3cb1cf2a 100644 --- a/pkg/worker/caged/libretro/nanoarch/input_test.go +++ b/pkg/worker/caged/libretro/nanoarch/input_test.go @@ -250,14 +250,15 @@ func TestKeyboardState_IsPressed(t *testing.T) { } // Press key + k := uint(42) ks.SetKey([]byte{0, 0, 0, 42, 1, 0, 0}) - if (ks.keys[42/64].Load()>>(42%64))&1 != 1 { + if (ks.keys[k/64].Load()>>(k%64))&1 != 1 { t.Error("key should be pressed") } // Release key ks.SetKey([]byte{0, 0, 0, 42, 0, 0, 0}) - if (ks.keys[42/64].Load()>>(42%64))&1 != 0 { + if (ks.keys[k/64].Load()>>(k%64))&1 != 0 { t.Error("key should be released") } } diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.go b/pkg/worker/caged/libretro/nanoarch/nanoarch.go index 0e36801e..eb6cd4ce 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.go +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.go @@ -6,6 +6,7 @@ import ( "maps" "path/filepath" "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -39,7 +40,7 @@ type Nanoarch struct { retropad InputState keyboardCb *C.struct_retro_keyboard_callback - LastFrameTime int64 + LastFrameTime time.Time LibCo bool meta Metadata options map[string]string @@ -53,7 +54,7 @@ type Nanoarch struct { i C.struct_retro_system_info api C.unsigned } - tickTime int64 + tickTime time.Duration cSaveDirectory *C.char cSystemDirectory *C.char cUserName *C.char @@ -72,11 +73,15 @@ type Nanoarch struct { hackSkipSameThreadSave bool limiter func(func()) log *logger.Logger + + audioBuffCb *C.struct_retro_audio_buffer_status_callback + lastFrameDelta float64 // most recent frame delta (ns) + audioDrift float64 // cumulative timing drift for frameskip } type Handlers struct { OnAudio func(ptr unsafe.Pointer, frames int) - OnVideo func(data []byte, delta int32, fi FrameInfo) + OnVideo func(data []byte, delta time.Duration, fi FrameInfo) OnDup func() OnSystemAvInfo func() } @@ -128,7 +133,7 @@ var Nan0 = Nanoarch{ limiter: func(fn func()) { fn() }, Handlers: Handlers{ OnAudio: func(unsafe.Pointer, int) {}, - OnVideo: func([]byte, int32, FrameInfo) {}, + OnVideo: func([]byte, time.Duration, FrameInfo) {}, OnDup: func() {}, }, } @@ -186,6 +191,8 @@ func (n *Nanoarch) CoreLoad(meta Metadata) { n.Video.gl.autoCtx = meta.AutoGlContext n.Video.gl.enabled = meta.IsGlAllowed + Nan0.audioBuffCb = nil + thread.SwitchGraphics(n.Video.gl.enabled) // hacks @@ -312,7 +319,7 @@ func (n *Nanoarch) LoadGame(path string) error { n.serializeSize = C.bridge_retro_serialize_size(retroSerializeSize) n.log.Info().Msgf("Save file size: %v", byteCountBinary(int64(n.serializeSize))) - Nan0.tickTime = int64(time.Second / time.Duration(n.sys.av.timing.fps)) + Nan0.tickTime = time.Duration(float64(time.Second) / float64(n.sys.av.timing.fps)) if n.vfr { n.log.Info().Msgf("variable framerate (VFR) is enabled") } @@ -345,12 +352,13 @@ func (n *Nanoarch) LoadGame(path string) error { } } - n.LastFrameTime = time.Now().UnixNano() + n.LastFrameTime = time.Now() return nil } func (n *Nanoarch) Shutdown() { + Nan0.audioBuffCb = nil if n.LibCo { thread.Main(func() { C.same_thread(retroUnloadGame) @@ -387,6 +395,7 @@ func (n *Nanoarch) Shutdown() { } n.options = nil n.options4rom = nil + Nan0.audioBuffCb = nil C.free(unsafe.Pointer(n.cUserName)) C.free(unsafe.Pointer(n.cSaveDirectory)) C.free(unsafe.Pointer(n.cSystemDirectory)) @@ -487,10 +496,10 @@ func printOpenGLDriverInfo() { var openGLInfo strings.Builder openGLInfo.Grow(128) version, vendor, renderrer, glsl := graphics.GLInfo() - openGLInfo.WriteString(fmt.Sprintf("\n[OpenGL] Version: %v\n", version)) - openGLInfo.WriteString(fmt.Sprintf("[OpenGL] Vendor: %v\n", vendor)) - openGLInfo.WriteString(fmt.Sprintf("[OpenGL] Renderer: %v\n", renderrer)) - openGLInfo.WriteString(fmt.Sprintf("[OpenGL] GLSL Version: %v", glsl)) + fmt.Fprintf(&openGLInfo, "\n[OpenGL] Version: %v\n", version) + fmt.Fprintf(&openGLInfo, "[OpenGL] Vendor: %v\n", vendor) + fmt.Fprintf(&openGLInfo, "[OpenGL] Renderer: %v\n", renderrer) + fmt.Fprintf(&openGLInfo, "[OpenGL] GLSL Version: %v", glsl) Nan0.log.Debug().Msg(openGLInfo.String()) } @@ -599,14 +608,7 @@ func byteCountBinary(b int64) string { return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) } -func (m Metadata) HasHack(h string) bool { - for _, n := range m.Hacks { - if h == n { - return true - } - } - return false -} +func (m Metadata) HasHack(h string) bool { return slices.Contains(m.Hacks, h) } var ( retroAPIVersion unsafe.Pointer @@ -645,10 +647,14 @@ func coreVideoRefresh(data unsafe.Pointer, width, height uint, packed uint) { // this is useful only for cores with variable framerate, for the fixed framerate cores this adds stutter // !to find docs on Libretro refresh sync and frame times dt := Nan0.tickTime + // Update frame timing only when needed: VFR mode or audio buffer callback. + if Nan0.vfr || Nan0.audioBuffCb != nil { + now := time.Now() + Nan0.lastFrameDelta = float64(now.Sub(Nan0.LastFrameTime)) + Nan0.LastFrameTime = now + } if Nan0.vfr { - t := time.Now().UnixNano() - dt = t - Nan0.LastFrameTime - Nan0.LastFrameTime = t + dt = time.Duration(Nan0.lastFrameDelta) } // when the core returns a duplicate frame @@ -679,7 +685,7 @@ func coreVideoRefresh(data unsafe.Pointer, width, height uint, packed uint) { // also we have an option of xN output frame magnification // so, it may be rescaled - Nan0.Handlers.OnVideo(data_, int32(dt), FrameInfo{W: width, H: height, Stride: packed}) + Nan0.Handlers.OnVideo(data_, dt, FrameInfo{W: width, H: height, Stride: packed}) } //export coreAudioSampleBatch @@ -688,6 +694,28 @@ func coreAudioSampleBatch(data unsafe.Pointer, frames C.size_t) C.size_t { return frames } Nan0.Handlers.OnAudio(data, int(frames)<<1) + + // Audio buffer status for the core (e.g. mGBA frameskip). + if Nan0.audioBuffCb != nil { + rawRate := int(Nan0.sys.av.timing.sample_rate) + expectedNs := float64(frames) / float64(rawRate) * 1e9 + d := Nan0.lastFrameDelta - expectedNs + Nan0.audioDrift += d + + occupancy := min(max(50-int(Nan0.audioDrift/(expectedNs*4)*50), 0), 100) + + underrun := false + if Nan0.audioDrift > expectedNs*4 { + underrun = true + Nan0.audioDrift = 0 + } + if Nan0.audioDrift < -expectedNs*4 { + Nan0.audioDrift = -expectedNs * 4 + } + + C.call_audio_buffer_status(Nan0.audioBuffCb, true, C.unsigned(occupancy), C.bool(underrun)) + } + return frames } @@ -813,7 +841,7 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { break } cInfo := strings.Builder{} - cInfo.WriteString(fmt.Sprintf("Controller [%v] ", c)) + fmt.Fprintf(&cInfo, "Controller [%v] ", c) cd := (*[32]C.struct_retro_controller_description)(tp) delim := ", " n := int(controller.num_types) @@ -821,7 +849,7 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { if i == n-1 { delim = "" } - cInfo.WriteString(fmt.Sprintf("%v: %v%s", cd[i].id, C.GoString(cd[i].desc), delim)) + fmt.Fprintf(&cInfo, "%v: %v%s", cd[i].id, C.GoString(cd[i].desc), delim) } //Nan0.log.Debug().Msgf("%v", cInfo.String()) } @@ -830,6 +858,21 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { Nan0.log.Debug().Msgf("Keyboard event callback was set") Nan0.keyboardCb = (*C.struct_retro_keyboard_callback)(data) return true + case C.RETRO_ENVIRONMENT_SET_AUDIO_BUFFER_STATUS_CALLBACK: + if data == nil { + Nan0.audioBuffCb = nil + return true + } + + src := (*C.struct_retro_audio_buffer_status_callback)(data) + Nan0.audioBuffCb = new(C.struct_retro_audio_buffer_status_callback) + *Nan0.audioBuffCb = *src + Nan0.log.Info().Msg("Audio buffer status callback registered") + return true + case C.RETRO_ENVIRONMENT_SET_MINIMUM_AUDIO_LATENCY: + latency := *(*C.unsigned)(data) + Nan0.log.Info().Uint("latency_ms", uint(latency)).Msg("audio latency requested") + return true } return false } diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.h b/pkg/worker/caged/libretro/nanoarch/nanoarch.h index d8e09265..2479d0cf 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.h +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.h @@ -35,4 +35,10 @@ void same_thread(void *f); void *same_thread_with_args2(void *f, int type, void *arg1, void *arg2); void same_thread_stop(); +static inline void call_audio_buffer_status( + struct retro_audio_buffer_status_callback *cb, + bool active, unsigned occupancy, bool underrun) { + if (cb && cb->callback) cb->callback(active, occupancy, underrun); +} + #endif diff --git a/pkg/worker/caged/libretro/recording.go b/pkg/worker/caged/libretro/recording.go index 64734536..eb84d6ee 100644 --- a/pkg/worker/caged/libretro/recording.go +++ b/pkg/worker/caged/libretro/recording.go @@ -2,6 +2,7 @@ package libretro import ( "time" + "unsafe" "github.com/giongto35/cloud-game/v3/pkg/config" "github.com/giongto35/cloud-game/v3/pkg/logger" @@ -33,7 +34,7 @@ func WithRecording(fe Emulator, rec bool, user string, game string, conf config. func (r *RecordingFrontend) SetAudioCb(fn func(app.Audio)) { r.Emulator.SetAudioCb(func(audio app.Audio) { if r.IsRecording() { - pcm := audio.Data + pcm := unsafe.Slice((*int16)(unsafe.Pointer(unsafe.SliceData(audio.Data))), len(audio.Data)>>1) // example: 1600 = x / 1000 * 48000 * 2 l := time.Duration(float64(len(pcm)) / float64(r.AudioSampleRate()<<1) * 1000000000) r.rec.WriteAudio(recorder.Audio{Samples: pcm, Duration: l}) diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go index 2c403bf2..6972cc73 100644 --- a/pkg/worker/coordinatorhandlers.go +++ b/pkg/worker/coordinatorhandlers.go @@ -110,7 +110,7 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou } game := games.GameMetadata(gameInfo) - r = room.NewRoom[*room.GameSession](uid, nil, w.router.Users(), nil) + r = room.NewRoom(uid, nil, w.router.Users(), nil) r.HandleClose = func() { c.CloseRoom(uid) c.log.Debug().Msgf("room close request %v sent", uid) @@ -134,18 +134,16 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou r.SetApp(app) - m := media.NewWebRtcMediaPipe(w.conf.Encoder.Audio, w.conf.Encoder.Video, w.log) + m := media.NewGstreamer(w.conf.Encoder, w.log) // recreate the video encoder app.VideoChangeCb(func() { app.ViewportRecalculate() m.VideoW, m.VideoH = app.ViewportSize() - m.VideoScale = app.Scale() + m.VideoScale, m.ScaleMethod = app.Scale() - if m.IsInitialized() { - if err := m.Reinit(); err != nil { - c.log.Error().Err(err).Msgf("reinit fail") - } + if err := m.Reinit(); err != nil { + c.log.Error().Err(err).Msgf("reinit fail") } data, err := api.Wrap(api.Out{ @@ -154,8 +152,10 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou W: m.VideoW, H: m.VideoH, A: app.AspectRatio(), - S: int(app.Scale()), + S: int(m.VideoScale), + SM: m.ScaleMethod, Flip: app.Flipped(), + Rot: app.Rotation(), }}) if err != nil { c.log.Error().Err(err).Msgf("wrap") @@ -172,9 +172,15 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou } m.AudioSrcHz = app.AudioSampleRate() - m.AudioFrames = w.conf.Encoder.Audio.Frames m.VideoW, m.VideoH = app.ViewportSize() - m.VideoScale = app.Scale() + m.VideoScale, m.ScaleMethod = app.Scale() + coreConf := w.conf.Emulator.GetLibretroCoreConfig(game.System) + m.MaxThreads = coreConf.MaxThreads + m.VideoFPS = app.FPS() + m.VideoVFR = coreConf.VFR + + m.SetPixFmt(app.PixFormat()) + m.SetRot(app.Rotation()) r.SetMedia(m) @@ -185,15 +191,10 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou return api.EmptyPacket } - m.SetPixFmt(app.PixFormat()) - m.SetRot(app.Rotation()) - - r.BindAppMedia() + r.InitMedia() r.StartApp() } - c.log.Debug().Msg("Start session input poll") - needsKbMouse := r.App().KbMouseSupport() s := room.WithWebRTC(user.Session) @@ -210,14 +211,18 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest, w *Worker) api.Ou Record: w.conf.Recording.Enabled, KbMouse: needsKbMouse, } + if r.App().AspectEnabled() { ww, hh := r.App().ViewportSize() + scale, scaleM := r.App().Scale() response.AV = &api.AppVideoInfo{ W: ww, H: hh, A: r.App().AspectRatio(), - S: int(r.App().Scale()), + S: int(scale), + SM: scaleM, Flip: r.App().Flipped(), + Rot: r.App().Rotation(), } } diff --git a/pkg/worker/media/buffer.go b/pkg/worker/media/buffer.go deleted file mode 100644 index e13bb1f0..00000000 --- a/pkg/worker/media/buffer.go +++ /dev/null @@ -1,143 +0,0 @@ -package media - -import ( - "errors" - - "github.com/giongto35/cloud-game/v3/pkg/resampler" -) - -type ResampleAlgo uint8 - -const ( - ResampleNearest ResampleAlgo = iota - ResampleLinear - ResampleSpeex -) - -type buffer struct { - raw samples - scratch samples - buckets []bucket - srcHz int - dstHz int - bi int - algo ResampleAlgo - - resampler *resampler.Resampler -} - -type bucket struct { - mem samples - ms float32 - p int - dst int -} - -func newBuffer(frames []float32, hz int) (*buffer, error) { - if hz < 2000 || len(frames) == 0 { - return nil, errors.New("invalid params") - } - - buckets := make([]bucket, len(frames)) - var total int - for i, ms := range frames { - n := stereoSamples(hz, ms) - buckets[i] = bucket{ms: ms, dst: n} - total += n - } - if total == 0 { - return nil, errors.New("zero buffer size") - } - - raw := make(samples, total) - for i, off := 0, 0; i < len(buckets); i++ { - buckets[i].mem = raw[off : off+buckets[i].dst] - off += buckets[i].dst - } - - return &buffer{ - raw: raw, - scratch: make(samples, 5760), - buckets: buckets, - srcHz: hz, - dstHz: hz, - bi: len(buckets) - 1, - }, nil -} - -func (b *buffer) close() { - if b.resampler != nil { - b.resampler.Destroy() - b.resampler = nil - } -} - -func (b *buffer) resample(hz int, algo ResampleAlgo) error { - b.algo, b.dstHz = algo, hz - for i := range b.buckets { - b.buckets[i].dst = stereoSamples(hz, b.buckets[i].ms) - } - if algo == ResampleSpeex { - var err error - b.resampler, err = resampler.Init(2, b.srcHz, hz, resampler.QualityMax) - return err - } - return nil -} - -func (b *buffer) write(s samples, onFull func(samples, float32)) int { - n := len(s) - for i := 0; i < n; { - cur := &b.buckets[b.bi] - c := copy(cur.mem[cur.p:], s[i:]) - i += c - cur.p += c - if cur.p == len(cur.mem) { - onFull(b.stretch(cur.mem, cur.dst), cur.ms) - b.choose(n - i) - b.buckets[b.bi].p = 0 - } - } - return n -} - -func (b *buffer) choose(rem int) { - for i := len(b.buckets) - 1; i >= 0; i-- { - if rem >= len(b.buckets[i].mem) { - b.bi = i - return - } - } - b.bi = 0 -} - -func (b *buffer) stretch(src samples, size int) samples { - if len(src) == size { - return src - } - - if cap(b.scratch) < size { - b.scratch = make(samples, size) - } - out := b.scratch[:size] - - if b.algo == ResampleSpeex && b.resampler != nil { - if n, _ := b.resampler.Process(out, src); n > 0 { - for i := n; i < size; i += 2 { - out[i], out[i+1] = out[n-2], out[n-1] - } - return out - } - } - - if b.algo == ResampleNearest { - resampler.Nearest(out, src) - } else { - resampler.Linear(out, src) - } - return out -} - -func stereoSamples(hz int, ms float32) int { - return int(float32(hz)*ms/1000+0.5) * 2 -} diff --git a/pkg/worker/media/buffer_test.go b/pkg/worker/media/buffer_test.go deleted file mode 100644 index 6c8d300a..00000000 --- a/pkg/worker/media/buffer_test.go +++ /dev/null @@ -1,318 +0,0 @@ -package media - -import ( - "reflect" - "testing" - - "github.com/giongto35/cloud-game/v3/pkg/resampler" -) - -func mustBuffer(t *testing.T, frames []float32, hz int) *buffer { - t.Helper() - buf, err := newBuffer(frames, hz) - if err != nil { - t.Fatalf("failed to create buffer: %v", err) - } - return buf -} - -func samplesOf(v int16, n int) samples { - s := make(samples, n) - for i := range s { - s[i] = v - } - return s -} - -func ramp(pairs int) samples { - s := make(samples, pairs*2) - for i := range pairs { - s[i*2], s[i*2+1] = int16(i), int16(i) - } - return s -} - -func TestNewBuffer(t *testing.T) { - tests := []struct { - name string - frames []float32 - hz int - wantErr bool - }{ - {"valid single", []float32{10}, 48000, false}, - {"valid multi", []float32{10, 20}, 48000, false}, - {"hz too low", []float32{10}, 1999, true}, - {"empty frames", []float32{}, 48000, true}, - {"nil frames", nil, 48000, true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf, err := newBuffer(tt.frames, tt.hz) - if (err != nil) != tt.wantErr { - t.Errorf("err = %v, wantErr %v", err, tt.wantErr) - } - if buf != nil { - buf.close() - } - }) - } -} - -func TestBufferBucketSizes(t *testing.T) { - buf := mustBuffer(t, []float32{10, 20}, 48000) - defer buf.close() - - if len(buf.buckets) != 2 { - t.Fatalf("got %d buckets, want 2", len(buf.buckets)) - } - if n := len(buf.buckets[0].mem); n != 960 { - t.Errorf("bucket[0] = %d, want 960", n) - } - if n := len(buf.buckets[1].mem); n != 1920 { - t.Errorf("bucket[1] = %d, want 1920", n) - } -} - -func TestBufferClose(t *testing.T) { - buf := mustBuffer(t, []float32{10}, 48000) - buf.close() - buf.close() // idempotent - if buf.resampler != nil { - t.Error("resampler should be nil after close") - } -} - -func TestBufferWrite(t *testing.T) { - tests := []struct { - name string - writes []struct { - v int16 - n int - } - want samples - }{ - { - name: "overflow triggers callback", - writes: []struct { - v int16 - n int - }{{1, 10}, {2, 20}, {3, 30}}, - want: samples{ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - }, - }, - { - name: "partial fill", - writes: []struct { - v int16 - n int - }{{1, 3}, {2, 18}, {3, 2}}, - want: samples{1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := mustBuffer(t, []float32{10, 5}, 2000) - defer buf.close() - - var got samples - for _, w := range tt.writes { - buf.write(samplesOf(w.v, w.n), func(s samples, _ float32) { got = s }) - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("\ngot: %v\nwant: %v", got, tt.want) - } - }) - } -} - -func TestBufferWriteExact(t *testing.T) { - buf := mustBuffer(t, []float32{10}, 2000) // 40 samples - defer buf.close() - - calls := 0 - buf.write(samplesOf(1, 40), func(_ samples, ms float32) { - calls++ - if ms != 10 { - t.Errorf("ms = %v, want 10", ms) - } - }) - if calls != 1 { - t.Errorf("calls = %d, want 1", calls) - } -} - -func TestBufferWriteReturn(t *testing.T) { - buf := mustBuffer(t, []float32{10}, 2000) - defer buf.close() - - if n := buf.write(samplesOf(1, 100), func(samples, float32) {}); n != 100 { - t.Errorf("return = %d, want 100", n) - } -} - -func TestBufferChoose(t *testing.T) { - buf := mustBuffer(t, []float32{20, 10, 5}, 48000) // 1920, 960, 480 - defer buf.close() - - tests := []struct{ rem, want int }{ - {10000, 2}, {500, 2}, {479, 0}, {0, 0}, - } - for _, tt := range tests { - buf.choose(tt.rem) - if buf.bi != tt.want { - t.Errorf("choose(%d) = %d, want %d", tt.rem, buf.bi, tt.want) - } - } -} - -func TestStereoSamples(t *testing.T) { - tests := []struct { - hz int - ms float32 - want int - }{ - {16000, 5, 160}, - {32768, 10, 656}, - {32768, 2.5, 164}, - {32768, 5, 328}, - {44100, 10, 882}, - {48000, 10, 960}, - {48000, 2.5, 240}, - } - for _, tt := range tests { - if got := stereoSamples(tt.hz, tt.ms); got != tt.want { - t.Errorf("stereoSamples(%d, %.0f) = %d, want %d", tt.hz, tt.ms, got, tt.want) - } - } -} - -func TestStretchPassthrough(t *testing.T) { - buf := mustBuffer(t, []float32{10}, 48000) - defer buf.close() - - src := samples{1, 2, 3, 4} - if res := buf.stretch(src, 4); &res[0] != &src[0] { - t.Error("expected zero-copy when sizes match") - } -} - -func TestLinear(t *testing.T) { - t.Run("interpolation", func(t *testing.T) { - out := make(samples, 8) - resampler.Linear(out, samples{0, 0, 100, 100}) - if out[2] <= 0 || out[2] >= 100 { - t.Errorf("middle value %d not interpolated", out[2]) - } - }) - - t.Run("sizes", func(t *testing.T) { - cases := []struct{ srcPairs, dstSize int }{ - {4, 16}, {8, 8}, {4, 8}, - } - for _, tc := range cases { - out := make(samples, tc.dstSize) - resampler.Linear(out, ramp(tc.srcPairs)) - if len(out) != tc.dstSize { - t.Errorf("len = %d, want %d", len(out), tc.dstSize) - } - } - }) -} - -func TestNearest(t *testing.T) { - tests := []struct { - src samples - want samples - }{ - {samples{10, 20, 30, 40}, samples{10, 20, 10, 20, 30, 40, 30, 40}}, - {samples{10, 20, 30, 40, 50, 60, 70, 80}, samples{10, 20, 50, 60}}, - } - for _, tt := range tests { - out := make(samples, len(tt.want)) - resampler.Nearest(out, tt.src) - if !reflect.DeepEqual(out, tt.want) { - t.Errorf("nearest(%v) = %v, want %v", tt.src, out, tt.want) - } - } -} - -func TestSpeex(t *testing.T) { - buf := mustBuffer(t, []float32{10}, 48000) - defer buf.close() - - if err := buf.resample(24000, ResampleSpeex); err != nil { - t.Fatal(err) - } - - t.Run("stretch", func(t *testing.T) { - res := buf.stretch(samplesOf(1000, 960), 480) - if len(res) != 480 { - t.Errorf("len = %d, want 480", len(res)) - } - for _, s := range res { - if s != 0 { - return - } - } - t.Error("output is silent") - }) - - t.Run("write", func(t *testing.T) { - calls := 0 - buf.write(samplesOf(5000, 960), func(s samples, ms float32) { - calls++ - if len(s) != 480 { - t.Errorf("len = %d, want 480", len(s)) - } - if ms != 10 { - t.Errorf("ms = %v, want 10", ms) - } - }) - if calls != 1 { - t.Errorf("calls = %d, want 1", calls) - } - }) -} - -func BenchmarkStretch(b *testing.B) { - src := samplesOf(1000, 1920) // 20ms @ 48kHz - - b.Run("speex", func(b *testing.B) { - buf, _ := newBuffer([]float32{20}, 48000) - defer buf.close() - _ = buf.resample(24000, ResampleSpeex) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.stretch(src, 960) - } - }) - - b.Run("linear", func(b *testing.B) { - buf, _ := newBuffer([]float32{20}, 48000) - defer buf.close() - _ = buf.resample(24000, ResampleLinear) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.stretch(src, 960) - } - }) - - b.Run("nearest", func(b *testing.B) { - buf, _ := newBuffer([]float32{20}, 48000) - defer buf.close() - _ = buf.resample(24000, ResampleNearest) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.stretch(src, 960) - } - }) -} diff --git a/pkg/worker/media/gstreamer.go b/pkg/worker/media/gstreamer.go new file mode 100644 index 00000000..fcd32b3b --- /dev/null +++ b/pkg/worker/media/gstreamer.go @@ -0,0 +1,579 @@ +package media + +import ( + "cmp" + "fmt" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/giongto35/cloud-game/v3/pkg/config" + "github.com/giongto35/cloud-game/v3/pkg/logger" + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/klauspost/cpuid/v2" +) + +/* +#cgo pkg-config: gstreamer-video-1.0 gstreamer-app-1.0 +#include +#include +#include +#include + +static inline GstVideoFormat gstVideoFormatFromString(const gchar *s) { + return gst_video_format_from_string(s); +} + +static inline void doubleToFraction(const gdouble src, gint *num, gint *den) { + gst_util_double_to_fraction(src, num, den); +} + +// pushVideoBuf wraps frame memory directly. +static inline void pushVideoBuf(GstAppSrc *src, void *data, gsize len, + GstVideoFormat fmt, guint w, guint h, gint stride) { + GstBuffer *buf = gst_buffer_new_wrapped_full( + GST_MEMORY_FLAG_READONLY, data, len, 0, len, NULL, NULL); + if (!buf) return; + if (stride != 0) { + gsize offset[GST_VIDEO_MAX_PLANES] = {0}; + gint strides[GST_VIDEO_MAX_PLANES] = {stride}; + gst_buffer_add_video_meta_full(buf, GST_VIDEO_FRAME_FLAG_NONE, fmt, w, h, 1, offset, strides); + } + gst_app_src_push_buffer(src, buf); +} + +// pushAudioBuf copies audio data into a new buffer. +static inline void pushAudioBuf(GstAppSrc *src, void *data, gsize len) { + GstBuffer *buf = gst_buffer_new_allocate(NULL, len, NULL); + if (!buf) return; + gst_buffer_fill(buf, 0, data, len); + gst_app_src_push_buffer(src, buf); +} + +// pullMappedBuffer pulls a sample from the appsink, gets its buffer, +// and maps it for reading. Returns the buffer (with ref held) on success, +// or NULL if no sample/buffer is available. +// The caller must call unmapAndUnref to release. +static inline GstBuffer* pullMappedBuffer(GstAppSink *sink, GstMapInfo *mapInfo) { + GstSample *sample = gst_app_sink_pull_sample(sink); + if (!sample) return NULL; + GstBuffer *buffer = gst_sample_get_buffer(sample); + if (buffer) gst_buffer_ref(buffer); + gst_sample_unref(sample); + if (!buffer) return NULL; + if (!gst_buffer_map(buffer, mapInfo, GST_MAP_READ)) { + gst_buffer_unref(buffer); + return NULL; + } + return buffer; +} + +static inline void unmapAndUnref(GstBuffer *buffer, GstMapInfo *mapInfo) { + gst_buffer_unmap(buffer, mapInfo); + gst_buffer_unref(buffer); +} +*/ +import "C" + +const ( + pixFmtBGRx uint32 = 0 + pixFmtBGRA uint32 = 1 + pixFmtRGB16 uint32 = 2 + + defaultAudioFrameMs = 20 * time.Millisecond + + maxForcedKeyframes = 3 +) + +var forceKeyframeEvent *gst.Event +var cachedSegment *gst.Segment +var cpuCores int +var pixFmtToGst = map[uint32]string{ + pixFmtBGRx: "BGRx", + pixFmtBGRA: "BGRA", + pixFmtRGB16: "RGB16", +} +var pixFmtCache = map[string]uint32{} + +func init() { + gst.Init(nil) + + s := gst.NewStructure("GstForceKeyUnit") + s.SetValue("all-headers", true) + forceKeyframeEvent = gst.NewCustomEvent(gst.EventTypeCustomDownstream, s) + + cpuCores = cmp.Or(cpuid.CPU.PhysicalCores, 4) - 1 + + cachedSegment = gst.NewSegment() + cachedSegment.Init(gst.FormatTime) +} + +// GstMediaPipe a video and audio pipline based on GStreamer. +// Very unsafe. +// +// Video encoding is done in a single goroutine to avoid races. +// Audio is pulled from the appsink on GStreamer's own audio thread. +// +// Goroutines (3): +// - video worker x1 (push+pull loop for video encoding) +// - bus messages x2 (one per pipeline, bus message logging) +type GstMediaPipe struct { + a, v *pipe + + onAudio func([]byte, time.Duration) + + conf config.Encoder + + pixFmt uint32 + + VideoW, VideoH int + VideoScale float64 + ScaleMethod string + MaxThreads int + AudioSrcHz int + VideoVFR bool + VideoFPS float64 + + oldPf uint32 + oldRot uint + bpp int + vidFmt uint32 // cached GstVideoFormat enum + + frameI int // for forced keyframes + keyI int // periodic keyframe counter + kfi int // 0=GStreamer auto, >0=force keyframe every N frames + aSegSent bool // for Opusenc bug + + // used for reinit + videoCh chan videoJob + videoDone chan struct{} + reinit atomic.Bool + mu sync.Mutex + + log *logger.Logger +} + +type videoJob struct { + data []byte + w, h, stride int + dur time.Duration + cb func([]byte, time.Duration) +} + +type Video struct { + Frame RawFrame + Duration time.Duration +} + +type RawFrame struct { + Data []byte + Stride int + W, H int +} + +type pipe struct { + pipeline *gst.Pipeline + source *app.Source + sink *app.Sink + srcPad *gst.Pad + frameDur time.Duration + stale atomic.Bool +} + +func (p *pipe) src() *C.GstAppSrc { return (*C.GstAppSrc)(unsafe.Pointer(p.source.Instance())) } +func (p *pipe) sinkPtr() *C.GstAppSink { return (*C.GstAppSink)(unsafe.Pointer(p.sink.Instance())) } +func (p *pipe) stop() { + if p != nil && p.pipeline != nil { + p.pipeline.SetState(gst.StateNull) + } +} + +func (p *pipe) destroy() { + if p == nil { + return + } + if p.pipeline != nil { + p.pipeline.GetPipelineBus().Post(gst.NewEOSMessage(p.pipeline)) + p.pipeline.SetState(gst.StateNull) + p.pipeline = nil + } + p.sink = nil + p.source = nil +} + +type builder struct{ strings.Builder } + +func (b *builder) write(s string) { b.Builder.WriteString(s) } +func (b *builder) writef(s string, args ...any) { fmt.Fprintf(&b.Builder, s, args...) } +func (b *builder) writeIfSet(s string) { + if s != "" { + b.write(s) + } +} + +func NewGstreamer(conf config.Encoder, log *logger.Logger) *GstMediaPipe { + return &GstMediaPipe{conf: conf, bpp: 4, vidFmt: gstVideoFormat("BGRx"), log: log} +} + +func (g *GstMediaPipe) Init() error { + if err := g.initVideo(); err != nil { + return fmt.Errorf("gst video init: %w", err) + } + if err := g.initAudio(); err != nil { + return fmt.Errorf("gst audio init: %w", err) + } + return nil +} + +func (g *GstMediaPipe) initAudio() (err error) { + srcHz := g.AudioSrcHz + g.a, err = buildAudioPipeline(srcHz, g.conf, g.log) + if err != nil { + return + } + g.a.sink.SetCallbacks(&app.SinkCallbacks{NewSampleFunc: g.pullAudio}) + return g.a.pipeline.SetState(gst.StatePlaying) +} + +func (g *GstMediaPipe) initVideo() (err error) { + w, h, scale := g.VideoW, g.VideoH, g.VideoScale + sw, sh := round(w, scale), round(h, scale) + if g.oldRot%180 != 0 { + w, h = h, w + } + g.v, err = buildVideoPipeline(w, h, sw, sh, g.pixFmt, g.ScaleMethod, g.MaxThreads, g.conf, + g.log, g.VideoFPS, g.VideoVFR) + if err != nil { + return + } + + // if params contain keyframe-mode=disabled, we force periodic keyframes + // ourselves instead of relying on GStreamer's auto mode + if opts, _ := g.conf.VideoSettings(); opts != nil && strings.Contains(opts.Params, "keyframe-mode=disabled") { + g.kfi = cmp.Or(g.conf.Video.KeyframeInterval, 120) + } else { + g.kfi = 0 + } + + p := g.v + fmt := g.vidFmt + g.videoCh = make(chan videoJob, 1) + g.videoDone = make(chan struct{}) + go g.videoWorker(p, fmt, g.videoCh, g.videoDone) + + return p.pipeline.SetState(gst.StatePlaying) +} + +func (g *GstMediaPipe) SetPixFmt(f uint32) { + g.oldPf, g.pixFmt = f, f + g.vidFmt = gstVideoFormat(cmp.Or(pixFmtToGst[f], "RGBA")) + if f == pixFmtRGB16 { + g.bpp = 2 + } else { + g.bpp = 4 + } +} +func (g *GstMediaPipe) SetRot(r uint) { g.oldRot = r } + +func (g *GstMediaPipe) Destroy() { + g.mu.Lock() + defer g.mu.Unlock() + + g.reinit.Store(true) + + if g.videoCh != nil { + g.v.stop() + close(g.videoCh) + <-g.videoDone + } + + g.v.destroy() + g.a.destroy() + g.aSegSent = false + g.keyI = 0 +} + +func (g *GstMediaPipe) ProcessAudio(audio []byte, cb func([]byte, time.Duration)) { + g.onAudio = cb + if !g.aSegSent { + g.aSegSent = true + g.a.srcPad.PushEvent(gst.NewSegmentEvent(cachedSegment)) + } + C.pushAudioBuf(g.a.src(), unsafe.Pointer(&audio[0]), C.gsize(len(audio))) +} + +// pullAudio pulls audio buffers from the appsink when they are available. +func (g *GstMediaPipe) pullAudio(_ *app.Sink) gst.FlowReturn { + var mapInfo C.GstMapInfo + buf := C.pullMappedBuffer(g.a.sinkPtr(), &mapInfo) + if buf == nil { + return gst.FlowEOS + } + defer C.unmapAndUnref(buf, &mapInfo) + if g.onAudio != nil { + g.onAudio(unsafe.Slice((*byte)(unsafe.Pointer(mapInfo.data)), int(mapInfo.size)), g.a.frameDur) + } + return gst.FlowOK +} + +// videoWorker is the single goroutine that does push+pull for video. +func (g *GstMediaPipe) videoWorker(v *pipe, vidFmt uint32, ch <-chan videoJob, done chan<- struct{}) { + defer close(done) + for job := range ch { + if v.stale.Load() { + continue + } + C.pushVideoBuf(v.src(), + unsafe.Pointer(&job.data[0]), C.gsize(len(job.data)), + C.GstVideoFormat(vidFmt), C.guint(job.w), C.guint(job.h), C.gint(job.stride)) + + var mi C.GstMapInfo + buf := C.pullMappedBuffer(v.sinkPtr(), &mi) + if buf == nil { + continue + } + data := unsafe.Slice((*byte)(unsafe.Pointer(mi.data)), int(mi.size)) + job.cb(data, job.dur) + C.unmapAndUnref(buf, &mi) + } +} + +func (g *GstMediaPipe) ProcessVideo(v Video, cb func([]byte, time.Duration)) { + if g.reinit.Load() { + return + } + + // when someone starts watching, force the first few keyframes to avoid stalling + // and if scene-change detection is disabled (kfi > 0) + if g.frameI < maxForcedKeyframes || (g.kfi > 0 && g.keyI%g.kfi == 0) { + g.v.srcPad.PushEvent(forceKeyframeEvent) + if g.frameI < maxForcedKeyframes { + g.frameI++ + } + } + g.keyI++ + + w, h := v.Frame.W, v.Frame.H + stride := v.Frame.Stride + + data := v.Frame.Data[:stride*h] + if stride == w*g.bpp { + stride = 0 + } + + select { + case g.videoCh <- videoJob{data: data, w: w, h: h, stride: stride, dur: v.Duration, cb: cb}: + default: + // if busy the frame is dropped + // or maybe handle it with gst? + } +} + +func (g *GstMediaPipe) Reinit() error { + // prevent concurrent reinits + g.mu.Lock() + defer g.mu.Unlock() + + if g.v == nil { + return nil + } + + g.reinit.Store(true) + defer g.reinit.Store(false) + + // drop stale frames (for the new pipe) + g.v.stale.Store(true) + + // stop the video worker + if g.videoCh != nil { + g.v.stop() + close(g.videoCh) + <-g.videoDone + } + + // we rebuild new pipeline every time because + // changing the piplne parts proves to be unreliable + g.v.destroy() + if err := g.initVideo(); err != nil { + return fmt.Errorf("gst video reinit: %w", err) + } + + g.keyI = 0 + return nil +} + +func buildVideoPipeline(w, h, scaledW, scaledH int, pixFmt uint32, scaleMethod string, + maxThreads int, conf config.Encoder, log *logger.Logger, fps float64, vfr bool) (*pipe, error) { + format := cmp.Or(pixFmtToGst[pixFmt], "RGBA") + kfi := cmp.Or(conf.Video.KeyframeInterval, 120) + opts, codec := conf.VideoSettings() + if opts == nil { + return nil, fmt.Errorf("unsupported codec: %s", conf.Video.Codec) + } + + build := builder{} + build.Grow(512) + build.write("appsrc name=video_src is-live=true ") + var fn, fd C.gint + C.doubleToFraction(C.gdouble(fps), &fn, &fd) + if vfr { + build.writef(`caps="video/x-raw,format=%s,width=%d,height=%d,framerate=0/1,max-framerate=%d/%d" `, format, w, h, int(fn), int(fd)) + } else { + build.writef(`caps="video/x-raw,format=%s,width=%d,height=%d,framerate=%d/%d" `, format, w, h, int(fn), int(fd)) + } + build.writef("! videoconvertscale name=video_scale chroma-resampler=cubic method=%s n-threads=2 ", cmp.Or(scaleMethod, "nearest-neighbour")) + build.write("! queue name=video_q max-size-buffers=1 ") + build.writef("! capsfilter name=video_caps caps=\"video/x-raw,format=I420,color-range=0_255,width=%d,height=%d ", scaledW, scaledH) + if conf.Video.Colorimetry != "" { + build.writef(",colorimetry=%s", conf.Video.Colorimetry) + } + build.write("\" ") + build.write("! ") + switch codec { + // h264 - requires gstreamer1.0-plugins-ugly + x264 (GPL) + // case "h264": + // return fmt.Sprintf("%s threads=%d key-int-max=%d", + // cmp.Or(gst.Encoder, "x264enc"), + // cmp.Or(conf.Video.Threads, 0), + // kfi, + // ) + case "vp8", "vp9": + encoder := cmp.Or(opts.Encoder, fmt.Sprintf("vp%cenc", codec[len(codec)-1])) + threads := cmp.Or(conf.Video.Threads, cpuCores) + if maxThreads > 0 && threads > maxThreads { + threads = maxThreads + } + build.writef("%s name=video_enc threads=%d keyframe-max-dist=%d ", encoder, threads, kfi) + } + build.writeIfSet(opts.Params) + if opts.Caps != "" { + build.write(" ! " + opts.Caps) + } + build.write(" ! queue name=video_q2 max-size-buffers=1 ") + build.write("! appsink name=video_sink sync=false max-buffers=1 drop=true") + + pipeDef := build.String() + + log.Debug().Msgf("Gstreamer [video]: %s", pipeDef) + + pipeline, err := gst.NewPipelineFromString(pipeDef) + if err != nil { + return nil, err + } + srcEl, err := pipeline.GetElementByName("video_src") + if err != nil { + return nil, err + } + sinkEl, err := pipeline.GetElementByName("video_sink") + if err != nil { + return nil, err + } + go watchBus(pipeline, log) + return &pipe{ + pipeline: pipeline, + source: app.SrcFromElement(srcEl), + sink: app.SinkFromElement(sinkEl), + srcPad: srcEl.GetStaticPad("src"), + }, nil +} + +func buildAudioPipeline(srcHz int, conf config.Encoder, log *logger.Logger) (*pipe, error) { + opts := conf.AudioSettings() + if opts == nil { + return nil, fmt.Errorf("unsupported audio codec: %s", conf.Audio.Codec) + } + + build := builder{} + build.Grow(384) + build.write("appsrc name=audio_src is-live=true ") + build.writef("caps=audio/x-raw,format=S16LE,rate=%d,channels=2,layout=interleaved ", srcHz) + if srcHz != 48_000 { + m, q := "kaiser", 10 + switch conf.Audio.Resampler { + case 0: + m, q = "nearest", 0 + case 1: + m, q = "linear", 4 + } + build.write("! audioresample sinc-filter-mode=full ") + build.writef("resample-method=%s quality=%d ! audio/x-raw,rate=48000 ", m, q) + } + build.writef("! %s name=audio_enc %s ", cmp.Or(opts.Encoder, "opusenc"), opts.Params) + build.write("! queue name=audio_q max-size-buffers=1 ") + build.write("! appsink name=audio_sink sync=false max-buffers=1 drop=true") + + pipeDef := build.String() + log.Debug().Msgf("Gstreamer [audio]: %s", pipeDef) + + pipeline, err := gst.NewPipelineFromString(pipeDef) + if err != nil { + return nil, err + } + srcEl, _ := pipeline.GetElementByName("audio_src") + sinkEl, _ := pipeline.GetElementByName("audio_sink") + encEl, _ := pipeline.GetElementByName("audio_enc") + + frameMs := defaultAudioFrameMs + if encEl != nil { + if v, e := encEl.GetProperty("frame-size"); e == nil { + if ms := v.(int); ms > 0 { + frameMs = time.Duration(ms) * time.Millisecond + } + } + } + + go watchBus(pipeline, log) + return &pipe{ + pipeline: pipeline, + source: app.SrcFromElement(srcEl), + sink: app.SinkFromElement(sinkEl), + srcPad: srcEl.GetStaticPad("src"), + frameDur: frameMs, + }, nil +} + +func gstVideoFormat(format string) uint32 { + if v, ok := pixFmtCache[format]; ok { + return v + } + pix := unsafe.Pointer(unsafe.StringData(format + "\x00")) + var p runtime.Pinner + p.Pin(pix) + defer p.Unpin() + v := uint32(C.gstVideoFormatFromString((*C.gchar)(pix))) + pixFmtCache[format] = v + return v +} + +func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) &^ 1 } + +func watchBus(pipeline *gst.Pipeline, log *logger.Logger) { + bus := pipeline.GetPipelineBus() + + for { + msg := bus.TimedPop(gst.ClockTimeNone) + if msg == nil { + return + } + switch msg.Type() { + case gst.MessageError: + gerr := msg.ParseError() + log.Error().Str("debug", gerr.DebugString()).Err(gerr).Msg("gst pipeline error") + case gst.MessageWarning: + gerr := msg.ParseWarning() + if strings.Contains(gerr.Error(), "invalid video buffer") { + log.Debug().Str("debug", gerr.DebugString()).Err(gerr).Msg("gst pipeline warning") + } else { + log.Warn().Str("debug", gerr.DebugString()).Err(gerr).Msg("gst pipeline warning") + } + case gst.MessageEOS: + return + default: + // log.Debug().Msg("gst pipeline message: " + msg.String()) + } + } +} diff --git a/pkg/worker/media/gstreamer_test.go b/pkg/worker/media/gstreamer_test.go new file mode 100644 index 00000000..f3e898fb --- /dev/null +++ b/pkg/worker/media/gstreamer_test.go @@ -0,0 +1,253 @@ +package media + +import ( + "strings" + "testing" + "time" + + "github.com/giongto35/cloud-game/v3/pkg/config" + "github.com/giongto35/cloud-game/v3/pkg/logger" +) + +var testLog = logger.New(false) + +func TestRound(t *testing.T) { + tests := []struct { + x int + scale float64 + want int + }{ + {640, 1.0, 640}, + {640, 2.0, 1280}, + {640, 1.5, 960}, + {640, 0.5, 320}, + {481, 1.0, 482}, // odd rounds up to even + {481, 2.0, 962}, + {0, 2.0, 0}, + } + for _, tt := range tests { + got := round(tt.x, tt.scale) + if got != tt.want { + t.Errorf("round(%d, %v) = %d, want %d", tt.x, tt.scale, got, tt.want) + } + } +} + +func TestBuilder(t *testing.T) { + var b builder + b.write("abc ") + b.writef("x=%d ", 1) + b.writeIfSet("") + b.writeIfSet("! foo") + + s := b.String() + if !strings.Contains(s, "abc") { + t.Errorf("missing 'abc' in %q", s) + } + if !strings.Contains(s, "x=1") { + t.Errorf("missing 'x=1' in %q", s) + } + if !strings.Contains(s, "! foo") { + t.Errorf("missing '! foo' in %q", s) + } + if strings.Contains(s, "!!") { + t.Errorf("writeIfSet wrote empty string: %q", s) + } +} + +func TestNewGstreamerDefaults(t *testing.T) { + g := NewGstreamer(config.Encoder{}, testLog) + if g == nil { + t.Fatal("NewGstreamer returned nil") + } + if g.bpp != 4 { + t.Errorf("bpp = %d, want 4", g.bpp) + } + if g.vidFmt == 0 { + t.Error("vidFmt should not be zero (BGRx)") + } +} + +func TestSetRot(t *testing.T) { + g := NewGstreamer(config.Encoder{}, testLog) + g.SetRot(90) + if g.oldRot != 90 { + t.Errorf("oldRot = %d, want 90", g.oldRot) + } + g.SetRot(180) + if g.oldRot != 180 { + t.Errorf("oldRot = %d, want 180", g.oldRot) + } +} + +func TestPixFmtMapping(t *testing.T) { + if pixFmtToGst[pixFmtBGRx] != "BGRx" { + t.Errorf("BGRx format mismatch: %q", pixFmtToGst[pixFmtBGRx]) + } + if pixFmtToGst[pixFmtBGRA] != "BGRA" { + t.Errorf("BGRA format mismatch: %q", pixFmtToGst[pixFmtBGRA]) + } + if pixFmtToGst[pixFmtRGB16] != "RGB16" { + t.Errorf("RGB16 format mismatch: %q", pixFmtToGst[pixFmtRGB16]) + } +} + +var ( + testVideoCodec = config.Encoder{ + Video: config.Video{Codec: "vp8"}, + List: map[string]config.CodecSettings{"vp8": {}}, + } + testAudioCodec = config.Encoder{ + Audio: config.Audio{Codec: "opus"}, + List: map[string]config.CodecSettings{"opus": {}}, + } + testFullCodec = config.Encoder{ + Video: config.Video{Codec: "vp8"}, + Audio: config.Audio{Codec: "opus"}, + List: map[string]config.CodecSettings{"vp8": {}, "opus": {}}, + } +) + +func TestBuildVideoPipeline(t *testing.T) { + p, err := buildVideoPipeline(640, 480, 640, 480, pixFmtBGRx, "bilinear2", 0, testVideoCodec, testLog, 0, false) + if err != nil { + t.Fatal(err) + } + p.destroy() +} + +func TestBuildAudioPipeline(t *testing.T) { + p, err := buildAudioPipeline(48000, testAudioCodec, testLog) + if err != nil { + t.Fatal(err) + } + p.destroy() +} + +func TestInitAndDestroy(t *testing.T) { + g := NewGstreamer(testFullCodec, testLog) + g.VideoW, g.VideoH = 64, 64 + g.VideoScale = 1 + g.AudioSrcHz = 48000 + + if err := g.Init(); err != nil { + t.Fatal(err) + } + g.Destroy() +} + +func TestProcessVideo(t *testing.T) { + g := NewGstreamer(testFullCodec, testLog) + g.VideoW, g.VideoH = 64, 64 + g.VideoScale = 1 + g.AudioSrcHz = 48000 + + if err := g.Init(); err != nil { + t.Fatal(err) + } + defer g.Destroy() + + // Build a 64×64 BGRx solid-blue frame. + w, h := 64, 64 + stride := w * 4 + frame := make([]byte, stride*h) + for i := range h { + for j := range w { + off := i*stride + j*4 + frame[off+0] = 255 // B + frame[off+1] = 0 // G + frame[off+2] = 0 // R + frame[off+3] = 255 // x + } + } + + done := make(chan struct{}) + g.ProcessVideo(Video{ + Frame: RawFrame{Data: frame, W: w, H: h, Stride: stride}, + Duration: 16 * time.Millisecond, + }, func(data []byte, dur time.Duration) { + if len(data) == 0 { + t.Error("encoded video is empty") + } + if dur != 16*time.Millisecond { + t.Errorf("duration = %v, want 16ms", dur) + } + close(done) + }) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for encoded video") + } +} + +func TestProcessAudio(t *testing.T) { + g := NewGstreamer(testFullCodec, testLog) + g.VideoW, g.VideoH = 64, 64 + g.VideoScale = 1 + g.AudioSrcHz = 48000 + + if err := g.Init(); err != nil { + t.Fatal(err) + } + defer g.Destroy() + + // Feed 20ms of 48kHz S16LE stereo silence → 1920 samples = 3840 bytes. + samples := 48000 * 2 * 20 / 1000 + audio := make([]byte, samples*2) // S16LE = 2 bytes per sample + + done := make(chan struct{}) + g.ProcessAudio(audio, func(data []byte, dur time.Duration) { + if len(data) == 0 { + t.Error("encoded audio is empty") + } + close(done) + }) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for encoded audio") + } +} + +func TestReinit(t *testing.T) { + g := NewGstreamer(testFullCodec, testLog) + g.VideoW, g.VideoH = 64, 64 + g.VideoScale = 1 + g.AudioSrcHz = 48000 + + if err := g.Init(); err != nil { + t.Fatal(err) + } + defer g.Destroy() + + // Change scale and reinit. + g.VideoScale = 2 + if err := g.Reinit(); err != nil { + t.Fatal(err) + } + + // Push a frame through the new pipeline. + w, h := 64, 64 + stride := w * 4 + frame := make([]byte, stride*h) + + done := make(chan struct{}) + g.ProcessVideo(Video{ + Frame: RawFrame{Data: frame, W: w, H: h, Stride: stride}, + Duration: 16 * time.Millisecond, + }, func(data []byte, dur time.Duration) { + if len(data) == 0 { + t.Error("encoded video after reinit is empty") + } + close(done) + }) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for encoded video after reinit") + } +} diff --git a/pkg/worker/media/media.go b/pkg/worker/media/media.go deleted file mode 100644 index 1e79c272..00000000 --- a/pkg/worker/media/media.go +++ /dev/null @@ -1,191 +0,0 @@ -package media - -import ( - "fmt" - "sync" - "time" - - "github.com/giongto35/cloud-game/v3/pkg/config" - "github.com/giongto35/cloud-game/v3/pkg/encoder" - "github.com/giongto35/cloud-game/v3/pkg/encoder/opus" - "github.com/giongto35/cloud-game/v3/pkg/logger" - "github.com/giongto35/cloud-game/v3/pkg/worker/caged/app" -) - -const audioHz = 48000 - -type samples []int16 - -var ( - encoderOnce = sync.Once{} - opusCoder *opus.Encoder -) - -func DefaultOpus() (*opus.Encoder, error) { - var err error - encoderOnce.Do(func() { opusCoder, err = opus.NewEncoder(audioHz) }) - if err != nil { - return nil, err - } - if err = opusCoder.Reset(); err != nil { - return nil, err - } - return opusCoder, nil -} - -type WebrtcMediaPipe struct { - a *opus.Encoder - v *encoder.Video - onAudio func([]byte, float32) - audioBuf *buffer - log *logger.Logger - - mua sync.RWMutex - muv sync.RWMutex - - aConf config.Audio - vConf config.Video - - AudioSrcHz int - AudioFrames []float32 - VideoW, VideoH int - VideoScale float64 - - initialized bool - - // keep the old settings for reinit - oldPf uint32 - oldRot uint -} - -func NewWebRtcMediaPipe(ac config.Audio, vc config.Video, log *logger.Logger) *WebrtcMediaPipe { - return &WebrtcMediaPipe{log: log, aConf: ac, vConf: vc} -} - -func (wmp *WebrtcMediaPipe) SetAudioCb(cb func([]byte, int32)) { - wmp.onAudio = func(bytes []byte, ms float32) { - cb(bytes, int32(time.Duration(ms)*time.Millisecond)) - } -} -func (wmp *WebrtcMediaPipe) Destroy() { - v := wmp.Video() - if v != nil { - v.Stop() - } -} -func (wmp *WebrtcMediaPipe) PushAudio(audio []int16) { - wmp.audioBuf.write(audio, wmp.encodeAudio) -} - -func (wmp *WebrtcMediaPipe) Init() error { - if err := wmp.initAudio(wmp.AudioSrcHz, wmp.AudioFrames); err != nil { - return err - } - if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil { - return err - } - - a := wmp.Audio() - v := wmp.Video() - - if v == nil || a == nil { - return fmt.Errorf("could intit the encoders, v=%v a=%v", v != nil, a != nil) - } - - wmp.log.Debug().Msgf("%v", v.Info()) - wmp.initialized = true - return nil -} - -func (wmp *WebrtcMediaPipe) initAudio(srcHz int, frameSizes []float32) error { - au, err := DefaultOpus() - if err != nil { - return fmt.Errorf("opus fail: %w", err) - } - wmp.log.Debug().Msgf("Opus: %v", au.GetInfo()) - wmp.SetAudio(au) - buf, err := newBuffer(frameSizes, srcHz) - if err != nil { - return err - } - wmp.log.Debug().Msgf("Opus frames (ms): %v", frameSizes) - dstHz, _ := au.SampleRate() - if srcHz != dstHz { - buf.resample(dstHz, ResampleAlgo(wmp.aConf.Resampler)) - wmp.log.Debug().Msgf("Resample %vHz -> %vHz", srcHz, dstHz) - } - wmp.audioBuf = buf - return nil -} - -func (wmp *WebrtcMediaPipe) encodeAudio(pcm samples, ms float32) { - data, err := wmp.Audio().Encode(pcm) - if err != nil { - wmp.log.Error().Err(err).Msgf("opus encode fail") - return - } - wmp.onAudio(data, ms) -} - -func (wmp *WebrtcMediaPipe) initVideo(w, h int, scale float64, conf config.Video) (err error) { - sw, sh := round(w, scale), round(h, scale) - enc, err := encoder.NewVideoEncoder(w, h, sw, sh, scale, conf, wmp.log) - if err != nil { - return err - } - if enc == nil { - return fmt.Errorf("broken video encoder init") - } - wmp.SetVideo(enc) - wmp.log.Debug().Msgf("media scale: %vx%v -> %vx%v", w, h, sw, sh) - return err -} - -func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) & ^1 } - -func (wmp *WebrtcMediaPipe) ProcessVideo(v app.Video) []byte { - return wmp.Video().Encode(encoder.InFrame(v.Frame)) -} - -func (wmp *WebrtcMediaPipe) Reinit() error { - if !wmp.initialized { - return nil - } - - wmp.Video().Stop() - if err := wmp.initVideo(wmp.VideoW, wmp.VideoH, wmp.VideoScale, wmp.vConf); err != nil { - return err - } - // restore old - wmp.SetPixFmt(wmp.oldPf) - wmp.SetRot(wmp.oldRot) - return nil -} - -func (wmp *WebrtcMediaPipe) IsInitialized() bool { return wmp.initialized } -func (wmp *WebrtcMediaPipe) SetPixFmt(f uint32) { wmp.oldPf = f; wmp.v.SetPixFormat(f) } -func (wmp *WebrtcMediaPipe) SetRot(r uint) { wmp.oldRot = r; wmp.v.SetRot(r) } - -func (wmp *WebrtcMediaPipe) Video() *encoder.Video { - wmp.muv.RLock() - defer wmp.muv.RUnlock() - return wmp.v -} - -func (wmp *WebrtcMediaPipe) SetVideo(e *encoder.Video) { - wmp.muv.Lock() - wmp.v = e - wmp.muv.Unlock() -} - -func (wmp *WebrtcMediaPipe) Audio() *opus.Encoder { - wmp.mua.RLock() - defer wmp.mua.RUnlock() - return wmp.a -} - -func (wmp *WebrtcMediaPipe) SetAudio(e *opus.Encoder) { - wmp.mua.Lock() - wmp.a = e - wmp.mua.Unlock() -} diff --git a/pkg/worker/media/media_test.go b/pkg/worker/media/media_test.go deleted file mode 100644 index 16fb758f..00000000 --- a/pkg/worker/media/media_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package media - -import ( - "image" - "math/rand/v2" - "testing" - - "github.com/giongto35/cloud-game/v3/pkg/config" - "github.com/giongto35/cloud-game/v3/pkg/encoder" - "github.com/giongto35/cloud-game/v3/pkg/logger" -) - -var l = logger.New(false) - -func TestEncoders(t *testing.T) { - tests := []struct { - n int - w, h int - codec encoder.VideoCodec - frames int - }{ - {n: 3, w: 1920, h: 1080, codec: encoder.H264, frames: 60}, - {n: 3, w: 1920, h: 1080, codec: encoder.VP8, frames: 60}, - } - - for _, test := range tests { - a := genTestImage(test.w, test.h, rand.Float32()) - b := genTestImage(test.w, test.h, rand.Float32()) - for i := 0; i < test.n; i++ { - run(test.w, test.h, test.codec, test.frames, a, b, t) - } - } -} - -func BenchmarkH264(b *testing.B) { run(640, 480, encoder.H264, b.N, nil, nil, b) } -func BenchmarkVP8(b *testing.B) { run(1920, 1080, encoder.VP8, b.N, nil, nil, b) } - -func run(w, h int, cod encoder.VideoCodec, count int, a *image.RGBA, b *image.RGBA, backend testing.TB) { - conf := config.Video{ - Codec: string(cod), - Threads: 0, - KeyframeInterval: 120, - H264: struct { - Mode string - Crf uint8 - MaxRate int - BufSize int - LogLevel int32 - Preset string - Profile string - Tune string - }{ - Crf: 30, - LogLevel: 0, - Preset: "ultrafast", - Profile: "baseline", - Tune: "zerolatency", - }, - Vpx: struct { - Bitrate uint - CpuUsed int - TileColumns int - Tune string - }{ - Bitrate: 1000, - }, - } - - logger.SetGlobalLevel(logger.Disabled) - ve, err := encoder.NewVideoEncoder(w, h, w, h, 1, conf, l) - if err != nil { - backend.Error(err) - return - } - defer ve.Stop() - - if a == nil { - a = genTestImage(w, h, rand.Float32()) - } - if b == nil { - b = genTestImage(w, h, rand.Float32()) - } - - for i := range count { - im := a - if i%2 == 0 { - im = b - } - out := ve.Encode(encoder.InFrame{ - Data: im.Pix, - Stride: im.Stride, - W: im.Bounds().Dx(), - H: im.Bounds().Dy(), - }) - if out == nil { - backend.Fatalf("encoder closed abnormally") - } - } -} - -func genTestImage(w, h int, seed float32) *image.RGBA { - img := image.NewRGBA(image.Rectangle{Max: image.Point{X: w, Y: h}}) - for x := range w { - for y := range h { - i := img.PixOffset(x, y) - s := img.Pix[i : i+4 : i+4] - s[0] = uint8(seed * 255) - s[1] = uint8(seed * 255) - s[2] = uint8(seed * 255) - s[3] = 0xff - } - } - return img -} diff --git a/pkg/worker/room/room.go b/pkg/worker/room/room.go index 88380683..9ff9576e 100644 --- a/pkg/worker/room/room.go +++ b/pkg/worker/room/room.go @@ -3,8 +3,10 @@ package room import ( "iter" "sync" + "time" "github.com/giongto35/cloud-game/v3/pkg/worker/caged/app" + "github.com/giongto35/cloud-game/v3/pkg/worker/media" ) type MediaPipe interface { @@ -14,14 +16,10 @@ type MediaPipe interface { Init() error // Reinit initializes video and audio pipes with the new settings. Reinit() error - // PushAudio pushes the 16bit PCM audio frames into an encoder. - // Because we need to fill the buffer, the SetAudioCb should be - // used in order to get the result. - PushAudio([]int16) - // ProcessVideo returns encoded video frame. - ProcessVideo(app.Video) []byte - // SetAudioCb sets a callback for encoded audio data with its frame duration (ns). - SetAudioCb(func(data []byte, duration int32)) + // ProcessAudio pushes 16bit PCM audio frames into the encoder. + ProcessAudio([]byte, func([]byte, time.Duration)) + // ProcessVideo pushes a video frame into the encoder. + ProcessVideo(media.Video, func([]byte, time.Duration)) } type SessionManager[T Session] interface { @@ -36,8 +34,8 @@ type SessionManager[T Session] interface { type Session interface { Disconnect() - SendAudio([]byte, int32) - SendVideo([]byte, int32) + SendAudio([]byte, time.Duration) + SendVideo([]byte, time.Duration) SendData([]byte) } @@ -57,34 +55,34 @@ type Room[T Session] struct { } func NewRoom[T Session](id string, app app.App, um SessionManager[T], media MediaPipe) *Room[T] { - room := &Room[T]{id: id, app: app, users: um, media: media} - if app != nil && media != nil { - room.InitVideo() - room.InitAudio() - } - return room + return &Room[T]{id: id, app: app, users: um, media: media} } -func (r *Room[T]) InitAudio() { - r.app.SetAudioCb(func(a app.Audio) { r.media.PushAudio(a.Data) }) - r.media.SetAudioCb(func(d []byte, l int32) { - for u := range r.users.Values() { - u.SendAudio(d, l) - } +func (r *Room[T]) InitMedia() { + r.app.SetAudioCb(func(a app.Audio) { + r.media.ProcessAudio(a.Data, r.sendAudio) }) -} - -func (r *Room[T]) InitVideo() { r.app.SetVideoCb(func(v app.Video) { - data := r.media.ProcessVideo(v) - for u := range r.users.Values() { - u.SendVideo(data, v.Duration) - } + r.media.ProcessVideo(media.Video{ + Frame: media.RawFrame{Data: v.Frame.Data, W: v.Frame.W, H: v.Frame.H, Stride: v.Frame.Stride}, + Duration: v.Duration, + }, r.sendVideo) }) } +func (r *Room[T]) sendAudio(data []byte, dur time.Duration) { + for u := range r.users.Values() { + u.SendAudio(data, dur) + } +} + +func (r *Room[T]) sendVideo(data []byte, dur time.Duration) { + for u := range r.users.Values() { + u.SendVideo(data, dur) + } +} + func (r *Room[T]) App() app.App { return r.app } -func (r *Room[T]) BindAppMedia() { r.InitAudio(); r.InitVideo() } func (r *Room[T]) Id() string { return r.id } func (r *Room[T]) SetApp(app app.App) { r.app = app } func (r *Room[T]) SetMedia(m MediaPipe) { r.media = m } diff --git a/pkg/worker/room/room_test.go b/pkg/worker/room/room_test.go index 7a537d69..54d79c05 100644 --- a/pkg/worker/room/room_test.go +++ b/pkg/worker/room/room_test.go @@ -1,6 +1,7 @@ package room import ( + "encoding/binary" "flag" "fmt" "hash/crc32" @@ -9,6 +10,7 @@ import ( "image/draw" "image/png" "log" + "math" "os" "path/filepath" "runtime" @@ -18,10 +20,6 @@ import ( "github.com/giongto35/cloud-game/v3/pkg/com" "github.com/giongto35/cloud-game/v3/pkg/config" - "github.com/giongto35/cloud-game/v3/pkg/encoder" - "github.com/giongto35/cloud-game/v3/pkg/encoder/color/bgra" - "github.com/giongto35/cloud-game/v3/pkg/encoder/color/rgb565" - "github.com/giongto35/cloud-game/v3/pkg/encoder/color/rgba" "github.com/giongto35/cloud-game/v3/pkg/games" "github.com/giongto35/cloud-game/v3/pkg/logger" "github.com/giongto35/cloud-game/v3/pkg/worker/caged" @@ -46,7 +44,12 @@ type testRoom struct { started bool } -type codec = encoder.VideoCodec +type codec = string + +const ( + VP8 = "vp8" + VP9 = "vp9" +) type conf struct { roomName string @@ -111,7 +114,7 @@ func TestMain(m *testing.M) { func TestRoom(t *testing.T) { tests := []testParams{ - {game: alwas, codecs: []codec{encoder.H264, encoder.VP8, encoder.VP9}, frames: 300}, + {game: alwas, codecs: []codec{VP8, VP9}, frames: 300}, } for _, test := range tests { @@ -135,7 +138,7 @@ func TestAll(t *testing.T) { for _, test := range tests { var frame app.RawFrame - room := room(conf{game: test.game, codec: encoder.VP8, autoGlContext: autoGlContext, autoAppStart: false}) + room := room(conf{game: test.game, codec: VP8, autoGlContext: autoGlContext, autoAppStart: false}) flip := test.system == "gl" thread.Main(func() { frame = room.WaitFrame(test.frames) }) room.Close() @@ -143,17 +146,17 @@ func TestAll(t *testing.T) { if renderFrames { rect := image.Rect(0, 0, frame.W, frame.H) var src image.Image - src1 := bgra.NewBGRA(rect) + src1 := NewBGRA(rect) src1.Pix = frame.Data src1.Stride = frame.Stride src = src1 if test.color == 2 { - src2 := rgb565.NewRGB565(rect) + src2 := NewRGB565(rect) src2.Pix = frame.Data src2.Stride = frame.Stride src = src2 } - dst := rgba.ToRGBA(src, flip) + dst := ToRGBA(src, flip) tag := fmt.Sprintf("%v-%v-0x%08x", runtime.GOOS, test.game.Type, crc32.Checksum(frame.Data, crc32q)) dumpCanvas(dst, tag, fmt.Sprintf("%v [%v]", tag, test.frames), outputPath) } @@ -203,7 +206,7 @@ func room(cfg conf) testRoom { conf.Emulator.LocalPath = expand("tests", conf.Emulator.LocalPath) conf.Emulator.Storage = expand("tests", "storage") - conf.Encoder.Video.Codec = string(cfg.codec) + conf.Encoder.Video.Codec = cfg.codec l := logger.NewConsole(conf.Worker.Debug, "w", false) if cfg.noLog { @@ -227,16 +230,16 @@ func room(cfg conf) testRoom { l.Fatal().Err(err).Msgf("couldn't load the game %v", cfg.game) } - m := media.NewWebRtcMediaPipe(conf.Encoder.Audio, conf.Encoder.Video, l) + m := media.NewGstreamer(conf.Encoder, l) m.AudioSrcHz = emu.AudioSampleRate() - m.AudioFrames = conf.Encoder.Audio.Frames m.VideoW, m.VideoH = emu.ViewportSize() - m.VideoScale = emu.Scale() + m.VideoScale, _ = emu.Scale() if err := m.Init(); err != nil { l.Fatal().Err(err).Msgf("no init") } - room := NewRoom[*GameSession](id, emu, &com.NetMap[SessionKey, *GameSession]{}, m) + room := NewRoom(id, emu, &com.NetMap[SessionKey, *GameSession]{}, m) + room.InitMedia() if cfg.autoAppStart { room.StartApp() } @@ -249,15 +252,15 @@ func room(cfg conf) testRoom { func BenchmarkRoom(b *testing.B) { benches := []testParams{ // warm up - {system: "gba", game: sushi, codecs: []codec{encoder.VP8, encoder.VP9}, frames: 50}, - {system: "gba", game: sushi, codecs: []codec{encoder.VP8, encoder.H264}, frames: 100}, - {system: "nes", game: alwas, codecs: []codec{encoder.VP8, encoder.H264}, frames: 100}, + {system: "gba", game: sushi, codecs: []codec{VP8, VP9}, frames: 50}, + {system: "gba", game: sushi, codecs: []codec{VP8}, frames: 100}, + {system: "nes", game: alwas, codecs: []codec{VP8}, frames: 100}, } for _, bench := range benches { for _, cod := range bench.codecs { b.Run(fmt.Sprintf("%s-%v-%d", bench.system, cod, bench.frames), func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { b.StopTimer() room := room(conf{game: bench.game, codec: cod, noLog: true}) b.StartTimer() @@ -275,3 +278,125 @@ func expand(p ...string) string { ph, _ := filepath.Abs(filepath.FromSlash(filepath.Join(p...))) return ph } + +type BGRA struct { + image.RGBA +} + +var BGRAModel = color.ModelFunc(func(c color.Color) color.Color { + if _, ok := c.(BGRAColor); ok { + return c + } + r, g, b, a := c.RGBA() + return BGRAColor{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)} +}) + +// BGRAColor represents a BGRA color. +type BGRAColor struct { + R, G, B, A uint8 +} + +func (c BGRAColor) RGBA() (r, g, b, a uint32) { + r = uint32(c.B) + r |= r << 8 + g = uint32(c.G) + g |= g << 8 + b = uint32(c.R) + b |= b << 8 + a = uint32(255) //uint32(c.A) + a |= a << 8 + return +} + +func NewBGRA(r image.Rectangle) *BGRA { + return &BGRA{*image.NewRGBA(r)} +} + +func (p *BGRA) ColorModel() color.Model { return BGRAModel } +func (p *BGRA) At(x, y int) color.Color { + i := p.PixOffset(x, y) + s := p.Pix[i : i+4 : i+4] + return BGRAColor{s[0], s[1], s[2], s[3]} +} + +func (p *BGRA) Set(x, y int, c color.Color) { + i := p.PixOffset(x, y) + c1 := BGRAModel.Convert(c).(BGRAColor) + s := p.Pix[i : i+4 : i+4] + s[0] = c1.R + s[1] = c1.G + s[2] = c1.B + s[3] = 255 +} + +func ToRGBA(img image.Image, flipped bool) *image.RGBA { + bounds := img.Bounds() + sw, sh := bounds.Dx(), bounds.Dy() + dst := image.NewRGBA(image.Rect(0, 0, sw, sh)) + for y := range sh { + yy := y + if flipped { + yy = sh - y + } + for x := range sw { + px := img.At(x, y) + rgba := color.RGBAModel.Convert(px).(color.RGBA) + dst.Set(x, yy, rgba) + } + } + return dst +} + +// RGB565 is an in-memory image whose At method returns RGB565 values. +type RGB565 struct { + // Pix holds the image's pixels, as RGB565 values in big-endian format. The pixel at + // (x, y) starts at Pix[(y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*2]. + Pix []uint8 + // Stride is the Pix stride (in bytes) between vertically adjacent pixels. + Stride int + // Rect is the image's bounds. + Rect image.Rectangle +} + +// Model is the model for RGB565 colors. +var Model = color.ModelFunc(func(c color.Color) color.Color { + //if _, ok := c.(Color); ok { + // return c + //} + r, g, b, _ := c.RGBA() + return Color(uint16((r<<8)&rMask | (g<<3)&gMask | (b>>3)&bMask)) +}) + +const ( + rMask = 0b1111100000000000 + gMask = 0b0000011111100000 + bMask = 0b0000000000011111 +) + +// Color represents an RGB565 color. +type Color uint16 + +func (c Color) RGBA() (r, g, b, a uint32) { + return uint32(math.Round(float64(c&rMask>>11)*255.0/31.0)) << 8, + uint32(math.Round(float64(c&gMask>>5)*255.0/63.0)) << 8, + uint32(math.Round(float64(c&bMask)*255.0/31.0)) << 8, + 0xffff +} + +func NewRGB565(r image.Rectangle) *RGB565 { + return &RGB565{Pix: make([]uint8, r.Dx()*r.Dy()<<1), Stride: r.Dx() << 1, Rect: r} +} + +func (p *RGB565) Bounds() image.Rectangle { return p.Rect } +func (p *RGB565) ColorModel() color.Model { return Model } +func (p *RGB565) PixOffset(x, y int) int { return (x-p.Rect.Min.X)<<1 + (y-p.Rect.Min.Y)*p.Stride } + +func (p *RGB565) At(x, y int) color.Color { + i := p.PixOffset(x, y) + return Color(binary.LittleEndian.Uint16(p.Pix[i : i+2])) +} + +func (p *RGB565) Set(x, y int, c color.Color) { + i := p.PixOffset(x, y) + binary.LittleEndian.PutUint16(p.Pix[i:i+2], uint16(Model.Convert(c).(Color))) +} diff --git a/pkg/worker/room/router_test.go b/pkg/worker/room/router_test.go index d4f2e621..1c9858c3 100644 --- a/pkg/worker/room/router_test.go +++ b/pkg/worker/room/router_test.go @@ -2,6 +2,7 @@ package room import ( "testing" + "time" "github.com/giongto35/cloud-game/v3/pkg/com" ) @@ -15,12 +16,12 @@ type tSession struct { connected bool } -func (t *tSession) SendAudio([]byte, int32) {} -func (t *tSession) SendVideo([]byte, int32) {} -func (t *tSession) SendData([]byte) {} -func (t *tSession) Connect() { t.connected = true } -func (t *tSession) Disconnect() { t.connected = false } -func (t *tSession) Id() sKey { return t.id } +func (t *tSession) SendAudio([]byte, time.Duration) {} +func (t *tSession) SendVideo([]byte, time.Duration) {} +func (t *tSession) SendData([]byte) {} +func (t *tSession) Connect() { t.connected = true } +func (t *tSession) Disconnect() { t.connected = false } +func (t *tSession) Id() sKey { return t.id } type lookMap struct { com.NetMap[sKey, *tSession] diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 0da257b2..856cdfa5 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -131,6 +131,7 @@ func (w *Worker) Start(done chan struct{}) { } func (w *Worker) Stop() error { + w.Reset() var err error for _, s := range w.services { if s != nil { diff --git a/web/js/stream.js b/web/js/stream.js index 69364161..5b1ed1b2 100644 --- a/web/js/stream.js +++ b/web/js/stream.js @@ -30,33 +30,34 @@ const state = { autoplayWait: false, }; -const flip = (flip) => (videoEl.style.transform = flip ? "scaleY(-1)" : ""); const mute = (mute) => (videoEl.muted = mute); +const applyTransform = () => { + const rot = state.rot ? `rotate(${-state.rot}deg)` : ""; + const flip = state.flip ? "scaleY(-1)" : ""; + mirrorEl.style.transform = rot; + videoEl.style.transform = [rot, flip].filter(Boolean).join(" "); +}; + const onPlay = () => { state.ready = true; videoEl.poster = ""; - flip(state.flip); + applyTransform(); resize(state.w, state.h, state.aspect, state.fit); useCustomScreen(options.mirrorMode === "mirror"); }; -const play = () => { - const promise = videoEl.play(); - - if (promise === undefined) { - log.error("oh no, the video is not a promise!"); - return; - } - - promise.then(onPlay).catch((error) => { - if (error.name === "NotAllowedError") { - showPlayButton(); - } else { - log.error("Playback fail", error); - } - }); -}; +const play = () => + videoEl + .play() + ?.then(onPlay) + .catch((error) => { + if (error.name === "NotAllowedError") { + showPlayButton(); + } else { + log.error("Playback fail", error); + } + }); const toggle = (show) => state.screen.toggleAttribute("hidden", show === undefined ? show : !show); @@ -72,6 +73,37 @@ const resize = (w, h, aspect, fit) => { if (fit !== undefined) { state.screen.style["object-fit"] = fit; } + + // properly size the element for rotated view + if (state.rot && Math.abs(state.rot % 180) > 1) { + const fullscreen = document.fullscreenElement !== null; + const ch = fullscreen + ? window.innerHeight + : state.screen.parentElement.clientHeight; + if (ch && state.aspect) { + const availableH = ch - (fullscreen ? 14 : 0); + const fw = availableH; + const fh = Math.round(availableH * state.aspect); + const shift = (availableH - fh) / 2; + const rot = `rotate(${-state.rot}deg)`; + const flp = + state.flip && state.screen === videoEl ? "scaleY(-1)" : ""; + state.screen.style.width = fw + "px"; + state.screen.style.height = fh + "px"; + state.screen.style.objectFit = "fill"; + state.screen.style.position = "relative"; + state.screen.style.top = "0"; + state.screen.style.transform = + `translateY(${shift}px) ${rot} ${flp}`.trim(); + } + } else { + state.screen.style.width = ""; + state.screen.style.height = ""; + state.screen.style.position = ""; + state.screen.style.top = ""; + // hack: cover 1px edge artifact + state.screen.style.marginLeft = state.flip ? "1px" : ""; + } }; const showPlayButton = () => { @@ -105,19 +137,31 @@ videoEl.onerror = (e) => log.error("Playback error", e); const onFullscreen = (fullscreen) => { const el = document.fullscreenElement; + state.screen.parentElement.style.overflow = fullscreen ? "visible" : ""; + if (fullscreen) { // timeout is due to a chrome bug setTimeout(() => { - // aspect ratio calc - const w = window.screen.width ?? window.innerWidth; - const hh = el.innerHeight || el.clientHeight || 0; - const dw = (w - hh * state.aspect) / 2; - state.screen.style.padding = `0 ${dw}px`; + // aspect ratio calc (skip for rotated - resize handles it) + if (!(state.rot && Math.abs(state.rot % 180) > 1)) { + const w = window.screen.width ?? window.innerWidth; + const hh = el.innerHeight || el.clientHeight || 0; + const dw = (w - hh * state.aspect) / 2; + state.screen.style.padding = `0 ${dw}px`; + } else { + // clear any leftover padding for rotated content + state.screen.style.padding = "0"; + } state.screen.classList.toggle("with-footer"); + // re-apply transform in case fullscreen transition reset it + applyTransform(); + resize(state.w, state.h, state.aspect, state.fit); }, 1); } else { state.screen.style.padding = "0"; state.screen.classList.toggle("with-footer"); + applyTransform(); + resize(state.w, state.h, state.aspect, state.fit); } if (el === videoEl) { @@ -155,11 +199,20 @@ const useCustomScreen = (use) => { toggle(false); state.screen = mirrorEl; - resize(videoEl.videoWidth, videoEl.videoHeight); + applyTransform(); + resize( + videoEl.videoWidth, + videoEl.videoHeight, + state.aspect, + state.fit, + ); - // stretch depending on the video orientation - const isPortrait = videoEl.videoWidth < videoEl.videoHeight; - state.screen.style.width = isPortrait ? "auto" : videoEl.videoWidth; + // stretch depending on the video orientation (skip when rotated - + // resize handles the sizing) + if (!(state.rot && Math.abs(state.rot % 180) > 1)) { + const isPortrait = videoEl.videoWidth < videoEl.videoHeight; + state.screen.style.width = isPortrait ? "auto" : videoEl.videoWidth; + } let surface = state.screen.getContext("2d"); if (state.ready) { @@ -179,6 +232,7 @@ const useCustomScreen = (use) => { clearInterval(state.timerId); toggle(false); state.screen = videoEl; + mirrorEl.style.transform = ""; if (state.ready) { toggle(true); } @@ -199,12 +253,11 @@ const init = () => { }; sub(APP_VIDEO_CHANGED, (payload) => { - const { w, h, a, s, flip } = payload; + const { w, h, a, s, flip, rot } = payload; - if (flip !== undefined) { - state.flip = flip; - if (state.ready) flip(); - } + if (flip !== undefined) state.flip = flip; + if (rot !== undefined) state.rot = rot; + if (state.ready) applyTransform(); const scale = !s ? 1 : s; const ww = w * scale; @@ -212,15 +265,13 @@ sub(APP_VIDEO_CHANGED, (payload) => { state.aspect = a; - const a2 = (ww / hh).toFixed(6); - state.h = hh; state.w = Math.floor(hh * a); resize( ww, hh, state.aspect, - a > 1 && a.toFixed(6) !== a2 ? "fill" : "contain", + a > 1 && a.toFixed(6) !== (ww / hh).toFixed(6) ? "fill" : "contain", ); recalculateSize(); });