diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 00000000..036d6ece --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,161 @@ +name: Backend Tests + +on: + push: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + pull_request: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + workflow_dispatch: + inputs: + full_suite: + description: Run the full backend test suite + type: boolean + default: true + +permissions: + contents: read + packages: read + +concurrency: + group: backend-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + plan: + name: Plan test groups + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.resolve.outputs.labels }} + has_tests: ${{ steps.resolve.outputs.has_tests }} + base_image: ${{ steps.base_image.outputs.image }} + sync_python_deps: ${{ steps.base_image.outputs.sync_python_deps }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Collect changed paths + id: changed + shell: bash + run: | + set -euo pipefail + : > /tmp/changed_paths.txt + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ inputs.full_suite }}" = "true" ]; then + : > /tmp/changed_paths.txt + echo "mode=full" >> "$GITHUB_OUTPUT" + else + git diff --name-only HEAD~1 HEAD > /tmp/changed_paths.txt || true + echo "mode=diff" >> "$GITHUB_OUTPUT" + fi + elif [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin "${{ github.base_ref }}" + git diff --name-only "origin/${{ github.base_ref }}...HEAD" > /tmp/changed_paths.txt + echo "mode=pr" >> "$GITHUB_OUTPUT" + else + if [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then + git ls-files > /tmp/changed_paths.txt + echo "mode=initial-push" >> "$GITHUB_OUTPUT" + else + git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" > /tmp/changed_paths.txt + echo "mode=push" >> "$GITHUB_OUTPUT" + fi + fi + + - name: Select base image + id: base_image + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + TARGET_BRANCH="${{ github.base_ref }}" + else + TARGET_BRANCH="${{ github.ref_name }}" + fi + if [ "$TARGET_BRANCH" = "main" ]; then + TAG="base" + else + TAG="base-dev" + fi + REPO_OWNER="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]')" + echo "image=ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" >> "$GITHUB_OUTPUT" + if grep -qx 'pyproject.toml' /tmp/changed_paths.txt || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "sync_python_deps=true" >> "$GITHUB_OUTPUT" + else + echo "sync_python_deps=false" >> "$GITHUB_OUTPUT" + fi + echo "Using base image: ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" + + - name: Resolve Django test labels + id: resolve + env: + FULL_SUITE: ${{ github.event_name == 'workflow_dispatch' && inputs.full_suite == true && 'true' || 'false' }} + run: | + set -euo pipefail + LABELS=$(python scripts/ci_backend_test_labels.py < /tmp/changed_paths.txt) + echo "labels=${LABELS}" >> "$GITHUB_OUTPUT" + if [ "${LABELS}" = "[]" ]; then + echo "has_tests=false" >> "$GITHUB_OUTPUT" + else + echo "has_tests=true" >> "$GITHUB_OUTPUT" + fi + echo "Selected labels: ${LABELS}" + + test: + name: ${{ matrix.label }} + needs: plan + if: needs.plan.outputs.has_tests == 'true' + runs-on: ubuntu-latest + container: + image: ${{ needs.plan.outputs.base_image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --entrypoint "" + strategy: + max-parallel: 6 + fail-fast: false + matrix: + label: ${{ fromJSON(needs.plan.outputs.labels) }} + env: + DISPATCHARR_ENV: aio + DJANGO_SECRET_KEY: ci-test-secret-key + POSTGRES_DB: dispatcharr + POSTGRES_USER: dispatch + POSTGRES_PASSWORD: secret + DISPATCHARR_LOG_LEVEL: WARNING + SYNC_PYTHON_DEPS: ${{ needs.plan.outputs.sync_python_deps }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Run tests in base image + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: bash scripts/ci_bootstrap_backend.sh "${{ matrix.label }}" -v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d19b94..a97f5158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,152 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.27.2] - 2026-06-30 + +### Added + +- **New proxy setting: Client Connect Grace Period (`channel_client_wait_period`, default 5s).** Adds a dedicated timeout for channels that have filled their buffer but still have no viewers (`waiting_for_clients`). Previously that window reused `channel_shutdown_delay` (default 0s), so those channels were torn down almost immediately. + +### Changed + +- **Proxy grace-period settings are now split into three distinct timeouts.** The cleanup watchdog already applied `channel_init_grace_period` while a channel was still connecting (buffer not ready) and reused `channel_shutdown_delay` once `connection_ready_time` was set, including for `waiting_for_clients` with zero viewers. With the default `channel_shutdown_delay` of 0s, a buffered channel waiting for its first viewer was stopped almost immediately; raising shutdown delay was the only workaround, but that also delayed teardown after real disconnects. Behaviour is now: + - **`channel_init_grace_period` (default 60s, max 300s):** how long the proxy may spend connecting and cycling failover streams before giving up on startup. + - **`channel_client_wait_period` (default 5s):** how long a ready channel with no viewers stays up waiting for the first client (the original grace-period use case). + - **`channel_shutdown_delay` (default 0s):** delay after the last client disconnects only; no longer applies when the buffer is ready but no viewer has connected yet. +- **Migration 0026 bumps `channel_init_grace_period` to 60s when the stored value is below 60.** Existing installs on the old 5s default (or any custom value under 60) are raised automatically. Values already at 60s or higher are unchanged. If you previously raised `channel_shutdown_delay` to keep buffered channels alive with no viewers, set `channel_client_wait_period` instead (Settings → Proxy → Advanced). +- **Proxy settings UI: less-used options moved under Advanced.** Settings → Proxy now shows the day-to-day tuning fields by default (`buffering_timeout`, `buffering_speed`, `channel_shutdown_delay`, `new_client_behind_seconds`). **Buffer Chunk TTL**, **Channel Initialization Timeout**, and **Client Connect Grace Period** are tucked under **Show Advanced Settings**. + +## [0.27.1] - 2026-06-25 + +### Security + +- Updated `Django` 6.0.5 → 6.0.6, resolving the following CVEs: + - **CVE-2026-6873**: Signed cookie salt namespace collision in `HttpRequest.get_signed_cookie()`. + - **CVE-2026-7666**: Potential unencrypted email transmission via STARTTLS in the SMTP backend. + - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. + - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. + - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. +- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (1 low, 2 moderate, 1 high): + - Updated `vite` 7.3.2 → 7.3.5, resolving **moderate** NTLMv2 hash disclosure via UNC path handling on Windows ([GHSA-v6wh-96g9-6wx3](https://github.com/advisories/GHSA-v6wh-96g9-6wx3)) and **high** `server.fs.deny` bypass on Windows alternate paths ([GHSA-fx2h-pf6j-xcff](https://github.com/advisories/GHSA-fx2h-pf6j-xcff)) + - Updated `js-yaml` 4.1.1 → 5.1.0, resolving **moderate** quadratic-complexity DoS in merge key handling via repeated aliases ([GHSA-h67p-54hq-rp68](https://github.com/advisories/GHSA-h67p-54hq-rp68)) + - Updated `esbuild` 0.27.3 → 0.28.1, resolving **low** arbitrary file read when running the development server on Windows ([GHSA-g7r4-m6w7-qqqr](https://github.com/advisories/GHSA-g7r4-m6w7-qqqr)) + +### Added + +- **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. + +### Performance + +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **XMLTV EPG export is faster and no longer balloons worker memory.** `generate_epg()` was reworked end-to-end for large guides. (Fixes #1366) + - Streams incrementally: on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker); repeat requests within 300s stream chunks back from Redis. `malloc_trim` runs after cold builds. + - Channel streams are prefetched once (only `id`/`name`) instead of one query per custom-dummy channel; dummy `EPGData` programme existence is bulk-checked in a single query. + - The primary channel id is escaped once per `epg_id` group instead of once per programme (~750k fewer `html.escape` calls on a large guide). + - The channel query no longer JOINs multi-MB `programme_index` blobs per channel (~13s saved on a ~2000-channel guide; indices live in `EPGSourceIndex`). + - Programme export uses `(epg_id, id)` keyset pagination with a per-source `start_time` sort; a matching composite index on `ProgramData` (created `CONCURRENTLY` on PostgreSQL) lets each chunk use an ordered index range scan instead of re-sorting every chunk. +- **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. + +### Changed + +- **`programme_index` moved off `EPGSource` into a dedicated `EPGSourceIndex` table.** The multi-MB byte-offset index was repeatedly pulled into web and Celery workers by ordinary `EPGSource` queries and `select_related` JOINs (which ignore manager-level `defer()`). It now lives in a one-to-one `EPGSourceIndex` row, read only when explicitly accessed through the `EPGSource.programme_index` property, so no list, detail, or JOIN query can load it by accident. Migration `0026` copies existing indices across. No API or index-build behavior change. + +- **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. + +- **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. + +- Dependency updates: + - `Django` 6.0.5 → 6.0.6 (security patch; see Security section) + - `requests` 2.33.1 → 2.34.2 + - `gevent` 26.4.0 → 26.5.0 + - `torch` 2.11.0+cpu → 2.12.1+cpu + - `sentence-transformers` 5.4.1 → 5.6.0 + - `lxml` 6.1.0 → 6.1.1 + + +### Fixed + +- **Channel Initialization Grace Period is honoured during live stream startup.** Preview and playback no longer abort after a hardcoded 10s while the channel is still connecting with an empty buffer; the TS generator init-wait stall check and upstream health monitor now use the configured `channel_init_grace_period` (same as the server cleanup watchdog) instead of `CONNECTION_TIMEOUT`. (Fixes #1380) +- **Channels are marked `active` as soon as the buffer threshold is met and a client is streaming.** Once the initial buffer fills, state is set to `active` immediately when viewers are attached, or `waiting_for_clients` when the buffer is ready but no client is connected yet (e.g. proxy API warmup). The cleanup watchdog no longer waits an extra grace period before promoting to `active`. Proxy settings copy now describes `channel_init_grace_period` as an initialization buffer timeout. +- **DVR recording playback auth is complete for native video, HLS segments, and redirects.** Completed recordings use `/file/` with native `