diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 00000000..036d6ece --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,161 @@ +name: Backend Tests + +on: + push: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + pull_request: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + workflow_dispatch: + inputs: + full_suite: + description: Run the full backend test suite + type: boolean + default: true + +permissions: + contents: read + packages: read + +concurrency: + group: backend-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + plan: + name: Plan test groups + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.resolve.outputs.labels }} + has_tests: ${{ steps.resolve.outputs.has_tests }} + base_image: ${{ steps.base_image.outputs.image }} + sync_python_deps: ${{ steps.base_image.outputs.sync_python_deps }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Collect changed paths + id: changed + shell: bash + run: | + set -euo pipefail + : > /tmp/changed_paths.txt + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ inputs.full_suite }}" = "true" ]; then + : > /tmp/changed_paths.txt + echo "mode=full" >> "$GITHUB_OUTPUT" + else + git diff --name-only HEAD~1 HEAD > /tmp/changed_paths.txt || true + echo "mode=diff" >> "$GITHUB_OUTPUT" + fi + elif [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin "${{ github.base_ref }}" + git diff --name-only "origin/${{ github.base_ref }}...HEAD" > /tmp/changed_paths.txt + echo "mode=pr" >> "$GITHUB_OUTPUT" + else + if [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then + git ls-files > /tmp/changed_paths.txt + echo "mode=initial-push" >> "$GITHUB_OUTPUT" + else + git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" > /tmp/changed_paths.txt + echo "mode=push" >> "$GITHUB_OUTPUT" + fi + fi + + - name: Select base image + id: base_image + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + TARGET_BRANCH="${{ github.base_ref }}" + else + TARGET_BRANCH="${{ github.ref_name }}" + fi + if [ "$TARGET_BRANCH" = "main" ]; then + TAG="base" + else + TAG="base-dev" + fi + REPO_OWNER="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]')" + echo "image=ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" >> "$GITHUB_OUTPUT" + if grep -qx 'pyproject.toml' /tmp/changed_paths.txt || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "sync_python_deps=true" >> "$GITHUB_OUTPUT" + else + echo "sync_python_deps=false" >> "$GITHUB_OUTPUT" + fi + echo "Using base image: ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" + + - name: Resolve Django test labels + id: resolve + env: + FULL_SUITE: ${{ github.event_name == 'workflow_dispatch' && inputs.full_suite == true && 'true' || 'false' }} + run: | + set -euo pipefail + LABELS=$(python scripts/ci_backend_test_labels.py < /tmp/changed_paths.txt) + echo "labels=${LABELS}" >> "$GITHUB_OUTPUT" + if [ "${LABELS}" = "[]" ]; then + echo "has_tests=false" >> "$GITHUB_OUTPUT" + else + echo "has_tests=true" >> "$GITHUB_OUTPUT" + fi + echo "Selected labels: ${LABELS}" + + test: + name: ${{ matrix.label }} + needs: plan + if: needs.plan.outputs.has_tests == 'true' + runs-on: ubuntu-latest + container: + image: ${{ needs.plan.outputs.base_image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --entrypoint "" + strategy: + max-parallel: 6 + fail-fast: false + matrix: + label: ${{ fromJSON(needs.plan.outputs.labels) }} + env: + DISPATCHARR_ENV: aio + DJANGO_SECRET_KEY: ci-test-secret-key + POSTGRES_DB: dispatcharr + POSTGRES_USER: dispatch + POSTGRES_PASSWORD: secret + DISPATCHARR_LOG_LEVEL: WARNING + SYNC_PYTHON_DEPS: ${{ needs.plan.outputs.sync_python_deps }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Run tests in base image + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: bash scripts/ci_bootstrap_backend.sh "${{ matrix.label }}" -v2 diff --git a/scripts/ci_backend_test_labels.py b/scripts/ci_backend_test_labels.py new file mode 100644 index 00000000..e69356c4 --- /dev/null +++ b/scripts/ci_backend_test_labels.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Map changed repository paths to Django test package labels for CI.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import PurePosixPath + +# Keep in sync with dispatcharr.test_runner package discovery. +ALL_LABELS: tuple[str, ...] = ( + "apps.accounts.tests", + "apps.backups.tests", + "apps.channels.tests", + "apps.connect.tests", + "apps.dashboard.tests", + "apps.epg.tests", + "apps.m3u.tests", + "apps.output.tests", + "apps.plugins.tests", + "apps.timeshift.tests", + "apps.proxy.live_proxy.tests", + "apps.proxy.vod_proxy.tests", + "core.tests", + "tests", +) + +# Longest-prefix wins; order matters for overlapping rules. +_PATH_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("apps/channels/", ("apps.channels.tests",)), + ("apps/epg/", ("apps.epg.tests",)), + ("apps/m3u/", ("apps.m3u.tests",)), + ("apps/proxy/", ("apps.proxy.live_proxy.tests", "apps.proxy.vod_proxy.tests")), + ("apps/connect/", ("apps.connect.tests",)), + ("apps/output/", ("apps.output.tests",)), + ("apps/accounts/", ("apps.accounts.tests",)), + ("apps/backups/", ("apps.backups.tests",)), + ("apps/dashboard/", ("apps.dashboard.tests",)), + ("apps/plugins/", ("apps.plugins.tests",)), + ("apps/timeshift/", ("apps.timeshift.tests",)), + ("apps/vod/", ("apps.output.tests",)), + ("apps/hdhr/", ("apps.output.tests", "apps.channels.tests")), + ("apps/api/", ALL_LABELS), + ("core/", ("core.tests", "tests")), + ("tests/", ("tests",)), +) + +_SHARED_PREFIXES: tuple[str, ...] = ( + "dispatcharr/", + "pyproject.toml", + "manage.py", + "version.py", + "scripts/ci_backend_test_labels.py", + "scripts/ci_bootstrap_backend.sh", + ".github/workflows/backend-tests.yml", +) + + +def _normalize(path: str) -> str: + return PurePosixPath(path.strip().replace("\\", "/")).as_posix() + + +def labels_for_paths(paths: list[str], *, full_suite: bool = False) -> list[str]: + if full_suite: + return list(ALL_LABELS) + + labels: set[str] = set() + for raw in paths: + path = _normalize(raw) + if not path: + continue + if any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in _SHARED_PREFIXES): + return list(ALL_LABELS) + for prefix, group_labels in _PATH_RULES: + if path.startswith(prefix): + labels.update(group_labels) + break + + return sorted(labels) + + +def _read_paths(argv: list[str]) -> list[str]: + if argv: + return argv + if not sys.stdin.isatty(): + return [line for line in sys.stdin.read().splitlines() if line.strip()] + return [] + + +def main(argv: list[str] | None = None) -> int: + argv = argv if argv is not None else sys.argv[1:] + full_suite = os.environ.get("FULL_SUITE", "").lower() in {"1", "true", "yes"} + if full_suite: + print(json.dumps(list(ALL_LABELS))) + return 0 + paths = _read_paths(argv) + labels = labels_for_paths(paths, full_suite=False) + print(json.dumps(labels)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_bootstrap_backend.sh b/scripts/ci_bootstrap_backend.sh new file mode 100644 index 00000000..d13e0218 --- /dev/null +++ b/scripts/ci_bootstrap_backend.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Bootstrap internal Postgres/Redis (AIO-style) and run Django tests in the base image. +set -euo pipefail + +REPO_ROOT="${GITHUB_WORKSPACE:-$(pwd)}" +cd "${REPO_ROOT}" + +export DISPATCHARR_ENV="${DISPATCHARR_ENV:-aio}" +export PUID="${PUID:-1000}" +export PGID="${PGID:-1000}" +export POSTGRES_DB="${POSTGRES_DB:-dispatcharr}" +export POSTGRES_USER="${POSTGRES_USER:-dispatch}" +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-secret}" +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" +export POSTGRES_DIR="${POSTGRES_DIR:-/tmp/dispatcharr-ci/db}" +export REDIS_HOST="${REDIS_HOST:-localhost}" +export REDIS_PORT="${REDIS_PORT:-6379}" +export REDIS_DB="${REDIS_DB:-0}" +export DJANGO_SECRET_KEY="${DJANGO_SECRET_KEY:-ci-test-secret-key}" +export DISPATCHARR_LOG_LEVEL="${DISPATCHARR_LOG_LEVEL:-WARNING}" +export PATH="/dispatcharrpy/bin:${PATH}" +export PG_VERSION +PG_VERSION="$(ls /usr/lib/postgresql/ | sort -V | tail -n 1)" +export PG_BINDIR="/usr/lib/postgresql/${PG_VERSION}/bin" + +if [[ "$DISPATCHARR_ENV" == "aio" ]]; then + export POSTGRES_HOST="${POSTGRES_HOST:-/var/run/postgresql}" +else + export POSTGRES_HOST="${POSTGRES_HOST:-localhost}" +fi + +if [[ "${SYNC_PYTHON_DEPS:-}" == "true" ]]; then + echo "Syncing Python dependencies with uv..." + uv sync --python /dispatcharrpy/bin/python --no-install-project --no-dev +fi + +echo "Setting up CI user and PostgreSQL data directory..." +# shellcheck source=docker/init/01-user-setup.sh +. "${REPO_ROOT}/docker/init/01-user-setup.sh" +mkdir -p "${POSTGRES_DIR}" +chown "${PUID}:${PGID}" "${POSTGRES_DIR}" +chmod 700 "${POSTGRES_DIR}" +# shellcheck source=docker/init/02-postgres.sh +. "${REPO_ROOT}/docker/init/02-postgres.sh" + +echo "Starting internal PostgreSQL..." +prepare_pg_socket_dir +su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_ctl -D ${POSTGRES_DIR} start -w -t 120 -o '-c port=${POSTGRES_PORT}'" +until su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do + sleep 1 +done +set +e +promote_app_role +_promote_status=$? +ensure_app_database +_ensure_status=$? +set -e +if [ "$_promote_status" -ne 0 ] || [ "$_ensure_status" -ne 0 ]; then + echo "Failed to configure PostgreSQL role/database (promote=${_promote_status}, ensure=${_ensure_status})" + exit 1 +fi + +echo "Starting internal Redis..." +if redis-cli -p "${REDIS_PORT}" ping >/dev/null 2>&1; then + echo "Redis already listening on port ${REDIS_PORT}" +else + redis-server --daemonize yes --protected-mode no --bind 127.0.0.1 --port "${REDIS_PORT}" +fi +python "${REPO_ROOT}/scripts/wait_for_redis.py" + +cleanup() { + redis-cli -p "${REDIS_PORT}" shutdown nosave >/dev/null 2>&1 || true + su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_ctl -D ${POSTGRES_DIR} stop -m fast" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +exec python manage.py test --keepdb "$@"