Mirrors/Dispatcharr
1
0
Fork 1
mirror of https://github.com/Dispatcharr/Dispatcharr.git synced 2026-07-18 00:55:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'dev' of https://github.com/Dispatcharr/Dispatcharr into pr/firestaerter3/1125

Browse source
This commit is contained in:
SergeantPanda 2026-04-13 17:42:43 -05:00
commit d73071b20f
209 changed files with 28228 additions and 6493 deletions
Download patch file Download diff file Expand all files Collapse all files

15
.github/FUNDING.yml vendored Normal file
View file

@ -0,0 +1,15 @@
# These are supported funding model platforms
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username
open_collective: dispatcharr # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
polar: # Replace with a single Polar username
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
thanks_dev: # Replace with a single thanks.dev username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

20
.github/workflows/base-image.yml vendored
View file

@ -28,7 +28,7 @@ jobs:
branch_tag: ${{ steps.meta.outputs.branch_tag }}
timestamp: ${{ steps.timestamp.outputs.timestamp }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -74,7 +74,7 @@ jobs:
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -85,17 +85,17 @@ jobs:
git config user.email "actions@github.com"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
@ -103,7 +103,7 @@ jobs:
- name: Extract metadata for Docker
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v6
with:
images: |
ghcr.io/${{ needs.prepare.outputs.repo_owner }}/${{ needs.prepare.outputs.repo_name }}
@ -124,7 +124,7 @@ jobs:
build_version=DispatcharrBase version: ${{ needs.prepare.outputs.branch_tag }}-${{ needs.prepare.outputs.timestamp }}
- name: Build and push Docker base image
uses: docker/build-push-action@v4
uses: docker/build-push-action@v7
with:
context: .
file: ./docker/DispatcharrBase
@ -149,17 +149,17 @@ jobs:
if: ${{ github.event_name != 'pull_request' }}
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}

25
.github/workflows/ci.yml vendored
View file

@ -25,7 +25,7 @@ jobs:
timestamp: ${{ steps.timestamp.outputs.timestamp }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -83,7 +83,7 @@ jobs:
runs-on: ${{ matrix.runner }}
# no per-job outputs here; shared metadata comes from the `prepare` job
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -103,17 +103,18 @@ jobs:
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
@ -121,7 +122,7 @@ jobs:
- name: Extract metadata for Docker
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v6
with:
images: |
ghcr.io/${{ needs.prepare.outputs.repo_owner }}/${{ needs.prepare.outputs.repo_name }}
@ -142,7 +143,7 @@ jobs:
build_version=Dispatcharr version: ${{ needs.prepare.outputs.version }}-${{ needs.prepare.outputs.timestamp }}
- name: Build and push Docker image
uses: docker/build-push-action@v4
uses: docker/build-push-action@v7
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
@ -155,8 +156,8 @@ jobs:
tags: |
ghcr.io/${{ needs.prepare.outputs.repo_owner }}/${{ needs.prepare.outputs.repo_name }}:${{ needs.prepare.outputs.branch_tag }}-${{ matrix.platform }}
ghcr.io/${{ needs.prepare.outputs.repo_owner }}/${{ needs.prepare.outputs.repo_name }}:${{ needs.prepare.outputs.version }}-${{ needs.prepare.outputs.timestamp }}-${{ matrix.platform }}
docker.io/${{ secrets.DOCKERHUB_ORGANIZATION }}/${{ needs.prepare.outputs.repo_name }}:${{ needs.prepare.outputs.branch_tag }}-${{ matrix.platform }}
docker.io/${{ secrets.DOCKERHUB_ORGANIZATION }}/${{ needs.prepare.outputs.repo_name }}:${{ needs.prepare.outputs.version }}-${{ needs.prepare.outputs.timestamp }}-${{ matrix.platform }}
${{ github.event_name != 'pull_request' && format('docker.io/{0}/{1}:{2}-{3}', secrets.DOCKERHUB_ORGANIZATION, needs.prepare.outputs.repo_name, needs.prepare.outputs.branch_tag, matrix.platform) || '' }}
${{ github.event_name != 'pull_request' && format('docker.io/{0}/{1}:{2}-{3}-{4}', secrets.DOCKERHUB_ORGANIZATION, needs.prepare.outputs.repo_name, needs.prepare.outputs.version, needs.prepare.outputs.timestamp, matrix.platform) || '' }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
REPO_OWNER=${{ needs.prepare.outputs.repo_owner }}
@ -174,17 +175,17 @@ jobs:
if: ${{ github.event_name != 'pull_request' }}
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}

8
.github/workflows/docker-build.yml vendored
View file

@ -15,13 +15,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@v4
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -42,7 +42,7 @@ jobs:
run: echo "REPO_NAME=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
- name: Build and push Docker image
uses: docker/build-push-action@v5
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile

2
.github/workflows/frontend-tests.yml vendored
View file

@ -22,7 +22,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v6
- name: Setup Node.js
uses: actions/setup-node@v4

22
.github/workflows/release.yml vendored
View file

@ -27,7 +27,7 @@ jobs:
repo_name: ${{ steps.meta.outputs.repo_name }}
timestamp: ${{ steps.timestamp.outputs.timestamp }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -83,7 +83,7 @@ jobs:
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
@ -95,17 +95,17 @@ jobs:
git config user.email "actions@github.com"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
@ -113,7 +113,7 @@ jobs:
- name: Extract metadata for Docker
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v6
with:
images: |
ghcr.io/${{ needs.prepare.outputs.repo_owner }}/${{ needs.prepare.outputs.repo_name }}
@ -134,7 +134,7 @@ jobs:
build_version=Dispatcharr version: ${{ needs.prepare.outputs.new_version }} Build date: ${{ needs.prepare.outputs.timestamp }}
- name: Build and push Docker image
uses: docker/build-push-action@v4
uses: docker/build-push-action@v7
with:
context: .
push: true
@ -157,17 +157,17 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v4
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
@ -228,7 +228,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
with:
tag_name: v${{ needs.prepare.outputs.new_version }}
name: Release v${{ needs.prepare.outputs.new_version }}

202
CHANGELOG.md
View file

@ -7,6 +7,174 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Security
- Set `DEFAULT_PERMISSION_CLASSES` to `IsAdmin` in the DRF configuration. All viewsets and function-based views that require non-admin or unauthenticated access were explicitly annotated: proxy streaming endpoints (`stream_ts`, `stream_xc`, `stream_vod`, `head_vod`, `stream_xc_movie`, `stream_xc_episode`) use `@permission_classes([AllowAny])` (access is controlled by the per-stream-type network allow-list inside the view body); the `UserAgentViewSet`, `StreamProfileViewSet`, `CoreSettingsViewSet`, and `ProxySettingsViewSet` gained `get_permissions()` methods mapping read actions to `IsStandardUser` and write actions to `IsAdmin`; and `AuthViewSet.logout` was updated to return `[Authenticated()]`.
- Fixed missing `network_access_allowed` checks in the VOD proxy. `stream_vod`, `head_vod`, `stream_xc_movie`, and `stream_xc_episode` were not checking the `STREAMS` network policy, unlike the equivalent TS proxy endpoints.
- Explicitly marked the HDHomeRun discovery endpoints (`DiscoverAPIView`, `LineupAPIView`, `LineupStatusAPIView`, `HDHRDeviceXMLAPIView`) and the version endpoint with `permission_classes = [AllowAny]` to document their intentionally public access now that the global default is `IsAdmin`.
- Fixed path traversal vulnerability in file uploads. The M3U account upload (`apps/m3u/api_views.py`), logo upload (`apps/channels/api_views.py`), and backup upload (`apps/backups/api_views.py`) all used the uploaded filename directly without sanitization. `os.path.join()` discards all preceding components when it encounters an absolute path segment, and `pathlib`'s `/` operator behaves identically; a relative `../` sequence also escapes via OS path resolution at `open()` time. All three upload paths now strip directory components via `Path(name).name` and validate the resolved path remains within the intended upload directory. Exploiting any of these required admin credentials.
- Prevented users from setting `xc_password` (and other admin-managed keys) on their own account via the `PATCH /api/accounts/users/me/` endpoint.
- Hardened the HLS proxy `change_stream` endpoint by converting it from a plain Django view to a DRF `@api_view` with `@permission_classes([IsAdmin])`, ensuring the endpoint actually enforces admin-only access. The previous decorator arrangement (`@csrf_exempt` + `@permission_classes`) had no effect on a plain Django view.
- Added rate limiting to the login endpoint (`POST /api/accounts/token/`) using DRF's built-in throttling. A `LoginRateThrottle` (3 requests/minute per IP, sliding window) is applied to the `TokenObtainPairView`. Repeated failed attempts from the same IP receive `429 Too Many Requests`.
- Extended rate limiting to the session-auth login alias (`POST /api/accounts/auth/login/`). It now delegates entirely to `TokenObtainPairView`, inheriting its throttle, network access check, and audit logging, and returns JWT tokens instead of a session cookie (the session-based response was unusable since `SessionAuthentication` is not in `DEFAULT_AUTHENTICATION_CLASSES`). Both endpoints share the same `"login"` throttle scope, so attempts across either path count against the same per-IP limit.
- Removed `CORS_ALLOW_CREDENTIALS = True` from CORS configuration. Dispatcharr authenticates via JWT `Authorization` headers and API keys — not cookies — so credentials are never sent cross-origin by browsers. The setting was also redundant: browsers reject `Access-Control-Allow-Credentials: true` when `Access-Control-Allow-Origin` is a wildcard (`*`), so it had no effect in practice.
- Updated frontend npm dependencies to resolve 6 audit vulnerabilities (6 high):
- Updated `@xmldom/xmldom` 0.8.11 → 0.8.12, resolving **high** XML injection via unsafe CDATA serialization allowing attacker-controlled markup insertion ([GHSA-wh4c-j3r5-mjhp](https://github.com/advisories/GHSA-wh4c-j3r5-mjhp))
- Updated `lodash` 4.17.23 → 4.18.1, resolving **high** Code Injection via `_.template` imports key names ([GHSA-r5fr-rjxr-66jc](https://github.com/advisories/GHSA-r5fr-rjxr-66jc)) and **high** Prototype Pollution via array path bypass in `_.unset` and `_.omit` ([GHSA-f23m-r3pf-42rh](https://github.com/advisories/GHSA-f23m-r3pf-42rh))
- Updated `vite` 7.3.1 → 7.3.2, resolving **high** Path Traversal in optimized deps `.map` handling ([GHSA-4w7w-66w2-5vf9](https://github.com/advisories/GHSA-4w7w-66w2-5vf9)), **high** `server.fs.deny` bypass with queries ([GHSA-v2wj-q39q-566r](https://github.com/advisories/GHSA-v2wj-q39q-566r)), and **high** Arbitrary File Read via dev server WebSocket ([GHSA-p9ff-h696-f583](https://github.com/advisories/GHSA-p9ff-h696-f583))
- Updated `Django` 6.0.3 → 6.0.4, resolving the following CVEs:
- **CVE-2026-33033**: Potential DoS via `MultiPartParser` through crafted multipart uploads.
- **CVE-2026-33034**: SGI requests with a missing or understated `Content-Length` header could bypass the `DATA_UPLOAD_MAX_MEMORY_SIZE` limit.
- **CVE-2026-4292**: Privilege abuse in `ModelAdmin.list_editable`.
- **CVE-2026-3902**: ASGI header spoofing via underscore/hyphen conflation.
- **CVE-2026-4277**: Privilege abuse in `GenericInlineModelAdmin`.
### Added
- **EPG historical data window**: the EPG XML output and XC EPG API now support a `prev_days` URL parameter (e.g. `&prev_days=3`) to include past programs in the EPG response. This allows third-party players that request historical program schedules to receive the data they need. The EPG URL builder in the Channels page exposes "Days forward" and "Days back" controls. Per-user defaults for both values (`epg_days` / `epg_prev_days`) can be configured in the User settings modal and are applied automatically when no URL parameter is present. (Closes #1154)
- **Plugin Hub**: administrators can now browse, install, and update plugins directly from remote repositories via a new Plugin Hub page in Settings. (Closes #393) — Thanks [@sethwv](https://github.com/sethwv)
- Install plugins directly from the hub: the release zip is downloaded, SHA256 integrity is verified, and the plugin is installed atomically.
- Update managed plugins when a newer version is available from their source repo. Version compatibility constraints (`min_dispatcharr_version` / `max_dispatcharr_version`) are enforced at install time.
- Browse available plugins from all enabled repos with name, description, version, author, and icon.
- Plugins installed from a repo are tracked as "managed": source repo, slug, installed version, prerelease flag, and deprecated status are all persisted and surfaced in the UI.
- Add plugin repositories by manifest URL. The official Dispatcharr Plugins repository is pre-configured; third-party repos are supported by supplying an optional GPG public key.
- Manifest signatures are verified via GPG; the official repo uses a bundled public key. Signature status is displayed per-repo.
- Preview a repository URL before adding it - validates the manifest and reports plugin count and signature status without saving anything.
- Configurable automatic manifest refresh interval (in hours; 0 to disable) runs as a Celery background task.
### Removed
- Removed dead `VODConnectionManager` class (`apps/proxy/vod_proxy/connection_manager.py`) and its associated helpers, which had been superseded by `MultiWorkerVODConnectionManager`. All active code already used the multi-worker implementation. Removed the unused `VODConnectionManager` import from `vod_proxy/views.py`, the unscheduled `cleanup_vod_connections` task from `apps/proxy/tasks.py`, and the unscheduled `cleanup_vod_persistent_connections` task from `core/tasks.py`.
- Removed dead VOD URL routes: `VODPlaylistView` (playlist generation), `VODPositionView` (position tracking), and the class-based `VODStatsView` (replaced by the existing function-based `vod_stats` view).
- Removed dead `updateVODPosition()` API method from `frontend/src/api.js`, which called the now-removed position tracking endpoint.
### Fixed
- Fixed manual stream selection from the Stats page not enforcing M3U profile connection limits in multi-worker deployments. When a non-owning worker handled the `change_stream` request it correctly packaged `stream_id` and `m3u_profile_id` into the Redis pubsub message, but the owning worker's pubsub handler only consumed `url` and `user_agent` silently dropping both IDs before calling `stream_manager.update_url()`. Because `update_url` only calls `update_stream_profile()` when a `stream_id` is provided, the `profile_connections` counter was never updated after the switch, causing subsequent capacity checks to see incorrect counts and bypass the full-profile guard. The handler now extracts `stream_id` and `m3u_profile_id` from the event and forwards them to `update_url()`. The bug did not affect single-worker / dev-mode deployments because the owning worker handles those requests directly without pubsub.
- Fixed the `next_stream` rotation endpoint applying the same class of bug: `get_stream_info_for_switch()` was called and returned `m3u_profile_id`, but the result was dropped when forwarding to `ChannelService.change_stream_url()`, so `update_stream_profile()` was never called and `profile_connections` counters were not updated after an automatic stream rotation.
- Fixed stream switch metadata (`url`, `user_agent`, `stream_id`, `m3u_profile`) being written to Redis before the switch was confirmed to succeed. If the switch failed, URL unchanged or exception during teardown, Redis described a URL not actually in use. Metadata is now written only after `update_url()` returns `True`; on failure the owner writes `stream_manager.url` back as the ground truth. The non-owner no longer pre-writes metadata at all, all needed info is carried in the pubsub payload and written by the owner after confirmation.
- Fixed the Stats page "Active Stream" dropdown not updating when a stream switch occurs. The card was matching the active stream by comparing the URL stored in Redis against stream URLs from the database, which failed silently when the stored URL was a transformed/rewritten value that didn't substring-match the original. The dropdown now matches by `stream_id` (the authoritative value already present in the stats payload) and re-runs only when `stream_id` changes, so the normal polling interval drives updates with no extra renders.
- Fixed the XC Password field in the User modal being editable by standard users despite the backend (`PATCH /api/accounts/users/me/`) stripping `xc_password` from `custom_properties` for non-admin users, causing the change to silently revert on save. The field and its generate button are now disabled with an explanatory description when the current user is not an administrator.
- Fixed uploading a local M3U file with no expiration date set sending the string `"null"` as the `exp_date` field in the `FormData` request, causing a 400 validation error from the API. Null/undefined values are now skipped when building the `FormData` body, matching the behaviour already present in the update path.
- Fixed `PATCH /api/channels/channels/edit/bulk/` returning a 500 error when the request body included a `streams` list. The bulk edit handler was iterating `validated_data` directly and calling `setattr(channel, "streams", value)`, which Django prohibits on ManyToMany fields. Also added an `@extend_schema` decorator so the Swagger UI correctly documents the endpoint as accepting a JSON array and shows the `streams` field. (Fixes #883)
- Fixed several incorrect or incomplete OpenAPI (`@extend_schema`) schemas across the API:
- `POST /api/epg/import/` — request body was undocumented; now correctly shows the `id` field. Description updated from "import" to "refresh" to match frontend and backend terminology.
- `DELETE /api/channels/logos/bulk-delete/` — `delete_files` boolean was missing from the documented request body.
- `POST /api/channels/channels/batch-set-epg/` — `epg_data_id` inside each association object was not marked `allow_null`/`required=False`, even though passing `null` is the correct way to remove an EPG link.
- `PUT /api/connect/integrations/{id}/subscriptions/set/` — endpoint had no `@extend_schema` at all; now documents that the request body is a JSON array of subscription objects.
### Changed
- Improved the EPG response cache key. Previously it was based on the raw query string and username, meaning a user default of `epg_days=7` and an explicit `&days=7` URL parameter produced different cache entries for identical output. The key is now built from all resolved effective parameter values (`days`, `prev_days`, `cachedlogos`, `tvg_id_source`) so semantically equivalent requests always share the same cache entry.
- Improved the HDHR, M3U, and EPG URL builder popovers in the Channels table: each popover now opens with a brief intro sentence describing its purpose. Toggle switches were refactored to use Mantine's native `label` and `description` props (replacing the previous manual `Group`/`Stack`/`Text` layout), giving each switch a properly styled description line beneath its label. Switch alignment was also corrected. Toggles now appear on the left with the label and description stacked to the right, consistent with standard Mantine form layout.
- Redesigned the User settings modal with a tabbed layout: **Account** (username, email, name, password), **Permissions** (user level, stream limit, channel profiles, mature content filter - admin only), **EPG Defaults** (days forward/back), and **API & XC** (XC password, API key management). Fields are now logically grouped rather than split across two ad-hoc columns.
- EPG channel scanning now automatically removes stale `EPGData` entries. tvg-ids that were present in a previous scan but are no longer found in the upstream source, provided they are not mapped to any channel. This prevents unbounded database bloat over time. Entries mapped to at least one channel are always preserved.
- Rewrote the M3U line parser as an `iter_m3u_entries` generator that owns the full per-entry state machine. Intermediate directive lines between `#EXTINF` and the stream URL are now handled correctly rather than corrupting the pending entry or being silently misassigned. A `#EXTINF` with no following URL is discarded with a warning instead of carrying over a `url`-less entry into batch processing. Attribute keys are normalised to lowercase during parsing (provider attribute names remain case-insensitive end-to-end). The `#EXTINF` attribute regex is pre-compiled at module load, and attribute lookups use O(1) `dict.get()` instead of linear scans — approximately 10% faster parsing on large M3U files.
- Added support for the `#EXTGRP` directive in M3U files. When a `group-title` attribute is absent from the `#EXTINF` line, the value from a following `#EXTGRP:` line is used as the group. An explicit `group-title` attribute always takes priority. (Closes #1088)
- Added accumulation of `#EXTVLCOPT` directives per entry. Options are stored as a list under `vlc_opts` inside the stream's `custom_properties`, available for downstream use (e.g. passing VLC-specific options to the player). This is for a planned future enhancement and can also be utlized with the API.
- M3U stream name parsing now uses the comma text (the canonical display title per the base `#EXTINF` spec) as the primary stream name, falling back to `tvc-guide-title`, then `tvg-name`, rather than preferring `tvg-name` first. Providers that use `tvg-name` as an EPG key and put the human-readable title after the comma will now display the correct name. Providers that duplicate the same value in both fields are unaffected. (Fixes #1081)
- FloatingVideo player: the native video controls (timeline, play/pause, volume) are now hidden by default when a live stream starts and only appear when the user hovers over the player.
- Enhanced Swagger UI authorization dialog: registered a custom `OpenApiAuthenticationExtension` for `ApiKeyAuthentication` so drf-spectacular now generates an `ApiKeyAuth (apiKey)` entry alongside `jwtAuth`. Both entries include descriptive text linking to the relevant endpoints (`/api/accounts/token/`, `/api/accounts/api-keys/generate/`, `/api/accounts/api-keys/revoke/`).
- Refactored frontend form components (`AccountInfoModal`, `AssignChannelNumbers`, `Channel`, `ChannelBatch`, `ChannelGroup`, `Connection`, `CronBuilder`, `DummyEPG`, and `EPG`) to extract business logic into dedicated utility modules under `src/utils/forms/`. Each extracted module is covered by unit tests. Mantine compound component references (`Table.Tbody`, `Popover.Target`, `Accordion.Item`, etc.) have been updated to use flat named imports. — Thanks [@nick4810](https://github.com/nick4810)
- Improved the EPG BOM fix from v0.22.1: replaced the `lstrip(b'\xef\xbb\xbf')` / `startswith` approach with `start.find(b'<?xml')`, which locates the XML declaration regardless of any leading bytes BOM, whitespace, or other encoding markers without needing to know what those bytes are.
- Dependency updates:
- `Django` 6.0.3 → 6.0.4 (security patch; see Security section)
- `djangorestframework` 3.16.1 → 3.17.1
- `requests` 2.33.0 → 2.33.1
- `gevent` 25.9.1 → 26.4.0
- `rapidfuzz` 3.14.3 → 3.14.5
- `sentence-transformers` 5.3.0 → 5.4.0
- `lxml` 6.0.2 → 6.0.3
- Added `python-gnupg` for GPG signature verification of official and third-party plugin repository manifests.
## [0.22.1] - 2026-04-05
### Fixed
- Fixed EPG sources that emit a UTF-8 BOM (e.g. ErsatzTV, EPGShare, WebGrab+Plus) parsing 0 channels and 0 programmes after the HTML entity fix introduced in v0.22.0. `bytes.lstrip()` only strips ASCII whitespace, leaving the three BOM bytes (`EF BB BF`) in place, so `stripped.startswith(b'<?xml')` returned `False`. The function fell through to the no-declaration branch and prepended the HTML entity DOCTYPE block _before_ the BOM and XML declaration, producing invalid XML that lxml silently discarded under `recover=True`. Fixed by stripping the BOM explicitly before the whitespace strip: `start.lstrip(b'\xef\xbb\xbf').lstrip()`. BOM-free files are unaffected. (Closes #1173) — Thanks [@dwot](https://github.com/dwot) for the fix!
## [0.22.0] - 2026-04-01
### Security
- Updated `requests` 2.32.5 → 2.33.0, resolving the following CVE:
- **CVE-2026-25645** (moderate): Insecure temp file reuse in `extract_zipped_paths()` utility function.
- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (2 moderate, 2 high):
- Updated `brace-expansion` 5.0.2 → 5.0.5, resolving **moderate** zero-step sequence causing process hang and memory exhaustion ([GHSA-f886-m6hf-6m8v](https://github.com/advisories/GHSA-f886-m6hf-6m8v))
- Updated `flatted` 3.4.1 → 3.4.2, resolving **high** Prototype Pollution via `parse()` in NodeJS flatted ([GHSA-rf6f-7fwh-wjgh](https://github.com/advisories/GHSA-rf6f-7fwh-wjgh))
- Updated `picomatch` 4.0.3 → 4.0.4, resolving **high** method injection in POSIX character classes causing incorrect glob matching ([GHSA-3v7f-55p6-f55p](https://github.com/advisories/GHSA-3v7f-55p6-f55p)) and a ReDoS vulnerability via extglob quantifiers ([GHSA-c2c7-rcm5-vvqj](https://github.com/advisories/GHSA-c2c7-rcm5-vvqj))
- Updated `yaml` 1.10.2 → 1.10.3, resolving **moderate** stack overflow via deeply nested YAML collections ([GHSA-48c2-rrv3-qjmp](https://github.com/advisories/GHSA-48c2-rrv3-qjmp))
### Added
- Connection cards on the Stats page now show the **username** of the connected user. For live channel connections a new User column appears between IP Address and Connected; for VOD connections the username is shown inline next to the IP address in the Client summary row. The username is resolved from the user store using the `user_id` stored in Redis client metadata. (Closes #766, Closes #586)
- `ip_address` and `user_id` were not included in the client info returned by `get_detailed_channel_info()` despite being available in the Redis hash. Both fields are now extracted and returned. `user_id` is now also included in the VOD stats response.
- Web UI stream preview now sends an `Authorization: Bearer` header with each mpegts.js request, identifying the logged-in user. Live channel previews initiated from the web UI now appear on the Stats page with the correct username rather than as unknown user.
- `client_connect` and `client_disconnect` system events now include the **username** of the connected user. The username is stored alongside the client metadata in Redis and included in the event payload for `log_system_event` calls (making it available to webhook and script integrations).
- Donate button added to the sidebar footer. A heart icon links to the project's Open Collective page, visible in both expanded and collapsed states. Hovering shows a "Support Dispatcharr" tooltip. The version string is also now clickable to copy it to the clipboard.
- User stream limits: administrators can now set a maximum number of concurrent streams per user account. When a user reaches their limit, the system can automatically terminate an existing stream to free a slot based on configurable rules. Limit enforcement applies to both live channels and VOD. (Closes #544)
- Each user account has a new **Stream Limit** field (0 = unlimited) configurable from the user edit form in Settings → Users.
- Global enforcement behaviour is configurable in Settings → User Limits:
- **Terminate on Limit Exceeded**: automatically stop an existing stream when the user's limit is reached (vs. rejecting the new connection).
- **Terminate Oldest**: prefer terminating the oldest stream when freeing a slot; disable to prefer the newest.
- **Prioritize Single-Client Channels**: prefer terminating streams on channels that only this user is watching.
- **Ignore Same-Channel Connections**: count multiple connections to the same live channel as one stream toward the limit. Same-channel reconnects are always allowed through. When this is enabled and a channel must be freed, all connections to the chosen channel are terminated together so that the unique-channel count actually decreases. VOD is explicitly excluded from this bypass since VOD connections are not shared upstream.
- TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) — Thanks [@CodeBormen](https://github.com/CodeBormen)
- Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312)
### Changed
- M3U Profile form (XC accounts): added a **Simple / Advanced** mode toggle for credential-based URL rewriting. In Simple mode users enter just a new username and password; the search and replace patterns are built automatically from the account's current credentials. In Advanced mode the full regex fields are shown as before. The selected mode is saved to `custom_properties.xcMode` and auto-detected on existing profiles (a profile whose search pattern matches the account's current `username/password` is recognised as Simple automatically). The Live Regex Demonstration panel is hidden in Simple mode.
- XtreamCodes VOD endpoints (`/movie/` and `/series/`) no longer redirect clients to a UUID-based proxy URL. Requests are now handled directly in the proxy layer via `stream_xc_movie` and `stream_xc_episode`, which call `stream_vod()` internally. The original XC path is preserved for the client throughout the stream.
- `CustomTable` column layout now supports flexible (`grow`) columns alongside fixed-width ones:
- Column definitions accept a `grow` property (boolean or number) to opt into flex layout. A numeric value sets the flex-grow weight, allowing relative sizing between grow columns (e.g. `grow: 2` gives a column twice the share of spare space as `grow: 1`).
- `maxSize` is now respected on grow columns, capping how wide they expand via `maxWidth`.
- The wrapper's `minWidth` calculation now uses `minSize` (not TanStack's 150px default) for grow columns, preventing the table from overflowing its container when columns would otherwise be sized larger than available space.
- Dependency updates:
- `requests` 2.32.5 → 2.33.0 (security patch; see Security section)
- `celery` 5.6.2 → 5.6.3
- `torch` 2.10.0+cpu → 2.11.0+cpu
- `sentence-transformers` 5.2.3 → 5.3.0
- `yt-dlp` 2026.3.13 → 2026.3.17
- Docker base image cleanup: removed `python-is-python3`, `python3-pip`, and `streamlink` from the apt package list in `DispatcharrBase`. `python3-pip` and `streamlink` were pulling outdated system Python packages (e.g. `requests 2.31.0`, `cryptography 41.0.7`, `lxml 5.2.1`) into the system Python's site-packages despite the app running entirely in the uv-managed venv at `/dispatcharrpy`. `streamlink` is already installed in the venv via `pyproject.toml`. `python-is-python3` is unnecessary as `PATH` resolves bare `python` to the venv binary.
- M3U table **Max Streams** column now reflects the combined limit across all active profiles. When a playlist has multiple active profiles, the column displays their summed total (or ∞ if any profile is unlimited) and a hover tooltip lists each profile's individual limit by name. (Closes #816)
- Toggling an M3U profile's active state now immediately updates the playlist store (including the `playlists` array), so the **Max Streams** total in the M3U table reflects the change without a page reload.
- M3U account form: **Max Streams** field changed from a plain text input to a number input with increment/decrement controls, consistent with other integer fields.
- M3U account form: removed unused `useMantineTheme` import and `theme` variable.
- Moved `guideUtils.js` from `frontend/src/pages/` to `frontend/src/utils/` to be consistent with other utility modules (e.g. `networkUtils.js`). Updated all imports across `GuideRow.jsx`, `HourTimeline.jsx`, `ProgramDetailModal.jsx`, `RecordingCardUtils.js`, `Guide.jsx`, and related test files.
- Frontend cleanup: removed unused imports from `M3UGroupFilter`, `LiveGroupFilter`, and `VODCategoryFilter` (`Yup`, `M3UProfiles`, several unused Mantine components, dead `OptionWithTooltip` component, duplicate lucide-react imports, and `Divider` in `VODCategoryFilter`). No behaviour changes.
- Network Access settings: leaving a field blank no longer shows a validation error. The default CIDR range for that field is saved automatically and a "Defaults Restored" warning is displayed listing which fields were reset. (Closes #726)
### Fixed
- M3U profile URL rewriting now uses the `regex` module instead of `re` across all URL transform code paths (`url_utils.transform_url`, `core/views.py`, `vod_proxy/_transform_url`, `tasks.get_transformed_credentials`, and the WebSocket live-preview handler in `consumers.py`). The `regex` module natively accepts JavaScript/PCRE-style named capture groups (`(?<name>...)`) without any conversion, eliminating the root cause of patterns that matched in the frontend live preview but failed on the backend with a `re.error`. As a further improvement, `regex` also supports variable-length lookbehind assertions (e.g. `(?<=a+)`), which `re` rejects with an error; patterns using these will now work correctly on the backend as well. Replace-pattern JS tokens are still normalised before calling `regex.sub`: `$<name>` → `\g<name>` and `$1`/`$2`/… → `\1`/`\2`/… (Python replacement syntax). Also fixed a bug in the WebSocket preview handler where a pattern error was incorrectly returning the search pattern string as the preview output instead of the original URL. (Fixes #1005)
- Web UI stream preview (`FloatingVideo`) was calling `mpegts.createPlayer()` with all `Config` options (e.g. `enableWorker`, `liveSync`, `headers`) merged into the first `MediaDataSource` argument. mpegts.js only reads `Config` from the optional second argument; unrecognised fields in the first are silently ignored. As a result all player configuration was effectively the library defaults — worker offloading was disabled, latency management had no effect, and the `Authorization: Bearer` header (required for user identification) was never sent. Fixed by splitting into the correct two-argument call. Both `liveBufferLatencyChasing` and `liveSync` have been disabled, eliminating playback-rate fluctuations that caused audible stuttering on live streams. SourceBuffer cleanup thresholds were also relaxed from 10s/5s to 120s/60s to prevent frequent SourceBuffer pauses.
- HTML named entities in XMLTV EPG files are now correctly preserved during lxml parsing. Some EPG providers (particularly French and other European sources) use HTML named entities like `&eacute;`, `&icirc;`, `&uuml;` in channel names, program titles, and metadata. These are not valid XML entities — lxml 6.0.2 with `recover=True` silently drops them, causing characters to go missing (e.g., "Chaîne Télé" becomes "Chane Tl"). This is now fixed by injecting an XML `<!DOCTYPE tv [...]>` internal subset declaring all 252 HTML 4 named entities directly into the byte stream that lxml reads, using a lightweight in-memory wrapper (`_PrependStream`) with zero disk I/O. libxml2 resolves the entities during its normal C-level parse pass — no Python-level preprocessing or temporary files are involved. The DOCTYPE block (~8 KB) is built once at module load from Python's stdlib `html.entities.name2codepoint` and reused for every parse. Files that already declare their own `<!DOCTYPE>` are passed through unchanged. (Closes #1095) — Thanks [@CodeBormen](https://github.com/CodeBormen) for helping with this!
- Duplicate recordings created when EPG sources refresh and re-evaluate series rules (Fixes #940) — Thanks [@CodeBormen](https://github.com/CodeBormen):
- **Program ID instability**: `parse_programs_for_source()` deletes and recreates all `ProgramData` rows with new auto-increment IDs on every EPG refresh. The dedup set used these IDs, so it never matched after a refresh. Deduplication now uses a stable `(tvg_id, start_time, end_time)` composite key sourced from `Recording.custom_properties.program`.
- **Secondary guard using wrong times**: The DB guard compared unadjusted program times against offset-adjusted `Recording.start_time`/`end_time`, so it never matched when any DVR pre/post offset was configured. It now queries `custom_properties__program__start_time/end_time` (the original, unadjusted program times stored at recording creation).
- **No concurrency guard**: Each EPG source refresh fired `evaluate_series_rules.delay()` independently. Concurrent tasks loaded the dedup set before others committed, allowing races. Evaluation is now serialized with `acquire_task_lock` (reusing the existing EPG task pattern). Gracefully degrades if Redis is unavailable — the primary and secondary dedup guards still protect.
- EPG refresh tasks (`refresh_epg_data`) were being killed mid-transaction on large EPG sources. The `soft_time_limit=1700s` introduced in v0.21.0 raised `SoftTimeLimitExceeded`, a subclass of `Exception`, which was swallowed by the existing `except Exception` handler in `parse_programs_for_source`, leaving the database in a partial state with no logged error. `soft_time_limit` has been removed from `refresh_epg_data` and `time_limit` raised to 14400s (4 hours) as a true last-resort ceiling; the existing `TaskLockRenewer` daemon thread continues to renew the Redis lock every 120s for legitimately long-running tasks.
## [0.21.1] - 2026-03-18
### Fixed
- Docker container initialization fixes for PUID/PGID handling — Thanks [@CodeBormen](https://github.com/CodeBormen):
- Backups failing on previous installations where `/data/backups` already existed: `/data/backups` was missing from the `DATA_DIRS` list in the init script, causing the PUID/PGID ownership migration to skip the directory and leave it with incorrect permissions.
- Container startup failure on upgrade when data directories reside on external mounts (NFS, SMB/CIFS, FUSE): `chown` failures under `set -e` were crashing the container, breaking setups that worked fine on the previous image. Failures are now collected per-directory and reported as a consolidated warning; the container continues to start and Django reports at runtime if it cannot write a specific directory.
- Upgrading users running as UID 102 (the internal PostgreSQL system user) instead of the expected UID 1000: the PUID/PGID auto-detect introduced in v0.21.0 read ownership from `/data/db`, which was UID 102 in pre-PUID images, causing Django, file creation, and comskip to all run as the wrong user. PUID/PGID now default to 1000 (matching the original Django UID) rather than auto-detecting from data directory ownership.
## [0.21.0] - 2026-03-17
### Security
- Updated frontend npm dependencies to resolve 1 high-severity vulnerability:
- Updated `flatted` to 3.4.1, resolving **high** unbounded recursion DoS in the `parse()` revive phase ([GHSA-25h7-pfq9-p65f](https://github.com/advisories/GHSA-25h7-pfq9-p65f))
- Updated `Django` to 6.0.3 and `django-celery-beat` to 2.9.0, resolving new security vulnerabilities:
- [CVE-2026-25673](https://www.cve.org/CVERecord?id=CVE-2026-25673): Potential denial-of-service vulnerability in URLField via Unicode normalization on Windows (March 3, 2026)
- [CVE-2026-25674](https://www.cve.org/CVERecord?id=CVE-2026-25674): Potential incorrect permissions on newly created file system objects (March 3, 2026)
### Added
- Configurable sidebar navigation ordering and visibility — Thanks [@jcasimir](https://github.com/jcasimir)
@ -16,13 +184,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Order and visibility are saved per-user with optimistic updates and automatic rollback on failure. Changes appear in the sidebar immediately without a page reload.
- Admin users see a grouped navigation: flat items (`Channels`, `VODs`, `M3U & EPG Manager`, `TV Guide`, `DVR`, `Stats`, `Plugins`) plus collapsible `Integrations` (Connections, Logs) and `System` (Users, Logo Manager, Settings) groups. The `System` group cannot be hidden.
- Non-admin users see `Channels`, `TV Guide`, and `Settings`, with the `Settings` item not hideable.
- Unit tests for `NotificationCenter`, `NotificationCenterUtils`, and `M3URefreshNotification` components. — Thanks [@nick4810](https://github.com/nick4810)
- Unit tests for `NotificationCenter`, `NotificationCenterUtils`, and `M3URefreshNotification` components, and for settings form components `DvrSettingsForm`, `NetworkAccessForm`, `ProxySettingsForm`, `StreamSettingsForm`, `SystemSettingsForm`, `UiSettingsForm`. — Thanks [@nick4810](https://github.com/nick4810)
- Unit tests for DVR port resolution (`build_dvr_candidates`) and selective Redis flush behavior in modular mode. — Thanks [@CodeBormen](https://github.com/CodeBormen)
- Floating video player improvements
- **Title display**: The channel, stream, or VOD title is now shown in the player header bar. Title is passed through from all preview entry points: channel table, stream table, stream connection card, guide, DVR, recording cards, recording details modal, VOD modal, and series modal.
- **Persistent state**: Size, position, volume level, and mute state are now saved across sessions using a single `dispatcharr-player-prefs` localStorage key. Size and position are restored on next open (clamped to the current viewport); volume and mute are restored when the player initialises.
- New Client Buffer proxy setting: new clients joining an active channel are now positioned a configurable number of seconds behind live rather than a fixed chunk count. The start position is determined by wall-clock chunk receive time (stored as a Redis sorted set alongside the buffer), so the buffer depth is consistent in seconds regardless of stream bitrate. Setting the value to `0` starts clients at live with no buffer. Defaults to 5 seconds. Existing chunk-count gating for the first client connecting to a channel is unchanged. The setting is exposed in Settings → Proxy as "New Client Buffer (seconds)".
- Channel table filter for channels that have stale streams: A new "Has Stale Streams" filter option in the channel table header menu highlights and filters channels containing at least one stale stream. Channels with stale streams are visually distinguished with an orange tint. The filter is mutually exclusive with "Only Empty Channels". - Thanks [@JCBird1012](https://github.com/JCBird1012)
- "Next Highest Channel" numbering mode when creating channels from streams: A new `Next Highest` option is available alongside `Provider`, `Auto`, and `Custom` when creating channels from the Streams table. Selecting it assigns channel numbers starting one above the current highest channel number; the next available number is fetched from the backend at selection time. (Closes #1000) — Thanks [@JCBird1012](https://github.com/JCBird1012)
- TV Guide program cards now display richer metadata — Thanks [@CodeBormen](https://github.com/CodeBormen)
- **Season/episode badges** (e.g. `S12E06`) extracted from EPG `<episode-num>` elements, `onscreen` episode strings (e.g. `S12 E6`, `S3E21`, `S8 E8 P2/2`), and as a last-resort fallback from description text patterns at parse time (3-tier pipeline). (Closes #1065)
- **Episode subtitle** shown below the program title on guide cards; falls back to the short description when no subtitle is available.
@ -41,6 +210,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Search and filter controls**: Added search and filter controls to the recordings list.
- **Stream generator throttling**: Cached the `ProxyServer` singleton reference per client and throttled Redis resource checks (1 s) and non-owner health checks (2 s), eliminating 3+ Redis round-trips per stream loop iteration.
- **Automatic crash recovery on worker restart**: A `worker_ready` Celery signal now fires `recover_recordings_on_startup` automatically when the worker starts, so recordings stuck in "recording" status are recovered without manual intervention.
- Account expiration tracking and notifications for M3U profiles
- A new `exp_date` field on `M3UAccountProfile` stores the account expiration date as a proper `DateTimeField`. For Xtream Codes accounts the field is auto-synced from `custom_properties.user_info.exp_date` on every save (supports both Unix timestamps and ISO date strings). For non-XC M3U accounts the date can be entered manually via the account or profile form.
- The M3U accounts table now shows an **Expiration** column displaying the earliest expiration date across all profiles for that account (color-coded: red = expired, orange = expiring soon, green = OK). Hovering the cell shows a tooltip with per-profile expiration details including inactive-profile labels.
- A daily Celery Beat task (`check_xc_account_expirations`) checks all active profiles with an expiration date and manages system notifications: a normal-priority warning is raised for profiles expiring within 7 days; a high-priority alert is raised once the profile has already expired. Warning and expired notifications use separate keys so dismissing the 7-day warning does not suppress the expiration alert.
- Notifications are also updated immediately when a profile is saved: if the expiration date is cleared or pushed beyond the 7-day window, any existing warning/expired notifications are deleted; if the date falls within the window or is already past, the matching notification is updated in place.
- Non-XC accounts expose a `DateTimePicker` on both the M3U account form and the profile form.
### Changed
@ -59,16 +234,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Removed a dead `onLogout` handler in `Sidebar` that called `logout()` and `window.location.reload()` but was never wired to any UI element.
- EPG output when no `days` parameter is specified now excludes already-ended programs instead of returning all historical data.
### Security
- Updated frontend npm dependencies to resolve 1 high-severity vulnerability:
- Updated `flatted` to 3.4.1, resolving **high** unbounded recursion DoS in the `parse()` revive phase ([GHSA-25h7-pfq9-p65f](https://github.com/advisories/GHSA-25h7-pfq9-p65f))
- Updated `Django` to 6.0.3 and `django-celery-beat` to 2.9.0, resolving new security vulnerabilities:
- [CVE-2026-25673](https://www.cve.org/CVERecord?id=CVE-2026-25673): Potential denial-of-service vulnerability in URLField via Unicode normalization on Windows (March 3, 2026)
- [CVE-2026-25674](https://www.cve.org/CVERecord?id=CVE-2026-25674): Potential incorrect permissions on newly created file system objects (March 3, 2026)
### Fixed
- Single-stream channel creation modal not opening correctly when clicking the channel-creation button on an individual stream row in the Streams table. — Thanks [@JCBird1012](https://github.com/JCBird1012)
- DVR series rule creation failing with a 500 error when the stored `series_rules` data contained corrupted (non-dict) entries. Added type guards on the getter, setter, and generic settings serializer to filter invalid entries on read and write. Hardened the EPG ignore list getters (`prefixes`, `suffixes`, `custom`) with the same pattern. Frontend settings parse and save now validate `series_rules` with `Array.isArray()`, matching the existing EPG field pattern, preventing corrupted data from being round-tripped back to the database. (Fixes #1059) — Thanks [@CodeBormen](https://github.com/CodeBormen)
- TS proxy clients stuck indefinitely in keepalive mode when a stream fails and never recovers (Fixes #1102, #1103) — Thanks [@cmcpherson274](https://github.com/cmcpherson274) & [@CodeBormen](https://github.com/CodeBormen)
- **Keepalive duration cap**: Non-owner worker clients sending keepalive packets to hold a connection open during failover can now be held at most `MAX_KEEPALIVE_DURATION` seconds (default 300 s). If no real stream data has been received within that window, the client is disconnected with a warning log. The timer resets each time real data resumes, so independent stalls do not accumulate.
- **`last_active` tracking**: `last_active` is now updated on every keepalive packet and on every real data chunk, so clients actively waiting during a failover are not incorrectly evicted as ghost clients by the heartbeat thread. The heartbeat thread now only refreshes the Redis TTL rather than updating `last_active`, ensuring the ghost-detection check reflects true client activity rather than heartbeat activity.
- **Buffer reset on stream transition**: A new `reset_buffer_position()` method on `StreamBuffer` clears the in-memory write buffer and partial-packet accumulator when switching between FFmpeg processes. Without this, a partial 188-byte TS packet from the dying FFmpeg process was being prepended to the first bytes from the new FFmpeg process, producing a corrupted TS packet boundary that broke audio decoder sync on the client side. Redis-stored chunks already consumed by clients are unaffected.
- **`add_chunk()` locking hardened**: The lock scope in `add_chunk()` was expanded to cover the entire partial-packet merge and write-buffer accumulation phase, preventing a race condition between `add_chunk()` and the new `reset_buffer_position()` call.
- uWSGI segfaults caused by mixing threading and gevent concurrency models. The dev and debug uWSGI configs had `threads` and `enable-threads = true` set alongside gevent, which triggers segmentation faults particularly on ARM64/Python 3.13. Removed those options to match the already-correct production config. — Thanks [@jcasimir](https://github.com/jcasimir)
- `Stream.last_seen` and `ChannelGroupM3UAccount.last_seen` model defaults now use `django.utils.timezone.now` instead of `datetime.datetime.now`, eliminating spurious `RuntimeWarning: DateTimeField received a naive datetime` warnings emitted during test database creation and on new record creation when `USE_TZ=True`.
- EPG programme parsing crash when an XMLTV source contains programme titles exceeding 255 characters. Previously, a single oversized title would cause the entire `bulk_create` batch to fail with a database truncation error, silently dropping all programmes in that batch. Titles are now truncated to 255 characters before being saved. (Fixes #1039)
- Container startup failure when `PUID`/`PGID` is set, caused by `/data/db` ownership conflicts between the `postgres` system user (UID 102) and the configured PUID/PGID. PostgreSQL now runs as the PUID/PGID user in AIO mode, eliminating all `chown`-to-UID-102 operations and unifying `/data` ownership. (Fixes #1078) — Thanks [@CodeBormen](https://github.com/CodeBormen)
- Existing installations where PUID/PGID differs from the current `/data/db` owner are migrated automatically on first startup; a sentinel file prevents redundant recursive `chown` on subsequent boots.
@ -80,7 +256,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Resize stuck after releasing mouse outside window**: The `mouseup` event is not delivered when the pointer leaves the viewport, leaving the `mousemove` listener active indefinitely. Fixed by checking `event.buttons === 0` at the top of `handleResizeMove`; when no button is held the resize session is torn down immediately.
- **Drag stuck after releasing mouse outside window**: Same root cause as the resize bug. Fixed by detecting `event.buttons === 0` in the `onDrag` handler and dispatching a synthetic `mouseup` event so react-draggable cleanly ends the drag session.
- **Player draggable off screen**: The player could be dragged off any edge, making the header (and drag handle) unreachable. The player is now fully bounded: left and top edges are clamped to `x ≥ 0` / `y ≥ 0` so the header is always reachable, and right/bottom edges are clamped to the viewport. Size and position are also re-clamped automatically when the browser window is resized, with proportional scale-down if the saved size exceeds the new viewport.
- **Security**: Any authenticated user could self-escalate privileges by sending `user_level`, `is_staff`, or `is_superuser` in a `PATCH /api/accounts/users/me/` request. The `/me/` endpoint now enforces an allowlist (`custom_properties`, `first_name`, `last_name`, `email`, `password`); any other fields return HTTP 400. Privilege-sensitive fields can only be changed via the admin-only `PATCH /api/accounts/users/{id}/` endpoint.
- Double error notification when saving user preferences: `API.updateMe` was catching errors internally and displaying a notification before re-throwing, causing callers to display a second notification for the same failure.
- Navigation preference saves from concurrent sessions could overwrite each other due to a double-merge race: the frontend was pre-merging `custom_properties` before sending, then the backend merged again against the DB value, causing the second session's write to silently drop keys set by the first. The frontend now sends only the delta; the backend merges authoritatively against the stored value.
- Stale nav item IDs (e.g. from a previous nav structure) are now scrubbed from `navOrder` and `hiddenNav` on the next preference save, preventing unbounded growth of the `custom_properties` JSON field.
@ -110,6 +285,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **`update_stream_profile()` uses a Redis pipeline** for the old-profile DECR + key update + new-profile INCR sequence, preventing counter drift if the process crashes between operations.
- **`stream_generator._cleanup()`** now falls back to `Stream.objects.get()` when the channel UUID resolves to a preview flow rather than a normal channel, rather than silently skipping the slot release.
- **VOD `cleanup_persistent_connection()`** fallback DECR is now conditional: it only decrements the profile counter when the connection tracking key had already expired by TTL (i.e., `remove_connection()` would have skipped the DECR), preventing double-decrements when the key is still present.
- Ghost clients and channels stuck in `INITIALIZING` state in the TS proxy (Fixes #695, #669) — Thanks [@CodeBormen](https://github.com/CodeBormen)
- **`INITIALIZING` added to cleanup grace period monitoring**: channels stuck in `INITIALIZING` are now surfaced by the cleanup task and torn down, preventing indefinite hangs when stream startup fails.
- **Orphaned channel cleanup validates client SET entries**: the cleanup task now cross-checks client SET members against actual metadata hashes; ghost SET entries are removed and the channel is torn down cleanly when no real clients remain.
- **Stats page self-heals**: ghost client SET entries are detected and removed when reading channel stats, preventing stale entries from inflating the active-client count.
- **`remove_ghost_clients()` extracted to `ClientManager`**: ghost-detection logic is now a single authoritative helper, callable with an optional pre-fetched `client_ids` set to eliminate a redundant Redis `SMEMBERS` round-trip when the caller already holds the set.
- **Ownership TTL fallback**: the error-state writer now triggers via ownership check _or_ state guard fallback, so a channel stuck in a pre-active state is correctly marked `ERROR` even when the ownership TTL expired during retries.
- **Missing `SOURCE_BITRATE` / `FFMPEG_BITRATE` metadata constants** added to `ChannelMetadataField`, preventing `AttributeError` on detailed channel stats reads.
- TS proxy client stream lag recovery now only bumps clients forward when their next required chunk has genuinely expired from Redis (TTL), rather than unconditionally jumping if they fell more than 50 chunks behind. Clients are repositioned to the oldest available chunk (minimum data loss) using an atomic server-side Lua binary search, falling back to near the buffer head if nothing is available.
- TS proxy streams dying after 30–200 seconds in multi-worker uWSGI/Celery deployments, caused by three interrelated bugs. (Fixes #992, #980) - Thanks [@PFalko](https://github.com/PFalko)
- **Double ProxyServer instantiation**: `ProxyConfig.ready()` called `TSProxyServer()` directly while `TSProxyConfig.ready()` also called `TSProxyServer.get_instance()`, creating two instances per worker — each with its own cleanup thread. The orphaned thread could not extend ownership because it had no entries in `stream_managers`. Fixed by using `TSProxyServer.get_instance()` in `ProxyConfig.ready()`.

586
Plugin_repo.md Normal file
View file

@ -0,0 +1,586 @@
# Dispatcharr Plugin Repository Specification
How to create and host a plugin repository that Dispatcharr can consume.
For writing plugins themselves, see [Plugins.md](Plugins.md).
---
## Overview
Dispatcharr discovers plugins from remote repositories using a two-level manifest system:
1. **Repo manifest** - a JSON file listing all plugins in the repo with basic metadata.
2. **Per-plugin manifest** (optional) - a JSON file per plugin with full version history, checksums, and compatibility info.
Users add a repo by its manifest URL. Dispatcharr fetches and caches the repo manifest periodically (default: every 6 hours, configurable). The UI displays all plugins from enabled repos in a browsable store.
---
## Repo Manifest
The repo manifest is the entry point. Dispatcharr fetches this URL and caches the response.
### Minimal Example (no signing)
```json
{
"registry_name": "My Plugin Repo",
"plugins": [
{
"slug": "my_plugin",
"name": "My Plugin",
"description": "Does something useful",
"author": "Your Name",
"latest_version": "1.0.0",
"latest_url": "https://example.com/releases/my_plugin-1.0.0.zip"
}
]
}
```
This is the simplest valid repo manifest - one plugin with enough info to show in the store and install.
### Full Example (with signing)
```json
{
"manifest": {
"registry_name": "My Plugin Repo",
"registry_url": "https://github.com/myorg/my-plugins",
"root_url": "https://raw.githubusercontent.com/myorg/my-plugins/releases",
"plugins": [
{
"slug": "weather_display",
"name": "Weather Display",
"description": "Shows weather info on the dashboard",
"author": "Acme Labs",
"license": "MIT",
"latest_version": "1.2.5",
"last_updated": "2025-01-20T15:30:00Z",
"manifest_url": "plugins/weather_display/manifest.json",
"latest_url": "plugins/weather_display/releases/weather_display-1.2.5.zip",
"latest_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"icon_url": "plugins/weather_display/logo.png",
"min_dispatcharr_version": "2.5.0",
"max_dispatcharr_version": null
}
]
},
"signature": "-----BEGIN PGP SIGNATURE-----\n..."
}
```
### Accepted Formats
Dispatcharr accepts two top-level shapes:
**Wrapped (supports signing):**
```json
{
"manifest": { "plugins": [...], ... },
"signature": "..."
}
```
**Flat (no signing):**
```json
{
"plugins": [...],
"registry_name": "...",
"root_url": "..."
}
```
The wrapped format is required for signing. If you don't need signing, the flat format works and is simpler.
### Name Restrictions
`registry_name` is required. Dispatcharr rejects repos that are missing it.
Third-party repos must not use names that could be confused with an official Dispatcharr repo. The following words are blocked in `registry_name` (case-insensitive):
- "official"
- "dispatcharr plugins"
- "dispatcharr repo"
- "dispatcharr official"
If the name contains any of these, the repo will be rejected on add and skipped during refresh.
---
## Repo Manifest Fields
### Top-Level Metadata
| Field | Required | Description |
|-------|----------|-------------|
| `registry_name` | **Yes** | Display name for the repo. Must not contain words like "official" or "dispatcharr" that could be mistaken for an official repo (see [Name Restrictions](#name-restrictions)). |
| `registry_url` | No | URL to the repo's home page (e.g. GitHub). Used as a fallback for generating icon URLs. |
| `root_url` | No | Base URL for resolving relative URLs in plugin entries. Trailing slashes are stripped. |
| `plugins` | **Yes** | Array of plugin entry objects. |
### Plugin Entry Fields
| Field | Required | Description |
|-------|----------|-------------|
| `slug` | **Yes** | Unique identifier. Alphanumeric, dashes, and underscores. Used as the install directory name (lowercased, dashes converted to underscores). |
| `name` | **Yes** | Human-readable display name. |
| `description` | No | Short description shown on the plugin card. |
| `author` | No | Author or organization name. |
| `license` | No | SPDX license identifier (e.g. `MIT`, `GPL-3.0`). Displayed as a link to the SPDX license page. |
| `latest_version` | No | Current latest version string (semver: `1.2.3` or `v1.2.3`). Drives update detection. |
| `last_updated` | No | ISO 8601 timestamp of the latest release. Shown as "Built" date in the detail view. |
| `manifest_url` | No | URL (or relative path) to the per-plugin manifest with full version history. See [Per-Plugin Manifest](#per-plugin-manifest). |
| `latest_url` | No | Direct download URL (or relative path) to the latest release zip. |
| `latest_sha256` | No | SHA256 checksum of the latest release zip (lowercase hex, 64 chars). |
| `latest_md5` | No | MD5 checksum of the latest release zip. Informational only - not validated by Dispatcharr. |
| `icon_url` | No | URL (or relative path) to a logo image (PNG recommended). |
| `min_dispatcharr_version` | No | Minimum Dispatcharr version required. Install is blocked if the running version is older. |
| `max_dispatcharr_version` | No | Maximum Dispatcharr version supported. Install is blocked if the running version is newer. |
Extra fields in a plugin entry are passed through to the frontend as-is, so you can include custom metadata (e.g. `homepage`, `tags`) without breaking anything.
### URL Resolution
If `root_url` is set and a URL field (`manifest_url`, `latest_url`, `icon_url`) does not start with `http://` or `https://`, it is treated as relative and resolved as:
```
{root_url}/{field_value}
```
This lets you keep plugin entries compact:
```json
{
"root_url": "https://raw.githubusercontent.com/myorg/my-plugins/releases",
"plugins": [
{
"slug": "my_plugin",
"latest_url": "plugins/my_plugin/my_plugin-1.0.0.zip",
"icon_url": "plugins/my_plugin/logo.png",
"manifest_url": "plugins/my_plugin/manifest.json"
}
]
}
```
**Icon fallback:** If `icon_url` is missing and `registry_url` is set, Dispatcharr generates a fallback URL by converting the GitHub URL to a raw content URL:
```
{registry_url => raw.githubusercontent.com}/refs/heads/main/plugins/{slug}/logo.png
```
---
## Per-Plugin Manifest (Optional)
The per-plugin manifest provides full version history. It is fetched on-demand when a user clicks "More Info" on a plugin card. It is **not required** - if `manifest_url` is absent, the UI builds a detail view from the repo-level fields instead.
Include a per-plugin manifest if you want to:
- Offer multiple downloadable versions
- Show per-version compatibility ranges
- Display build timestamps and commit links for each version
- Provide detailed author/license info beyond what's in the repo manifest
### Accepted Formats
Same as the root manifest - both flat and wrapped formats are accepted:
**Flat (no signing):**
```json
{
"slug": "...",
"versions": [...]
}
```
**Wrapped (supports signing):**
```json
{
"manifest": {
"slug": "...",
"versions": [...]
},
"signature": "-----BEGIN PGP SIGNATURE-----\n..."
}
```
Use the wrapped format if you want to GPG-sign the per-plugin manifest.
### Example
```json
{
"slug": "weather_display",
"name": "Weather Display",
"description": "Shows weather information on the Dispatcharr dashboard",
"author": "Acme Labs",
"license": "MIT",
"latest_version": "1.2.5",
"versions": [
{
"version": "1.2.5",
"url": "releases/weather_display-1.2.5.zip",
"checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"build_timestamp": "2025-01-20T15:30:00Z",
"commit_sha": "4e8f1b108c1e84f60520710d13e54eb2fb519648",
"commit_sha_short": "4e8f1b1",
"min_dispatcharr_version": "2.5.0",
"max_dispatcharr_version": null
},
{
"version": "1.2.5-rc.1",
"url": "releases/weather_display-1.2.5-rc.1.zip",
"checksum_sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
"prerelease": true,
"build_timestamp": "2025-01-18T09:00:00Z",
"min_dispatcharr_version": "2.5.0"
},
{
"version": "1.2.4",
"url": "releases/weather_display-1.2.4.zip",
"checksum_sha256": "d4d967a67a4947e55183308cece206b30dda3e1b4fe00aae60f45a49c83b7ed6",
"build_timestamp": "2025-01-15T10:00:00Z",
"min_dispatcharr_version": "2.4.0"
}
],
"latest": {
"version": "1.2.5",
"url": "releases/weather_display-1.2.5.zip",
"checksum_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"build_timestamp": "2025-01-20T15:30:00Z",
"min_dispatcharr_version": "2.5.0"
}
}
```
### Per-Plugin Manifest Fields
| Field | Required | Description |
|-------|----------|-------------|
| `slug` | No | Plugin identifier (should match the repo entry). |
| `name` | No | Display name. |
| `description` | No | Full description shown in the detail modal. |
| `author` | No | Author/org name shown in the detail modal. |
| `license` | No | SPDX license identifier. |
| `latest_version` | No | Latest version string. |
| `versions` | No | Array of version objects (newest first recommended). |
| `latest` | No | Object mirroring the latest version entry for quick access. |
### Version Object Fields
| Field | Required | Description |
|-------|----------|-------------|
| `version` | **Yes** | Version string (`1.2.3` or `v1.2.3`). |
| `url` | **Yes** | Download URL for the zip. Relative URLs are resolved against the repo's `root_url`. |
| `checksum_sha256` | No | SHA256 hex checksum. **Strongly recommended.** Validated on install - mismatch blocks the install. |
| `prerelease` | No | Boolean. When `true`, marks this version as a pre-release (alpha, beta, RC, etc.). If the installed version is a prerelease, Dispatcharr will not suggest updating to the latest stable version - the user must install a new version manually. The latest version in the root manifest is always assumed to be stable, so this field only needs to appear in the per-plugin manifest. Omit or set to `false` for stable releases. |
| `build_timestamp` | No | ISO 8601 build timestamp. Shown as "Built" in the version detail. |
| `commit_sha` | No | Full Git commit SHA. Used to build a commit link if `registry_url` is set. |
| `commit_sha_short` | No | Abbreviated commit SHA. Displayed in the version detail table as a clickable link. |
| `min_dispatcharr_version` | No | Minimum compatible Dispatcharr version. |
| `max_dispatcharr_version` | No | Maximum compatible Dispatcharr version. |
Relative `url` values in versions are resolved the same way as repo-level URLs: `{root_url}/{url}`.
---
## Without a Per-Plugin Manifest
If you omit `manifest_url` from a plugin entry, the store still works. When a user clicks "More Info", the UI builds a detail view from the repo-level fields:
- `description`, `author`, `license` from the plugin entry
- A single version entry built from `latest_version`, `latest_url`, `latest_sha256`, `min_dispatcharr_version`, `max_dispatcharr_version`, and `last_updated`
This is the simplest path for third-party repos that only publish one version at a time. You lose version history and per-version release dates, but install, update detection, and everything else works the same.
---
## Signing
Signing your repo manifest lets Dispatcharr verify it hasn't been tampered with. Signing is **optional** - unsigned repos work fine but show an "unverified" badge in the UI.
### How It Works
1. You generate a GPG keypair.
2. You sign the manifest JSON and include the detached signature in the response.
3. When adding the repo in Dispatcharr, the user pastes your public key.
4. Dispatcharr verifies the signature on every manifest fetch.
### Key Format
Standard PGP/GPG armored keys:
```
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBG...
...
-----END PGP PUBLIC KEY BLOCK-----
```
### Signing Convention
The signature is computed over the **canonical JSON** representation of the `manifest` object (not the entire response), plus a trailing newline:
```bash
# Canonical format: compact JSON (no spaces) + trailing newline
jq -c '.manifest' manifest.json | gpg --armor --detach-sign
```
In code terms:
```python
import json
canonical = json.dumps(manifest_obj, separators=(",", ":")) + "\n"
```
> **Important:** The signing input must be `json.dumps(obj, separators=(",", ":")) + "\n"` - compact JSON with no whitespace, followed by exactly one newline. Any difference (pretty-printing, trailing spaces, key ordering changes) will cause verification to fail.
### Manifest Structure for Signing
Use the wrapped format so the signature sits alongside the manifest:
```json
{
"manifest": {
"registry_name": "...",
"plugins": [...]
},
"signature": "-----BEGIN PGP SIGNATURE-----\n...\n-----END PGP SIGNATURE-----"
}
```
### Verification Results
| Result | Meaning | UI Badge |
|--------|---------|----------|
| `true` | Valid signature | Green checkmark |
| `false` | Invalid signature or verification error | Red X |
| `null` | Not attempted (no signature, no key, or `python-gnupg` not installed) | Gray/neutral |
### Signing Workflow Example
```bash
# Generate a keypair (one-time)
gpg --gen-key
# Export your public key (give this to repo users)
gpg --armor --export "your@email.com" > my-repo.pub
# Build your manifest
cat > manifest.json << 'EOF'
{
"manifest": {
"registry_name": "My Repo",
"root_url": "https://example.com/releases",
"plugins": [
{
"slug": "my_plugin",
"name": "My Plugin",
"latest_version": "1.0.0",
"latest_url": "plugins/my_plugin/my_plugin-1.0.0.zip"
}
]
}
}
EOF
# Sign the manifest object (canonical JSON + newline)
jq -c '.manifest' manifest.json | gpg --armor --detach-sign > manifest.sig
# Combine into final output
jq --arg sig "$(cat manifest.sig)" '.signature = $sig' manifest.json > signed_manifest.json
```
### Third-Party Key Management
When a user adds your repo URL, they can paste your public key. Dispatcharr stores the key per-repo and uses it for verification. Users can update the key at any time from the repo management UI.
If you don't provide a key and the repo is not the official Dispatcharr repo, signature verification is skipped (result: `null`).
---
## Release Zip Format
Each plugin release is a `.zip` archive.
### Requirements
- Must contain a `plugin.py` with a `Plugin` class, **or** a Python package with `__init__.py` exporting a `Plugin` class.
- Files can be at the top level of the zip or inside a single subdirectory.
- Optionally include `plugin.json` for metadata discovery without code execution.
- Optionally include `logo.png` for the plugin icon.
### Size Limits
- Maximum 2000 files per archive.
- Maximum total size: 200 MB (configurable via `MAX_PLUGIN_IMPORT_BYTES` setting).
### Recommended Structure
```
my_plugin-1.0.0.zip
plugin.py
plugin.json
logo.png
(any other files your plugin needs)
```
Or with a subdirectory:
```
my_plugin-1.0.0.zip
my_plugin/
plugin.py
plugin.json
logo.png
utils.py
```
---
## Install Flow
When a user installs a plugin from the store:
1. **Version compatibility check** - if `min_dispatcharr_version` or `max_dispatcharr_version` is set, the running Dispatcharr version is compared. Install is blocked if out of range.
2. **Download** - the zip is streamed from `download_url` (max 200 MB).
3. **SHA256 integrity check** - if `sha256` was provided, the download is hashed and compared. Mismatch blocks the install.
4. **Extraction** - the zip is extracted to a temp directory, validated, then moved to `/data/plugins/{plugin_key}/`. If the plugin already exists, the old version is backed up and restored on failure (atomic rollback).
5. **Registration** - a `PluginConfig` record is created or updated, linking the plugin to its source repo and slug.
6. **Discovery reload** - the plugin loader re-scans all plugin directories.
The plugin is installed **disabled** by default. The user can enable it from the post-install dialog or the My Plugins page.
---
## Update Detection
Dispatcharr detects updates by comparing `installed_version` (stored in the database) against `latest_version` from the repo manifest. This uses repo-level fields only - per-plugin manifests are not needed for update detection.
A plugin shows "Update Available" when:
- It is managed (installed from a repo)
- Its `installed_version` differs from `latest_version`
- It was installed from the same repo
---
## Hosting Options
A plugin repo manifest is just a JSON file served over HTTPS. Some options:
### GitHub Pages / Raw Content
Host your manifest and release zips in a GitHub repo. Use raw.githubusercontent.com URLs:
```
https://raw.githubusercontent.com/myorg/my-plugins/main/manifest.json
```
Use `root_url` pointing to your releases branch/path so version URLs stay relative.
### Static File Server
Any web server that serves JSON works. Dispatcharr fetches manifests server-side, so CORS is not needed.
### GitHub Releases
You can host release zips as GitHub Release assets and reference them with absolute URLs in your manifest. The manifest itself can live in the repo's default branch.
---
## Refresh Behavior
- Manifests are refreshed automatically at a configurable interval (default: 6 hours, setting: `refresh_interval_hours`, 0 = disabled).
- Users can force a refresh from the repo management UI.
- A new repo is refreshed immediately when added.
- On refresh, if a plugin's `slug` disappears from the manifest, its `PluginConfig` is unlinked from the repo (becomes "unmanaged") but the installed files are not deleted.
---
## Checklist: Publishing a Plugin Repo
### Minimum Viable Repo
- [ ] Host a JSON file at a stable, public URL
- [ ] Set `registry_name` (required, must not sound official)
- [ ] Include at least one plugin entry with `slug`, `name`, and `latest_version`
- [ ] Host a downloadable `.zip` for each plugin and set `latest_url`
- [ ] Share the manifest URL with users
### Recommended
- [ ] Set `root_url` so plugin URLs can be relative
- [ ] Include `description`, `author`, and `icon_url` per plugin
- [ ] Include `latest_sha256` for integrity verification
- [ ] Include `license` (SPDX identifier)
- [ ] Include `last_updated` timestamps
- [ ] Add a per-plugin `manifest_url` with version history
- [ ] Include `sha256` in every version object
- [ ] Include `min_dispatcharr_version` where applicable
- [ ] Include `plugin.json` in each release zip
### Optional
- [ ] Sign your manifest with GPG and publish your public key
- [ ] Set `registry_url` to enable automatic icon fallback
- [ ] Set `max_dispatcharr_version` if a plugin is incompatible with newer releases
---
## Quick Reference: Repo Manifest Schema
```json
{
"manifest": {
"registry_name": "string (required)",
"registry_url": "string (optional)",
"root_url": "string (optional)",
"plugins": [
{
"slug": "string (required)",
"name": "string (required)",
"description": "string",
"author": "string",
"license": "string (SPDX)",
"latest_version": "string (semver)",
"last_updated": "string (ISO 8601)",
"manifest_url": "string (URL or relative path)",
"latest_url": "string (URL or relative path)",
"latest_sha256": "string (64-char hex)",
"icon_url": "string (URL or relative path)",
"min_dispatcharr_version": "string (semver)",
"max_dispatcharr_version": "string (semver) or null"
}
]
},
"signature": "string (armored PGP signature, optional)"
}
```
## Quick Reference: Per-Plugin Manifest Schema
```json
{
"slug": "string",
"name": "string",
"description": "string",
"author": "string",
"license": "string (SPDX)",
"latest_version": "string (semver)",
"versions": [
{
"version": "string (required)",
"url": "string (required, URL or relative path)",
"checksum_sha256": "string (64-char hex)",
"build_timestamp": "string (ISO 8601)",
"commit_sha": "string",
"commit_sha_short": "string",
"min_dispatcharr_version": "string (semver)",
"max_dispatcharr_version": "string (semver) or null"
}
],
"latest": {
"version": "string",
"url": "string",
"checksum_sha256": "string",
"build_timestamp": "string",
"min_dispatcharr_version": "string",
"max_dispatcharr_version": "string or null"
}
}
```

45
README.md
View file

@ -8,7 +8,7 @@
## 📖 What is Dispatcharr?
Dispatcharr is an **open-source powerhouse** for managing IPTV streams, EPG data, and VOD content with elegance and control.\
Dispatcharr (pronounced like "dispatcher") is an **open-source powerhouse** for managing IPTV streams, EPG data, and VOD content with elegance and control.\
Born from necessity and built with passion, it started as a personal project by **[OkinawaBoss](https://github.com/OkinawaBoss)** and evolved with contributions from legends like **[dekzter](https://github.com/dekzter)**, **[SergeantPanda](https://github.com/SergeantPanda)** and **Bucatini**.
> Think of Dispatcharr as the \*arr family's IPTV cousin — simple, smart, and designed for streamers who want reliability and flexibility.
@ -118,12 +118,9 @@ If you are running a Debian-based OS, use the `debian_install.sh` script. For ot
## 🤝 Want to Contribute?
We welcome **PRs, issues, ideas, and suggestions**!\
Here's how you can join the party:
We welcome **PRs, issues, ideas, and suggestions**!
- Follow our coding style and best practices.
- Be respectful, helpful, and open-minded.
- Respect the **GNU AGPL v3.0 license**.
- Prior to contributing, please read the [CONTRIBUTING.md](https://github.com/Dispatcharr/Dispatcharr/blob/main/CONTRIBUTING.md)
> Whether it's writing docs, squashing bugs, or building new features, your contribution matters! 🙋
@ -140,7 +137,6 @@ Here's how you can join the party:
- 👥 **Enhanced User Management** — Customizable XC API output per user account
- 🔄 **Output Stream Profiles** — Different clients with different stream profiles (bandwidth control, quality tiers)
- 🔌 **Fallback Videos** — Automatic fallback content when channels are unavailable
- 📡 **Webhooks** — Event-driven integrations and automations
---
@ -150,14 +146,6 @@ A huge thank you to all the incredible open-source projects and libraries that p
---
## ⚖️ License
> Dispatcharr is licensed under **GNU AGPL v3.0**:
For full license details, see [LICENSE](https://www.gnu.org/licenses/agpl-3.0.html).
---
## ✉️ Connect With Us
Have a question? Want to suggest a feature? Just want to say hi?\
@ -165,8 +153,27 @@ Have a question? Want to suggest a feature? Just want to say hi?\
---
## 💖 Support Dispatcharr
[![Dispatcharr Open Collective](https://opencollective.com/dispatcharr/tiers/badge.svg)](https://opencollective.com/dispatcharr/contribute)
Open Collective provides a transparent way for anyone who finds value in Dispatcharr to support things like:
• Infrastructure costs (Domains, Servers, etc.)
• Apple Developer Program and Google Play Developer accounts
• Helping contributors dedicate more time to improving the project
Support is completely optional, and Dispatcharr will always remain free and open-source.
[Contribute here](https://opencollective.com/dispatcharr/contribute)
---
## ⚖️ License & Legal
Dispatcharr is licensed under **GNU AGPL v3.0**: For full license details, see [LICENSE](https://www.gnu.org/licenses/agpl-3.0.html).
Dispatcharr is a trademark of the Dispatcharr project. Use of the Dispatcharr name or logo requires permission from the maintainers.
---
### 🚀 _Happy Streaming! The Dispatcharr Team_
Dispatcharr is a trademark of the Dispatcharr project.
Use of the Dispatcharr name or logo requires permission from the maintainers.

83
apps/accounts/api_views.py
View file

@ -6,6 +6,7 @@ from django.views.decorators.csrf import csrf_exempt
from rest_framework.decorators import api_view, permission_classes, action
from rest_framework.response import Response
from rest_framework import viewsets, status, serializers
from rest_framework.throttling import AnonRateThrottle
from drf_spectacular.utils import extend_schema, OpenApiParameter, inline_serializer
from drf_spectacular.types import OpenApiTypes
import json
@ -20,9 +21,14 @@ from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView
logger = logging.getLogger(__name__)
class LoginRateThrottle(AnonRateThrottle):
scope = "login"
class TokenObtainPairView(TokenObtainPairView):
throttle_classes = [LoginRateThrottle]
def post(self, request, *args, **kwargs):
# Custom logic here
if not network_access_allowed(request, "UI"):
# Log blocked login attempt due to network restrictions
from core.utils import log_system_event
@ -153,12 +159,11 @@ class AuthViewSet(viewsets.ViewSet):
Login doesn't require auth, but logout does
"""
if self.action == 'logout':
from rest_framework.permissions import IsAuthenticated
return [IsAuthenticated()]
return [Authenticated()]
return []
@extend_schema(
description="Authenticate and log in a user",
description="Alias for POST /api/accounts/token/ — returns JWT access and refresh tokens.",
request=inline_serializer(
name="LoginRequest",
fields={
@ -168,55 +173,10 @@ class AuthViewSet(viewsets.ViewSet):
),
)
def login(self, request):
"""Logs in a user and returns user details"""
username = request.data.get("username")
password = request.data.get("password")
user = authenticate(request, username=username, password=password)
# Get client info for logging
from core.utils import log_system_event
client_ip = request.META.get('REMOTE_ADDR', 'unknown')
user_agent = request.META.get('HTTP_USER_AGENT', 'unknown')
logger.debug(f"Login attempt via session: user={username} ip={client_ip}")
if user:
login(request, user)
# Update last_login timestamp
from django.utils import timezone
user.last_login = timezone.now()
user.save(update_fields=['last_login'])
# Log successful login
log_system_event(
event_type='login_success',
user=username,
client_ip=client_ip,
user_agent=user_agent,
)
logger.info(f"Login success via session: user={username} ip={client_ip}")
return Response(
{
"message": "Login successful",
"user": {
"id": user.id,
"username": user.username,
"email": user.email,
"groups": list(user.groups.values_list("name", flat=True)),
},
}
)
# Log failed login attempt
log_system_event(
event_type='login_failed',
user=username or 'unknown',
client_ip=client_ip,
user_agent=user_agent,
reason='Invalid credentials',
)
logger.info(f"Login failed via session: user={username} ip={client_ip}")
return Response({"error": "Invalid credentials"}, status=400)
"""Delegates to TokenObtainPairView (JWT login). Throttling, logging, and
network access checks are handled there."""
view = TokenObtainPairView.as_view()
return view(request._request)
@extend_schema(
description="Log out the current user",
@ -287,11 +247,18 @@ class UserViewSet(viewsets.ModelViewSet):
if request.method == "PATCH":
ALLOWED_FIELDS = {"custom_properties", "first_name", "last_name", "email", "password"}
disallowed = set(request.data.keys()) - ALLOWED_FIELDS
if disallowed:
return Response(
{"detail": f"Fields not allowed for self-update: {', '.join(disallowed)}"},
status=400,
)
for key in disallowed:
request.data.pop(key, None)
# Strip admin-managed keys from custom_properties so users cannot
# set their own XC credentials via this endpoint.
ADMIN_ONLY_PROPS = {"xc_password"}
cp = request.data.get("custom_properties")
if isinstance(cp, dict):
for key in ADMIN_ONLY_PROPS:
cp.pop(key, None)
serializer = UserSerializer(user, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()

37
apps/accounts/authentication.py
View file

@ -1,9 +1,46 @@
from rest_framework import authentication
from rest_framework import exceptions
from django.conf import settings
from drf_spectacular.extensions import OpenApiAuthenticationExtension
from .models import User
class JWTAuthenticationScheme(OpenApiAuthenticationExtension):
target_class = "rest_framework_simplejwt.authentication.JWTAuthentication"
name = "jwtAuth"
def get_security_definition(self, auto_schema):
return {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT",
"description": (
"JWT Bearer authentication.\n\n"
"Obtain a token pair via `POST /api/accounts/token/` using your username and password, "
"then paste the **access token** here — Swagger adds the `Bearer ` prefix automatically.\n\n"
"Access tokens expire after 30 minutes. Refresh using `POST /api/accounts/token/refresh/`."
),
}
class ApiKeyAuthenticationScheme(OpenApiAuthenticationExtension):
target_class = "apps.accounts.authentication.ApiKeyAuthentication"
name = "ApiKeyAuth"
def get_security_definition(self, auto_schema):
return {
"type": "apiKey",
"in": "header",
"name": "X-API-Key",
"description": (
"API key authentication.\n\n"
"Pass your personal API key in the `X-API-Key` request header. "
"Keys can be generated via `POST /api/accounts/api-keys/generate/` "
"and revoked via `POST /api/accounts/api-keys/revoke/`."
),
}
class ApiKeyAuthentication(authentication.BaseAuthentication):
"""
Accepts header `Authorization: ApiKey <key>` or `X-API-Key: <key>`.

18
apps/accounts/migrations/0006_user_stream_limit.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.2.11 on 2026-03-19 13:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0005_alter_user_managers'),
]
operations = [
migrations.AddField(
model_name='user',
name='stream_limit',
field=models.IntegerField(default=0),
),
]

1
apps/accounts/models.py
View file

@ -30,6 +30,7 @@ class User(AbstractUser):
user_level = models.IntegerField(default=UserLevel.STREAMER)
custom_properties = models.JSONField(default=dict, blank=True, null=True)
api_key = models.CharField(max_length=200, blank=True, null=True, db_index=True)
stream_limit = models.IntegerField(default=0)
def __str__(self):
return self.username

1
apps/accounts/serializers.py
View file

@ -65,6 +65,7 @@ class UserSerializer(serializers.ModelSerializer):
"channel_profiles",
"custom_properties",
"avatar_config",
"stream_limit",
"is_staff",
"is_superuser",
"last_login",

13
apps/backups/api_views.py
View file

@ -13,6 +13,7 @@ from rest_framework.permissions import AllowAny
from apps.accounts.permissions import IsAdmin
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.response import Response
from core.utils import safe_upload_path
from . import services
from .tasks import create_backup_task, restore_backup_task
@ -267,10 +268,18 @@ def upload_backup(request):
try:
backup_dir = services.get_backup_dir()
filename = uploaded.name or "uploaded-backup.zip"
# Sanitize filename: strip directory components to prevent path traversal
filename = Path(uploaded.name or "uploaded-backup.zip").name
if not filename:
filename = "uploaded-backup.zip"
try:
safe_upload_path(filename, str(backup_dir))
except ValueError:
return Response({"detail": "Invalid filename."}, status=status.HTTP_400_BAD_REQUEST)
# Ensure unique filename
backup_file = backup_dir / filename
backup_file = (backup_dir / filename).resolve()
counter = 1
while backup_file.exists():
name_parts = filename.rsplit(".", 1)

30
apps/backups/services.py
View file

@ -28,10 +28,36 @@ def _is_postgresql() -> bool:
def _get_pg_env() -> dict:
"""Get environment variables for PostgreSQL commands."""
"""Get environment variables for PostgreSQL commands.
Includes PGPASSWORD for password auth and PGSSL* variables for TLS.
Reads TLS config from DATABASES['default']['OPTIONS'], which is
populated by settings.py when POSTGRES_SSL=true.
"""
db_config = settings.DATABASES["default"]
env = os.environ.copy()
env["PGPASSWORD"] = db_config.get("PASSWORD", "")
password = db_config.get("PASSWORD", "")
if password:
env["PGPASSWORD"] = password
else:
env.pop("PGPASSWORD", None)
# Propagate TLS configuration from Django OPTIONS to libpq env vars.
options = db_config.get("OPTIONS", {})
_ssl_env_map = {
"sslmode": "PGSSLMODE",
"sslrootcert": "PGSSLROOTCERT",
"sslcert": "PGSSLCERT",
"sslkey": "PGSSLKEY",
}
# Always strip inherited PGSSL* vars first, then set only what is explicitly configured
for opt_key, env_key in _ssl_env_map.items():
env.pop(env_key, None)
value = options.get(opt_key)
if value:
env[env_key] = value
return env

90
apps/backups/tests.py
View file

@ -15,6 +15,96 @@ from . import services
User = get_user_model()
class PgEnvTlsTestCase(TestCase):
"""Test that _get_pg_env includes TLS and password env vars correctly."""
databases = []
@patch('apps.backups.services.settings')
def test_pg_env_includes_ssl_vars_when_tls_enabled(self, mock_settings):
mock_settings.DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "testdb",
"USER": "testuser",
"PASSWORD": "testpass",
"HOST": "localhost",
"PORT": 5432,
"OPTIONS": {
"sslmode": "verify-full",
"sslrootcert": "/certs/ca.crt",
"sslcert": "/certs/client.crt",
"sslkey": "/certs/client.key",
},
}
}
env = services._get_pg_env()
self.assertEqual(env["PGSSLMODE"], "verify-full")
self.assertEqual(env["PGSSLROOTCERT"], "/certs/ca.crt")
self.assertEqual(env["PGSSLCERT"], "/certs/client.crt")
self.assertEqual(env["PGSSLKEY"], "/certs/client.key")
self.assertEqual(env["PGPASSWORD"], "testpass")
@patch('apps.backups.services.settings')
def test_pg_env_no_ssl_vars_when_tls_disabled(self, mock_settings):
mock_settings.DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "testdb",
"USER": "testuser",
"PASSWORD": "testpass",
"HOST": "localhost",
"PORT": 5432,
}
}
env = services._get_pg_env()
self.assertNotIn("PGSSLMODE", env)
self.assertNotIn("PGSSLROOTCERT", env)
self.assertNotIn("PGSSLCERT", env)
self.assertNotIn("PGSSLKEY", env)
self.assertEqual(env["PGPASSWORD"], "testpass")
@patch('apps.backups.services.settings')
def test_pg_env_no_password_when_empty(self, mock_settings):
"""Cert-only auth: PGPASSWORD must not be set when password is empty."""
mock_settings.DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "testdb",
"USER": "testuser",
"PASSWORD": "",
"HOST": "localhost",
"PORT": 5432,
"OPTIONS": {"sslmode": "verify-full"},
}
}
env = services._get_pg_env()
self.assertNotIn("PGPASSWORD", env)
self.assertEqual(env["PGSSLMODE"], "verify-full")
@patch('apps.backups.services.settings')
def test_pg_env_partial_ssl_options(self, mock_settings):
"""Server-only TLS: only sslmode and CA cert, no client cert/key."""
mock_settings.DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "testdb",
"USER": "testuser",
"PASSWORD": "pass",
"HOST": "localhost",
"PORT": 5432,
"OPTIONS": {
"sslmode": "verify-ca",
"sslrootcert": "/certs/ca.crt",
},
}
}
env = services._get_pg_env()
self.assertEqual(env["PGSSLMODE"], "verify-ca")
self.assertEqual(env["PGSSLROOTCERT"], "/certs/ca.crt")
self.assertNotIn("PGSSLCERT", env)
self.assertNotIn("PGSSLKEY", env)
class BackupServicesTestCase(TestCase):
"""Test cases for backup services"""

157
apps/channels/api_views.py
View file

@ -24,12 +24,13 @@ from apps.accounts.permissions import (
)
from core.models import UserAgent, CoreSettings
from core.utils import RedisClient
from core.utils import RedisClient, safe_upload_path
from .models import (
Stream,
Channel,
ChannelGroup,
ChannelStream,
Logo,
ChannelProfile,
ChannelProfileMembership,
@ -630,6 +631,53 @@ class ChannelViewSet(viewsets.ModelViewSet):
context["include_streams"] = include_streams
return context
@extend_schema(
methods=["PATCH"],
description=(
"Bulk edit multiple channels in a single request. "
"Accepts a JSON array of channel update objects. Each object must include `id` (the channel's primary key). "
"All other fields are optional and support partial updates. "
"The `streams` field accepts a list of stream IDs and will replace the channel's current stream assignments. "
"All updates are validated before any changes are applied and executed in a single database transaction."
),
request=inline_serializer(
name="ChannelBulkEditRequest",
fields={
"id": serializers.IntegerField(help_text="ID of the channel to update (required)."),
"name": serializers.CharField(required=False),
"channel_number": serializers.FloatField(required=False),
"channel_group_id": serializers.IntegerField(required=False, allow_null=True),
"streams": serializers.ListField(
child=serializers.IntegerField(),
required=False,
help_text="List of stream IDs to assign to this channel (replaces existing assignments).",
),
"stream_profile_id": serializers.IntegerField(required=False, allow_null=True),
"logo_id": serializers.IntegerField(required=False, allow_null=True),
"tvg_id": serializers.CharField(required=False, allow_blank=True),
"tvc_guide_stationid": serializers.CharField(required=False, allow_blank=True),
"epg_data_id": serializers.IntegerField(required=False, allow_null=True),
"user_level": serializers.IntegerField(required=False),
"is_adult": serializers.BooleanField(required=False),
},
many=True,
),
responses={
200: inline_serializer(
name="ChannelBulkEditResponse",
fields={
"message": serializers.CharField(),
"channels": ChannelSerializer(many=True),
},
),
400: inline_serializer(
name="ChannelBulkEditErrorResponse",
fields={
"errors": serializers.ListField(child=serializers.DictField()),
},
),
},
)
@action(detail=False, methods=["patch"], url_path="edit/bulk")
def edit_bulk(self, request):
"""
@ -709,19 +757,24 @@ class ChannelViewSet(viewsets.ModelViewSet):
# Apply all updates in a transaction
with transaction.atomic():
streams_updates = []
for channel, validated_data in validated_updates:
# Pop streams before setattr loop — M2M fields can't be set via setattr
streams = validated_data.pop("streams", None)
if streams is not None:
streams_updates.append((channel, streams))
for key, value in validated_data.items():
setattr(channel, key, value)
# Single bulk_update query instead of individual saves
channels_to_update = [channel for channel, _ in validated_updates]
if channels_to_update:
# Collect all unique field names from all updates
# Collect all unique field names from all updates (streams already popped)
all_fields = set()
for _, validated_data in validated_updates:
all_fields.update(validated_data.keys())
# Only call bulk_update if there are fields to update
# Only call bulk_update if there are non-M2M fields to update
if all_fields:
Channel.objects.bulk_update(
channels_to_update,
@ -729,6 +782,32 @@ class ChannelViewSet(viewsets.ModelViewSet):
batch_size=100
)
# Handle streams M2M updates separately
for channel, streams in streams_updates:
normalized_ids = [
stream.id if hasattr(stream, "id") else stream for stream in streams
]
current_links = {
cs.stream_id: cs for cs in channel.channelstream_set.all()
}
existing_ids = set(current_links.keys())
new_ids = set(normalized_ids)
to_remove = existing_ids - new_ids
if to_remove:
channel.channelstream_set.filter(stream_id__in=to_remove).delete()
for order, stream_id in enumerate(normalized_ids):
if stream_id in current_links:
cs = current_links[stream_id]
if cs.order != order:
cs.order = order
cs.save(update_fields=["order"])
else:
ChannelStream.objects.create(
channel=channel, stream_id=stream_id, order=order
)
# Return the updated objects (already in memory)
serialized_channels = ChannelSerializer(
[channel for channel, _ in validated_updates],
@ -1078,6 +1157,10 @@ class ChannelViewSet(viewsets.ModelViewSet):
elif channel_number == 0:
# Special case: 0 means ignore provider numbers and auto-assign
channel_number = None
elif channel_number == -1:
# Special case: -1 means assign the number after the current highest
highest = Channel.objects.order_by('-channel_number').values_list('channel_number', flat=True).first()
channel_number = (int(highest) + 1) if highest is not None else 1
if channel_number is None:
# Still None, auto-assign the next available channel number
@ -1482,7 +1565,11 @@ class ChannelViewSet(viewsets.ModelViewSet):
name="EpgAssociation",
fields={
"channel_id": serializers.IntegerField(),
"epg_data_id": serializers.IntegerField(),
"epg_data_id": serializers.IntegerField(
required=False,
allow_null=True,
help_text="EPG data ID to link. Pass null to remove EPG linkage.",
),
},
),
)
@ -1658,7 +1745,12 @@ class BulkDeleteLogosAPIView(APIView):
"logo_ids": serializers.ListField(
child=serializers.IntegerField(),
help_text="Logo IDs to delete",
)
),
"delete_files": serializers.BooleanField(
required=False,
default=False,
help_text="Whether to also delete local logo files from disk.",
),
},
),
)
@ -1897,10 +1989,13 @@ class LogoViewSet(viewsets.ModelViewSet):
{"error": str(e)}, status=status.HTTP_400_BAD_REQUEST
)
file_name = file.name
file_path = os.path.join("/data/logos", file_name)
# Sanitize filename: strip directory components to prevent path traversal
try:
file_path = safe_upload_path(file.name, "/data/logos")
except ValueError:
return Response({"error": "Invalid filename."}, status=status.HTTP_400_BAD_REQUEST)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
os.makedirs("/data/logos", exist_ok=True)
with open(file_path, "wb+") as destination:
for chunk in file.chunks():
destination.write(chunk)
@ -1920,7 +2015,7 @@ class LogoViewSet(viewsets.ModelViewSet):
# Get custom name from request data, fallback to filename
custom_name = request.data.get('name', '').strip()
logo_name = custom_name if custom_name else file_name
logo_name = custom_name if custom_name else os.path.basename(file_path)
logo, _ = Logo.objects.get_or_create(
url=file_path,
@ -2670,7 +2765,49 @@ class RecordingViewSet(viewsets.ModelViewSet):
recording_id = instance.pk
channel_name = instance.channel.name
# Capture state before the DB row is deleted
# Attempt to close the DVR client connection for this channel if active
try:
channel_uuid = str(instance.channel.uuid)
# Lazy imports to avoid module overhead if proxy isn't used
from core.utils import RedisClient
from apps.proxy.ts_proxy.redis_keys import RedisKeys
from apps.proxy.ts_proxy.services.channel_service import ChannelService
r = RedisClient.get_client()
if r:
client_set_key = RedisKeys.clients(channel_uuid)
client_ids = r.smembers(client_set_key) or []
stopped = 0
for cid in client_ids:
try:
meta_key = RedisKeys.client_metadata(channel_uuid, cid)
ua = r.hget(meta_key, "user_agent")
# Identify DVR recording client by its user agent
if ua and "Dispatcharr-DVR" in ua:
try:
ChannelService.stop_client(channel_uuid, cid)
stopped += 1
except Exception as inner_e:
logger.debug(f"Failed to stop DVR client {cid} for channel {channel_uuid}: {inner_e}")
except Exception as inner:
logger.debug(f"Error while checking client metadata: {inner}")
if stopped:
logger.info(f"Stopped {stopped} DVR client(s) for channel {channel_uuid} due to recording cancellation")
# If no clients remain after stopping DVR clients, proactively stop the channel
try:
remaining = r.scard(client_set_key) or 0
except Exception:
remaining = 0
if remaining == 0:
try:
ChannelService.stop_channel(channel_uuid)
logger.info(f"Stopped channel {channel_uuid} (no clients remain)")
except Exception as sc_e:
logger.debug(f"Unable to stop channel {channel_uuid}: {sc_e}")
except Exception as e:
logger.debug(f"Unable to stop DVR clients for cancelled recording: {e}")
# Capture paths before deletion
cp = instance.custom_properties or {}
rec_status = cp.get("status", "")
file_path = cp.get("file_path")

4
apps/channels/migrations/0005_stream_channel_group_stream_last_seen_and_more.py
View file

@ -1,7 +1,7 @@
# Generated by Django 5.1.6 on 2025-03-19 16:33
import datetime
import django.db.models.deletion
import django.utils.timezone
import uuid
from django.db import migrations, models
@ -22,7 +22,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='stream',
name='last_seen',
field=models.DateTimeField(db_index=True, default=datetime.datetime.now),
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='channel',

4
apps/channels/migrations/0031_channelgroupm3uaccount_is_stale_and_more.py
View file

@ -1,6 +1,6 @@
# Generated by Django 5.2.9 on 2026-01-09 18:19
import datetime
import django.utils.timezone
from django.db import migrations, models
@ -19,7 +19,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='channelgroupm3uaccount',
name='last_seen',
field=models.DateTimeField(db_index=True, default=datetime.datetime.now, help_text='Last time this group was seen in the M3U source during a refresh'),
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, help_text='Last time this group was seen in the M3U source during a refresh'),
),
migrations.AddField(
model_name='stream',

159
apps/channels/models.py
View file

@ -7,7 +7,7 @@ from apps.proxy.ts_proxy.redis_keys import RedisKeys
from apps.proxy.ts_proxy.constants import ChannelMetadataField
import logging
import uuid
from datetime import datetime
from django.utils import timezone
import hashlib
import json
from apps.epg.models import EPGData
@ -95,7 +95,7 @@ class Stream(models.Model):
help_text="Unique hash for this stream from the M3U account",
db_index=True,
)
last_seen = models.DateTimeField(db_index=True, default=datetime.now)
last_seen = models.DateTimeField(db_index=True, default=timezone.now)
is_stale = models.BooleanField(
default=False,
db_index=True,
@ -204,7 +204,7 @@ class Stream(models.Model):
return stream_profile
def get_stream(self):
def get_stream(self, requester=None):
"""
Finds an available profile for this stream and reserves a connection slot.
@ -400,6 +400,144 @@ class Channel(models.Model):
return stream_profile
def _pick_channel_to_preempt(
self,
profile_id,
requester_level,
redis_client,
exclude_channel_ids=None,
cooldown_seconds=30,
):
"""
Pick the lowest-impact channel to terminate on the given profile.
Returns: Optional[int] channel_id to preempt
"""
exclude_channel_ids = set(exclude_channel_ids or [])
candidates = []
# 1) Try to get active channel IDs for this profile from an index set if available
ch_set_key = f"ts_proxy:profile:{profile_id}:channels"
try:
ch_ids = { (int(x) if not isinstance(x, int) else x) for x in (redis_client.smembers(ch_set_key) or set()) }
except Exception:
ch_ids = set()
logger.debug("Candidate channels for preemption:")
logger.debug(ch_ids)
# 2) Fallback: scan metadata keys and filter by m3u_profile == profile_id
if not ch_ids:
cursor = 0
pattern = "ts_proxy:channel:*:metadata"
while True:
cursor, keys = redis_client.scan(cursor=cursor, match=pattern, count=500)
if keys:
# Prefer HGET m3u_profile if metadata is a hash
pipe = redis_client.pipeline()
for k in keys:
pipe.hget(k, "m3u_profile")
prof_vals = pipe.execute()
for k, prof_val in zip(keys, prof_vals):
try:
pid = int(prof_val) if prof_val is not None else None
except Exception:
pid = None
if pid == profile_id:
parts = k.split(":") # ts_proxy:channel:{id}:metadata
if len(parts) >= 4:
try:
ch_ids.add(int(parts[2]))
except Exception:
pass
if cursor == 0:
break
logger.debug("Candidate channels for preemption:")
logger.debug(ch_ids)
if not ch_ids:
return None
# 3) Score candidates
for ch_id in ch_ids:
if ch_id in exclude_channel_ids:
continue
# Skip if recently preempted
last_preempt_key = f"ts_proxy:channel:{ch_id}:last_preempt"
try:
last_preempt = float(redis_client.get(last_preempt_key) or 0.0)
except Exception:
last_preempt = 0.0
if last_preempt and (time.time() - last_preempt) < cooldown_seconds:
continue
# Clients and their levels
clients_key = f"ts_proxy:channel:{ch_id}:clients"
member_ids = list(redis_client.smembers(clients_key) or [])
viewer_count = len(member_ids)
max_viewer_level = 0
if viewer_count:
pipe = redis_client.pipeline()
for cid in member_ids:
pipe.hget(f"ts_proxy:channel:{ch_id}:clients:{cid}", "user_level")
levels_raw = pipe.execute()
levels = []
for lv in levels_raw:
try:
levels.append(int(lv or 0))
except Exception:
levels.append(0)
max_viewer_level = max(levels or [0])
# Only preempt if requester strictly outranks this channel's viewers
if requester_level <= max_viewer_level:
continue
# Metadata (protected/recording/started_at_ts)
meta_key = f"ts_proxy:channel:{ch_id}:metadata"
try:
protected, recording, started_at_ts = redis_client.hmget(
meta_key, "protected", "recording", "started_at_ts"
)
except Exception:
protected = recording = started_at_ts = None
protected = str(protected or "0") in ("1", "true", "True")
recording = str(recording or "0") in ("1", "true", "True")
if protected or recording:
continue
try:
started_at_ts = float(started_at_ts) if started_at_ts is not None else None
except Exception:
started_at_ts = None
if started_at_ts is None:
started_at_ts = time.time() # treat unknown as newest
# Score: lower is safer to terminate
has_viewers = 1 if viewer_count > 0 else 0
score = (has_viewers, max_viewer_level, viewer_count, started_at_ts)
candidates.append((score, ch_id))
logger.debug("Candidate channels after scoring:")
logger.debug(candidates)
if not candidates:
return None
candidates.sort(key=lambda x: x[0])
victim_id = candidates[0][1]
# Mark preempt timestamp to avoid thrashing
try:
redis_client.set(f"ts_proxy:channel:{victim_id}:last_preempt", str(time.time()), ex=3600)
except Exception:
pass
return victim_id
def _check_and_reserve_profile_slot(self, profile, redis_client):
"""
Atomically check and reserve a connection slot for the given profile.
@ -432,7 +570,7 @@ class Channel(models.Model):
redis_client.decr(profile_connections_key)
return (False, new_count - 1)
def get_stream(self):
def get_stream(self, requester=None):
"""
Finds an available stream for the requested channel and returns the selected stream and profile.
@ -513,6 +651,17 @@ class Channel(models.Model):
None,
) # Return newly assigned stream and matched profile
else:
# At capacity: try to preempt a lower-impact channel on this profile
victim_channel_id = self._pick_channel_to_preempt(
profile_id=profile.id,
requester_level=requester.user_level if requester else 100,
redis_client=redis_client,
exclude_channel_ids=None,
)
if victim_channel_id:
logger.info(f"Preempting channel {victim_channel_id} for new stream on profile {profile.id}")
# return self.id, profile.id, victim_channel_id
# This profile is at max connections
has_streams_but_maxed_out = True
logger.debug(
@ -739,7 +888,7 @@ class ChannelGroupM3UAccount(models.Model):
help_text='Starting channel number for auto-created channels in this group'
)
last_seen = models.DateTimeField(
default=datetime.now,
default=timezone.now,
db_index=True,
help_text='Last time this group was seen in the M3U source during a refresh'
)

86
apps/channels/tasks.py
View file

@ -19,6 +19,7 @@ from rapidfuzz import fuzz
from apps.channels.models import Channel
from apps.epg.models import EPGData
from core.models import CoreSettings
from core.utils import acquire_task_lock, release_task_lock
from django.db import OperationalError, close_old_connections
from channels.layers import get_channel_layer
@ -1070,12 +1071,39 @@ def match_single_channel_epg(channel_id):
def evaluate_series_rules_impl(tvg_id: str | None = None):
"""Synchronous implementation of series rule evaluation; returns details for debugging."""
result = {"scheduled": 0, "details": []}
# Serialize all invocations to prevent concurrent evaluations from
# racing to create duplicate recordings (e.g. multiple EPG sources
# refreshing simultaneously each firing evaluate_series_rules.delay()).
# If Redis is unavailable, proceed without lock — the primary and
# secondary dedup guards still prevent duplicates.
lock_acquired = False
try:
lock_acquired = acquire_task_lock('evaluate_series_rules', 'all')
if not lock_acquired:
result["details"].append({"status": "skipped", "reason": "concurrent evaluation in progress"})
return result
except (ConnectionError, OSError, AttributeError):
logger.warning("Could not acquire series rule evaluation lock (Redis unavailable), proceeding without lock")
try:
return _evaluate_series_rules_locked(tvg_id, result)
finally:
if lock_acquired:
try:
release_task_lock('evaluate_series_rules', 'all')
except (ConnectionError, OSError, AttributeError):
logger.warning("Could not release series rule evaluation lock")
def _evaluate_series_rules_locked(tvg_id, result):
"""Inner implementation of series rule evaluation, called under lock."""
from django.utils import timezone
from apps.channels.models import Recording, Channel
from apps.epg.models import EPGData, ProgramData
rules = CoreSettings.get_dvr_series_rules()
result = {"scheduled": 0, "details": []}
if not isinstance(rules, list) or not rules:
return result
@ -1089,14 +1117,23 @@ def evaluate_series_rules_impl(tvg_id: str | None = None):
now = timezone.now()
horizon = now + timedelta(days=7)
# Preload existing recordings' program ids to avoid duplicates
existing_program_ids = set()
for rec in Recording.objects.all().only("custom_properties"):
# Preload existing recordings keyed by stable program attributes that
# survive EPG refreshes (tvg_id + original start/end times stored in
# custom_properties). ProgramData.id changes on every EPG refresh so
# it cannot be used for deduplication. Only load future recordings
# to bound the set size — past recordings cannot collide with newly
# scheduled future programs.
existing_program_keys = set()
for cp in Recording.objects.filter(
end_time__gte=now,
).values_list("custom_properties", flat=True):
try:
pid = rec.custom_properties.get("program", {}).get("id") if rec.custom_properties else None
if pid is not None:
# Normalize to string for consistent comparisons
existing_program_ids.add(str(pid))
prog_data = (cp or {}).get("program", {})
tvg_id_val = prog_data.get("tvg_id")
st = prog_data.get("start_time")
et = prog_data.get("end_time")
if tvg_id_val and st and et:
existing_program_keys.add((str(tvg_id_val), str(st), str(et)))
except Exception:
continue
@ -1191,17 +1228,21 @@ def evaluate_series_rules_impl(tvg_id: str | None = None):
created_here = 0
for prog in unique_programs:
try:
# Skip if already scheduled by program id
if str(prog.id) in existing_program_ids:
# Skip if a recording already exists for this exact airing
# (keyed by tvg_id + original program times, which are stable
# across EPG refreshes unlike ProgramData.id).
prog_key = (str(prog.tvg_id), prog.start_time.isoformat(), prog.end_time.isoformat())
if prog_key in existing_program_keys:
continue
# Extra guard: skip if a recording exists for the same channel + timeslot
# Extra guard: DB query using the same stable attributes
# stored in custom_properties (unadjusted program times,
# not offset-adjusted Recording.start_time/end_time).
try:
from django.db.models import Q
if Recording.objects.filter(
channel=channel,
start_time=prog.start_time,
end_time=prog.end_time,
).filter(Q(custom_properties__program__id=prog.id) | Q(custom_properties__program__title=prog.title)).exists():
custom_properties__program__tvg_id=prog.tvg_id,
custom_properties__program__start_time=prog.start_time.isoformat(),
custom_properties__program__end_time=prog.end_time.isoformat(),
).exists():
continue
except Exception:
continue # already scheduled/recorded
@ -1245,7 +1286,7 @@ def evaluate_series_rules_impl(tvg_id: str | None = None):
}
},
)
existing_program_ids.add(str(prog.id))
existing_program_keys.add(prog_key)
created_here += 1
try:
prefetch_recording_artwork.apply_async(args=[rec.id], countdown=1)
@ -2452,15 +2493,12 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str):
metadata_key = RedisKeys.channel_metadata(str(channel.uuid))
md = r.hgetall(metadata_key)
if md:
def _gv(bkey):
return md.get(bkey.encode('utf-8'))
def _d(bkey, cast=str):
v = _gv(bkey)
v = md.get(bkey)
try:
if v is None:
return None
s = v.decode('utf-8')
s = v
return cast(s) if cast is not str else s
except Exception:
return None
@ -3338,6 +3376,10 @@ def bulk_create_channels_from_streams(self, stream_ids, channel_profile_ids=None
elif starting_channel_number == 0:
# Mode 2: Start from lowest available number
next_number = 1
elif starting_channel_number == -1:
# Mode 4: Start after the current highest channel number
highest = Channel.objects.order_by('-channel_number').values_list('channel_number', flat=True).first()
next_number = (int(highest) + 1) if highest is not None else 1
else:
# Mode 3: Start from specified number
next_number = starting_channel_number

718
apps/channels/tests/test_series_rule_dedup.py Normal file
View file

@ -0,0 +1,718 @@
"""Tests for series rule evaluation deduplication.
Unit tests verify the dedup logic in evaluate_series_rules_impl.
Integration tests exercise the full path: EPG refresh → series rule
evaluation → Recording creation → post_save signal chain.
"""
from datetime import timedelta
from unittest.mock import patch, MagicMock
from django.test import TestCase
from django.utils import timezone
from apps.channels.models import Channel, Recording
from apps.epg.models import EPGSource, EPGData, ProgramData
from core.models import CoreSettings
def _set_series_rules(rules):
"""Helper to store series rules in CoreSettings."""
CoreSettings.set_dvr_series_rules(rules)
def _set_dvr_offsets(pre_min=0, post_min=0):
"""Helper to store DVR pre/post offsets."""
CoreSettings._update_group("dvr_settings", "DVR Settings", {
"pre_offset_minutes": pre_min,
"post_offset_minutes": post_min,
})
class SeriesRuleDedupBaseTestCase(TestCase):
"""Shared setup for series rule dedup tests."""
def setUp(self):
self.now = timezone.now()
self.epg_source = EPGSource.objects.create(
name="Test EPG", source_type="xmltv"
)
self.epg = EPGData.objects.create(
tvg_id="test.channel.1",
name="Test Channel EPG",
epg_source=self.epg_source,
)
self.channel = Channel.objects.create(
channel_number=1, name="Test Channel", epg_data=self.epg
)
_set_series_rules([{
"tvg_id": "test.channel.1",
"mode": "all",
"title": "Test Show",
}])
_set_dvr_offsets(pre_min=0, post_min=0)
def _create_program(self, hours_from_now=1, title="Test Show",
sub_title="Episode 1", tvg_id="test.channel.1"):
"""Create a ProgramData at the given offset."""
start = self.now + timedelta(hours=hours_from_now)
end = start + timedelta(hours=1)
return ProgramData.objects.create(
epg=self.epg,
tvg_id=tvg_id,
start_time=start,
end_time=end,
title=title,
sub_title=sub_title,
)
def _simulate_epg_refresh(self, programs_data):
"""Delete all ProgramData and recreate with new IDs (simulates EPG refresh)."""
ProgramData.objects.filter(epg=self.epg).delete()
new_programs = []
for data in programs_data:
prog = ProgramData.objects.create(epg=self.epg, **data)
new_programs.append(prog)
return new_programs
def _program_data_for_refresh(self, prog):
"""Build the dict needed by _simulate_epg_refresh from a ProgramData."""
return {
"tvg_id": prog.tvg_id,
"start_time": prog.start_time,
"end_time": prog.end_time,
"title": prog.title,
"sub_title": prog.sub_title,
}
# ---------------------------------------------------------------------------
# Unit tests: dedup logic in evaluate_series_rules_impl
# ---------------------------------------------------------------------------
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class ProgramIdStabilityTests(SeriesRuleDedupBaseTestCase):
"""Verify dedup works after EPG refresh changes ProgramData IDs."""
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_no_duplicate_after_epg_refresh(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Same program should not be recorded twice after EPG refresh."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2)
old_id = prog.id
result1 = evaluate_series_rules_impl()
self.assertEqual(result1["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 1)
new_programs = self._simulate_epg_refresh(
[self._program_data_for_refresh(prog)]
)
self.assertNotEqual(old_id, new_programs[0].id)
result2 = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
self.assertEqual(result2["scheduled"], 0)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_no_duplicate_with_offsets_after_refresh(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Dedup works when DVR offsets shift Recording times away from program times."""
from apps.channels.tasks import evaluate_series_rules_impl
_set_dvr_offsets(pre_min=5, post_min=5)
prog = self._create_program(hours_from_now=2)
result1 = evaluate_series_rules_impl()
self.assertEqual(result1["scheduled"], 1)
rec = Recording.objects.first()
self.assertEqual(rec.start_time, prog.start_time - timedelta(minutes=5))
self.assertEqual(rec.end_time, prog.end_time + timedelta(minutes=5))
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
result2 = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_different_episodes_still_recorded(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Different episodes on the same channel should each get a recording."""
from apps.channels.tasks import evaluate_series_rules_impl
self._create_program(hours_from_now=2, sub_title="Episode 1")
self._create_program(hours_from_now=4, sub_title="Episode 2")
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 2)
self.assertEqual(Recording.objects.count(), 2)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_new_episode_after_refresh_is_recorded(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""A genuinely new episode appearing after EPG refresh should be recorded."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2, sub_title="Episode 1")
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
self._simulate_epg_refresh([
self._program_data_for_refresh(prog),
{
"tvg_id": "test.channel.1",
"start_time": prog.end_time,
"end_time": prog.end_time + timedelta(hours=1),
"title": "Test Show",
"sub_title": "Episode 2",
},
])
result2 = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 2)
self.assertEqual(result2["scheduled"], 1)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_multiple_epg_refreshes_no_duplicates(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Multiple consecutive EPG refreshes should not accumulate duplicates."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2)
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
for _ in range(5):
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class ConcurrencyGuardTests(SeriesRuleDedupBaseTestCase):
"""Verify the task lock prevents concurrent evaluation."""
def test_lock_acquired_and_released(self, mock_schedule, mock_artwork):
"""evaluate_series_rules_impl acquires and releases the task lock."""
from apps.channels.tasks import evaluate_series_rules_impl
self._create_program(hours_from_now=2)
with patch("apps.channels.tasks.acquire_task_lock", return_value=True) as mock_lock, \
patch("apps.channels.tasks.release_task_lock") as mock_release:
evaluate_series_rules_impl()
mock_lock.assert_called_once_with('evaluate_series_rules', 'all')
mock_release.assert_called_once_with('evaluate_series_rules', 'all')
def test_skips_when_lock_held(self, mock_schedule, mock_artwork):
"""Returns early with skip reason when lock is already held."""
from apps.channels.tasks import evaluate_series_rules_impl
self._create_program(hours_from_now=2)
with patch("apps.channels.tasks.acquire_task_lock", return_value=False):
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 0)
self.assertTrue(
any(d.get("reason") == "concurrent evaluation in progress"
for d in result["details"]),
)
self.assertEqual(Recording.objects.count(), 0)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_lock_released_on_exception(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Lock is released even if the inner implementation raises."""
from apps.channels.tasks import evaluate_series_rules_impl
with patch("apps.channels.tasks._evaluate_series_rules_locked",
side_effect=RuntimeError("test error")):
with self.assertRaises(RuntimeError):
evaluate_series_rules_impl()
mock_release.assert_called_once_with('evaluate_series_rules', 'all')
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class SecondaryGuardTests(SeriesRuleDedupBaseTestCase):
"""Verify the secondary DB guard uses stable program attributes."""
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_secondary_guard_catches_duplicate_with_offsets(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Secondary guard works with stale program IDs and DVR offsets."""
from apps.channels.tasks import evaluate_series_rules_impl
_set_dvr_offsets(pre_min=10, post_min=10)
prog = self._create_program(hours_from_now=2)
# Pre-existing recording with a stale program ID (from previous EPG refresh)
Recording.objects.create(
channel=self.channel,
start_time=prog.start_time - timedelta(minutes=10),
end_time=prog.end_time + timedelta(minutes=10),
custom_properties={
"program": {
"id": 99999,
"tvg_id": prog.tvg_id,
"title": prog.title,
"start_time": prog.start_time.isoformat(),
"end_time": prog.end_time.isoformat(),
}
},
)
result = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
self.assertEqual(result["scheduled"], 0)
# ---------------------------------------------------------------------------
# Integration tests: full path from EPG refresh through recording creation
# ---------------------------------------------------------------------------
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class IntegrationEPGRefreshTests(SeriesRuleDedupBaseTestCase):
"""End-to-end tests simulating the EPG refresh → evaluate → record flow.
These exercise the full signal chain: evaluate_series_rules_impl creates
a Recording, the post_save signal fires schedule_recording_task, and
subsequent evaluations (after EPG refresh) must not create duplicates.
"""
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_single_episode_no_duplicates(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Simulate: create rule → evaluate → EPG refresh → re-evaluate.
The full recording lifecycle must result in exactly 1 recording.
"""
from apps.channels.tasks import evaluate_series_rules_impl
# Initial EPG data
prog = self._create_program(hours_from_now=2, sub_title="Pilot")
# First evaluation creates the recording
result1 = evaluate_series_rules_impl()
self.assertEqual(result1["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 1)
# Verify the recording was created with correct program metadata
rec = Recording.objects.first()
self.assertEqual(rec.custom_properties["program"]["tvg_id"], "test.channel.1")
self.assertEqual(rec.custom_properties["program"]["title"], "Test Show")
self.assertEqual(
rec.custom_properties["program"]["start_time"],
prog.start_time.isoformat()
)
# Verify the post_save signal scheduled a task
mock_schedule.assert_called()
initial_schedule_count = mock_schedule.call_count
# Simulate EPG refresh (programs get new DB IDs)
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
# Re-evaluate after refresh (this is what EPG refresh triggers)
result2 = evaluate_series_rules_impl()
self.assertEqual(result2["scheduled"], 0)
self.assertEqual(Recording.objects.count(), 1)
# No additional task scheduling should have occurred
self.assertEqual(mock_schedule.call_count, initial_schedule_count)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_with_offsets_no_duplicates(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Full flow with DVR offsets: recording times differ from program times."""
from apps.channels.tasks import evaluate_series_rules_impl
_set_dvr_offsets(pre_min=5, post_min=10)
prog = self._create_program(hours_from_now=3, sub_title="Episode 1")
result1 = evaluate_series_rules_impl()
self.assertEqual(result1["scheduled"], 1)
rec = Recording.objects.first()
# Verify offset-adjusted recording times
self.assertEqual(rec.start_time, prog.start_time - timedelta(minutes=5))
self.assertEqual(rec.end_time, prog.end_time + timedelta(minutes=10))
# Verify original (unadjusted) program times in custom_properties
self.assertEqual(
rec.custom_properties["program"]["start_time"],
prog.start_time.isoformat()
)
self.assertEqual(
rec.custom_properties["program"]["end_time"],
prog.end_time.isoformat()
)
# EPG refresh + re-evaluate
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
result2 = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
self.assertEqual(result2["scheduled"], 0)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_multiple_episodes_across_refreshes(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""New episodes appear across multiple EPG refreshes; each recorded once."""
from apps.channels.tasks import evaluate_series_rules_impl
ep1 = self._create_program(hours_from_now=2, sub_title="Episode 1")
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
# EPG refresh adds episode 2 alongside episode 1
ep1_data = self._program_data_for_refresh(ep1)
ep2_start = ep1.end_time
ep2_data = {
"tvg_id": "test.channel.1",
"start_time": ep2_start,
"end_time": ep2_start + timedelta(hours=1),
"title": "Test Show",
"sub_title": "Episode 2",
}
self._simulate_epg_refresh([ep1_data, ep2_data])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 2)
# Another EPG refresh adds episode 3
ep3_start = ep2_start + timedelta(hours=1)
ep3_data = {
"tvg_id": "test.channel.1",
"start_time": ep3_start,
"end_time": ep3_start + timedelta(hours=1),
"title": "Test Show",
"sub_title": "Episode 3",
}
self._simulate_epg_refresh([ep1_data, ep2_data, ep3_data])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 3)
# Final EPG refresh with no new episodes — count must stay at 3
self._simulate_epg_refresh([ep1_data, ep2_data, ep3_data])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 3)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_multiple_series_rules(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Multiple series rules on different channels, each evaluated correctly."""
from apps.channels.tasks import evaluate_series_rules_impl
# Second channel with its own EPG
epg2 = EPGData.objects.create(
tvg_id="test.channel.2",
name="Channel 2 EPG",
epg_source=self.epg_source,
)
channel2 = Channel.objects.create(
channel_number=2, name="Test Channel 2", epg_data=epg2
)
_set_series_rules([
{"tvg_id": "test.channel.1", "mode": "all", "title": "Show A"},
{"tvg_id": "test.channel.2", "mode": "all", "title": "Show B"},
])
# Programs on both channels
start1 = self.now + timedelta(hours=2)
prog1 = ProgramData.objects.create(
epg=self.epg, tvg_id="test.channel.1",
start_time=start1, end_time=start1 + timedelta(hours=1),
title="Show A", sub_title="Episode 1",
)
start2 = self.now + timedelta(hours=3)
prog2 = ProgramData.objects.create(
epg=epg2, tvg_id="test.channel.2",
start_time=start2, end_time=start2 + timedelta(hours=1),
title="Show B", sub_title="Episode 1",
)
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 2)
self.assertEqual(Recording.objects.filter(channel=self.channel).count(), 1)
self.assertEqual(Recording.objects.filter(channel=channel2).count(), 1)
# EPG refresh for both channels
ProgramData.objects.filter(epg=self.epg).delete()
ProgramData.objects.filter(epg=epg2).delete()
ProgramData.objects.create(
epg=self.epg, tvg_id="test.channel.1",
start_time=start1, end_time=start1 + timedelta(hours=1),
title="Show A", sub_title="Episode 1",
)
ProgramData.objects.create(
epg=epg2, tvg_id="test.channel.2",
start_time=start2, end_time=start2 + timedelta(hours=1),
title="Show B", sub_title="Episode 1",
)
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 2,
"No duplicates across multiple series rules after EPG refresh")
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_rapid_epg_refreshes_simulate_user_report(
self, mock_release, mock_lock, mock_schedule, mock_artwork
):
"""Reproduce the user-reported scenario: series rule + multiple EPG refreshes
causing count to balloon from 6 to 25 and 5 simultaneous recordings.
Simulates 6 episodes with 5 EPG refreshes (each assigning new ProgramData IDs).
"""
from apps.channels.tasks import evaluate_series_rules_impl
# Create 6 episodes (the user had "next of 6")
episodes = []
for i in range(6):
start = self.now + timedelta(hours=2 + i * 2)
episodes.append({
"tvg_id": "test.channel.1",
"start_time": start,
"end_time": start + timedelta(hours=1),
"title": "Test Show",
"sub_title": f"Episode {i + 1}",
})
# Create initial ProgramData
for ep in episodes:
ProgramData.objects.create(epg=self.epg, **ep)
# First evaluation: should create exactly 6 recordings
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 6)
# Simulate 5 EPG refreshes (the user saw count balloon to 25)
for refresh_num in range(5):
self._simulate_epg_refresh(episodes)
result = evaluate_series_rules_impl()
self.assertEqual(
Recording.objects.count(), 6,
f"After EPG refresh #{refresh_num + 1}, expected 6 recordings "
f"but got {Recording.objects.count()}"
)
self.assertEqual(result["scheduled"], 0)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_recording_survives_program_removal_and_readd(
self, mock_release, mock_lock, mock_schedule, mock_artwork
):
"""Program temporarily disappears from EPG then reappears — no duplicate."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2, sub_title="Episode 1")
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
# EPG refresh removes the program entirely
self._simulate_epg_refresh([])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1,
"Existing recording preserved when program disappears from EPG")
# EPG refresh adds the program back (new ID)
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1,
"No duplicate when program reappears with new ID")
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_celery_task_wrapper_calls_impl(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""The @shared_task evaluate_series_rules delegates to _impl correctly."""
from apps.channels.tasks import evaluate_series_rules
self._create_program(hours_from_now=2)
result = evaluate_series_rules()
self.assertEqual(result["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 1)
# Call again (simulating a second EPG refresh trigger)
result2 = evaluate_series_rules()
self.assertEqual(result2["scheduled"], 0)
self.assertEqual(Recording.objects.count(), 1)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_tvg_id_scoped_evaluation(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Scoped evaluation (tvg_id parameter) still prevents duplicates."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2)
result1 = evaluate_series_rules_impl(tvg_id="test.channel.1")
self.assertEqual(result1["scheduled"], 1)
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
result2 = evaluate_series_rules_impl(tvg_id="test.channel.1")
self.assertEqual(result2["scheduled"], 0)
self.assertEqual(Recording.objects.count(), 1)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_full_flow_offset_change_between_refreshes(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Changing DVR offsets between EPG refreshes doesn't create duplicates.
Even though Recording.start_time/end_time change when offsets change,
the dedup key uses the original program times from custom_properties.
"""
from apps.channels.tasks import evaluate_series_rules_impl
_set_dvr_offsets(pre_min=5, post_min=5)
prog = self._create_program(hours_from_now=2)
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
rec = Recording.objects.first()
original_start = rec.start_time
original_end = rec.end_time
# Change offsets
_set_dvr_offsets(pre_min=10, post_min=15)
# EPG refresh
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
result = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1,
"Changing offsets between refreshes should not create duplicates")
self.assertEqual(result["scheduled"], 0)
# ---------------------------------------------------------------------------
# Edge case tests: Redis unavailability, non-series recordings, robustness
# ---------------------------------------------------------------------------
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class RedisUnavailabilityTests(SeriesRuleDedupBaseTestCase):
"""Verify evaluation works when Redis is unavailable (lock cannot be acquired)."""
def test_proceeds_when_redis_down(self, mock_schedule, mock_artwork):
"""Evaluation succeeds (with dedup guards) when Redis raises on lock acquire."""
from apps.channels.tasks import evaluate_series_rules_impl
self._create_program(hours_from_now=2)
with patch("apps.channels.tasks.acquire_task_lock",
side_effect=ConnectionError("Redis unavailable")):
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 1)
def test_dedup_still_works_without_lock(self, mock_schedule, mock_artwork):
"""Dedup guards prevent duplicates even when the lock is unavailable."""
from apps.channels.tasks import evaluate_series_rules_impl
prog = self._create_program(hours_from_now=2)
# First call: Redis down, proceeds without lock
with patch("apps.channels.tasks.acquire_task_lock",
side_effect=ConnectionError("Redis unavailable")):
evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1)
# EPG refresh
self._simulate_epg_refresh([self._program_data_for_refresh(prog)])
# Second call: Redis still down
with patch("apps.channels.tasks.acquire_task_lock",
side_effect=ConnectionError("Redis unavailable")):
result = evaluate_series_rules_impl()
self.assertEqual(Recording.objects.count(), 1,
"Dedup guards prevent duplicates even without lock")
self.assertEqual(result["scheduled"], 0)
def test_lock_not_released_when_not_acquired(self, mock_schedule, mock_artwork):
"""release_task_lock is not called if acquire raised an exception."""
from apps.channels.tasks import evaluate_series_rules_impl
self._create_program(hours_from_now=2)
with patch("apps.channels.tasks.acquire_task_lock",
side_effect=ConnectionError("Redis unavailable")), \
patch("apps.channels.tasks.release_task_lock") as mock_release:
evaluate_series_rules_impl()
mock_release.assert_not_called()
@patch("apps.channels.tasks.prefetch_recording_artwork")
@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id")
class NonSeriesRecordingTests(SeriesRuleDedupBaseTestCase):
"""Verify non-series recordings don't interfere with series rule dedup."""
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_manual_recording_without_program_data_ignored(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Recordings without custom_properties.program are skipped by dedup key builder."""
from apps.channels.tasks import evaluate_series_rules_impl
# Manual recording with no program metadata
Recording.objects.create(
channel=self.channel,
start_time=self.now + timedelta(hours=2),
end_time=self.now + timedelta(hours=3),
custom_properties={},
)
prog = self._create_program(hours_from_now=2)
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 2)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_recurring_rule_recording_does_not_interfere(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Recordings from recurring rules (custom_properties.rule) don't block series rules."""
from apps.channels.tasks import evaluate_series_rules_impl
Recording.objects.create(
channel=self.channel,
start_time=self.now + timedelta(hours=2),
end_time=self.now + timedelta(hours=3),
custom_properties={"rule": {"id": 1, "name": "Daily News"}},
)
prog = self._create_program(hours_from_now=2)
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 1)
self.assertEqual(Recording.objects.count(), 2)
@patch("apps.channels.tasks.acquire_task_lock", return_value=True)
@patch("apps.channels.tasks.release_task_lock")
def test_recording_with_null_custom_properties_ignored(self, mock_release, mock_lock,
mock_schedule, mock_artwork):
"""Recordings with None custom_properties don't crash the dedup key builder."""
from apps.channels.tasks import evaluate_series_rules_impl
Recording.objects.create(
channel=self.channel,
start_time=self.now + timedelta(hours=2),
end_time=self.now + timedelta(hours=3),
custom_properties=None,
)
prog = self._create_program(hours_from_now=2)
result = evaluate_series_rules_impl()
self.assertEqual(result["scheduled"], 1)

385
apps/channels/tests/test_ts_proxy_ghost_clients.py Normal file
View file

@ -0,0 +1,385 @@
"""Tests for ghost client detection and cleanup.
Covers:
- ClientManager.remove_ghost_clients() pipelined EXISTS logic
- channel_status detailed stats path removes ghost clients from Redis SET
- channel_status basic stats path removes ghost clients and corrects count
- _check_orphaned_metadata() validates client SET entries and cleans up
channels where all clients are ghosts
"""
from unittest.mock import MagicMock, patch, PropertyMock
from django.test import TestCase
from apps.proxy.ts_proxy.client_manager import ClientManager
from apps.proxy.ts_proxy.constants import ChannelMetadataField, ChannelState
from apps.proxy.ts_proxy.redis_keys import RedisKeys
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
CHANNEL_ID = "00000000-0000-0000-0000-000000000001"
def _make_proxy_server(redis_client=None):
"""Create a minimal mock ProxyServer with a redis_client."""
server = MagicMock()
server.redis_client = redis_client or MagicMock()
server.stream_managers = {}
server.client_managers = {}
server.worker_id = "test-worker-1"
return server
def _metadata_for_channel(state="active"):
"""Return a plausible channel metadata dict (bytes keys/values)."""
return {
ChannelMetadataField.STATE.encode(): state.encode(),
ChannelMetadataField.URL.encode(): b"http://example.com/stream",
ChannelMetadataField.STREAM_PROFILE.encode(): b"default",
ChannelMetadataField.OWNER.encode(): b"test-worker-1",
ChannelMetadataField.INIT_TIME.encode(): b"1773500000.0",
}
# ---------------------------------------------------------------------------
# Unit tests for ClientManager.remove_ghost_clients()
# ---------------------------------------------------------------------------
class RemoveGhostClientsTests(TestCase):
"""Directly exercises the static method that all callers rely on."""
def test_ghost_removed_and_returned(self):
"""Client ID in SET with no metadata hash should be SREM'd."""
redis = MagicMock()
redis.smembers.return_value = {b"ghost_001"}
pipe = MagicMock()
redis.pipeline.return_value = pipe
pipe.execute.return_value = [False] # EXISTS → False
result = ClientManager.remove_ghost_clients(redis, CHANNEL_ID)
self.assertEqual(result, [b"ghost_001"])
redis.srem.assert_called_once()
def test_live_client_preserved(self):
"""Client with valid metadata hash should NOT be removed."""
redis = MagicMock()
redis.smembers.return_value = {b"live_001"}
pipe = MagicMock()
redis.pipeline.return_value = pipe
pipe.execute.return_value = [True] # EXISTS → True
result = ClientManager.remove_ghost_clients(redis, CHANNEL_ID)
self.assertEqual(result, [])
redis.srem.assert_not_called()
def test_mixed_ghost_and_live(self):
"""Only ghost clients should be removed; live ones preserved."""
redis = MagicMock()
redis.smembers.return_value = {b"ghost_001", b"live_001"}
pipe = MagicMock()
redis.pipeline.return_value = pipe
# Order matches list(smembers), which is non-deterministic —
# map both IDs so the test is stable regardless of iteration order.
client_id_list = list(redis.smembers.return_value)
def exists_results():
return [
b"ghost_001" not in cid.decode() == False
for cid in client_id_list
]
# Simpler: mock based on key content
def pipe_exists(key):
pass # just enqueued; results come from execute()
pipe.exists.side_effect = pipe_exists
pipe.execute.return_value = [
"live" in cid.decode() for cid in client_id_list
]
result = ClientManager.remove_ghost_clients(redis, CHANNEL_ID)
self.assertEqual(len(result), 1)
self.assertTrue(any(b"ghost" in cid for cid in result))
redis.srem.assert_called_once()
def test_empty_set_returns_empty(self):
"""No clients means nothing to clean."""
redis = MagicMock()
redis.smembers.return_value = set()
result = ClientManager.remove_ghost_clients(redis, CHANNEL_ID)
self.assertEqual(result, [])
redis.pipeline.assert_not_called()
def test_pre_fetched_client_ids_skips_smembers(self):
"""When client_ids is passed, SMEMBERS should not be called."""
redis = MagicMock()
pipe = MagicMock()
redis.pipeline.return_value = pipe
pipe.execute.return_value = [False]
pre_fetched = {b"ghost_001"}
result = ClientManager.remove_ghost_clients(
redis, CHANNEL_ID, client_ids=pre_fetched
)
redis.smembers.assert_not_called()
self.assertEqual(len(result), 1)
# ---------------------------------------------------------------------------
# Detailed stats path: exercises get_detailed_channel_info()
# ---------------------------------------------------------------------------
@patch("apps.proxy.ts_proxy.channel_status.ProxyServer")
class DetailedStatsGhostClientTests(TestCase):
"""get_detailed_channel_info() should remove ghost clients whose metadata
hash has expired from the Redis client SET."""
def _setup_redis(self, mock_proxy_cls, client_ids, hgetall_side_effect):
"""Wire up a mock ProxyServer with controlled Redis responses."""
redis = MagicMock()
server = _make_proxy_server(redis)
mock_proxy_cls.get_instance.return_value = server
redis.hgetall.side_effect = hgetall_side_effect
redis.smembers.return_value = client_ids
# buffer_index, ttl, exists all need safe defaults
redis.get.return_value = b"10"
redis.ttl.return_value = 300
redis.exists.return_value = True
return redis
def test_ghost_client_removed_from_set(self, mock_proxy_cls):
"""Ghost client should be SREM'd and excluded from result."""
from apps.proxy.ts_proxy.channel_status import ChannelStatus
def hgetall_side_effect(key):
if "clients:" in key:
return {} # ghost — metadata expired
return _metadata_for_channel()
redis = self._setup_redis(
mock_proxy_cls, {b"ghost_001"}, hgetall_side_effect
)
result = ChannelStatus.get_detailed_channel_info(CHANNEL_ID)
self.assertEqual(result['client_count'], 0)
self.assertEqual(len(result['clients']), 0)
redis.srem.assert_called_once()
def test_live_client_preserved(self, mock_proxy_cls):
"""Client with valid metadata should appear in results."""
from apps.proxy.ts_proxy.channel_status import ChannelStatus
def hgetall_side_effect(key):
if "clients:" in key:
return {
b'user_agent': b'VLC/3.0',
b'worker_id': b'test-worker-1',
b'connected_at': b'1773500000.0',
}
return _metadata_for_channel()
redis = self._setup_redis(
mock_proxy_cls, {b"live_001"}, hgetall_side_effect
)
result = ChannelStatus.get_detailed_channel_info(CHANNEL_ID)
self.assertEqual(result['client_count'], 1)
self.assertEqual(len(result['clients']), 1)
redis.srem.assert_not_called()
def test_mixed_ghost_and_live(self, mock_proxy_cls):
"""Only ghost clients should be removed; live ones preserved."""
from apps.proxy.ts_proxy.channel_status import ChannelStatus
def hgetall_side_effect(key):
if "clients:" in key:
if "ghost" in key:
return {}
return {
b'user_agent': b'VLC/3.0',
b'worker_id': b'test-worker-1',
}
return _metadata_for_channel()
redis = self._setup_redis(
mock_proxy_cls, {b"ghost_001", b"live_001"}, hgetall_side_effect
)
result = ChannelStatus.get_detailed_channel_info(CHANNEL_ID)
self.assertEqual(result['client_count'], 1)
self.assertEqual(len(result['clients']), 1)
redis.srem.assert_called_once()
# ---------------------------------------------------------------------------
# Basic stats path: exercises get_basic_channel_info()
# ---------------------------------------------------------------------------
@patch("apps.proxy.ts_proxy.channel_status.ProxyServer")
class BasicStatsGhostClientTests(TestCase):
"""get_basic_channel_info() should call remove_ghost_clients(), skip
ghosts from display, and correct client_count."""
def _setup_redis(self, mock_proxy_cls, client_ids, ghost_ids):
"""Wire up mock ProxyServer. ghost_ids controls which EXISTS return False."""
redis = MagicMock()
server = _make_proxy_server(redis)
mock_proxy_cls.get_instance.return_value = server
redis.hgetall.return_value = _metadata_for_channel()
redis.get.return_value = b"10" # buffer_index
redis.scard.return_value = len(client_ids)
redis.smembers.return_value = client_ids
redis.hget.return_value = None # individual field lookups
# Pipeline for remove_ghost_clients
pipe = MagicMock()
redis.pipeline.return_value = pipe
client_id_list = list(client_ids)
pipe.execute.return_value = [
cid not in ghost_ids for cid in client_id_list
]
return redis
def test_ghost_removed_and_count_corrected(self, mock_proxy_cls):
"""Ghost client should be cleaned and client_count decremented."""
from apps.proxy.ts_proxy.channel_status import ChannelStatus
redis = self._setup_redis(
mock_proxy_cls,
client_ids={b"ghost_001"},
ghost_ids={b"ghost_001"},
)
result = ChannelStatus.get_basic_channel_info(CHANNEL_ID)
self.assertIsNotNone(result)
self.assertEqual(result['client_count'], 0)
redis.srem.assert_called_once()
def test_live_client_count_preserved(self, mock_proxy_cls):
"""Live clients should be counted correctly."""
from apps.proxy.ts_proxy.channel_status import ChannelStatus
redis = self._setup_redis(
mock_proxy_cls,
client_ids={b"live_001"},
ghost_ids=set(),
)
result = ChannelStatus.get_basic_channel_info(CHANNEL_ID)
self.assertIsNotNone(result)
self.assertEqual(result['client_count'], 1)
redis.srem.assert_not_called()
# ---------------------------------------------------------------------------
# Orphaned channel cleanup: exercises _check_orphaned_metadata()
# ---------------------------------------------------------------------------
@patch("apps.proxy.ts_proxy.channel_status.ProxyServer")
class OrphanedChannelGhostValidationTests(TestCase):
"""_check_orphaned_metadata() should validate client SET entries when
owner is dead and client_count > 0. If all clients are ghosts, it
should clean up the channel."""
def _make_server_for_orphan_check(self, mock_proxy_cls, channel_id,
client_ids, ghost_ids, owner="dead-worker"):
"""Build a mock ProxyServer whose Redis state simulates an orphaned channel."""
redis = MagicMock()
server = _make_proxy_server(redis)
mock_proxy_cls.get_instance.return_value = server
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = _metadata_for_channel()
metadata[ChannelMetadataField.OWNER.encode()] = owner.encode()
# scan returns the one channel metadata key
redis.scan.return_value = (0, [metadata_key.encode()])
redis.hgetall.return_value = metadata
redis.scard.return_value = len(client_ids)
redis.smembers.return_value = client_ids
# Owner heartbeat is dead
redis.exists.side_effect = lambda key: (
False if "heartbeat" in key else True
)
# Pipeline for remove_ghost_clients
pipe = MagicMock()
redis.pipeline.return_value = pipe
client_id_list = list(client_ids)
pipe.execute.return_value = [
cid not in ghost_ids for cid in client_id_list
]
return server, redis
def test_all_ghosts_triggers_cleanup(self, mock_proxy_cls):
"""When all clients are ghosts, channel should be cleaned up."""
from apps.proxy.ts_proxy.server import ProxyServer
channel_id = "00000000-0000-0000-0000-000000000005"
server, redis = self._make_server_for_orphan_check(
mock_proxy_cls, channel_id,
client_ids={b"ghost_001", b"ghost_002"},
ghost_ids={b"ghost_001", b"ghost_002"},
)
# Call the real method on a real-ish ProxyServer
# The method lives on the server instance, so invoke it directly.
# We need to call _check_orphaned_metadata on the actual server mock,
# but it's a MagicMock. Instead, test via remove_ghost_clients directly
# and verify the cleanup decision logic.
stale_ids = ClientManager.remove_ghost_clients(redis, channel_id)
real_count = max(0, len({b"ghost_001", b"ghost_002"}) - len(stale_ids))
self.assertEqual(len(stale_ids), 2)
self.assertEqual(real_count, 0)
redis.srem.assert_called_once()
def test_mixed_preserves_live_clients(self, mock_proxy_cls):
"""When some clients are live, real_count should be > 0."""
channel_id = "00000000-0000-0000-0000-000000000006"
server, redis = self._make_server_for_orphan_check(
mock_proxy_cls, channel_id,
client_ids={b"ghost_001", b"live_001"},
ghost_ids={b"ghost_001"},
)
stale_ids = ClientManager.remove_ghost_clients(redis, channel_id)
real_count = max(0, 2 - len(stale_ids))
self.assertEqual(len(stale_ids), 1)
self.assertEqual(real_count, 1)
def test_no_ghosts_no_cleanup(self, mock_proxy_cls):
"""When all clients are live, no SREM should be called."""
channel_id = "00000000-0000-0000-0000-000000000007"
server, redis = self._make_server_for_orphan_check(
mock_proxy_cls, channel_id,
client_ids={b"live_001"},
ghost_ids=set(),
)
stale_ids = ClientManager.remove_ghost_clients(redis, channel_id)
self.assertEqual(len(stale_ids), 0)
redis.srem.assert_not_called()

231
apps/channels/tests/test_ts_proxy_initializing.py Normal file
View file

@ -0,0 +1,231 @@
"""Tests for stuck INITIALIZING state fix.
Covers:
- stream_manager.run() finally block: ownership check + state guard fallback
- ChannelState.PRE_ACTIVE contains the correct states
- INITIALIZING is included in the cleanup task grace period check
"""
import time
import threading
from unittest.mock import MagicMock, patch
from django.test import TestCase
from apps.proxy.ts_proxy.constants import ChannelMetadataField, ChannelState
from apps.proxy.ts_proxy.redis_keys import RedisKeys
from apps.proxy.ts_proxy.stream_manager import StreamManager
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
CHANNEL_ID = "00000000-0000-0000-0000-000000000001"
def _make_stream_manager(tried_stream_ids=None, max_retries=3):
"""Build a StreamManager via __new__ (bypasses __init__) with the
minimum attributes required by the run() finally block."""
sm = StreamManager.__new__(StreamManager)
sm.channel_id = CHANNEL_ID
sm.worker_id = "worker-1"
sm.max_retries = max_retries
sm.tried_stream_ids = tried_stream_ids if tried_stream_ids is not None else set()
sm.running = False # while-loop exits immediately
sm.connected = False
sm.transcode_process_active = False
sm._buffer_check_timers = []
sm.url = "http://example.com/stream"
sm.url_switching = False
sm.url_switch_start_time = 0
sm.url_switch_timeout = 30
sm.stop_requested = False
sm.stopping = False
sm.socket = None
sm.transcode_process = None
sm.current_response = None
sm.current_session = None
sm.current_stream_id = None
buffer = MagicMock()
buffer.redis_client = MagicMock()
buffer.channel_id = CHANNEL_ID
sm.buffer = buffer
return sm
def _run_finally_block(sm, owner_value, current_state):
"""Invoke StreamManager.run() so its finally block executes against real code.
Patches threading.Thread and ConfigHelper so the try-block is inert
(self.running=False makes the while-loop exit immediately).
Returns True if the finally block wrote ERROR to Redis.
"""
redis = sm.buffer.redis_client
# Mock the owner key GET — the finally block calls redis.get(owner_key)
def get_side_effect(key):
if "owner" in key:
return owner_value
return None
redis.get.side_effect = get_side_effect
# Mock hget for state field lookup in the PRE_ACTIVE guard
if current_state is not None:
redis.hget.return_value = current_state.encode('utf-8')
else:
redis.hget.return_value = None
# Reset hset so we can detect whether ERROR was written
redis.hset.reset_mock()
redis.setex.reset_mock()
with patch.object(threading, 'Thread', return_value=MagicMock()):
with patch('apps.proxy.ts_proxy.stream_manager.ConfigHelper') as mock_cfg:
mock_cfg.max_stream_switches.return_value = 0
mock_cfg.max_retries.return_value = sm.max_retries
sm.run()
# Check if hset was called with ERROR state
if redis.hset.called:
mapping = redis.hset.call_args[1].get('mapping', {})
return mapping.get(ChannelMetadataField.STATE) == ChannelState.ERROR
return False
# ---------------------------------------------------------------------------
# stream_manager.run() finally block: ownership + state guard behavior
# ---------------------------------------------------------------------------
class StreamManagerFinallyBlockTests(TestCase):
"""The run() finally block writes ERROR if the worker is still the owner
(normal case) OR if ownership expired and the channel is still in a
pre-active state (no new owner has taken over)."""
# --- Owner still valid: always write ERROR ---
def test_owner_writes_error_regardless_of_state(self):
"""When we're still the owner, always write ERROR."""
sm = _make_stream_manager()
owner = sm.worker_id.encode('utf-8')
self.assertTrue(_run_finally_block(sm, owner, ChannelState.ACTIVE))
def test_owner_writes_error_on_initializing(self):
"""Owner + INITIALIZING = write ERROR."""
sm = _make_stream_manager()
owner = sm.worker_id.encode('utf-8')
self.assertTrue(_run_finally_block(sm, owner, ChannelState.INITIALIZING))
mapping = sm.buffer.redis_client.hset.call_args[1]['mapping']
self.assertEqual(mapping[ChannelMetadataField.STATE], ChannelState.ERROR)
# --- Ownership expired, no new owner: use state guard ---
def test_no_owner_initializing_writes_error(self):
"""Ownership expired + INITIALIZING = write ERROR."""
sm = _make_stream_manager()
self.assertTrue(_run_finally_block(sm, None, ChannelState.INITIALIZING))
def test_no_owner_connecting_writes_error(self):
"""Ownership expired + CONNECTING = write ERROR."""
sm = _make_stream_manager()
self.assertTrue(_run_finally_block(sm, None, ChannelState.CONNECTING))
def test_no_owner_buffering_writes_error(self):
"""Ownership expired + BUFFERING = write ERROR."""
sm = _make_stream_manager()
self.assertTrue(_run_finally_block(sm, None, ChannelState.BUFFERING))
def test_no_owner_waiting_for_clients_writes_error(self):
"""Ownership expired + WAITING_FOR_CLIENTS = write ERROR."""
sm = _make_stream_manager()
self.assertTrue(_run_finally_block(sm, None, ChannelState.WAITING_FOR_CLIENTS))
def test_no_owner_active_does_not_write(self):
"""Ownership expired + ACTIVE = do NOT write ERROR."""
sm = _make_stream_manager()
self.assertFalse(_run_finally_block(sm, None, ChannelState.ACTIVE))
def test_no_owner_error_does_not_write(self):
"""Ownership expired + already ERROR = do NOT write again."""
sm = _make_stream_manager()
self.assertFalse(_run_finally_block(sm, None, ChannelState.ERROR))
def test_no_owner_no_state_does_not_write(self):
"""Ownership expired + no state metadata = do NOT write."""
sm = _make_stream_manager()
self.assertFalse(_run_finally_block(sm, None, None))
# --- New owner took over: never clobber ---
def test_new_owner_initializing_does_not_write(self):
"""Another worker owns the channel — do NOT clobber."""
sm = _make_stream_manager()
self.assertFalse(_run_finally_block(sm, b"other-worker", ChannelState.INITIALIZING))
def test_new_owner_active_does_not_write(self):
"""Another worker owns the channel and is ACTIVE — do NOT write."""
sm = _make_stream_manager()
self.assertFalse(_run_finally_block(sm, b"other-worker", ChannelState.ACTIVE))
# --- Stopping key and error messages ---
def test_stopping_key_set_on_error_update(self):
"""When ERROR is written, stopping key must also be set."""
sm = _make_stream_manager()
_run_finally_block(sm, None, ChannelState.INITIALIZING)
sm.buffer.redis_client.setex.assert_called_once()
args = sm.buffer.redis_client.setex.call_args[0]
self.assertIn("stopping", args[0])
self.assertEqual(args[1], 60)
def test_error_message_includes_stream_count(self):
"""When multiple streams were tried, error message reflects that."""
sm = _make_stream_manager(tried_stream_ids={1, 2, 3})
_run_finally_block(sm, None, ChannelState.INITIALIZING)
mapping = sm.buffer.redis_client.hset.call_args[1]['mapping']
error_msg = mapping[ChannelMetadataField.ERROR_MESSAGE]
self.assertIn("3 stream options failed", error_msg)
def test_error_message_with_no_streams_tried(self):
"""When no alternate streams were tried, shows retry count."""
sm = _make_stream_manager(tried_stream_ids=set(), max_retries=5)
_run_finally_block(sm, None, ChannelState.INITIALIZING)
mapping = sm.buffer.redis_client.hset.call_args[1]['mapping']
error_msg = mapping[ChannelMetadataField.ERROR_MESSAGE]
self.assertIn("5", error_msg)
# ---------------------------------------------------------------------------
# ChannelState.PRE_ACTIVE: verify contents and immutability
# ---------------------------------------------------------------------------
class PreActiveStateTests(TestCase):
"""Verify PRE_ACTIVE contains the correct states and is immutable."""
def test_initializing_in_pre_active(self):
self.assertIn(ChannelState.INITIALIZING, ChannelState.PRE_ACTIVE)
def test_connecting_in_pre_active(self):
self.assertIn(ChannelState.CONNECTING, ChannelState.PRE_ACTIVE)
def test_buffering_in_pre_active(self):
self.assertIn(ChannelState.BUFFERING, ChannelState.PRE_ACTIVE)
def test_waiting_for_clients_in_pre_active(self):
self.assertIn(ChannelState.WAITING_FOR_CLIENTS, ChannelState.PRE_ACTIVE)
def test_active_not_in_pre_active(self):
self.assertNotIn(ChannelState.ACTIVE, ChannelState.PRE_ACTIVE)
def test_error_not_in_pre_active(self):
self.assertNotIn(ChannelState.ERROR, ChannelState.PRE_ACTIVE)
def test_pre_active_is_frozenset(self):
self.assertIsInstance(ChannelState.PRE_ACTIVE, frozenset)

7
apps/channels/tests/test_ts_proxy_keepalive.py
View file

@ -85,6 +85,13 @@ class NonOwnerWorkerKeepaliveTests(TestCase):
gen.stream_manager = None # non-owner worker
gen.consecutive_empty = consecutive_empty
# Attributes added by health-check throttling (set in __init__)
gen._last_health_check_time = 0.0
gen._last_health_check_result = False
gen._health_check_interval = 2.0
gen.proxy_server = None
return gen
def _mock_proxy_server(self, last_data_value):

195
apps/channels/tests/test_ts_proxy_keepalive_duration.py Normal file
View file

@ -0,0 +1,195 @@
"""
Unit tests for the keepalive duration cap in StreamGenerator._stream_data_generator.
Verifies that a client held in keepalive mode is disconnected after
MAX_KEEPALIVE_DURATION seconds, and that the timer resets when real data resumes.
"""
import time
from unittest.mock import MagicMock, patch, call
from django.test import TestCase
def _make_generator(consecutive_empty=10, local_index=10, buffer_index=10):
"""Minimal StreamGenerator stub for testing _stream_data_generator logic."""
from apps.proxy.ts_proxy.stream_generator import StreamGenerator
gen = StreamGenerator.__new__(StreamGenerator)
gen.channel_id = "00000000-0000-0000-0000-000000000099"
gen.client_id = "test-client-duration"
gen.consecutive_empty = consecutive_empty
gen.empty_reads = 0
gen.local_index = local_index
gen.bytes_sent = 0
gen.chunks_sent = 0
gen.last_yield_time = time.time()
gen.stream_start_time = time.time()
gen.last_stats_time = time.time()
gen.last_stats_bytes = 0
gen.current_rate = 0.0
gen.last_ttl_refresh = time.time()
gen.ttl_refresh_interval = 3
gen.is_owner_worker = False
gen.stream_manager = None
gen._last_health_check_time = 0.0
gen._last_health_check_result = False
gen._health_check_interval = 2.0
gen.proxy_server = None
buffer = MagicMock()
buffer.index = buffer_index
buffer.get_optimized_client_data.return_value = ([], local_index)
buffer.find_oldest_available_chunk.return_value = None
gen.buffer = buffer
return gen
class KeepaliveDurationCapTests(TestCase):
"""MAX_KEEPALIVE_DURATION cap disconnects clients stuck in keepalive mode."""
def _run_generator_to_break(self, gen, max_iterations=20):
"""Drive _stream_data_generator until it breaks or hits iteration limit."""
iterations = 0
for _ in gen._stream_data_generator():
iterations += 1
if iterations >= max_iterations:
break
return iterations
def test_cap_fires_after_max_duration_exceeded(self):
"""Generator exits when keepalive has run longer than MAX_KEEPALIVE_DURATION."""
gen = _make_generator()
with patch.object(gen, '_check_resources', return_value=True), \
patch.object(gen, '_should_send_keepalive', return_value=True), \
patch.object(gen, '_is_ghost_client', return_value=False), \
patch.object(gen, '_is_timeout', return_value=False), \
patch('apps.proxy.ts_proxy.stream_generator.create_ts_packet', return_value=b'\x00' * 188), \
patch('apps.proxy.ts_proxy.stream_generator.ProxyServer') as MockPS, \
patch('apps.proxy.ts_proxy.stream_generator.Config') as MockConfig, \
patch('apps.proxy.ts_proxy.stream_generator.gevent') as mock_gevent, \
patch('apps.proxy.ts_proxy.stream_generator.time') as mock_time:
MockPS.get_instance.return_value = None
MockConfig.KEEPALIVE_INTERVAL = 0
MockConfig.MAX_KEEPALIVE_DURATION = 30
# First call: keepalive_start_time not yet set (returns current)
# Second call: inside the cap check — simulate time elapsed > 30s
mock_time.time.side_effect = [
1000.0, # keepalive_start_time assignment
1031.0, # cap check: 31s elapsed > 30s limit
]
packets = list(gen._stream_data_generator())
# No packets should be yielded — cap fires before yield
self.assertEqual(len(packets), 0)
def test_cap_does_not_fire_before_max_duration(self):
"""Generator yields keepalive packets while within MAX_KEEPALIVE_DURATION."""
gen = _make_generator()
call_count = 0
def time_side_effect():
nonlocal call_count
call_count += 1
# keepalive_start_time set at t=1000; cap checks always see <30s elapsed
if call_count == 1:
return 1000.0 # keepalive_start_time
return 1010.0 # always 10s elapsed — under the 30s cap
with patch.object(gen, '_check_resources', side_effect=[True, True, False]), \
patch.object(gen, '_should_send_keepalive', return_value=True), \
patch.object(gen, '_is_ghost_client', return_value=False), \
patch.object(gen, '_is_timeout', return_value=False), \
patch('apps.proxy.ts_proxy.stream_generator.create_ts_packet', return_value=b'\x00' * 188), \
patch('apps.proxy.ts_proxy.stream_generator.ProxyServer') as MockPS, \
patch('apps.proxy.ts_proxy.stream_generator.Config') as MockConfig, \
patch('apps.proxy.ts_proxy.stream_generator.gevent'), \
patch('apps.proxy.ts_proxy.stream_generator.time') as mock_time:
MockPS.get_instance.return_value = None
MockConfig.KEEPALIVE_INTERVAL = 0
MockConfig.MAX_KEEPALIVE_DURATION = 30
mock_time.time.side_effect = time_side_effect
packets = list(gen._stream_data_generator())
# Two iterations with _check_resources=True should yield two keepalive packets
self.assertGreater(len(packets), 0)
def test_timer_resets_when_real_data_resumes(self):
"""keepalive_start_time is cleared to None when real chunks are received."""
gen = _make_generator()
chunk = b'\x47' * 188
real_chunks = ([chunk], gen.local_index + 1)
no_chunks = ([], gen.local_index)
# Sequence: no data (keepalive), then real data, then stop
gen.buffer.get_optimized_client_data.side_effect = [
no_chunks, # iteration 1: keepalive
real_chunks, # iteration 2: real data — should reset timer
no_chunks, # iteration 3: keepalive again — timer restarts fresh
]
captured_start_times = []
original_gen = gen
with patch.object(gen, '_check_resources', side_effect=[True, True, True, False]), \
patch.object(gen, '_should_send_keepalive', return_value=True), \
patch.object(gen, '_is_ghost_client', return_value=False), \
patch.object(gen, '_is_timeout', return_value=False), \
patch.object(gen, '_process_chunks', return_value=iter([chunk])), \
patch('apps.proxy.ts_proxy.stream_generator.create_ts_packet', return_value=b'\x00' * 188), \
patch('apps.proxy.ts_proxy.stream_generator.ProxyServer') as MockPS, \
patch('apps.proxy.ts_proxy.stream_generator.Config') as MockConfig, \
patch('apps.proxy.ts_proxy.stream_generator.gevent'), \
patch('apps.proxy.ts_proxy.stream_generator.time') as mock_time:
MockPS.get_instance.return_value = None
MockConfig.KEEPALIVE_INTERVAL = 0
MockConfig.MAX_KEEPALIVE_DURATION = 300
mock_time.time.return_value = 1000.0
list(gen._stream_data_generator())
# Test passes if no exception and generator completes normally —
# if the timer were NOT reset, the second keepalive block would
# carry over the old start time rather than starting fresh.
def test_cap_uses_config_value(self):
"""Cap threshold reads MAX_KEEPALIVE_DURATION from Config, not a hardcoded value."""
gen = _make_generator()
with patch.object(gen, '_check_resources', return_value=True), \
patch.object(gen, '_should_send_keepalive', return_value=True), \
patch.object(gen, '_is_ghost_client', return_value=False), \
patch.object(gen, '_is_timeout', return_value=False), \
patch('apps.proxy.ts_proxy.stream_generator.create_ts_packet', return_value=b'\x00' * 188), \
patch('apps.proxy.ts_proxy.stream_generator.ProxyServer') as MockPS, \
patch('apps.proxy.ts_proxy.stream_generator.Config') as MockConfig, \
patch('apps.proxy.ts_proxy.stream_generator.gevent'), \
patch('apps.proxy.ts_proxy.stream_generator.time') as mock_time:
MockPS.get_instance.return_value = None
MockConfig.KEEPALIVE_INTERVAL = 0
# Set a custom cap of 60s
MockConfig.MAX_KEEPALIVE_DURATION = 60
mock_time.time.side_effect = [
1000.0, # keepalive_start_time
1050.0, # cap check: 50s elapsed — under 60s, should NOT fire
1000.0, # last_yield_time update
1070.0, # cap check on next iteration: 70s elapsed — fires
]
packets = list(gen._stream_data_generator())
# First iteration: 50s < 60s cap — one keepalive yielded
# Second iteration: 70s > 60s cap — generator exits
self.assertEqual(len(packets), 1)

30
apps/connect/api_views.py
View file

@ -1,9 +1,10 @@
from rest_framework import viewsets, status
from rest_framework import viewsets, status, serializers
from rest_framework.pagination import PageNumberPagination
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.response import Response
from rest_framework.decorators import action
from django.utils import timezone
from drf_spectacular.utils import extend_schema, inline_serializer
from .models import Integration, EventSubscription, DeliveryLog
from .serializers import (
IntegrationSerializer,
@ -37,6 +38,33 @@ class IntegrationViewSet(viewsets.ModelViewSet):
serializer = EventSubscriptionSerializer(qs, many=True)
return Response(serializer.data)
@extend_schema(
methods=["PUT"],
description=(
"Replace the integration's event subscriptions with the provided list. "
"Accepts a JSON array of subscription objects. "
"Existing subscriptions not in the list will be deleted. "
"The 'payload_template' field is only relevant for webhook integrations."
),
request=inline_serializer(
name="SetSubscriptionsRequest",
fields={
"event": serializers.CharField(help_text="Event name (e.g. 'channel_start')."),
"enabled": serializers.BooleanField(required=False, default=True),
"payload_template": serializers.CharField(required=False, allow_blank=True, allow_null=True, help_text="Custom payload template (webhook integrations only)."),
},
many=True,
),
responses={200: inline_serializer(
name="SetSubscriptionsResponse",
fields={
"event": serializers.CharField(),
"enabled": serializers.BooleanField(),
"payload_template": serializers.CharField(allow_null=True),
},
many=True,
)},
)
@action(detail=True, methods=["put"], url_path=r"subscriptions/set")
def set_subscriptions(self, request, pk=None):
"""

10
apps/epg/api_views.py
View file

@ -427,7 +427,13 @@ class EPGImportAPIView(APIView):
return [Authenticated()]
@extend_schema(
description="Triggers an EPG data import",
description="Triggers an EPG data refresh for the given source.",
request=inline_serializer(
name="EPGImportRequest",
fields={
"id": serializers.IntegerField(help_text="ID of the EPG source to refresh."),
},
),
)
def post(self, request, format=None):
logger.info("EPGImportAPIView: Received request to import EPG data.")
@ -449,7 +455,7 @@ class EPGImportAPIView(APIView):
refresh_epg_data.delay(epg_id) # Trigger Celery task
logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.")
return Response(
{"success": True, "message": "EPG data import initiated."},
{"success": True, "message": "EPG data refresh initiated."},
status=status.HTTP_202_ACCEPTED,
)

189
apps/epg/tasks.py
View file

@ -2,6 +2,7 @@
import logging
import gzip
import html.entities
import os
import uuid
import requests
@ -15,7 +16,7 @@ import zipfile
from celery import shared_task
from django.conf import settings
from django.db import transaction
from django.db import connection, transaction
from django.utils import timezone
from apps.channels.models import Channel
from core.models import UserAgent, CoreSettings
@ -28,6 +29,103 @@ from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, se
logger = logging.getLogger(__name__)
# DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named
# entities so lxml/libxml2 can resolve references like &eacute; correctly
# instead of silently dropping them in recovery mode.
# The 5 XML-predefined entities (amp, lt, gt, quot, apos) are always
# recognised by the XML spec and must not be redeclared.
_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'})
def _build_html_entity_doctype() -> bytes:
"""Build a DOCTYPE internal subset declaring all HTML 4 named entities."""
lines = [b'<!DOCTYPE tv [\n']
for name, codepoint in sorted(html.entities.name2codepoint.items()):
if name not in _XML_ENTITIES:
# Numeric character references are always valid XML regardless of codepoint.
lines.append(f'<!ENTITY {name} "&#x{codepoint:X};">\n'.encode('ascii'))
lines.append(b']>\n')
return b''.join(lines)
_HTML_ENTITY_DOCTYPE = _build_html_entity_doctype()
class _PrependStream:
"""Wraps an open binary file and prepends a bytes prefix to its content.
Used by _open_xmltv_file to inject a DOCTYPE entity block before the
file content reaches lxml's iterparse, with zero disk I/O.
"""
__slots__ = ('_prefix', '_prefix_pos', '_file')
def __init__(self, prefix: bytes, file_obj):
self._prefix = prefix
self._prefix_pos = 0
self._file = file_obj
def read(self, size=-1):
prefix_len = len(self._prefix)
if self._prefix_pos >= prefix_len:
return self._file.read(size)
remaining = prefix_len - self._prefix_pos
if size < 0:
chunk = self._prefix[self._prefix_pos:] + self._file.read()
self._prefix_pos = prefix_len
return chunk
if size <= remaining:
chunk = self._prefix[self._prefix_pos:self._prefix_pos + size]
self._prefix_pos += size
return chunk
chunk = self._prefix[self._prefix_pos:]
self._prefix_pos = prefix_len
return chunk + self._file.read(size - remaining)
def close(self):
self._file.close()
def __enter__(self):
return self
def __exit__(self, *_):
self.close()
def _open_xmltv_file(file_path: str):
"""Open an XMLTV file for lxml iterparse, injecting an HTML entity DOCTYPE.
Prepends a <!DOCTYPE tv [...]> block that declares all 252 HTML 4 named
entities so lxml/libxml2 resolves references like &eacute; correctly
instead of silently dropping them in recovery mode. This involves zero
disk I/O — the DOCTYPE is streamed in-memory before the file content.
If the file already contains a <!DOCTYPE> declaration the file is returned
unchanged; a second DOCTYPE would be invalid XML.
The caller is responsible for closing the returned object.
"""
f = open(file_path, 'rb')
start = f.read(512)
# Do not inject if the file already declares a DOCTYPE.
if b'<!DOCTYPE' in start or b'<!doctype' in start.lower():
f.seek(0)
return f
# Insert the DOCTYPE after the XML declaration if one is present.
xml_pos = start.find(b'<?xml')
if xml_pos >= 0:
decl_end = start.find(b'?>', xml_pos)
if decl_end >= 0:
xml_decl = start[:decl_end + 2]
f.seek(decl_end + 2)
return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f)
# No XML declaration — insert DOCTYPE at the very start of the file.
f.seek(0)
return _PrependStream(_HTML_ENTITY_DOCTYPE, f)
def validate_icon_url_fast(icon_url, max_length=None):
"""
@ -146,7 +244,7 @@ def refresh_all_epg_data():
return "EPG data refreshed."
@shared_task(time_limit=1800, soft_time_limit=1700)
@shared_task(time_limit=14400)
def refresh_epg_data(source_id):
if not acquire_task_lock('refresh_epg_data', source_id):
logger.debug(f"EPG refresh for {source_id} already running")
@ -397,42 +495,41 @@ def fetch_xmltv(source):
# Download to temporary file
with open(temp_download_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=16384): # Increased chunk size for better performance
if chunk:
f.write(chunk)
for chunk in response.iter_content(chunk_size=16384):
f.write(chunk)
downloaded += len(chunk)
elapsed_time = time.time() - start_time
downloaded += len(chunk)
elapsed_time = time.time() - start_time
# Calculate download speed in KB/s
speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0
# Calculate download speed in KB/s
speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0
# Calculate progress percentage
if total_size and total_size > 0:
progress = min(100, int((downloaded / total_size) * 100))
else:
# If no content length header, estimate progress
progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown
# Calculate progress percentage
if total_size and total_size > 0:
progress = min(100, int((downloaded / total_size) * 100))
else:
# If no content length header, estimate progress
progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown
# Time remaining (in seconds)
time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0
# Time remaining (in seconds)
time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0
# Only send updates at specified intervals to avoid flooding
current_time = time.time()
if current_time - last_update_time >= update_interval and progress > 0:
last_update_time = current_time
send_epg_update(
source.id,
"downloading",
progress,
speed=round(speed, 2),
elapsed_time=round(elapsed_time, 1),
time_remaining=round(time_remaining, 1),
downloaded=f"{downloaded / (1024 * 1024):.2f} MB"
)
# Only send updates at specified intervals to avoid flooding
current_time = time.time()
if current_time - last_update_time >= update_interval and progress > 0:
last_update_time = current_time
send_epg_update(
source.id,
"downloading",
progress,
speed=round(speed, 2),
elapsed_time=round(elapsed_time, 1),
time_remaining=round(time_remaining, 1),
downloaded=f"{downloaded / (1024 * 1024):.2f} MB"
)
# Explicitly delete the chunk to free memory immediately
del chunk
# Explicitly delete the chunk to free memory immediately
del chunk
# Send completion notification
send_epg_update(source.id, "downloading", 100)
@ -526,6 +623,7 @@ def fetch_xmltv(source):
source.save(update_fields=['status'])
logger.info(f"Cached EPG file saved to {source.file_path}")
return True
except requests.exceptions.HTTPError as e:
@ -840,7 +938,8 @@ def parse_channels_only(source):
# Replace full dictionary load with more efficient lookup set
existing_tvg_ids = set()
existing_epgs = {} # Initialize the dictionary that will lazily load objects
existing_epgs = {}
scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup
last_id = 0
chunk_size = 5000
@ -888,7 +987,7 @@ def parse_channels_only(source):
# Open the file - no need to check file type since it's always XML now
logger.debug(f"Opening file for channel parsing: {file_path}")
source_file = open(file_path, 'rb')
source_file = _open_xmltv_file(file_path)
if process:
logger.debug(f"[parse_channels_only] Memory after opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB")
@ -908,6 +1007,7 @@ def parse_channels_only(source):
channel_count += 1
tvg_id = elem.get('id', '').strip()
if tvg_id:
scanned_tvg_ids.add(tvg_id)
display_name = None
icon_url = None
for child in elem:
@ -1055,6 +1155,16 @@ def parse_channels_only(source):
if epgs_to_update:
EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"])
logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries")
# Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel.
# Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list.
potentially_stale = existing_tvg_ids - scanned_tvg_ids
if potentially_stale:
stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True)
deleted_count, _ = stale_qs.delete()
if deleted_count:
logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel")
if process:
logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB")
@ -1121,6 +1231,9 @@ def parse_channels_only(source):
existing_epgs = None
epgs_to_create = None
epgs_to_update = None
if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None:
scanned_tvg_ids.clear()
scanned_tvg_ids = None
cleanup_memory(log_usage=should_log_memory, force_collection=True)
except Exception as e:
logger.warning(f"Cleanup error: {e}")
@ -1271,7 +1384,7 @@ def parse_programs_for_tvg_id(epg_id):
try:
# Open the file directly - no need to check compression
logger.debug(f"Opening file for parsing: {file_path}")
source_file = open(file_path, 'rb')
source_file = _open_xmltv_file(file_path)
# Stream parse the file using lxml's iterparse
program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True)
@ -1544,7 +1657,7 @@ def parse_programs_for_source(epg_source, tvg_id=None):
try:
logger.debug(f"Opening file for single-pass parsing: {file_path}")
source_file = open(file_path, 'rb')
source_file = _open_xmltv_file(file_path)
# Stream parse the file using lxml's iterparse
program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True)
@ -1657,6 +1770,10 @@ def parse_programs_for_source(epg_source, tvg_id=None):
batch_size = 1000
try:
with transaction.atomic():
# Kill any individual statement that hangs longer than 10 minutes.
# SET LOCAL automatically resets when this transaction ends (commit or rollback).
with connection.cursor() as cursor:
cursor.execute("SET LOCAL statement_timeout = '10min'")
# Delete existing programs for mapped EPGs
deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0]
logger.debug(f"Deleted {deleted_count} existing programs")

195
apps/epg/tests/test_entity_resolution.py Normal file
View file

@ -0,0 +1,195 @@
import os
import tempfile
from django.test import TestCase
from apps.epg.tasks import (
_NAMED_ENTITY_RE,
_detect_xml_encoding,
_replace_html_entity,
_resolve_html_entities,
)
class ReplaceHtmlEntityTests(TestCase):
"""Tests for the regex callback that resolves individual HTML entities."""
def _sub(self, text):
return _NAMED_ENTITY_RE.sub(_replace_html_entity, text)
def test_french_accented(self):
self.assertEqual(self._sub("Cha&icirc;ne T&eacute;l&eacute;"), "Chaîne Télé")
def test_german_umlauts(self):
self.assertEqual(self._sub("M&uuml;nchen &Uuml;bersicht &szlig;"), "München Übersicht ß")
def test_spanish(self):
self.assertEqual(self._sub("Espa&ntilde;a &iquest;Qu&eacute;?"), "España ¿Qué?")
def test_portuguese(self):
self.assertEqual(self._sub("Comunica&ccedil;&atilde;o"), "Comunicação")
def test_scandinavian(self):
self.assertEqual(self._sub("Norsk &oslash; &aring; &aelig;"), "Norsk ø å æ")
def test_greek_letters(self):
self.assertEqual(self._sub("&alpha;&beta;&gamma;"), "αβγ")
def test_currency_and_symbols(self):
self.assertEqual(self._sub("&copy; &euro; &pound; &yen;"), "© € £ ¥")
def test_preserves_xml_amp(self):
self.assertEqual(self._sub("A &amp; B"), "A &amp; B")
def test_preserves_xml_lt_gt(self):
self.assertEqual(self._sub("&lt;tag&gt;"), "&lt;tag&gt;")
def test_preserves_xml_quot_apos(self):
self.assertEqual(self._sub("&quot;hello&apos;"), "&quot;hello&apos;")
def test_preserves_uppercase_xml_entities(self):
"""&AMP;, &LT;, &GT;, &QUOT; resolve to XML-special chars; must not be replaced."""
self.assertEqual(self._sub("&AMP;"), "&AMP;")
self.assertEqual(self._sub("&LT;"), "&LT;")
self.assertEqual(self._sub("&GT;"), "&GT;")
self.assertEqual(self._sub("&QUOT;"), "&QUOT;")
def test_partial_entity_match_preserved(self):
"""html.unescape can partially match &amp inside &ampersand; — must not corrupt."""
self.assertEqual(self._sub("&ampersand;"), "&ampersand;")
def test_mixed_html_and_xml_entities(self):
self.assertEqual(
self._sub("R&eacute;sum&eacute; &amp; Co &lt;test&gt;"),
"Résumé &amp; Co &lt;test&gt;",
)
def test_plain_ascii_unchanged(self):
self.assertEqual(self._sub("Plain ASCII text"), "Plain ASCII text")
def test_direct_utf8_unchanged(self):
self.assertEqual(self._sub("日本語テレビ"), "日本語テレビ")
def test_unknown_entity_preserved(self):
self.assertEqual(self._sub("&zzfakeentity;"), "&zzfakeentity;")
class ResolveHtmlEntitiesFileTests(TestCase):
"""Tests for the file-level preprocessing function."""
def _make_file(self, content):
fd, path = tempfile.mkstemp(suffix=".xml")
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(content)
self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None)
return path
def test_resolves_entities_in_file(self):
path = self._make_file(
'<?xml version="1.0"?>\n<tv><channel><display-name>T&eacute;l&eacute;</display-name></channel></tv>'
)
_resolve_html_entities(path)
with open(path, "r", encoding="utf-8") as f:
content = f.read()
self.assertIn("Télé", content)
self.assertNotIn("&eacute;", content)
def test_preserves_xml_entities_in_file(self):
path = self._make_file("<tv><desc>A &amp; B &lt;C&gt;</desc></tv>")
_resolve_html_entities(path)
with open(path, "r", encoding="utf-8") as f:
content = f.read()
self.assertIn("&amp;", content)
self.assertIn("&lt;", content)
self.assertIn("&gt;", content)
def test_no_temp_file_left_on_success(self):
path = self._make_file("<tv>test</tv>")
_resolve_html_entities(path)
self.assertFalse(os.path.exists(path + ".entity_tmp"))
def test_plain_file_unchanged(self):
original = '<?xml version="1.0"?>\n<tv><channel><display-name>Plain</display-name></channel></tv>'
path = self._make_file(original)
_resolve_html_entities(path)
with open(path, "r", encoding="utf-8") as f:
content = f.read()
self.assertEqual(content, original)
def test_utf8_content_preserved(self):
original = "<tv><channel><display-name>日本語テレビ</display-name></channel></tv>"
path = self._make_file(original)
_resolve_html_entities(path)
with open(path, "r", encoding="utf-8") as f:
content = f.read()
self.assertIn("日本語テレビ", content)
def test_iso_8859_1_encoding(self):
"""Files declaring ISO-8859-1 should be read in that encoding."""
xml = '<?xml version="1.0" encoding="ISO-8859-1"?>\n<tv><channel><display-name>Cha&icirc;ne</display-name></channel></tv>'
fd, path = tempfile.mkstemp(suffix=".xml")
with os.fdopen(fd, "wb") as f:
f.write(xml.encode("iso-8859-1"))
self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None)
_resolve_html_entities(path)
with open(path, "r", encoding="iso-8859-1") as f:
content = f.read()
self.assertIn("Cha\u00eene", content)
self.assertNotIn("&icirc;", content)
def test_detect_encoding_utf8_default(self):
"""Headers without an encoding declaration default to UTF-8."""
self.assertEqual(_detect_xml_encoding(b'<?xml version="1.0"?>'), "utf-8")
def test_detect_encoding_iso_8859_1(self):
"""Encoding is read from the XML declaration."""
self.assertEqual(
_detect_xml_encoding(b'<?xml version="1.0" encoding="ISO-8859-1"?>'),
"ISO-8859-1",
)
def test_detect_encoding_single_quotes(self):
"""Encoding detection works with single-quoted attributes."""
self.assertEqual(
_detect_xml_encoding(b"<?xml version='1.0' encoding='windows-1252'?>"),
"windows-1252",
)
def test_detect_encoding_unknown_falls_back(self):
"""Unrecognized encoding falls back to UTF-8."""
self.assertEqual(
_detect_xml_encoding(b'<?xml version="1.0" encoding="x-fake-codec"?>'),
"utf-8",
)
def test_iso_8859_1_with_entities_roundtrip(self):
"""ISO-8859-1 file with entities: resolved without corrupting existing accented chars."""
# Mix of direct ISO-8859-1 chars and HTML entities
xml_str = '<?xml version="1.0" encoding="ISO-8859-1"?>\n<tv><channel><display-name>D\xe9j\xe0 &eacute;mission</display-name></channel></tv>'
fd, path = tempfile.mkstemp(suffix=".xml")
with os.fdopen(fd, "wb") as f:
f.write(xml_str.encode("iso-8859-1"))
self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None)
_resolve_html_entities(path)
with open(path, "r", encoding="iso-8859-1") as f:
content = f.read()
self.assertIn("D\xe9j\xe0", content, "Existing accented chars should be preserved")
self.assertIn("\xe9mission", content, "Entity should be resolved")
self.assertNotIn("&eacute;", content)
def test_mismatched_encoding_leaves_file_untouched(self):
"""File declaring UTF-8 but containing Latin-1 bytes is left alone."""
# \xe9 is valid ISO-8859-1 but invalid as a standalone UTF-8 byte
raw = b'<?xml version="1.0" encoding="UTF-8"?>\n<tv><channel><display-name>\xe9</display-name></channel></tv>'
fd, path = tempfile.mkstemp(suffix=".xml")
with os.fdopen(fd, "wb") as f:
f.write(raw)
self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None)
original_bytes = raw # save for comparison
_resolve_html_entities(path)
with open(path, "rb") as f:
result_bytes = f.read()
self.assertEqual(result_bytes, original_bytes, "File should be untouched on decode error")

5
apps/hdhr/api_views.py
View file

@ -1,6 +1,7 @@
from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.permissions import AllowAny
from apps.accounts.permissions import Authenticated, permission_classes_by_action
from django.http import JsonResponse, HttpResponseForbidden, HttpResponse
import logging
@ -46,6 +47,7 @@ class HDHRDeviceViewSet(viewsets.ModelViewSet):
# 🔹 2) Discover API
class DiscoverAPIView(APIView):
"""Returns device discovery information"""
permission_classes = [AllowAny]
@extend_schema(
description="Retrieve HDHomeRun device discovery information",
@ -98,6 +100,7 @@ class DiscoverAPIView(APIView):
# 🔹 3) Lineup API
class LineupAPIView(APIView):
"""Returns available channel lineup"""
permission_classes = [AllowAny]
@extend_schema(
description="Retrieve the available channel lineup",
@ -138,6 +141,7 @@ class LineupAPIView(APIView):
# 🔹 4) Lineup Status API
class LineupStatusAPIView(APIView):
"""Returns the current status of the HDHR lineup"""
permission_classes = [AllowAny]
@extend_schema(
description="Retrieve the HDHomeRun lineup status",
@ -155,6 +159,7 @@ class LineupStatusAPIView(APIView):
# 🔹 5) Device XML API
class HDHRDeviceXMLAPIView(APIView):
"""Returns HDHomeRun device configuration in XML"""
permission_classes = [AllowAny]
@extend_schema(
description="Retrieve the HDHomeRun device XML configuration",

19
apps/m3u/api_views.py
View file

@ -20,6 +20,7 @@ import json
from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile
from core.models import UserAgent
from core.utils import safe_upload_path
from apps.channels.models import ChannelGroupM3UAccount
from core.serializers import UserAgentSerializer
from apps.vod.models import M3UVODCategoryRelation
@ -40,7 +41,7 @@ class M3UAccountViewSet(viewsets.ModelViewSet):
queryset = M3UAccount.objects.select_related(
"refresh_task__crontab", "refresh_task__interval"
).prefetch_related("channel_group")
).prefetch_related("channel_group", "profiles")
serializer_class = M3UAccountSerializer
def get_permissions(self):
@ -54,10 +55,12 @@ class M3UAccountViewSet(viewsets.ModelViewSet):
file_path = None
if "file" in request.FILES:
file = request.FILES["file"]
file_name = file.name
file_path = os.path.join("/data/uploads/m3us", file_name)
try:
file_path = safe_upload_path(file.name, "/data/uploads/m3us")
except ValueError:
return Response({"detail": "Invalid filename."}, status=status.HTTP_400_BAD_REQUEST)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
os.makedirs("/data/uploads/m3us", exist_ok=True)
with open(file_path, "wb+") as destination:
for chunk in file.chunks():
destination.write(chunk)
@ -117,10 +120,12 @@ class M3UAccountViewSet(viewsets.ModelViewSet):
file_path = None
if "file" in request.FILES:
file = request.FILES["file"]
file_name = file.name
file_path = os.path.join("/data/uploads/m3us", file_name)
try:
file_path = safe_upload_path(file.name, "/data/uploads/m3us")
except ValueError:
return Response({"detail": "Invalid filename."}, status=status.HTTP_400_BAD_REQUEST)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
os.makedirs("/data/uploads/m3us", exist_ok=True)
with open(file_path, "wb+") as destination:
for chunk in file.chunks():
destination.write(chunk)

57
apps/m3u/migrations/0019_m3uaccountprofile_exp_date.py Normal file
View file

@ -0,0 +1,57 @@
# Generated by Django 6.0.3 on 2026-03-14 19:41
from datetime import datetime, timezone
from django.db import migrations, models
def populate_exp_date_from_custom_properties(apps, schema_editor):
"""Backfill exp_date from custom_properties['user_info']['exp_date']."""
M3UAccountProfile = apps.get_model('m3u', 'M3UAccountProfile')
profiles_to_update = []
for profile in M3UAccountProfile.objects.filter(
custom_properties__isnull=False,
).exclude(custom_properties={}):
user_info = profile.custom_properties.get('user_info', {})
raw_exp = user_info.get('exp_date')
if raw_exp is None:
continue
parsed = None
try:
if isinstance(raw_exp, (int, float)):
parsed = datetime.fromtimestamp(float(raw_exp), tz=timezone.utc)
elif isinstance(raw_exp, str):
try:
parsed = datetime.fromtimestamp(float(raw_exp), tz=timezone.utc)
except ValueError:
parsed = datetime.fromisoformat(raw_exp)
except (ValueError, TypeError, OSError):
pass
if parsed is not None:
profile.exp_date = parsed
profiles_to_update.append(profile)
if profiles_to_update:
M3UAccountProfile.objects.bulk_update(profiles_to_update, ['exp_date'], batch_size=500)
class Migration(migrations.Migration):
dependencies = [
('m3u', '0018_add_profile_custom_properties'),
]
operations = [
migrations.AddField(
model_name='m3uaccountprofile',
name='exp_date',
field=models.DateTimeField(blank=True, help_text='Account expiration date, auto-synced from custom_properties on save', null=True),
),
migrations.RunPython(
populate_exp_date_from_custom_properties,
reverse_code=migrations.RunPython.noop,
),
]

79
apps/m3u/models.py
View file

@ -1,3 +1,4 @@
from datetime import datetime, timezone
from django.db import models
from django.core.exceptions import ValidationError
from core.models import UserAgent
@ -264,11 +265,16 @@ class M3UAccountProfile(models.Model):
)
current_viewers = models.PositiveIntegerField(default=0)
custom_properties = models.JSONField(
default=dict,
blank=True,
null=True,
default=dict,
blank=True,
null=True,
help_text="Custom properties for storing account information from provider (e.g., XC account details, expiration dates)"
)
exp_date = models.DateTimeField(
null=True,
blank=True,
help_text="Account expiration date, auto-synced from custom_properties on save",
)
class Meta:
constraints = [
@ -280,36 +286,51 @@ class M3UAccountProfile(models.Model):
def __str__(self):
return f"{self.name} ({self.m3u_account.name})"
def get_account_expiration(self):
"""Get account expiration date from custom properties if available"""
def save(self, *args, **kwargs):
"""Auto-sync exp_date from custom_properties for XC accounts on every save.
For non-XC accounts, exp_date is set directly and left untouched here."""
parsed = self._parse_exp_date_from_custom_properties()
if parsed is not None:
# XC account with exp_date in custom_properties — always sync
self.exp_date = parsed
# else: keep whatever exp_date is already set (manual entry for non-XC)
super().save(*args, **kwargs)
@staticmethod
def _parse_exp_date(raw_value):
"""Parse a raw exp_date value (unix timestamp or ISO string) into a datetime."""
if raw_value is None:
return None
try:
if isinstance(raw_value, (int, float)):
return datetime.fromtimestamp(float(raw_value), tz=timezone.utc)
elif isinstance(raw_value, str):
try:
return datetime.fromtimestamp(float(raw_value), tz=timezone.utc)
except ValueError:
return datetime.fromisoformat(raw_value)
except (ValueError, TypeError, OSError):
pass
return None
def _parse_exp_date_from_custom_properties(self):
"""Extract exp_date from custom_properties JSON."""
if not self.custom_properties:
return None
user_info = self.custom_properties.get('user_info', {})
exp_date = user_info.get('exp_date')
if exp_date:
try:
from datetime import datetime
# XC exp_date is typically a Unix timestamp
if isinstance(exp_date, (int, float)):
return datetime.fromtimestamp(exp_date)
elif isinstance(exp_date, str):
# Try to parse as timestamp first, then as ISO date
try:
return datetime.fromtimestamp(float(exp_date))
except ValueError:
return datetime.fromisoformat(exp_date)
except (ValueError, TypeError):
pass
return None
return self._parse_exp_date(user_info.get('exp_date'))
def get_account_expiration(self):
"""Get account expiration date — uses the dedicated field if set, otherwise parses JSON."""
if self.exp_date:
return self.exp_date
return self._parse_exp_date_from_custom_properties()
def get_account_status(self):
"""Get account status from custom properties if available"""
if not self.custom_properties:
return None
user_info = self.custom_properties.get('user_info', {})
return user_info.get('status')
@ -317,7 +338,7 @@ class M3UAccountProfile(models.Model):
"""Get maximum connections from custom properties if available"""
if not self.custom_properties:
return None
user_info = self.custom_properties.get('user_info', {})
return user_info.get('max_connections')
@ -325,7 +346,7 @@ class M3UAccountProfile(models.Model):
"""Get active connections from custom properties if available"""
if not self.custom_properties:
return None
user_info = self.custom_properties.get('user_info', {})
return user_info.get('active_cons')
@ -333,7 +354,7 @@ class M3UAccountProfile(models.Model):
"""Get last refresh timestamp from custom properties if available"""
if not self.custom_properties:
return None
last_refresh = self.custom_properties.get('last_refresh')
if last_refresh:
try:
@ -341,7 +362,7 @@ class M3UAccountProfile(models.Model):
return datetime.fromisoformat(last_refresh)
except (ValueError, TypeError):
pass
return None

85
apps/m3u/serializers.py
View file

@ -7,6 +7,7 @@ from apps.channels.models import ChannelGroup, ChannelGroupM3UAccount
from apps.channels.serializers import (
ChannelGroupM3UAccountSerializer,
)
from datetime import timezone as dt_tz
import logging
import json
@ -52,12 +53,14 @@ class M3UAccountProfileSerializer(serializers.ModelSerializer):
"search_pattern",
"replace_pattern",
"custom_properties",
"exp_date",
"account",
]
read_only_fields = ["id", "account"]
extra_kwargs = {
'search_pattern': {'required': False, 'allow_blank': True},
'replace_pattern': {'required': False, 'allow_blank': True},
'exp_date': {'required': False, 'allow_null': True},
}
def create(self, validated_data):
@ -90,14 +93,14 @@ class M3UAccountProfileSerializer(serializers.ModelSerializer):
def update(self, instance, validated_data):
if instance.is_default:
# For default profiles, only allow updating name and custom_properties (for notes)
allowed_fields = {'name', 'custom_properties'}
# For default profiles, only allow updating name, custom_properties, and exp_date
allowed_fields = {'name', 'custom_properties', 'exp_date'}
# Remove any fields that aren't allowed for default profiles
disallowed_fields = set(validated_data.keys()) - allowed_fields
if disallowed_fields:
raise serializers.ValidationError(
f"Default profiles can only modify name and notes. "
f"Default profiles can only modify name, notes, and expiration. "
f"Cannot modify: {', '.join(disallowed_fields)}"
)
@ -117,6 +120,12 @@ class M3UAccountSerializer(serializers.ModelSerializer):
"""Serializer for M3U Account"""
filters = serializers.SerializerMethodField()
earliest_expiration = serializers.SerializerMethodField()
all_expirations = serializers.SerializerMethodField()
exp_date = serializers.DateTimeField(
required=False, allow_null=True, write_only=True,
help_text="Expiration date for the default profile (write-through)",
)
# Include user_agent as a mandatory field using its primary key.
user_agent = serializers.PrimaryKeyRelatedField(
queryset=UserAgent.objects.all(),
@ -172,6 +181,9 @@ class M3UAccountSerializer(serializers.ModelSerializer):
"auto_enable_new_groups_live",
"auto_enable_new_groups_vod",
"auto_enable_new_groups_series",
"earliest_expiration",
"all_expirations",
"exp_date",
]
extra_kwargs = {
"password": {
@ -200,9 +212,24 @@ class M3UAccountSerializer(serializers.ModelSerializer):
ct = instance.refresh_task.crontab
cron_expr = f"{ct.minute} {ct.hour} {ct.day_of_month} {ct.month_of_year} {ct.day_of_week}"
data["cron_expression"] = cron_expr
# Surface default profile's exp_date for the form.
# Use prefetch cache (obj.profiles.all()) to avoid an extra query per account.
# Always emit a Z-suffix UTC string so JS new Date() never misinterprets it as local.
default_profile = next((p for p in instance.profiles.all() if p.is_default), None)
exp = default_profile.exp_date if default_profile else None
if exp:
exp_utc = exp.astimezone(dt_tz.utc) if exp.tzinfo else exp.replace(tzinfo=dt_tz.utc)
data["exp_date"] = exp_utc.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
data["exp_date"] = None
return data
def update(self, instance, validated_data):
# Pop exp_date — it's written to the default profile, not the account
exp_date = validated_data.pop("exp_date", "__NOT_SET__")
# Pop cron_expression before it reaches model fields
# If not present (partial update), preserve the existing cron from the PeriodicTask
if "cron_expression" in validated_data:
@ -264,9 +291,26 @@ class M3UAccountSerializer(serializers.ModelSerializer):
memberships_to_update, ["enabled"]
)
# Write exp_date through to the default profile.
# Use a fresh DB query (not the prefetch cache) so we get the profile
# object AFTER the post_save signal (create_profile_for_m3u_account)
# has already updated max_streams, avoiding a stale-value overwrite.
if exp_date != "__NOT_SET__":
default_profile = instance.profiles.filter(is_default=True).first()
if default_profile:
default_profile.exp_date = exp_date
default_profile.save(update_fields=['exp_date'])
# Invalidate the profiles prefetch cache so to_representation
# sees the updated exp_date rather than the pre-request snapshot.
if '_prefetched_objects_cache' in instance.__dict__:
instance._prefetched_objects_cache.pop('profiles', None)
return instance
def create(self, validated_data):
# Pop exp_date — it's written to the default profile after creation
exp_date = validated_data.pop("exp_date", None)
# Pop cron_expression — it's not a model field
cron_expr = validated_data.pop("cron_expression", "")
@ -290,12 +334,47 @@ class M3UAccountSerializer(serializers.ModelSerializer):
instance = M3UAccount(**validated_data)
instance._cron_expression = cron_expr
instance.save()
# Write exp_date through to the default profile created by post_save signal
if exp_date is not None:
default_profile = instance.profiles.filter(is_default=True).first()
if default_profile:
default_profile.exp_date = exp_date
default_profile.save()
return instance
def get_filters(self, obj):
filters = obj.filters.order_by("order")
return M3UFilterSerializer(filters, many=True).data
def get_earliest_expiration(self, obj):
"""Return the soonest exp_date across all active profiles for this account."""
# Filter in Python over the prefetch cache to avoid an extra query per account.
expiring = [p.exp_date for p in obj.profiles.all() if p.is_active and p.exp_date]
if not expiring:
return None
exp = min(expiring)
exp_utc = exp.astimezone(dt_tz.utc) if exp.tzinfo else exp.replace(tzinfo=dt_tz.utc)
return exp_utc.strftime('%Y-%m-%dT%H:%M:%SZ')
def get_all_expirations(self, obj):
"""Return exp_date info for every profile that has one (for tooltip)."""
# Filter in Python over the prefetch cache to avoid an extra query per account.
profiles = sorted(
(p for p in obj.profiles.all() if p.exp_date),
key=lambda p: p.exp_date,
)
return [
{
"profile_id": p.id,
"profile_name": p.name,
"exp_date": (p.exp_date.astimezone(dt_tz.utc) if p.exp_date.tzinfo else p.exp_date.replace(tzinfo=dt_tz.utc)).strftime('%Y-%m-%dT%H:%M:%SZ'),
"is_active": p.is_active,
}
for p in profiles
]
class ServerGroupSerializer(serializers.ModelSerializer):
"""Serializer for Server Group"""

86
apps/m3u/signals.py
View file

@ -1,7 +1,7 @@
# apps/m3u/signals.py
from django.db.models.signals import post_save, post_delete, pre_save
from django.dispatch import receiver
from .models import M3UAccount
from .models import M3UAccount, M3UAccountProfile
from .tasks import refresh_single_m3u_account, refresh_m3u_groups, delete_m3u_refresh_task_by_id
from core.scheduling import create_or_update_periodic_task, delete_periodic_task
import json
@ -68,6 +68,62 @@ def create_or_update_refresh_task(sender, instance, created, update_fields=None,
if instance.refresh_task_id != task.id:
M3UAccount.objects.filter(id=instance.id).update(refresh_task=task)
@receiver(post_save, sender=M3UAccountProfile)
def update_profile_expiration_notification(sender, instance, created, update_fields=None, **kwargs):
"""
When a profile's exp_date is set or changed, immediately update its expiration notification
so the frontend reflects the new state without waiting for the daily celery task.
"""
# Only act when exp_date was involved in the save
if not created and update_fields is not None and "exp_date" not in update_fields:
return
try:
if not instance.exp_date:
# exp_date was cleared — remove any existing notifications immediately
from core.models import SystemNotification
from core.utils import send_notification_dismissed
keys = [f"xc-exp-warning-{instance.id}", f"xc-exp-expired-{instance.id}"]
deleted_keys = list(
SystemNotification.objects.filter(notification_key__in=keys)
.values_list("notification_key", flat=True)
)
SystemNotification.objects.filter(notification_key__in=deleted_keys).delete()
for key in deleted_keys:
send_notification_dismissed(key)
return
from apps.m3u.tasks import evaluate_profile_expiration_notification
evaluate_profile_expiration_notification(instance)
except Exception as e:
logger.error(f"Error updating expiration notification for profile {instance.id}: {str(e)}")
@receiver(post_delete, sender=M3UAccountProfile)
def cleanup_profile_notifications(sender, instance, **kwargs):
"""
Delete expiration notifications for a profile when it is deleted.
Handles both direct deletion and cascade deletion from M3UAccount.
"""
try:
from core.models import SystemNotification
from core.utils import send_notification_dismissed
keys = [f"xc-exp-warning-{instance.id}", f"xc-exp-expired-{instance.id}"]
deleted_keys = list(
SystemNotification.objects.filter(notification_key__in=keys)
.values_list("notification_key", flat=True)
)
if deleted_keys:
SystemNotification.objects.filter(notification_key__in=deleted_keys).delete()
for key in deleted_keys:
send_notification_dismissed(key)
logger.debug(f"Cleaned up {len(deleted_keys)} notifications for deleted profile {instance.id}")
except Exception as e:
logger.error(f"Error cleaning up notifications for profile {instance.id}: {str(e)}")
@receiver(post_delete, sender=M3UAccount)
def delete_refresh_task(sender, instance, **kwargs):
"""
@ -107,6 +163,34 @@ def update_status_on_active_change(sender, instance, **kwargs):
else:
# When deactivating, set status to disabled
instance.status = M3UAccount.Status.DISABLED
# Clean up any expiration notifications for all profiles of this account
try:
from core.models import SystemNotification
from core.utils import send_notification_dismissed
profile_ids = list(
M3UAccountProfile.objects.filter(m3u_account=instance)
.values_list("id", flat=True)
)
keys = [
key
for pid in profile_ids
for key in [f"xc-exp-warning-{pid}", f"xc-exp-expired-{pid}"]
]
if keys:
deleted_keys = list(
SystemNotification.objects.filter(notification_key__in=keys)
.values_list("notification_key", flat=True)
)
if deleted_keys:
SystemNotification.objects.filter(notification_key__in=deleted_keys).delete()
for key in deleted_keys:
send_notification_dismissed(key)
logger.debug(
f"Cleaned up {len(deleted_keys)} notifications for deactivated M3U account {instance.id}"
)
except Exception as notify_err:
logger.error(f"Error cleaning up notifications on account deactivation: {notify_err}")
except M3UAccount.DoesNotExist:
# New record, will use default status
pass

337
apps/m3u/tasks.py
View file

@ -1,6 +1,7 @@
# apps/m3u/tasks.py
import logging
import re
import regex
import requests
import os
import gc
@ -11,7 +12,7 @@ from celery.result import AsyncResult
from celery import shared_task, current_app, group
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db import models, transaction
from .models import M3UAccount
from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount
from asgiref.sync import async_to_sync
@ -38,6 +39,8 @@ logger = logging.getLogger(__name__)
BATCH_SIZE = 1500 # Optimized batch size for threading
m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u")
_EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2')
def fetch_m3u_lines(account, use_cache=False):
os.makedirs(m3u_dir, exist_ok=True)
@ -484,18 +487,13 @@ def parse_extinf_line(line: str) -> dict:
return None
content = line[len("#EXTINF:") :].strip()
# Single pass: extract all attributes AND track the last attribute position
# This regex matches both key="value" and key='value' patterns
# Single pass: extract all attributes AND track the last attribute position.
# Keys are normalised to lowercase so downstream code can use plain dict.get()
attrs = {}
last_attr_end = 0
# Use a single regex that handles both quote types.
# Keys must stop at '=' so values like base64-padded URLs ending with '=='
# don't get folded into the preceding attribute name.
for match in re.finditer(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2', content):
key = match.group(1)
value = match.group(3)
attrs[key] = value
for match in _EXTINF_ATTR_RE.finditer(content):
attrs[match.group(1).lower()] = match.group(3)
last_attr_end = match.end()
# Everything after the last attribute (skipping leading comma and whitespace) is the display name
@ -513,15 +511,80 @@ def parse_extinf_line(line: str) -> dict:
else:
display_name = content.strip()
# Use tvg-name attribute if available; otherwise try tvc-guide-title, then fall back to display name.
name = get_case_insensitive_attr(attrs, "tvg-name", None)
if not name:
name = get_case_insensitive_attr(attrs, "tvc-guide-title", None)
if not name:
name = display_name
# Per the base #EXTINF spec, the comma text is the canonical human-readable title.
# Fall back to tvc-guide-title, then tvg-name (which some providers use as an EPG key,
# not a display label), and finally the raw content if everything else is empty.
name = display_name or attrs.get("tvc-guide-title") or attrs.get("tvg-name") or content.strip()
return {"attributes": attrs, "display_name": display_name, "name": name}
def iter_m3u_entries(lines):
"""
Generator that yields fully-assembled M3U stream entries from raw lines.
Each yielded dict is guaranteed to contain a ``url`` key in addition to the
fields produced by :func:`parse_extinf_line` (``attributes``, ``display_name``,
``name``). Recognised extended-tag lines that appear *between* an ``#EXTINF``
and its URL are accumulated into the pending entry so they are available for
downstream processing:
- ``#EXTGRP`` — sets ``attributes["group-title"]`` when no ``group-title``
attribute was present on the ``#EXTINF`` line (explicit attribute wins).
- ``#EXTVLCOPT`` — stored as a list under the ``vlc_opts`` key.
Unknown directives (``#KODIPROP``, etc.) and blank lines are
silently skipped while keeping the pending entry intact. A second ``#EXTINF``
before a URL discards the first entry with a warning. A trailing ``#EXTINF``
at end-of-file with no URL is also discarded.
Adding support for a new directive requires only a new ``elif`` branch here;
no other code needs to change.
"""
pending = None
pending_line_num = None
for line_num, raw_line in enumerate(lines, 1):
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXTINF"):
if pending is not None:
logger.warning(
f"Line {pending_line_num}: #EXTINF had no URL (next #EXTINF at line {line_num}); "
f"discarding entry: {list(pending['attributes'].items())[:3]}"
)
parsed = parse_extinf_line(line)
if parsed is None:
logger.warning(f"Line {line_num}: Failed to parse #EXTINF: {line[:200]}")
pending = parsed # None if malformed; URL branch guards on `pending is not None`
pending_line_num = line_num
elif line.startswith("#EXTGRP:"):
# Only apply when group-title is absent — explicit attribute wins.
if pending is not None and "group-title" not in pending["attributes"]:
pending["attributes"]["group-title"] = line[len("#EXTGRP:"):].strip()
# else: #EXTGRP outside an entry, or group-title already set — silently skip
elif line.startswith("#EXTVLCOPT:"):
if pending is not None:
pending.setdefault("vlc_opts", []).append(line[len("#EXTVLCOPT:"):])
# else: #EXTVLCOPT outside an entry — silently skip
elif pending is not None and line.startswith(("http", "rtsp", "rtp", "udp")):
pending["url"] = normalize_stream_url(line) if line.startswith("udp") else line
yield pending
pending = None
pending_line_num = None
# else: unknown directive or bare content — skip, keeping pending intact
if pending is not None:
logger.warning(
f"Line {pending_line_num}: #EXTINF at end of file had no URL; "
f"discarding entry: {list(pending['attributes'].items())[:3]}"
)
@shared_task
def refresh_m3u_accounts():
"""Queue background parse for all active M3UAccounts."""
@ -1114,7 +1177,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys):
"m3u_account": account,
"channel_group_id": int(groups.get(group_title)),
"stream_hash": stream_hash,
"custom_properties": stream_info["attributes"],
"custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"],
"is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)),
"is_stale": False,
"stream_id": provider_stream_id,
@ -1482,7 +1545,6 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_sta
release_task_lock("refresh_m3u_account_groups", account_id)
return error_msg, None
else:
# Here's the key change - use the success flag from fetch_m3u_lines
lines, success = fetch_m3u_lines(account, use_cache)
if not success:
# If fetch failed, don't continue processing
@ -1493,71 +1555,20 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_sta
# Log basic file structure for debugging
logger.debug(f"Processing {len(lines)} lines from M3U file")
line_count = 0
extinf_count = 0
url_count = 0
valid_stream_count = 0
problematic_lines = []
for line_index, line in enumerate(lines):
line_count += 1
line = line.strip()
for entry in iter_m3u_entries(lines):
valid_stream_count += 1
group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "")
if group_title_attr and group_title_attr not in groups:
logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'")
groups[group_title_attr] = {}
extinf_data.append(entry)
if line.startswith("#EXTINF"):
extinf_count += 1
parsed = parse_extinf_line(line)
if parsed:
group_title_attr = get_case_insensitive_attr(
parsed["attributes"], "group-title", ""
)
if group_title_attr:
group_name = group_title_attr
# Log new groups as they're discovered
if group_name not in groups:
logger.debug(
f"Found new group for M3U account {account_id}: '{group_name}'"
)
groups[group_name] = {}
if valid_stream_count % 1000 == 0:
logger.debug(f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}")
extinf_data.append(parsed)
else:
# Log problematic EXTINF lines
logger.warning(
f"Failed to parse EXTINF at line {line_index+1}: {line[:200]}"
)
problematic_lines.append((line_index + 1, line[:200]))
elif extinf_data and (line.startswith("http") or line.startswith("rtsp") or line.startswith("rtp") or line.startswith("udp")):
url_count += 1
# Normalize UDP URLs only (e.g., remove VLC-specific @ prefix)
normalized_url = normalize_stream_url(line) if line.startswith("udp") else line
# Associate URL with the last EXTINF line
extinf_data[-1]["url"] = normalized_url
valid_stream_count += 1
# Periodically log progress for large files
if valid_stream_count % 1000 == 0:
logger.debug(
f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}"
)
# Log summary statistics
logger.info(
f"M3U parsing complete - Lines: {line_count}, EXTINF: {extinf_count}, URLs: {url_count}, Valid streams: {valid_stream_count}"
)
if problematic_lines:
logger.warning(
f"Found {len(problematic_lines)} problematic lines during parsing"
)
for i, (line_num, content) in enumerate(
problematic_lines[:10]
): # Log max 10 examples
logger.warning(f"Problematic line #{i+1} at line {line_num}: {content}")
if len(problematic_lines) > 10:
logger.warning(
f"... and {len(problematic_lines) - 10} more problematic lines"
)
logger.info(f"M3U parsing complete - Valid streams: {valid_stream_count}")
# Log group statistics
logger.info(
@ -2399,11 +2410,13 @@ def get_transformed_credentials(account, profile=None):
# Apply profile-specific transformations if profile is provided
if profile and profile.search_pattern and profile.replace_pattern:
try:
# Handle backreferences in the replacement pattern
safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', profile.replace_pattern)
# Handle backreferences: convert JS-style $<name> -> \g<name>, $1 -> \1
# regex module accepts JS-style (?<name>...) named groups natively
safe_replace_pattern = regex.sub(r'\$<([^>]+)>', r'\\g<\1>', profile.replace_pattern)
safe_replace_pattern = regex.sub(r'\$(\d+)', r'\\\1', safe_replace_pattern)
# Apply transformation to the complete URL
transformed_complete_url = re.sub(profile.search_pattern, safe_replace_pattern, complete_url)
transformed_complete_url = regex.sub(profile.search_pattern, safe_replace_pattern, complete_url)
logger.info(f"Transformed complete URL: {complete_url} -> {transformed_complete_url}")
# Extract components from the transformed URL
@ -3185,3 +3198,163 @@ def send_m3u_update(account_id, action, progress, **kwargs):
# Explicitly clear data reference to help garbage collection
data = None
def evaluate_profile_expiration_notification(profile):
"""
Evaluate a single M3UAccountProfile's expiration date and create, update,
or delete the corresponding SystemNotification accordingly.
Returns the notification key that should remain active (warning or expired),
or None if the profile is not expiring soon and any stale notifications were removed.
This return value is used by the bulk task to track active keys for stale cleanup.
"""
from core.models import SystemNotification
from core.utils import send_websocket_notification, send_notification_dismissed
exp = profile.exp_date
if not exp:
return None
now = timezone.now()
warning_threshold = now + timezone.timedelta(days=7)
warning_key = f"xc-exp-warning-{profile.id}"
expired_key = f"xc-exp-expired-{profile.id}"
if exp <= now:
# Already expired — delete warning, create/update expired notification
deleted_warning = list(
SystemNotification.objects.filter(notification_key=warning_key)
.values_list("notification_key", flat=True)
)
SystemNotification.objects.filter(notification_key=warning_key).delete()
for key in deleted_warning:
send_notification_dismissed(key)
notification, created = SystemNotification.objects.update_or_create(
notification_key=expired_key,
defaults={
"notification_type": SystemNotification.NotificationType.WARNING,
"priority": SystemNotification.Priority.HIGH,
"title": f"Account Expired: {profile.name}",
"message": (
f'Profile "{profile.name}" on M3U account '
f'"{profile.m3u_account.name}" has expired '
f"(expired {exp.strftime('%Y-%m-%d %H:%M UTC')})."
),
"action_data": {
"profile_id": profile.id,
"account_id": profile.m3u_account.id,
"account_name": profile.m3u_account.name,
"profile_name": profile.name,
"exp_date": exp.isoformat(),
},
"is_active": True,
"admin_only": True,
},
)
send_websocket_notification(notification)
return expired_key
elif exp <= warning_threshold:
# Expiring within 7 days — delete expired notification, create/update warning
deleted_expired = list(
SystemNotification.objects.filter(notification_key=expired_key)
.values_list("notification_key", flat=True)
)
SystemNotification.objects.filter(notification_key=expired_key).delete()
for key in deleted_expired:
send_notification_dismissed(key)
days_left = (exp - now).days
if days_left == 0:
expires_in_str = "today"
elif days_left == 1:
expires_in_str = "in 1 day"
else:
expires_in_str = f"in {days_left} days"
notification, created = SystemNotification.objects.update_or_create(
notification_key=warning_key,
defaults={
"notification_type": SystemNotification.NotificationType.WARNING,
"priority": SystemNotification.Priority.NORMAL,
"title": f"Account Expiring: {profile.name}",
"message": (
f'Profile "{profile.name}" on M3U account '
f'"{profile.m3u_account.name}" expires {expires_in_str} '
f"(expires {exp.strftime('%Y-%m-%d %H:%M UTC')})."
),
"action_data": {
"profile_id": profile.id,
"account_id": profile.m3u_account.id,
"account_name": profile.m3u_account.name,
"profile_name": profile.name,
"exp_date": exp.isoformat(),
},
"is_active": True,
"admin_only": True,
},
)
send_websocket_notification(notification)
return warning_key
else:
# Not expiring soon — delete any stale notifications
deleted_keys = list(
SystemNotification.objects.filter(
notification_key__in=[warning_key, expired_key]
).values_list("notification_key", flat=True)
)
SystemNotification.objects.filter(
notification_key__in=[warning_key, expired_key]
).delete()
for key in deleted_keys:
send_notification_dismissed(key)
return None
@shared_task
def check_account_expirations():
"""
Daily task: check all account profiles for upcoming expirations.
Creates/updates SystemNotifications for profiles expiring within 7 days.
Uses separate notification keys for warning vs expired so users can
dismiss the 7-day warning and still receive the expired notification.
"""
from apps.m3u.models import M3UAccountProfile
from core.models import SystemNotification
from core.utils import send_notification_dismissed
# Find all active profiles with an exp_date that is set
expiring_profiles = (
M3UAccountProfile.objects.filter(
m3u_account__is_active=True,
is_active=True,
exp_date__isnull=False,
)
.select_related("m3u_account")
)
active_notification_keys = set()
for profile in expiring_profiles:
active_key = evaluate_profile_expiration_notification(profile)
if active_key:
active_notification_keys.add(active_key)
# Delete stale notifications for profiles whose expiration was extended
stale = SystemNotification.objects.filter(
is_active=True,
).filter(
models.Q(notification_key__startswith="xc-exp-warning-") |
models.Q(notification_key__startswith="xc-exp-expired-")
).exclude(notification_key__in=active_notification_keys)
stale_keys = list(stale.values_list("notification_key", flat=True))
stale.delete()
for key in stale_keys:
send_notification_dismissed(key)
logger.info(
f"Account expiration check complete: {len(active_notification_keys)} active notifications"
)

216
apps/m3u/tests/test_expiration_notifications.py Normal file
View file

@ -0,0 +1,216 @@
"""
Tests for evaluate_profile_expiration_notification.
Covers all four branches:
- no exp_date → returns None, touches nothing
- already expired → creates/updates expired notification, removes warning
- expiring within 7d → creates/updates warning notification, removes expired
- not expiring soon → removes any stale notifications, returns None
"""
from datetime import timedelta
from unittest.mock import patch, MagicMock
from django.test import SimpleTestCase
from django.utils import timezone
def _make_profile(exp_date, profile_id=1, profile_name="Test Profile",
account_id=10, account_name="Test Account"):
"""Return a minimal mock M3UAccountProfile."""
profile = MagicMock()
profile.id = profile_id
profile.name = profile_name
profile.exp_date = exp_date
profile.m3u_account.id = account_id
profile.m3u_account.name = account_name
return profile
class EvaluateProfileExpirationNotificationTests(SimpleTestCase):
def setUp(self):
# These three names are local imports inside evaluate_profile_expiration_notification,
# so we must patch them at their source modules rather than on apps.m3u.tasks.
self.mock_sn = MagicMock()
self.mock_send_ws = patch("core.utils.send_websocket_notification").start()
self.mock_dismissed = patch("core.utils.send_notification_dismissed").start()
patch("core.models.SystemNotification", self.mock_sn).start()
def tearDown(self):
patch.stopall()
def _run(self, profile):
from apps.m3u.tasks import evaluate_profile_expiration_notification
return evaluate_profile_expiration_notification(profile)
# ------------------------------------------------------------------ #
# No expiration date
# ------------------------------------------------------------------ #
def test_no_exp_date_returns_none(self):
profile = _make_profile(exp_date=None)
result = self._run(profile)
self.assertIsNone(result)
self.mock_sn.objects.update_or_create.assert_not_called()
self.mock_send_ws.assert_not_called()
# ------------------------------------------------------------------ #
# Already expired
# ------------------------------------------------------------------ #
@patch("apps.m3u.tasks.timezone")
def test_expired_creates_expired_notification(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now - timedelta(days=1))
# No existing warning notification to delete
self.mock_sn.objects.filter.return_value.values_list.return_value = []
notification = MagicMock()
self.mock_sn.objects.update_or_create.return_value = (notification, True)
result = self._run(profile)
self.assertEqual(result, f"xc-exp-expired-{profile.id}")
self.mock_sn.objects.update_or_create.assert_called_once()
call_kwargs = self.mock_sn.objects.update_or_create.call_args
self.assertEqual(call_kwargs.kwargs["notification_key"], f"xc-exp-expired-{profile.id}")
self.assertTrue(call_kwargs.kwargs["defaults"]["admin_only"])
self.mock_send_ws.assert_called_once_with(notification)
@patch("apps.m3u.tasks.timezone")
def test_expired_removes_stale_warning_notification(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now - timedelta(hours=1))
warning_key = f"xc-exp-warning-{profile.id}"
# Simulate an existing warning notification
self.mock_sn.objects.filter.return_value.values_list.return_value = [warning_key]
self.mock_sn.objects.update_or_create.return_value = (MagicMock(), False)
self._run(profile)
self.mock_dismissed.assert_any_call(warning_key)
# ------------------------------------------------------------------ #
# Expiring within 7 days
# ------------------------------------------------------------------ #
@patch("apps.m3u.tasks.timezone")
def test_warning_window_creates_warning_notification(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(days=3))
self.mock_sn.objects.filter.return_value.values_list.return_value = []
notification = MagicMock()
self.mock_sn.objects.update_or_create.return_value = (notification, True)
result = self._run(profile)
self.assertEqual(result, f"xc-exp-warning-{profile.id}")
call_kwargs = self.mock_sn.objects.update_or_create.call_args
self.assertEqual(call_kwargs.kwargs["notification_key"], f"xc-exp-warning-{profile.id}")
self.assertTrue(call_kwargs.kwargs["defaults"]["admin_only"])
self.mock_send_ws.assert_called_once_with(notification)
@patch("apps.m3u.tasks.timezone")
def test_warning_message_says_today_when_same_day(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(hours=2))
self.mock_sn.objects.filter.return_value.values_list.return_value = []
self.mock_sn.objects.update_or_create.return_value = (MagicMock(), True)
self._run(profile)
defaults = self.mock_sn.objects.update_or_create.call_args.kwargs["defaults"]
self.assertIn("today", defaults["message"])
@patch("apps.m3u.tasks.timezone")
def test_warning_message_says_1_day(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(hours=30))
self.mock_sn.objects.filter.return_value.values_list.return_value = []
self.mock_sn.objects.update_or_create.return_value = (MagicMock(), True)
self._run(profile)
defaults = self.mock_sn.objects.update_or_create.call_args.kwargs["defaults"]
self.assertIn("in 1 day", defaults["message"])
@patch("apps.m3u.tasks.timezone")
def test_warning_removes_stale_expired_notification(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(days=5))
expired_key = f"xc-exp-expired-{profile.id}"
self.mock_sn.objects.filter.return_value.values_list.return_value = [expired_key]
self.mock_sn.objects.update_or_create.return_value = (MagicMock(), False)
self._run(profile)
self.mock_dismissed.assert_any_call(expired_key)
# ------------------------------------------------------------------ #
# Not expiring soon (> 7 days away)
# ------------------------------------------------------------------ #
@patch("apps.m3u.tasks.timezone")
def test_not_expiring_soon_returns_none(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(days=30))
self.mock_sn.objects.filter.return_value.values_list.return_value = []
result = self._run(profile)
self.assertIsNone(result)
self.mock_sn.objects.update_or_create.assert_not_called()
self.mock_send_ws.assert_not_called()
@patch("apps.m3u.tasks.timezone")
def test_not_expiring_soon_removes_stale_notifications(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
profile = _make_profile(exp_date=now + timedelta(days=30))
warning_key = f"xc-exp-warning-{profile.id}"
self.mock_sn.objects.filter.return_value.values_list.return_value = [warning_key]
self._run(profile)
self.mock_dismissed.assert_called_once_with(warning_key)
# ------------------------------------------------------------------ #
# Boundary: exactly at the 7-day warning threshold
# ------------------------------------------------------------------ #
@patch("apps.m3u.tasks.timezone")
def test_exactly_7_days_away_triggers_warning(self, mock_tz):
now = timezone.now()
mock_tz.now.return_value = now
mock_tz.timedelta = timedelta
# exp_date == now + 7 days → exp <= warning_threshold → warning
profile = _make_profile(exp_date=now + timedelta(days=7))
self.mock_sn.objects.filter.return_value.values_list.return_value = []
self.mock_sn.objects.update_or_create.return_value = (MagicMock(), True)
result = self._run(profile)
self.assertEqual(result, f"xc-exp-warning-{profile.id}")

93
apps/output/views.py
View file

@ -194,7 +194,7 @@ def generate_m3u(request, profile_name=None, user=None):
epg_base_url = build_absolute_uri_with_port(request, reverse('output:epg_endpoint', args=[profile_name]) if profile_name else reverse('output:epg_endpoint'))
# Optionally preserve certain query parameters
preserved_params = ['tvg_id_source', 'cachedlogos', 'days']
preserved_params = ['tvg_id_source', 'cachedlogos', 'days', 'prev_days']
query_params = {k: v for k, v in request.GET.items() if k in preserved_params}
if query_params:
from urllib.parse import urlencode
@ -1268,7 +1268,28 @@ def generate_epg(request, profile_name=None, user=None):
"""
# Check cache for recent identical request (helps with double-GET from browsers)
from django.core.cache import cache
cache_params = f"{profile_name or 'all'}:{user.username if user else 'anonymous'}:{request.GET.urlencode()}"
# Resolve all effective parameter values once here so they are reused for both
# the cache key and inside epg_generator() via closure.
# The cache key is built from resolved values only — not from the raw query string —
# so equivalent requests (e.g. days=7 via URL param vs. user default of 7) share
# the same cache entry regardless of how the value was supplied.
user_custom = (user.custom_properties or {}) if user else {}
try:
num_days = int(request.GET.get('days', user_custom.get('epg_days', 0)))
num_days = max(0, min(num_days, 365))
except (ValueError, TypeError):
num_days = 0
try:
prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0)))
prev_days = max(0, min(prev_days, 30))
except (ValueError, TypeError):
prev_days = 0
use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false'
tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower()
cache_params = (
f"{profile_name or 'all'}:{user.username if user else 'anonymous'}"
f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}"
)
content_cache_key = f"epg_content:{cache_params}"
cached_content = cache.get(content_cache_key)
@ -1331,29 +1352,14 @@ def generate_epg(request, profile_name=None, user=None):
else:
channels = Channel.objects.all().order_by("channel_number")
# Check if the request wants to use direct logo URLs instead of cache
use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false'
# Get the source to use for tvg-id value
# Options: 'channel_number' (default), 'tvg_id', 'gracenote'
tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower()
# Get the number of days for EPG data
try:
# Default to 0 days (everything) for real EPG if not specified
days_param = request.GET.get('days', '0')
num_days = int(days_param)
# Set reasonable limits
num_days = max(0, min(num_days, 365)) # Between 0 and 365 days
except ValueError:
num_days = 0 # Default to all data if invalid value
# For dummy EPG, use either the specified value or default to 3 days
dummy_days = num_days if num_days > 0 else 3
# Calculate cutoff date for EPG data filtering (only if days > 0)
# Calculate cutoff dates for EPG data filtering
now = django_timezone.now()
cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None
lookback_cutoff = now - timedelta(days=prev_days)
# Build collision-free channel number mapping for XC clients (if user is authenticated)
# XC clients require integer channel numbers, so we need to ensure no conflicts
@ -1643,13 +1649,14 @@ def generate_epg(request, profile_name=None, user=None):
# For real EPG data - filter only if days parameter was specified
if num_days > 0:
programs_qs = channel.epg_data.programs.filter(
end_time__gte=now,
end_time__gte=lookback_cutoff,
start_time__lt=cutoff_date
).order_by('id') # Explicit ordering for consistent chunking
else:
# Return all non-expired programs if days=0 or not specified
# Return programs from lookback_cutoff onward (includes recent past
# for catch-up when prev_days > 0, otherwise current/future only)
programs_qs = channel.epg_data.programs.filter(
end_time__gte=now
end_time__gte=lookback_cutoff
).order_by('id')
# Process programs in chunks to avoid cursor timeout issues
@ -2332,6 +2339,20 @@ def xc_get_epg(request, user, short=False):
channel_num_int = channel_num_map.get(channel.id, int(channel.channel_number))
limit = int(request.GET.get('limit', 4))
user_custom = user.custom_properties or {}
try:
num_days = int(request.GET.get('days', user_custom.get('epg_days', 0)))
num_days = max(0, min(num_days, 365))
except (ValueError, TypeError):
num_days = 0
try:
prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0)))
prev_days = max(0, min(prev_days, 30))
except (ValueError, TypeError):
prev_days = 0
now = django_timezone.now()
lookback_cutoff = now - timedelta(days=prev_days)
forward_cutoff = now + timedelta(days=num_days) if num_days > 0 else None
if channel.epg_data:
# Check if this is a dummy EPG that generates on-demand
if channel.epg_data.epg_source and channel.epg_data.epg_source.source_type == 'dummy':
@ -2344,24 +2365,28 @@ def xc_get_epg(request, user, short=False):
)
else:
# Has stored programs, use them
if short == False:
if short:
# Short EPG: current and upcoming only (never historical), limited count
programs = channel.epg_data.programs.filter(
end_time__gt=django_timezone.now()
).order_by('start_time')
else:
programs = channel.epg_data.programs.filter(
end_time__gt=django_timezone.now()
end_time__gt=now
).order_by('start_time')[:limit]
else:
qs = channel.epg_data.programs.filter(end_time__gt=lookback_cutoff)
if forward_cutoff:
qs = qs.filter(start_time__lt=forward_cutoff)
programs = qs.order_by('start_time')
else:
# Regular EPG with stored programs
if short == False:
if short:
# Short EPG: current and upcoming only (never historical), limited count
programs = channel.epg_data.programs.filter(
end_time__gt=django_timezone.now()
).order_by('start_time')
end_time__gt=now
).order_by('start_time')[:limit]
else:
programs = channel.epg_data.programs.filter(
end_time__gt=django_timezone.now()
).order_by('start_time')[:limit]
qs = channel.epg_data.programs.filter(end_time__gt=lookback_cutoff)
if forward_cutoff:
qs = qs.filter(start_time__lt=forward_cutoff)
programs = qs.order_by('start_time')
else:
# No EPG data assigned, generate default dummy
programs = generate_dummy_programs(channel_id=channel_id, channel_name=channel.name, epg_source=None)

17
apps/plugins/api_urls.py
View file

@ -8,6 +8,14 @@ from .api_views import (
PluginImportAPIView,
PluginDeleteAPIView,
PluginLogoAPIView,
PluginRepoListCreateAPIView,
PluginRepoPreviewAPIView,
PluginRepoDetailAPIView,
PluginRepoRefreshAPIView,
AvailablePluginsAPIView,
PluginDetailManifestAPIView,
PluginInstallFromRepoAPIView,
PluginRepoSettingsAPIView,
)
app_name = "plugins"
@ -21,4 +29,13 @@ urlpatterns = [
path("plugins/<str:key>/run/", PluginRunAPIView.as_view(), name="run"),
path("plugins/<str:key>/enabled/", PluginEnabledAPIView.as_view(), name="enabled"),
path("plugins/<str:key>/logo/", PluginLogoAPIView.as_view(), name="logo"),
# Plugin repos (hub / store) - static paths first, then parametric
path("repos/", PluginRepoListCreateAPIView.as_view(), name="repo-list"),
path("repos/available/", AvailablePluginsAPIView.as_view(), name="available-plugins"),
path("repos/plugin-detail/", PluginDetailManifestAPIView.as_view(), name="plugin-detail-manifest"),
path("repos/install/", PluginInstallFromRepoAPIView.as_view(), name="repo-install"),
path("repos/settings/", PluginRepoSettingsAPIView.as_view(), name="repo-settings"),
path("repos/preview/", PluginRepoPreviewAPIView.as_view(), name="repo-preview"),
path("repos/<int:pk>/", PluginRepoDetailAPIView.as_view(), name="repo-detail"),
path("repos/<int:pk>/refresh/", PluginRepoRefreshAPIView.as_view(), name="repo-refresh"),
]

1220
apps/plugins/api_views.py
View file

File diff suppressed because it is too large Load diff

50
apps/plugins/apps.py
View file

@ -52,3 +52,53 @@ class PluginsConfig(AppConfig):
import logging
logging.getLogger(__name__).exception("Plugin discovery wiring failed during app ready")
# Register periodic task for refreshing plugin repo manifests
self._setup_repo_refresh_schedule()
# Refresh repo manifests once at startup so the UI always has current data
self._enqueue_startup_refresh()
def _enqueue_startup_refresh(self):
from dispatcharr.app_initialization import should_skip_initialization
if should_skip_initialization():
return
try:
from .tasks import refresh_plugin_repos
refresh_plugin_repos.apply_async(countdown=10)
except Exception:
import logging
logging.getLogger(__name__).debug(
"Could not enqueue startup plugin repo refresh (Celery may not be ready yet)"
)
def _setup_repo_refresh_schedule(self):
from dispatcharr.app_initialization import should_skip_initialization
if should_skip_initialization():
return
try:
from core.scheduling import create_or_update_periodic_task, delete_periodic_task
from core.models import CoreSettings
from .tasks import PLUGIN_REPO_REFRESH_TASK_NAME
interval = 6
try:
obj = CoreSettings.objects.get(key="plugin_repo_settings")
interval = obj.value.get("refresh_interval_hours", 6)
except CoreSettings.DoesNotExist:
pass
if interval == 0:
delete_periodic_task(PLUGIN_REPO_REFRESH_TASK_NAME)
else:
create_or_update_periodic_task(
task_name=PLUGIN_REPO_REFRESH_TASK_NAME,
celery_task_path="apps.plugins.tasks.refresh_plugin_repos",
interval_hours=interval,
enabled=True,
)
except Exception:
import logging
logging.getLogger(__name__).debug(
"Could not set up plugin repo refresh schedule (migrations may not have run yet)"
)

11
apps/plugins/keys/dispatcharr-plugins.pub Normal file
View file

@ -0,0 +1,11 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mDMEacgfABYJKwYBBAHaRw8BAQdAh1MuVNBxk+CExQPjOVDvAGvIk6BdGS2ce9/h
zB7lYtW0TERpc3BhdGNoYXJyIFBsdWdpbiBSZXBvIChkaXNwYXRjaGFyci1hdXRv
Z2VuZXJhdGVkKSA8cGx1Z2luc0BkaXNwYXRjaGFyci50dj6IrwQTFgoAVxYhBEap
MFaOD7nKg0zX+H7AOmtMIjTOBQJpyB8AGxSAAAAAAAQADm1hbnUyLDIuNSsxLjEy
LDAsMwIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRB+wDprTCI0zvNZ
AP9r3TpMpiI8BCNo9B5M9lJ+QLRo9ihPWIcqBzJ9eFCoSQEAgguiZsNy6aJzKjIb
yDvGuoZi3I2/GNM/f2qVzFtgPQk=
=Zf/y
-----END PGP PUBLIC KEY BLOCK-----

52
apps/plugins/loader.py
View file

@ -367,21 +367,35 @@ class PluginManager:
obj.save()
def list_plugins(self) -> List[Dict[str, Any]]:
from .models import PluginConfig
from .models import PluginConfig, PluginRepo
plugins: List[Dict[str, Any]] = []
with self._lock:
registry_snapshot = dict(self._registry)
try:
configs = {c.key: c for c in PluginConfig.objects.all()}
configs = {c.key: c for c in PluginConfig.objects.select_related("source_repo").all()}
except Exception as e:
# Database might not be migrated yet; fall back to registry only
logger.warning("PluginConfig table unavailable; listing registry only: %s", e)
configs = {}
# Build repo latest-version lookup from cached manifests
repo_latest = {} # slug -> latest_version
try:
for repo in PluginRepo.objects.filter(enabled=True):
manifest_data = repo.cached_manifest or {}
manifest = manifest_data.get("manifest", manifest_data)
for rp in manifest.get("plugins", []):
s = rp.get("slug", "")
if s:
repo_latest[s] = rp.get("latest_version", "")
except Exception:
pass
# First, include all discovered plugins
for key, lp in registry_snapshot.items():
conf = configs.get(key)
conf_slug = conf.slug if conf else ""
trusted = bool(conf and (conf.ever_enabled or conf.enabled))
logo_url = self._get_logo_url(key, path=lp.path)
plugins.append(
@ -393,7 +407,7 @@ class PluginManager:
"author": getattr(lp, "author", "") or "",
"help_url": getattr(lp, "help_url", "") or "",
"enabled": conf.enabled if conf else False,
"ever_enabled": getattr(conf, "ever_enabled", False) if conf else False,
"ever_enabled": conf.ever_enabled if conf else False,
"fields": lp.fields or [],
"settings": (conf.settings if conf else {}),
"actions": lp.actions or [],
@ -402,6 +416,22 @@ class PluginManager:
"loaded": bool(lp.loaded),
"legacy": bool(getattr(lp, "legacy", False)),
"logo_url": logo_url,
"source_repo": conf.source_repo_id if conf else None,
"source_repo_name": conf.source_repo.name if conf and conf.source_repo else None,
"is_official_repo": bool(conf and conf.source_repo and conf.source_repo.is_official),
"slug": conf_slug,
"is_managed": bool(conf and conf.source_repo_id),
"installed_version_is_prerelease": bool(
conf and conf.installed_version_is_prerelease
),
"update_available": bool(
conf_slug and conf and conf.source_repo_id
and not (conf and conf.installed_version_is_prerelease)
and repo_latest.get(conf_slug)
and lp.version != repo_latest.get(conf_slug)
),
"latest_version": repo_latest.get(conf_slug, ""),
"deprecated": conf.deprecated if conf else False,
}
)
@ -428,6 +458,22 @@ class PluginManager:
"loaded": False,
"legacy": False,
"logo_url": self._get_logo_url(key),
"source_repo": conf.source_repo_id,
"source_repo_name": conf.source_repo.name if conf.source_repo else None,
"is_official_repo": bool(conf.source_repo and conf.source_repo.is_official),
"slug": conf.slug,
"is_managed": bool(conf.source_repo_id),
"installed_version_is_prerelease": bool(
conf.installed_version_is_prerelease
),
"update_available": bool(
conf.slug and conf.source_repo_id
and not conf.installed_version_is_prerelease
and repo_latest.get(conf.slug)
and conf.version != repo_latest.get(conf.slug)
),
"latest_version": repo_latest.get(conf.slug or "", ""),
"deprecated": conf.deprecated,
}
)

84
apps/plugins/migrations/0002_pluginrepo.py Normal file
View file

@ -0,0 +1,84 @@
import django.db.models.deletion
from django.db import migrations, models
def seed_official_repo(apps, schema_editor):
PluginRepo = apps.get_model("plugins", "PluginRepo")
PluginRepo.objects.get_or_create(
url="https://raw.githubusercontent.com/Dispatcharr/Plugins/releases/manifest.json",
defaults={
"name": "Dispatcharr Official",
"is_official": True,
"enabled": True,
},
)
def unseed_official_repo(apps, schema_editor):
PluginRepo = apps.get_model("plugins", "PluginRepo")
PluginRepo.objects.filter(is_official=True).delete()
class Migration(migrations.Migration):
dependencies = [
("plugins", "0001_initial"),
]
operations = [
migrations.CreateModel(
name="PluginRepo",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=255)),
("url", models.URLField(unique=True)),
("is_official", models.BooleanField(default=False)),
("enabled", models.BooleanField(default=True)),
("cached_manifest", models.JSONField(blank=True, default=dict)),
("last_fetched", models.DateTimeField(blank=True, null=True)),
("public_key", models.TextField(blank=True, default="")),
("signature_verified", models.BooleanField(blank=True, default=None, null=True)),
("last_fetch_status", models.CharField(blank=True, default="", max_length=255)),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
],
options={
"ordering": ["-is_official", "name"],
},
),
migrations.RunPython(seed_official_repo, unseed_official_repo),
migrations.AddField(
model_name="pluginconfig",
name="source_repo",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="installed_plugins",
to="plugins.pluginrepo",
),
),
migrations.AddField(
model_name="pluginconfig",
name="slug",
field=models.CharField(blank=True, default="", max_length=128),
),
migrations.AddField(
model_name="pluginconfig",
name="installed_version_is_prerelease",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="pluginconfig",
name="deprecated",
field=models.BooleanField(default=False),
),
]

44
apps/plugins/models.py
View file

@ -12,8 +12,52 @@ class PluginConfig(models.Model):
# Tracks whether this plugin has ever been enabled at least once
ever_enabled = models.BooleanField(default=False)
settings = models.JSONField(default=dict, blank=True)
# Managed plugin fields (populated when installed from a repo)
source_repo = models.ForeignKey(
"PluginRepo",
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="installed_plugins",
)
slug = models.CharField(max_length=128, blank=True, default="")
installed_version_is_prerelease = models.BooleanField(default=False)
deprecated = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
@property
def is_managed(self):
return bool(self.source_repo_id)
def __str__(self) -> str:
return f"{self.name} ({self.key})"
OFFICIAL_REPO_URL = (
"https://raw.githubusercontent.com/Dispatcharr/Plugins/releases/manifest.json"
)
class PluginRepo(models.Model):
"""A remote plugin repository manifest URL."""
name = models.CharField(max_length=255)
url = models.URLField(unique=True)
is_official = models.BooleanField(default=False)
enabled = models.BooleanField(default=True)
cached_manifest = models.JSONField(default=dict, blank=True)
public_key = models.TextField(blank=True, default="")
signature_verified = models.BooleanField(null=True, blank=True, default=None)
last_fetched = models.DateTimeField(null=True, blank=True)
last_fetch_status = models.CharField(max_length=255, blank=True, default="")
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["-is_official", "name"]
def __str__(self) -> str:
return self.name

38
apps/plugins/serializers.py
View file

@ -1,4 +1,5 @@
from rest_framework import serializers
from .models import PluginRepo
class PluginActionSerializer(serializers.Serializer):
@ -46,3 +47,40 @@ class PluginSerializer(serializers.Serializer):
fields = PluginFieldSerializer(many=True)
settings = serializers.JSONField()
actions = PluginActionSerializer(many=True)
source_repo = serializers.IntegerField(required=False, allow_null=True)
slug = serializers.CharField(required=False, allow_blank=True)
is_managed = serializers.BooleanField(required=False)
deprecated = serializers.BooleanField(required=False)
class PluginRepoSerializer(serializers.ModelSerializer):
registry_url = serializers.SerializerMethodField()
plugin_count = serializers.SerializerMethodField()
class Meta:
model = PluginRepo
fields = [
"id",
"name",
"url",
"is_official",
"enabled",
"public_key",
"signature_verified",
"registry_url",
"plugin_count",
"last_fetched",
"last_fetch_status",
"created_at",
"updated_at",
]
read_only_fields = ["id", "name", "is_official", "signature_verified", "registry_url", "plugin_count", "last_fetched", "last_fetch_status", "created_at", "updated_at"]
def get_registry_url(self, obj):
manifest = (obj.cached_manifest or {}).get("manifest", obj.cached_manifest or {})
return manifest.get("registry_url", "") or ""
def get_plugin_count(self, obj):
manifest = (obj.cached_manifest or {}).get("manifest", obj.cached_manifest or {})
plugins = manifest.get("plugins", [])
return len(plugins) if isinstance(plugins, list) else 0

33
apps/plugins/tasks.py Normal file
View file

@ -0,0 +1,33 @@
import logging
from celery import shared_task
logger = logging.getLogger(__name__)
PLUGIN_REPO_REFRESH_TASK_NAME = "plugin-repo-refresh-task"
@shared_task
def refresh_plugin_repos():
"""Refresh cached manifests for all enabled plugin repos."""
from .models import PluginRepo
from .api_views import _fetch_manifest, _save_fetched_manifest_to_repo, _unmanage_dropped_slugs
from django.utils import timezone
repos = PluginRepo.objects.filter(enabled=True)
for repo in repos:
try:
key_text = repo.public_key if not repo.is_official else None
data, verified = _fetch_manifest(repo.url, public_key_text=key_text)
err = _save_fetched_manifest_to_repo(repo, data, verified)
if err:
logger.warning("Skipping repo '%s': %s", repo.name, err)
continue
_unmanage_dropped_slugs(repo, data)
logger.info("Refreshed plugin repo '%s'", repo.name)
except Exception as e:
resp = getattr(e, 'response', None)
status_str = str(resp.status_code) if resp is not None and hasattr(resp, 'status_code') else type(e).__name__
repo.last_fetch_status = status_str[:255]
repo.last_fetched = timezone.now()
repo.save(update_fields=["last_fetch_status", "last_fetched", "updated_at"])
logger.warning("Failed to refresh plugin repo '%s': %s", repo.name, e)

3
apps/proxy/config.py
View file

@ -99,7 +99,7 @@ class TSConfig(BaseConfig):
CLIENT_RECORD_TTL = 60 # How long client records persist in Redis (seconds). Client will be considered MIA after this time.
CLEANUP_CHECK_INTERVAL = 1 # How often to check for disconnected clients (seconds)
CLIENT_HEARTBEAT_INTERVAL = 5 # How often to send client heartbeats (seconds)
GHOST_CLIENT_MULTIPLIER = 6.0 # How many heartbeat intervals before client considered ghost (6 would mean 36 seconds if heartbeat interval is 6)
GHOST_CLIENT_MULTIPLIER = 10.0 # How many heartbeat intervals before client considered ghost (10 = 50s, must exceed STREAM_TIMEOUT + FAILOVER_GRACE_PERIOD = 40s)
CLIENT_WAIT_TIMEOUT = 30 # Seconds to wait for client to connect
# Stream health and recovery settings
@ -108,6 +108,7 @@ class TSConfig(BaseConfig):
MIN_STABLE_TIME_BEFORE_RECONNECT = 30 # Minimum seconds a stream must be stable to try reconnect
FAILOVER_GRACE_PERIOD = 20 # Extra time (seconds) to allow for stream switching before disconnecting clients
URL_SWITCH_TIMEOUT = 20 # Max time allowed for a stream switch operation
MAX_KEEPALIVE_DURATION = 300 # Keepalive packets prevent _is_timeout() from firing, so without this a permanently failed stream holds clients open indefinitely.

20
apps/proxy/hls_proxy/views.py
View file

@ -4,6 +4,8 @@ import logging
from django.http import StreamingHttpResponse, JsonResponse, HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from rest_framework.decorators import api_view, permission_classes
from apps.accounts.permissions import IsAdmin
from .server import ProxyServer, Config
logger = logging.getLogger(__name__)
@ -15,7 +17,7 @@ def stream_endpoint(request, channel_id):
"""Handle HLS manifest requests"""
if channel_id not in proxy_server.stream_managers:
return JsonResponse({'error': 'Channel not found'}, status=404)
response = proxy_server.stream_endpoint(channel_id)
return StreamingHttpResponse(
response[0],
@ -30,10 +32,10 @@ def get_segment(request, segment_name):
try:
segment_num = int(segment_name.split('.')[0])
buffer = proxy_server.stream_buffers.get(segment_num)
if not buffer:
return JsonResponse({'error': 'Segment not found'}, status=404)
return StreamingHttpResponse(
buffer,
content_type='video/MP2T'
@ -44,19 +46,19 @@ def get_segment(request, segment_name):
logger.error(f"Error serving segment: {e}")
return JsonResponse({'error': str(e)}, status=500)
@csrf_exempt
@require_http_methods(["POST"])
@api_view(["POST"])
@permission_classes([IsAdmin])
def change_stream(request, channel_id):
"""Change stream URL for existing channel"""
try:
if channel_id not in proxy_server.stream_managers:
return JsonResponse({'error': 'Channel not found'}, status=404)
data = json.loads(request.body)
new_url = data.get('url')
if not new_url:
return JsonResponse({'error': 'No URL provided'}, status=400)
manager = proxy_server.stream_managers[channel_id]
if manager.update_url(new_url):
return JsonResponse({
@ -64,7 +66,7 @@ def change_stream(request, channel_id):
'channel': channel_id,
'url': new_url
})
return JsonResponse({
'message': 'URL unchanged',
'channel': channel_id,
@ -85,7 +87,7 @@ def initialize_stream(request, channel_id):
url = data.get('url')
if not url:
return JsonResponse({'error': 'No URL provided'}, status=400)
proxy_server.initialize_channel(url, channel_id)
return JsonResponse({
'message': 'Stream initialized',

19
apps/proxy/tasks.py
View file

@ -1,16 +1,11 @@
# yourapp/tasks.py
from celery import shared_task
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync
import redis
import json
import logging
import re
import gc # Add import for garbage collection
import gc
from core.utils import RedisClient
from apps.proxy.ts_proxy.channel_status import ChannelStatus
from core.utils import send_websocket_update
from apps.proxy.vod_proxy.connection_manager import get_connection_manager
logger = logging.getLogger(__name__)
@ -31,7 +26,7 @@ def fetch_channel_stats():
while True:
cursor, keys = redis_client.scan(cursor, match=channel_pattern)
for key in keys:
channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8'))
channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key)
if channel_id_match:
ch_id = channel_id_match.group(1)
channel_info = ChannelStatus.get_basic_channel_info(ch_id)
@ -61,12 +56,4 @@ def fetch_channel_stats():
all_channels = None
gc.collect()
@shared_task
def cleanup_vod_connections():
"""Clean up stale VOD connections"""
try:
connection_manager = get_connection_manager()
connection_manager.cleanup_stale_connections(max_age_seconds=3600) # 1 hour
logger.info("VOD connection cleanup completed")
except Exception as e:
logger.error(f"Error in VOD connection cleanup: {e}", exc_info=True)

268
apps/proxy/ts_proxy/channel_status.py
View file

@ -6,6 +6,7 @@ from .redis_keys import RedisKeys
from .constants import TS_PACKET_SIZE, ChannelMetadataField
from redis.exceptions import ConnectionError, TimeoutError
from .utils import get_logger
from .client_manager import ClientManager
from django.db import DatabaseError # Add import for error handling
logger = get_logger()
@ -38,19 +39,19 @@ class ChannelStatus:
info = {
'channel_id': channel_id,
'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'),
'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'),
'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'),
'started_at': metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'),
'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'),
'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0,
'state': metadata.get(ChannelMetadataField.STATE, 'unknown'),
'url': metadata.get(ChannelMetadataField.URL, ''),
'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ''),
'started_at': metadata.get(ChannelMetadataField.INIT_TIME, '0'),
'owner': metadata.get(ChannelMetadataField.OWNER, 'unknown'),
'buffer_index': int(buffer_index_value) if buffer_index_value else 0,
}
# Add stream ID and name information
stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8'))
stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID)
if stream_id_bytes:
try:
stream_id = int(stream_id_bytes.decode('utf-8'))
stream_id = int(stream_id_bytes)
info['stream_id'] = stream_id
# Look up stream name from database
@ -65,10 +66,10 @@ class ChannelStatus:
logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}")
# Add M3U profile information
m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8'))
m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE)
if m3u_profile_id_bytes:
try:
m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8'))
m3u_profile_id = int(m3u_profile_id_bytes)
info['m3u_profile_id'] = m3u_profile_id
# Look up M3U profile name from database
@ -83,22 +84,22 @@ class ChannelStatus:
logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}")
# Add timing information
state_changed_field = ChannelMetadataField.STATE_CHANGED_AT.encode('utf-8')
state_changed_field = ChannelMetadataField.STATE_CHANGED_AT
if state_changed_field in metadata:
state_changed_at = float(metadata[state_changed_field].decode('utf-8'))
state_changed_at = float(metadata[state_changed_field])
info['state_changed_at'] = state_changed_at
info['state_duration'] = time.time() - state_changed_at
init_time_field = ChannelMetadataField.INIT_TIME.encode('utf-8')
init_time_field = ChannelMetadataField.INIT_TIME
if init_time_field in metadata:
created_at = float(metadata[init_time_field].decode('utf-8'))
created_at = float(metadata[init_time_field])
info['started_at'] = created_at
info['uptime'] = time.time() - created_at
# Add data throughput information
total_bytes_field = ChannelMetadataField.TOTAL_BYTES.encode('utf-8')
total_bytes_field = ChannelMetadataField.TOTAL_BYTES
if total_bytes_field in metadata:
total_bytes = int(metadata[total_bytes_field].decode('utf-8'))
total_bytes = int(metadata[total_bytes_field])
info['total_bytes'] = total_bytes
# Format total bytes in human-readable form
@ -127,43 +128,58 @@ class ChannelStatus:
client_ids = proxy_server.redis_client.smembers(client_set_key)
clients = []
stale_client_ids = []
for client_id in client_ids:
client_id_str = client_id.decode('utf-8')
client_id_str = client_id
client_key = RedisKeys.client_metadata(channel_id, client_id_str)
client_data = proxy_server.redis_client.hgetall(client_key)
if client_data:
client_info = {
'client_id': client_id_str,
'user_agent': client_data.get(b'user_agent', b'unknown').decode('utf-8'),
'worker_id': client_data.get(b'worker_id', b'unknown').decode('utf-8'),
}
if not client_data:
# Metadata hash expired but SET entry persists (ghost client).
stale_client_ids.append(client_id)
continue
if b'connected_at' in client_data:
connected_at = float(client_data[b'connected_at'].decode('utf-8'))
client_info['connected_at'] = connected_at
client_info['connection_duration'] = time.time() - connected_at
client_info = {
'client_id': client_id_str,
'user_agent': client_data.get('user_agent', 'unknown'),
'worker_id': client_data.get('worker_id', 'unknown'),
'ip_address': client_data.get('ip_address', 'unknown'),
'user_id': client_data.get('user_id', '0'),
}
if b'last_active' in client_data:
last_active = float(client_data[b'last_active'].decode('utf-8'))
client_info['last_active'] = last_active
client_info['last_active_ago'] = time.time() - last_active
if 'connected_at' in client_data:
connected_at = float(client_data['connected_at'])
client_info['connected_at'] = connected_at
client_info['connection_duration'] = time.time() - connected_at
# Add transfer rate statistics
if b'bytes_sent' in client_data:
client_info['bytes_sent'] = int(client_data[b'bytes_sent'].decode('utf-8'))
if 'last_active' in client_data:
last_active = float(client_data['last_active'])
client_info['last_active'] = last_active
client_info['last_active_ago'] = time.time() - last_active
# Add average transfer rate
if b'avg_rate_KBps' in client_data:
client_info['avg_rate_KBps'] = float(client_data[b'avg_rate_KBps'].decode('utf-8'))
elif b'transfer_rate_KBps' in client_data: # For backward compatibility
client_info['avg_rate_KBps'] = float(client_data[b'transfer_rate_KBps'].decode('utf-8'))
# Add transfer rate statistics
if 'bytes_sent' in client_data:
client_info['bytes_sent'] = int(client_data['bytes_sent'])
# Add current transfer rate
if b'current_rate_KBps' in client_data:
client_info['current_rate_KBps'] = float(client_data[b'current_rate_KBps'].decode('utf-8'))
# Add average transfer rate
if 'avg_rate_KBps' in client_data:
client_info['avg_rate_KBps'] = float(client_data['avg_rate_KBps'])
elif 'transfer_rate_KBps' in client_data: # For backward compatibility
client_info['avg_rate_KBps'] = float(client_data['transfer_rate_KBps'])
clients.append(client_info)
# Add current transfer rate
if 'current_rate_KBps' in client_data:
client_info['current_rate_KBps'] = float(client_data['current_rate_KBps'])
clients.append(client_info)
# Clean up stale SET entries so SCARD stays accurate.
if stale_client_ids:
proxy_server.redis_client.srem(client_set_key, *stale_client_ids)
logger.info(
f"Removed {len(stale_client_ids)} ghost client(s) from "
f"channel {channel_id} client set"
)
info['clients'] = clients
info['client_count'] = len(clients)
@ -235,7 +251,7 @@ class ChannelStatus:
while True:
cursor, keys = proxy_server.redis_client.scan(cursor, match=buffer_key_pattern, count=100)
if keys:
all_buffer_keys.extend([k.decode('utf-8') for k in keys])
all_buffer_keys.extend([k for k in keys])
if cursor == 0 or len(all_buffer_keys) >= 20: # Limit to 20 keys
break
@ -265,61 +281,64 @@ class ChannelStatus:
}
# Add FFmpeg stream information
video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8'))
video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC)
if video_codec:
info['video_codec'] = video_codec.decode('utf-8')
info['video_codec'] = video_codec
resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8'))
resolution = metadata.get(ChannelMetadataField.RESOLUTION)
if resolution:
info['resolution'] = resolution.decode('utf-8')
info['resolution'] = resolution
source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8'))
source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS)
if source_fps:
info['source_fps'] = float(source_fps.decode('utf-8'))
info['source_fps'] = source_fps
pixel_format = metadata.get(ChannelMetadataField.PIXEL_FORMAT.encode('utf-8'))
pixel_format = metadata.get(ChannelMetadataField.PIXEL_FORMAT)
if pixel_format:
info['pixel_format'] = pixel_format.decode('utf-8')
info['pixel_format'] = pixel_format
source_bitrate = metadata.get(ChannelMetadataField.SOURCE_BITRATE.encode('utf-8'))
source_bitrate = metadata.get(ChannelMetadataField.SOURCE_BITRATE)
if source_bitrate:
info['source_bitrate'] = float(source_bitrate.decode('utf-8'))
info['source_bitrate'] = source_bitrate
audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8'))
audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC)
if audio_codec:
info['audio_codec'] = audio_codec.decode('utf-8')
info['audio_codec'] = audio_codec
sample_rate = metadata.get(ChannelMetadataField.SAMPLE_RATE.encode('utf-8'))
sample_rate = metadata.get(ChannelMetadataField.SAMPLE_RATE)
if sample_rate:
info['sample_rate'] = int(sample_rate.decode('utf-8'))
info['sample_rate'] = sample_rate
audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8'))
audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS)
if audio_channels:
info['audio_channels'] = audio_channels.decode('utf-8')
info['audio_channels'] = audio_channels
audio_bitrate = metadata.get(ChannelMetadataField.AUDIO_BITRATE.encode('utf-8'))
audio_bitrate = metadata.get(ChannelMetadataField.AUDIO_BITRATE)
if audio_bitrate:
info['audio_bitrate'] = float(audio_bitrate.decode('utf-8'))
info['audio_bitrate'] = audio_bitrate
# Add FFmpeg performance stats
ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8'))
ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED)
if ffmpeg_speed:
info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8'))
info['ffmpeg_speed'] = ffmpeg_speed
ffmpeg_fps = metadata.get(ChannelMetadataField.FFMPEG_FPS.encode('utf-8'))
ffmpeg_fps = metadata.get(ChannelMetadataField.FFMPEG_FPS)
if ffmpeg_fps:
info['ffmpeg_fps'] = float(ffmpeg_fps.decode('utf-8'))
info['ffmpeg_fps'] = ffmpeg_fps
actual_fps = metadata.get(ChannelMetadataField.ACTUAL_FPS.encode('utf-8'))
actual_fps = metadata.get(ChannelMetadataField.ACTUAL_FPS)
if actual_fps:
info['actual_fps'] = float(actual_fps.decode('utf-8'))
info['actual_fps'] = actual_fps
ffmpeg_bitrate = metadata.get(ChannelMetadataField.FFMPEG_BITRATE.encode('utf-8'))
ffmpeg_bitrate = metadata.get(ChannelMetadataField.FFMPEG_BITRATE)
if ffmpeg_bitrate:
info['ffmpeg_bitrate'] = float(ffmpeg_bitrate.decode('utf-8'))
stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8'))
info['ffmpeg_bitrate'] = ffmpeg_bitrate
stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE)
if stream_type:
info['stream_type'] = stream_type.decode('utf-8')
info['stream_type'] = stream_type
return info
@ -364,33 +383,27 @@ class ChannelStatus:
client_count = proxy_server.redis_client.scard(client_set_key) or 0
# Calculate uptime
init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0')
created_at = float(init_time_bytes.decode('utf-8'))
init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME, '0')
created_at = float(init_time_bytes)
uptime = time.time() - created_at if created_at > 0 else 0
# Safely decode bytes or use defaults
def safe_decode(bytes_value, default="unknown"):
if bytes_value is None:
return default
return bytes_value.decode('utf-8')
# Simplified info
info = {
'channel_id': channel_id,
'state': safe_decode(metadata.get(ChannelMetadataField.STATE.encode('utf-8'))),
'url': safe_decode(metadata.get(ChannelMetadataField.URL.encode('utf-8')), ""),
'stream_profile': safe_decode(metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8')), ""),
'owner': safe_decode(metadata.get(ChannelMetadataField.OWNER.encode('utf-8'))),
'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0,
'state': metadata.get(ChannelMetadataField.STATE),
'url': metadata.get(ChannelMetadataField.URL, ""),
'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""),
'owner': metadata.get(ChannelMetadataField.OWNER),
'buffer_index': int(buffer_index_value) if buffer_index_value else 0,
'client_count': client_count,
'uptime': uptime
}
# Add stream ID and name information
stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8'))
stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID)
if stream_id_bytes:
try:
stream_id = int(stream_id_bytes.decode('utf-8'))
stream_id = int(stream_id_bytes)
info['stream_id'] = stream_id
# Look up stream name from database
@ -405,9 +418,9 @@ class ChannelStatus:
logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}")
# Add data throughput information to basic info
total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES.encode('utf-8'))
total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES)
if total_bytes_bytes:
total_bytes = int(total_bytes_bytes.decode('utf-8'))
total_bytes = int(total_bytes_bytes)
info['total_bytes'] = total_bytes
# Calculate and add bitrate
@ -430,42 +443,53 @@ class ChannelStatus:
clients = []
client_ids = proxy_server.redis_client.smembers(client_set_key)
# Process only if we have clients and keep it limited
if client_ids:
# Get up to 10 clients for the basic view
for client_id in list(client_ids)[:10]:
client_id_str = client_id.decode('utf-8')
client_key = RedisKeys.client_metadata(channel_id, client_id_str)
# Remove ghost SET entries before building the client list.
# Pass the already-fetched client_ids to avoid a redundant SMEMBERS.
stale_client_ids = ClientManager.remove_ghost_clients(
proxy_server.redis_client, channel_id, client_ids=client_ids
)
if stale_client_ids:
client_count = max(0, client_count - len(stale_client_ids))
# Build concise client list (up to 10) from remaining live clients.
if client_ids:
for client_id in list(client_ids)[:10]:
if client_id in stale_client_ids:
continue
client_key = RedisKeys.client_metadata(channel_id, client_id)
# Efficient way - just retrieve the essentials
client_info = {
'client_id': client_id_str,
'client_id': client_id,
}
# Safely get user_agent and ip_address
user_agent_bytes = proxy_server.redis_client.hget(client_key, 'user_agent')
client_info['user_agent'] = safe_decode(user_agent_bytes)
client_info['user_agent'] = user_agent_bytes
ip_address_bytes = proxy_server.redis_client.hget(client_key, 'ip_address')
if ip_address_bytes:
client_info['ip_address'] = safe_decode(ip_address_bytes)
client_info['ip_address'] = ip_address_bytes
# Just get connected_at for client age
connected_at_bytes = proxy_server.redis_client.hget(client_key, 'connected_at')
if connected_at_bytes:
connected_at = float(connected_at_bytes.decode('utf-8'))
connected_at = float(connected_at_bytes)
client_info['connected_since'] = time.time() - connected_at
user_id_bytes = proxy_server.redis_client.hget(client_key, 'user_id')
if user_id_bytes:
client_info['user_id'] = user_id_bytes
clients.append(client_info)
# Add clients to info
info['clients'] = clients
info['client_count'] = client_count
# Add M3U profile information
m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8'))
if m3u_profile_id_bytes:
m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE)
if m3u_profile_id:
try:
m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8'))
m3u_profile_id = int(m3u_profile_id)
info['m3u_profile_id'] = m3u_profile_id
# Look up M3U profile name from database
@ -477,32 +501,36 @@ class ChannelStatus:
except (ImportError, DatabaseError) as e:
logger.warning(f"Failed to get M3U profile name for ID {m3u_profile_id}: {e}")
except ValueError:
logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}")
logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}")
# Add stream info to basic info as well
video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8'))
video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC)
if video_codec:
info['video_codec'] = video_codec.decode('utf-8')
info['video_codec'] = video_codec
resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8'))
resolution = metadata.get(ChannelMetadataField.RESOLUTION)
if resolution:
info['resolution'] = resolution.decode('utf-8')
info['resolution'] = resolution
source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8'))
source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS)
if source_fps:
info['source_fps'] = float(source_fps.decode('utf-8'))
ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8'))
info['source_fps'] = float(source_fps)
ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED)
if ffmpeg_speed:
info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8'))
audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8'))
info['ffmpeg_speed'] = float(ffmpeg_speed)
audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC)
if audio_codec:
info['audio_codec'] = audio_codec.decode('utf-8')
audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8'))
info['audio_codec'] = audio_codec
audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS)
if audio_channels:
info['audio_channels'] = audio_channels.decode('utf-8')
stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8'))
info['audio_channels'] = audio_channels
stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE)
if stream_type:
info['stream_type'] = stream_type.decode('utf-8')
info['stream_type'] = stream_type
return info
except Exception as e:

72
apps/proxy/ts_proxy/client_manager.py
View file

@ -1,10 +1,8 @@
"""Client connection management for TS streams"""
import threading
import logging
import time
import json
import gevent
from typing import Set, Optional
from apps.proxy.config import TSConfig as Config
from redis.exceptions import ConnectionError, TimeoutError
@ -58,7 +56,8 @@ class ClientManager:
from django.conf import settings
redis_url = getattr(settings, 'REDIS_URL', 'redis://localhost:6379/0')
redis_client = redis.Redis.from_url(redis_url, decode_responses=True)
ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {})
redis_client = redis.Redis.from_url(redis_url, decode_responses=True, **ssl_params)
all_channels = []
cursor = 0
@ -129,7 +128,7 @@ class ClientManager:
# Check for stale activity using last_active field
last_active = self.redis_client.hget(client_key, "last_active")
if last_active:
last_active_time = float(last_active.decode('utf-8'))
last_active_time = float(last_active)
ghost_timeout = self.heartbeat_interval * getattr(Config, 'GHOST_CLIENT_MULTIPLIER', 5.0)
if current_time - last_active_time > ghost_timeout:
@ -158,9 +157,8 @@ class ClientManager:
if time_since_heartbeat < self.heartbeat_interval * 0.5: # Only heartbeat at half interval minimum
continue
# Only update clients that remain
# Only refresh TTL - do NOT update last_active
client_key = f"ts_proxy:channel:{self.channel_id}:clients:{client_id}"
pipe.hset(client_key, "last_active", str(current_time))
pipe.expire(client_key, self.client_ttl)
# Keep client in the set with TTL
@ -230,7 +228,7 @@ class ClientManager:
except Exception as e:
logger.error(f"Error notifying owner of client activity: {e}")
def add_client(self, client_id, client_ip, user_agent=None):
def add_client(self, client_id, client_ip, user_agent=None, user=None):
"""Add a client with duplicate prevention"""
if client_id in self._registered_clients:
logger.debug(f"Client {client_id} already registered, skipping")
@ -248,7 +246,9 @@ class ClientManager:
"ip_address": client_ip,
"connected_at": current_time,
"last_active": current_time,
"worker_id": self.worker_id or "unknown"
"worker_id": self.worker_id or "unknown",
"user_id": str(user.id) if user is not None else "0",
# "user_level": user.user_level if user is not None else 100, # default to a high value since no user means the non-user specific M3U/HDHR
}
try:
@ -278,7 +278,8 @@ class ClientManager:
"channel_id": self.channel_id,
"client_id": client_id,
"worker_id": self.worker_id or "unknown",
"timestamp": time.time()
"timestamp": time.time(),
"username": user.username if user is not None else "unknown"
}
if user_agent:
@ -309,8 +310,6 @@ class ClientManager:
def remove_client(self, client_id):
"""Remove a client from this channel and Redis"""
client_ip = None
with self.lock:
if client_id in self.clients:
self.clients.remove(client_id)
@ -321,13 +320,11 @@ class ClientManager:
self.last_active_time = time.time()
if self.redis_client:
# Get client IP before removing the data
# Get client data before removing the data
client_key = f"ts_proxy:channel:{self.channel_id}:clients:{client_id}"
client_data = self.redis_client.hgetall(client_key)
if client_data and b'ip_address' in client_data:
client_ip = client_data[b'ip_address'].decode('utf-8')
elif client_data and 'ip_address' in client_data:
client_ip = client_data['ip_address']
client_username = self.redis_client.hget(client_key, "username") or "unknown"
if isinstance(client_username, bytes):
client_username = client_username.decode("utf-8")
# Remove from channel's client set
self.redis_client.srem(self.client_set_key, client_id)
@ -368,7 +365,8 @@ class ClientManager:
"client_id": client_id,
"worker_id": self.worker_id or "unknown",
"timestamp": time.time(),
"remaining_clients": remaining
"remaining_clients": remaining,
"username": client_username
})
self.redis_client.publish(RedisKeys.events_channel(self.channel_id), event_data)
@ -413,3 +411,41 @@ class ClientManager:
self.redis_client.expire(self.client_set_key, self.client_ttl)
except Exception as e:
logger.error(f"Error refreshing client TTL: {e}")
@staticmethod
def remove_ghost_clients(redis_client, channel_id, client_ids=None):
"""Remove client SET entries whose metadata hash has expired.
Returns the list of removed (stale) client IDs, or an empty list
if none were found. Uses a pipelined EXISTS check for efficiency.
Args:
client_ids: Optional pre-fetched result of SMEMBERS for this
channel. Pass this to avoid a redundant SMEMBERS
call when the caller has already fetched it.
"""
client_set_key = RedisKeys.clients(channel_id)
if client_ids is None:
client_ids = redis_client.smembers(client_set_key)
if not client_ids:
return []
client_id_list = list(client_ids)
pipe = redis_client.pipeline()
for cid in client_id_list:
pipe.exists(RedisKeys.client_metadata(channel_id, cid))
results = pipe.execute()
stale_ids = [
cid for cid, exists in zip(client_id_list, results)
if not exists
]
if stale_ids:
redis_client.srem(client_set_key, *stale_ids)
logger.info(
f"Removed {len(stale_ids)} ghost client(s) from "
f"channel {channel_id} client set"
)
return stale_ids

6
apps/proxy/ts_proxy/constants.py
View file

@ -20,6 +20,10 @@ class ChannelState:
STOPPED = "stopped"
BUFFERING = "buffering"
# States before a channel is fully active. Used by the stream manager
# finally block to decide whether a failed stream can write ERROR.
PRE_ACTIVE = frozenset([INITIALIZING, CONNECTING, BUFFERING, WAITING_FOR_CLIENTS])
# Event types
class EventType:
STREAM_SWITCH = "stream_switch"
@ -71,6 +75,7 @@ class ChannelMetadataField:
FFMPEG_FPS = "ffmpeg_fps"
ACTUAL_FPS = "actual_fps"
FFMPEG_OUTPUT_BITRATE = "ffmpeg_output_bitrate"
FFMPEG_BITRATE = "ffmpeg_bitrate"
FFMPEG_STATS_UPDATED = "ffmpeg_stats_updated"
# Video stream info
@ -81,6 +86,7 @@ class ChannelMetadataField:
SOURCE_FPS = "source_fps"
PIXEL_FORMAT = "pixel_format"
VIDEO_BITRATE = "video_bitrate"
SOURCE_BITRATE = "source_bitrate"
# Audio stream info
AUDIO_CODEC = "audio_codec"

150
apps/proxy/ts_proxy/server.py
View file

@ -166,6 +166,7 @@ class ProxyServer:
redis_password = os.environ.get("REDIS_PASSWORD", getattr(settings, 'REDIS_PASSWORD', ''))
redis_user = os.environ.get("REDIS_USER", getattr(settings, 'REDIS_USER', ''))
ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {})
pubsub_client = redis.Redis(
host=redis_host,
port=redis_port,
@ -175,7 +176,9 @@ class ProxyServer:
socket_timeout=60,
socket_connect_timeout=10,
socket_keepalive=True,
health_check_interval=30
health_check_interval=30,
decode_responses=True,
**ssl_params
)
logger.info("Created fallback Redis PubSub client for event listener")
@ -196,8 +199,8 @@ class ProxyServer:
continue
try:
channel = message["channel"].decode("utf-8")
data = json.loads(message["data"].decode("utf-8"))
channel = message["channel"]
data = json.loads(message["data"])
event_type = data.get("event")
channel_id = data.get("channel_id")
@ -224,26 +227,29 @@ class ProxyServer:
# Handle stream switch request
new_url = data.get("url")
user_agent = data.get("user_agent")
event_stream_id = data.get("stream_id")
event_m3u_profile_id = data.get("m3u_profile_id")
if new_url and channel_id in self.stream_managers:
# Update metadata in Redis
# Mark the switch as in-progress in Redis so other workers know to wait
if self.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
self.redis_client.hset(metadata_key, "url", new_url)
if user_agent:
self.redis_client.hset(metadata_key, "user_agent", user_agent)
# Set switch status
status_key = RedisKeys.switch_status(channel_id)
self.redis_client.set(status_key, "switching")
# Perform the stream switch
# Perform the stream switch, forwarding stream_id and m3u_profile_id
stream_manager = self.stream_managers[channel_id]
success = stream_manager.update_url(new_url)
success = stream_manager.update_url(new_url, event_stream_id, event_m3u_profile_id)
if success:
logger.info(f"Stream switch initiated for channel {channel_id}")
# Confirm the URL in metadata now that the switch happened
if self.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
self.redis_client.hset(metadata_key, "url", new_url)
if user_agent:
self.redis_client.hset(metadata_key, "user_agent", user_agent)
# Publish confirmation
switch_result = {
"event": EventType.STREAM_SWITCHED, # Use constant instead of string
@ -263,6 +269,14 @@ class ProxyServer:
else:
logger.error(f"Failed to switch stream for channel {channel_id}")
# Roll back the URL in metadata to what the manager will
# actually reconnect to. The non-owner may have pre-written
# the desired URL; use stream_manager.url (the ground truth)
# so Redis is consistent with the live stream.
if self.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
self.redis_client.hset(metadata_key, "url", stream_manager.url)
# Publish failure
switch_result = {
"event": EventType.STREAM_SWITCHED,
@ -373,7 +387,7 @@ class ProxyServer:
if result is None:
return None
try:
return result.decode('utf-8')
return result
except (AttributeError, UnicodeDecodeError) as e:
logger.error(f"Error decoding channel owner for {channel_id}: {e}, raw={result!r}")
return None
@ -412,7 +426,7 @@ class ProxyServer:
current_owner = self._execute_redis_command(
lambda: self.redis_client.get(lock_key)
)
if current_owner and current_owner.decode('utf-8') == self.worker_id:
if current_owner and current_owner == self.worker_id:
# Refresh TTL
self._execute_redis_command(
lambda: self.redis_client.expire(lock_key, ttl)
@ -437,7 +451,7 @@ class ProxyServer:
# Only delete if we're the current owner to prevent race conditions
current = self.redis_client.get(lock_key)
if current and current.decode('utf-8') == self.worker_id:
if current and current == self.worker_id:
self.redis_client.delete(lock_key)
logger.info(f"Released ownership of channel {channel_id}")
@ -471,7 +485,7 @@ class ProxyServer:
return False
return False
if current.decode('utf-8') == self.worker_id:
if current == self.worker_id:
self.redis_client.expire(lock_key, ttl)
return True
@ -488,15 +502,15 @@ class ProxyServer:
metadata_key = RedisKeys.channel_metadata(channel_id)
if self.redis_client.exists(metadata_key):
metadata = self.redis_client.hgetall(metadata_key)
if b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if 'state' in metadata:
state = metadata['state']
active_states = [ChannelState.INITIALIZING, ChannelState.CONNECTING,
ChannelState.WAITING_FOR_CLIENTS, ChannelState.ACTIVE, ChannelState.BUFFERING]
if state in active_states:
logger.info(f"Channel {channel_id} already being initialized with state {state}")
# Create buffer and client manager only if we don't have them
if channel_id not in self.stream_buffers:
self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=self.redis_client)
self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer())
if channel_id not in self.client_managers:
self.client_managers[channel_id] = ClientManager(
channel_id,
@ -507,7 +521,7 @@ class ProxyServer:
# Create buffer and client manager instances (or reuse if they exist)
if channel_id not in self.stream_buffers:
buffer = StreamBuffer(channel_id, redis_client=self.redis_client)
buffer = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer())
self.stream_buffers[channel_id] = buffer
if channel_id not in self.client_managers:
@ -546,18 +560,18 @@ class ProxyServer:
# If no url was passed, try to get from Redis
if not url and existing_metadata:
url_bytes = existing_metadata.get(b'url')
url_bytes = existing_metadata.get('url')
if url_bytes:
channel_url = url_bytes.decode('utf-8')
channel_url = url_bytes
ua_bytes = existing_metadata.get(b'user_agent')
ua_bytes = existing_metadata.get('user_agent')
if ua_bytes:
channel_user_agent = ua_bytes.decode('utf-8')
channel_user_agent = ua_bytes
# Get stream ID from metadata if not provided
if not channel_stream_id and b'stream_id' in existing_metadata:
if not channel_stream_id and 'stream_id' in existing_metadata:
try:
channel_stream_id = int(existing_metadata[b'stream_id'].decode('utf-8'))
channel_stream_id = int(existing_metadata['stream_id'])
logger.debug(f"Found stream_id {channel_stream_id} in metadata for channel {channel_id}")
except (ValueError, TypeError) as e:
logger.debug(f"Could not parse stream_id from metadata: {e}")
@ -572,7 +586,7 @@ class ProxyServer:
# Create buffer but not stream manager (only if not already exists)
if channel_id not in self.stream_buffers:
buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client)
buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer())
self.stream_buffers[channel_id] = buffer
# Create client manager with channel_id and redis_client (only if not already exists)
@ -595,7 +609,7 @@ class ProxyServer:
# Create buffer but not stream manager (only if not already exists)
if channel_id not in self.stream_buffers:
buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client)
buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer())
self.stream_buffers[channel_id] = buffer
# Create client manager with channel_id and redis_client (only if not already exists)
@ -634,12 +648,12 @@ class ProxyServer:
# Verify the stream_id was set correctly in Redis
stream_id_value = self.redis_client.hget(metadata_key, "stream_id")
if stream_id_value:
logger.info(f"Verified stream_id {stream_id_value.decode('utf-8')} is set in Redis for channel {channel_id}")
logger.info(f"Verified stream_id {stream_id_value} is set in Redis for channel {channel_id}")
else:
logger.warning(f"Failed to set stream_id in Redis for channel {channel_id}")
# Create stream buffer
buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client)
buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer())
logger.debug(f"Created StreamBuffer for channel {channel_id}")
self.stream_buffers[channel_id] = buffer
@ -733,8 +747,8 @@ class ProxyServer:
metadata = self.redis_client.hgetall(metadata_key)
# Get channel state and owner
state = metadata.get(b'state', b'unknown').decode('utf-8')
owner = metadata.get(b'owner', b'').decode('utf-8')
state = metadata.get('state', 'unknown')
owner = metadata.get('owner', '')
# States that indicate the channel is running properly or shutting down
valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS,
@ -772,8 +786,8 @@ class ProxyServer:
return False
else:
# Unknown or initializing state, check how long it's been in this state
if b'state_changed_at' in metadata:
state_changed_at = float(metadata[b'state_changed_at'].decode('utf-8'))
if 'state_changed_at' in metadata:
state_changed_at = float(metadata['state_changed_at'])
state_age = time.time() - state_changed_at
# If in initializing state for too long, consider it stale
@ -811,8 +825,8 @@ class ProxyServer:
# If we have metadata, log details for debugging
if metadata:
state = metadata.get(b'state', b'unknown').decode('utf-8')
owner = metadata.get(b'owner', b'unknown').decode('utf-8')
state = metadata.get('state', 'unknown')
owner = metadata.get('owner', 'unknown')
logger.info(f"Zombie channel details - state: {state}, owner: {owner}")
# Clean up Redis keys
@ -937,16 +951,16 @@ class ProxyServer:
metadata = self.redis_client.hgetall(metadata_key)
if metadata:
# Calculate runtime from init_time
if b'init_time' in metadata:
if 'init_time' in metadata:
try:
init_time = float(metadata[b'init_time'].decode('utf-8'))
init_time = float(metadata['init_time'])
runtime = round(time.time() - init_time, 2)
except Exception:
pass
# Get total bytes transferred
if b'total_bytes' in metadata:
if 'total_bytes' in metadata:
try:
total_bytes = int(metadata[b'total_bytes'].decode('utf-8'))
total_bytes = int(metadata['total_bytes'])
except Exception:
pass
@ -1057,8 +1071,8 @@ class ProxyServer:
if self.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = self.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
channel_state = metadata[b'state'].decode('utf-8')
if metadata and 'state' in metadata:
channel_state = metadata['state']
# Check if channel has any clients left
total_clients = 0
@ -1080,7 +1094,7 @@ class ProxyServer:
logger.info(f"Channel {channel_id} has {total_clients} clients, state: {channel_state}")
# If in connecting or waiting_for_clients state, check grace period
if channel_state in [ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS]:
if channel_state in [ChannelState.INITIALIZING, ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS]:
# Check if channel is already stopping
if self.redis_client:
stop_key = RedisKeys.channel_stopping(channel_id)
@ -1090,9 +1104,9 @@ class ProxyServer:
# Get connection_ready_time from metadata (indicates if channel reached ready state)
connection_ready_time = None
if metadata and b'connection_ready_time' in metadata:
if metadata and 'connection_ready_time' in metadata:
try:
connection_ready_time = float(metadata[b'connection_ready_time'].decode('utf-8'))
connection_ready_time = float(metadata['connection_ready_time'])
except (ValueError, TypeError):
pass
@ -1104,15 +1118,15 @@ class ProxyServer:
attempt_value = self.redis_client.get(attempt_key)
if attempt_value:
try:
connection_attempt_time = float(attempt_value.decode('utf-8'))
connection_attempt_time = float(attempt_value)
except (ValueError, TypeError):
pass
# Also get init time as a fallback
init_time = None
if metadata and b'init_time' in metadata:
if metadata and 'init_time' in metadata:
try:
init_time = float(metadata[b'init_time'].decode('utf-8'))
init_time = float(metadata['init_time'])
except (ValueError, TypeError):
pass
@ -1183,7 +1197,7 @@ class ProxyServer:
disconnect_value = self.redis_client.get(disconnect_key)
if disconnect_value:
try:
disconnect_time = float(disconnect_value.decode('utf-8'))
disconnect_time = float(disconnect_value)
except (ValueError, TypeError) as e:
logger.error(f"Invalid disconnect time for channel {channel_id}: {e}")
@ -1304,7 +1318,7 @@ class ProxyServer:
for key in channel_keys:
try:
channel_id = key.decode('utf-8').split(':')[2]
channel_id = key.split(':')[2]
# Check if this channel has an owner
owner = self.get_channel_owner(channel_id)
@ -1349,7 +1363,7 @@ class ProxyServer:
for key in channel_keys:
try:
channel_id = key.decode('utf-8').split(':')[2]
channel_id = key.split(':')[2]
# Get metadata first
metadata = self.redis_client.hgetall(key)
@ -1364,7 +1378,7 @@ class ProxyServer:
continue
# Get owner
owner = metadata.get(b'owner', b'').decode('utf-8') if b'owner' in metadata else ''
owner = metadata.get('owner', '') if 'owner' in metadata else ''
# Check if owner is still alive
owner_alive = False
@ -1378,7 +1392,7 @@ class ProxyServer:
# If no owner and no clients, clean it up
if not owner_alive and client_count == 0:
state = metadata.get(b'state', b'unknown').decode('utf-8') if b'state' in metadata else 'unknown'
state = metadata.get('state', 'unknown')
logger.warning(f"Found orphaned metadata for channel {channel_id} (state: {state}, owner: {owner}, clients: {client_count}) - cleaning up")
# If we have it locally, stop it properly to clean up transcode/proxy processes
@ -1389,8 +1403,30 @@ class ProxyServer:
# Just clean up Redis keys for remote channels
self._clean_redis_keys(channel_id)
elif not owner_alive and client_count > 0:
# Owner is gone but clients remain - just log for now
logger.warning(f"Found orphaned channel {channel_id} with {client_count} clients but no owner - may need ownership takeover")
# SCARD may include ghost entries from a dead worker's
# expired metadata hashes. Validate before deciding.
stale_ids = ClientManager.remove_ghost_clients(
self.redis_client, channel_id
)
real_count = max(0, client_count - len(stale_ids))
if real_count <= 0:
# No real clients remain — safe to clean up.
state = metadata.get('state', 'unknown')
logger.warning(
f"Orphaned channel {channel_id} (state: {state}, "
f"owner: {owner}) had {client_count} ghost client(s) "
f"- cleaning up"
)
if channel_id in self.stream_managers or channel_id in self.client_managers:
self.stop_channel(channel_id)
else:
self._clean_redis_keys(channel_id)
else:
logger.warning(
f"Orphaned channel {channel_id} still has "
f"{real_count} live client(s) after ghost removal "
f"- may need ownership takeover"
)
except Exception as e:
logger.error(f"Error processing metadata key {key}: {e}", exc_info=True)
@ -1470,8 +1506,8 @@ class ProxyServer:
# Get current state for logging
current_state = None
metadata = self.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
current_state = metadata[b'state'].decode('utf-8')
if metadata and 'state' in metadata:
current_state = metadata['state']
# Only update if state is actually changing
if current_state == new_state:

52
apps/proxy/ts_proxy/services/channel_service.py
View file

@ -61,7 +61,7 @@ class ChannelService:
# Verify the stream_id was set
stream_id_value = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID)
if stream_id_value:
logger.debug(f"Verified stream_id {stream_id_value.decode('utf-8')} is now set in Redis")
logger.debug(f"Verified stream_id {stream_id_value} is now set in Redis")
else:
logger.error(f"Failed to set stream_id {stream_id} in Redis before initialization")
@ -131,7 +131,7 @@ class ChannelService:
try:
# This is inefficient but used for diagnostics - in production would use more targeted checks
redis_keys = proxy_server.redis_client.keys(f"ts_proxy:*:{channel_id}*")
redis_keys = [k.decode('utf-8') for k in redis_keys] if redis_keys else []
redis_keys = [k for k in redis_keys] if redis_keys else []
except Exception as e:
logger.error(f"Error checking Redis keys: {e}")
@ -168,15 +168,6 @@ class ChannelService:
else:
result = {'status': 'success'}
# Update metadata in Redis regardless of ownership
if proxy_server.redis_client:
try:
ChannelService._update_channel_metadata(channel_id, new_url, user_agent, stream_id, m3u_profile_id)
result['metadata_updated'] = True
except Exception as e:
logger.error(f"Error updating Redis metadata: {e}", exc_info=True)
result['metadata_updated'] = False
# If we're the owner, update directly
if proxy_server.am_i_owner(channel_id) and channel_id in proxy_server.stream_managers:
logger.info(f"This worker is the owner, changing stream URL for channel {channel_id}")
@ -187,14 +178,33 @@ class ChannelService:
success = manager.update_url(new_url, stream_id, m3u_profile_id)
logger.info(f"Stream URL changed from {old_url} to {new_url}, result: {success}")
# Update Redis metadata based on the actual outcome.
# On success, write the new values. On failure, restore whatever URL
# the manager will actually reconnect to (may be old_url if the
# exception happened before self.url was reassigned, or new_url if it
# happened after) so Redis never describes a URL that isn't in use.
if proxy_server.redis_client:
try:
if success:
ChannelService._update_channel_metadata(channel_id, new_url, user_agent, stream_id, m3u_profile_id)
else:
ChannelService._update_channel_metadata(channel_id, manager.url, user_agent)
result['metadata_updated'] = True
except Exception as e:
logger.error(f"Error updating Redis metadata: {e}", exc_info=True)
result['metadata_updated'] = False
result.update({
'direct_update': True,
'success': success,
'worker_id': proxy_server.worker_id
})
else:
# If we're not the owner, publish an event for the owner to pick up
logger.info(f"Not the owner, requesting URL change via Redis PubSub")
# Not the owner: publish the switch event. The owner will update metadata
# after the actual switch attempt succeeds (or roll back on failure).
# All needed info (url, user_agent, stream_id, m3u_profile_id) is carried
# in the pubsub message, so there is no reason to pre-write metadata here.
logger.debug(f"This worker is not the owner, publishing stream switch event for channel {channel_id}")
if proxy_server.redis_client:
ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent, stream_id, m3u_profile_id)
result.update({
@ -236,8 +246,8 @@ class ChannelService:
metadata_key = RedisKeys.channel_metadata(channel_id)
try:
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if metadata and 'state' in metadata:
state = metadata['state']
channel_info = {"state": state}
# Immediately mark as stopping in metadata so clients detect it faster
@ -382,8 +392,8 @@ class ChannelService:
metadata = proxy_server.redis_client.hgetall(metadata_key)
# Extract state and owner
state = metadata.get(ChannelMetadataField.STATE.encode(), b'unknown').decode('utf-8')
owner = metadata.get(ChannelMetadataField.OWNER.encode(), b'unknown').decode('utf-8')
state = metadata.get(ChannelMetadataField.STATE, 'unknown')
owner = metadata.get(ChannelMetadataField.OWNER, 'unknown')
# Valid states indicate channel is running properly
valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING]
@ -409,7 +419,7 @@ class ChannelService:
}
if last_data:
last_data_time = float(last_data.decode('utf-8'))
last_data_time = float(last_data)
data_age = time.time() - last_data_time
details["last_data_age"] = data_age
@ -432,13 +442,13 @@ class ChannelService:
try:
# Use factory to parse the line based on stream type
parsed_data = LogParserFactory.parse(stream_type, stream_info_line)
if not parsed_data:
return
# Update Redis and database with parsed data
ChannelService._update_stream_info_in_redis(
channel_id,
channel_id,
parsed_data.get('video_codec'),
parsed_data.get('resolution'),
parsed_data.get('width'),
@ -579,7 +589,7 @@ class ChannelService:
metadata_key = RedisKeys.channel_metadata(channel_id)
# First check if the key exists and what type it is
key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8')
key_type = proxy_server.redis_client.type(metadata_key)
logger.debug(f"Redis key {metadata_key} is of type: {key_type}")
# Build metadata update dict

73
apps/proxy/ts_proxy/stream_buffer.py
View file

@ -76,27 +76,28 @@ class StreamBuffer:
if not hasattr(self, '_partial_packet'):
self._partial_packet = bytearray()
# Combine with any previous partial packet
combined_data = bytearray(self._partial_packet) + bytearray(chunk)
# Calculate complete packets
complete_packets_size = (len(combined_data) // self.TS_PACKET_SIZE) * self.TS_PACKET_SIZE
if complete_packets_size == 0:
# Not enough data for a complete packet
self._partial_packet = combined_data
return True
# Split into complete packets and remainder
complete_packets = combined_data[:complete_packets_size]
self._partial_packet = combined_data[complete_packets_size:]
# Add completed packets to write buffer
self._write_buffer.extend(complete_packets)
# Only write to Redis when we have enough data for an optimized chunk
# Lock the full operation to prevent race with reset_buffer_position
writes_done = 0
with self.lock:
# Combine with any previous partial packet
combined_data = bytearray(self._partial_packet) + bytearray(chunk)
# Calculate complete packets
complete_packets_size = (len(combined_data) // self.TS_PACKET_SIZE) * self.TS_PACKET_SIZE
if complete_packets_size == 0:
# Not enough data for a complete packet
self._partial_packet = combined_data
return True
# Split into complete packets and remainder
complete_packets = combined_data[:complete_packets_size]
self._partial_packet = combined_data[complete_packets_size:]
# Add completed packets to write buffer
self._write_buffer.extend(complete_packets)
# Only write to Redis when we have enough data for an optimized chunk
while len(self._write_buffer) >= self.target_chunk_size:
# Extract a full chunk
chunk_data = self._write_buffer[:self.target_chunk_size]
@ -132,6 +133,40 @@ class StreamBuffer:
logger.error(f"Error adding chunk to buffer: {e}")
return False
def reset_buffer_position(self):
"""
Reset internal buffers for a clean stream transition (failover).
Called by stream_manager.update_url() when switching between FFmpeg
processes. Without this, _partial_packet from the old FFmpeg gets
concatenated with the first bytes from the new FFmpeg, creating
corrupted TS packets that break audio decoder sync in the client.
"""
try:
with self.lock:
old_write_size = len(self._write_buffer)
old_partial_size = len(getattr(self, '_partial_packet', b''))
self._write_buffer = bytearray()
if hasattr(self, '_partial_packet'):
self._partial_packet = bytearray()
if old_write_size > 0 or old_partial_size > 0:
logger.info(
f"Reset buffer position for channel {self.channel_id}: "
f"cleared {old_write_size} bytes from write buffer, "
f"{old_partial_size} bytes from partial packet"
)
else:
logger.debug(
f"Reset buffer position for channel {self.channel_id}: "
f"buffers were already clean"
)
except Exception as e:
logger.error(
f"Error resetting buffer position for channel {self.channel_id}: {e}"
)
def get_chunks(self, start_index=None):
"""Get chunks from the buffer with detailed logging"""
try:

56
apps/proxy/ts_proxy/stream_generator.py
View file

@ -25,7 +25,7 @@ class StreamGenerator:
data delivery, and cleanup.
"""
def __init__(self, channel_id, client_id, client_ip, client_user_agent, channel_initializing=False):
def __init__(self, channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None):
"""
Initialize the stream generator with client and channel details.
@ -35,12 +35,14 @@ class StreamGenerator:
client_ip: Client's IP address
client_user_agent: User agent string from client
channel_initializing: Whether the channel is still initializing
user: Authenticated user making the request
"""
self.channel_id = channel_id
self.client_id = client_id
self.client_ip = client_ip
self.client_user_agent = client_user_agent
self.channel_initializing = channel_initializing
self.user = user
# Performance and state tracking
self.stream_start_time = time.time()
@ -112,7 +114,8 @@ class StreamGenerator:
channel_name=channel_obj.name,
client_ip=self.client_ip,
client_id=self.client_id,
user_agent=self.client_user_agent[:100] if self.client_user_agent else None
user_agent=self.client_user_agent[:100] if self.client_user_agent else None,
username=self.user.username if self.user else None
)
except Exception as e:
logger.error(f"Could not log client connect event: {e}")
@ -141,13 +144,13 @@ class StreamGenerator:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if metadata and 'state' in metadata:
state = metadata['state']
if state in ['waiting_for_clients', 'active']:
logger.info(f"[{self.client_id}] Channel {self.channel_id} now ready (state={state})")
return True
elif state in ['error', 'stopped', 'stopping']: # Added 'stopping' to error states
error_message = metadata.get(b'error_message', b'Unknown error').decode('utf-8')
error_message = metadata.get('error_message', 'Unknown error')
logger.error(f"[{self.client_id}] Channel {self.channel_id} in error state: {state}, message: {error_message}")
# Send error packet before giving up
yield create_ts_packet('error', f"Error: {error_message}")
@ -155,9 +158,9 @@ class StreamGenerator:
else:
# Improved logging to track initialization progress
init_time = "unknown"
if b'init_time' in metadata:
if 'init_time' in metadata:
try:
init_time_float = float(metadata[b'init_time'].decode('utf-8'))
init_time_float = float(metadata['init_time'])
init_duration = time.time() - init_time_float
init_time = f"{init_duration:.1f}s ago"
except:
@ -255,6 +258,10 @@ class StreamGenerator:
def _stream_data_generator(self):
"""Generate stream data chunks based on buffer contents."""
# Keepalive packets refresh last_yield_time, so _is_timeout() never fires
# during sustained stream failure. This timer enforces a wall-clock cap.
keepalive_start_time = None
# Main streaming loop
while True:
# Check if resources still exist
@ -265,6 +272,7 @@ class StreamGenerator:
chunks, next_index = self.buffer.get_optimized_client_data(self.local_index)
if chunks:
keepalive_start_time = None # Each recovery restarts the cap independently.
yield from self._process_chunks(chunks, next_index)
self.local_index = next_index
self.last_yield_time = time.time()
@ -306,12 +314,28 @@ class StreamGenerator:
continue # Retry immediately with the new position
if self._should_send_keepalive(self.local_index):
if keepalive_start_time is None:
keepalive_start_time = time.time()
max_keepalive = getattr(Config, 'MAX_KEEPALIVE_DURATION', 300)
if time.time() - keepalive_start_time > max_keepalive:
logger.warning(
f"[{self.client_id}] Keepalive duration exceeded {max_keepalive}s "
f"with no stream recovery, disconnecting"
)
break
keepalive_packet = create_ts_packet('keepalive')
logger.debug(f"[{self.client_id}] Sending keepalive packet while waiting at buffer head")
yield keepalive_packet
self.bytes_sent += len(keepalive_packet)
self.last_yield_time = time.time()
self.consecutive_empty = 0 # Reset consecutive counter but keep total empty_reads
# Update last_active so clients waiting during failover aren't flagged as ghosts
proxy_server = ProxyServer.get_instance()
if proxy_server and proxy_server.redis_client:
client_key = RedisKeys.client_metadata(self.channel_id, self.client_id)
proxy_server.redis_client.hset(client_key, "last_active", str(time.time()))
gevent.sleep(Config.KEEPALIVE_INTERVAL) # Replace time.sleep
else:
# Standard wait with backoff
@ -369,8 +393,8 @@ class StreamGenerator:
# Channel state in metadata
metadata_key = RedisKeys.channel_metadata(self.channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if metadata and 'state' in metadata:
state = metadata['state']
if state in ['error', 'stopped', 'stopping']:
logger.info(f"[{self.client_id}] Channel in {state} state, terminating stream")
return False
@ -428,7 +452,8 @@ class StreamGenerator:
ChannelMetadataField.BYTES_SENT: str(self.bytes_sent),
ChannelMetadataField.AVG_RATE_KBPS: str(round(avg_rate, 1)),
ChannelMetadataField.CURRENT_RATE_KBPS: str(round(self.current_rate, 1)),
ChannelMetadataField.STATS_UPDATED_AT: str(current_time)
ChannelMetadataField.STATS_UPDATED_AT: str(current_time),
"last_active": str(current_time)
}
proxy_server.redis_client.hset(client_key, mapping=stats)
@ -533,8 +558,6 @@ class StreamGenerator:
if metadata:
stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID)
if stream_id_bytes:
stream_id = int(stream_id_bytes.decode('utf-8'))
# Check if we're the last client
if self.channel_id in proxy_server.client_managers:
client_count = proxy_server.client_managers[self.channel_id].get_total_client_count()
@ -573,7 +596,8 @@ class StreamGenerator:
client_id=self.client_id,
user_agent=self.client_user_agent[:100] if self.client_user_agent else None,
duration=round(elapsed, 2),
bytes_sent=self.bytes_sent
bytes_sent=self.bytes_sent,
username=self.user.username if self.user else None
)
except Exception as e:
logger.error(f"Could not log client disconnect event: {e}")
@ -608,10 +632,10 @@ class StreamGenerator:
gevent.spawn(delayed_shutdown)
def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False):
def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None):
"""
Factory function to create a new stream generator.
Returns a function that can be passed to StreamingHttpResponse.
"""
generator = StreamGenerator(channel_id, client_id, client_ip, client_user_agent, channel_initializing)
return generator.generate
generator = StreamGenerator(channel_id, client_id, client_ip, client_user_agent, channel_initializing, user=user)
return generator.generate

50
apps/proxy/ts_proxy/stream_manager.py
View file

@ -90,7 +90,7 @@ class StreamManager:
# Try to get stream_id specifically
stream_id_bytes = buffer.redis_client.hget(metadata_key, "stream_id")
if stream_id_bytes:
self.current_stream_id = int(stream_id_bytes.decode('utf-8'))
self.current_stream_id = int(stream_id_bytes)
self.tried_stream_ids.add(self.current_stream_id)
logger.info(f"Loaded stream ID {self.current_stream_id} from Redis for channel {buffer.channel_id}")
else:
@ -401,24 +401,44 @@ class StreamManager:
# Close all connections
self._close_all_connections()
# Update channel state in Redis to prevent clients from waiting indefinitely
# Transition to ERROR so clients stop waiting. Ownership may have
# expired during retries, so fall back to a state guard when no
# owner exists — but never clobber a new owner's active stream.
if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client:
try:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
# Check if we're the owner before updating state
owner_key = RedisKeys.channel_owner(self.channel_id)
current_owner = self.buffer.redis_client.get(owner_key)
# Use the worker_id that was passed in during initialization
if current_owner and self.worker_id and current_owner.decode('utf-8') == self.worker_id:
# Determine the appropriate error message based on retry failures
is_owner = (
current_owner
and self.worker_id
and current_owner == self.worker_id
)
no_owner = current_owner is None
should_update = is_owner
if not should_update and no_owner:
current_state_bytes = self.buffer.redis_client.hget(
metadata_key, ChannelMetadataField.STATE
)
current_state = (
current_state_bytes
if current_state_bytes else None
)
should_update = current_state in ChannelState.PRE_ACTIVE
if not should_update and current_state:
logger.info(
f"Channel {self.channel_id} has no owner but "
f"state is {current_state} — skipping ERROR update"
)
if should_update:
if self.tried_stream_ids and len(self.tried_stream_ids) > 0:
error_message = f"All {len(self.tried_stream_ids)} stream options failed"
else:
error_message = f"Connection failed after {self.max_retries} attempts"
# Update metadata to indicate error state
update_data = {
ChannelMetadataField.STATE: ChannelState.ERROR,
ChannelMetadataField.STATE_CHANGED_AT: str(time.time()),
@ -426,9 +446,13 @@ class StreamManager:
ChannelMetadataField.ERROR_TIME: str(time.time())
}
self.buffer.redis_client.hset(metadata_key, mapping=update_data)
logger.info(f"Updated channel {self.channel_id} state to ERROR in Redis after stream failure")
logger.info(
f"Updated channel {self.channel_id} state to ERROR "
f"in Redis after stream failure "
f"(owner={'self' if is_owner else 'expired'})"
)
# Also set stopping key to ensure clients disconnect
# Signal clients to disconnect
stop_key = RedisKeys.channel_stopping(self.channel_id)
self.buffer.redis_client.setex(stop_key, 60, "true")
except Exception as e:
@ -1479,9 +1503,9 @@ class StreamManager:
current_state = None
try:
metadata = redis_client.hgetall(metadata_key)
state_field = ChannelMetadataField.STATE.encode('utf-8')
state_field = ChannelMetadataField.STATE
if metadata and state_field in metadata:
current_state = metadata[state_field].decode('utf-8')
current_state = metadata[state_field]
except Exception as e:
logger.error(f"Error checking current state: {e}")
@ -1686,4 +1710,4 @@ class StreamManager:
"""Safely reset the URL switching state if it gets stuck"""
self.url_switching = False
self.url_switch_start_time = 0
logger.info(f"Reset URL switching state for channel {self.channel_id}")
logger.info(f"Reset URL switching state for channel {self.channel_id}")

19
apps/proxy/ts_proxy/url_utils.py
View file

@ -3,7 +3,7 @@ Utilities for handling stream URLs and transformations.
"""
import logging
import re
import regex
from typing import Optional, Tuple, List
from django.shortcuts import get_object_or_404
from apps.channels.models import Channel, Stream
@ -146,13 +146,14 @@ def transform_url(input_url: str, search_pattern: str, replace_pattern: str) ->
logger.debug(f" base URL: {input_url}")
logger.debug(f" search: {search_pattern}")
# Handle backreferences in the replacement pattern
safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace_pattern)
# Convert JS-style backreferences in replace pattern: $<name> -> \g<name>, $1 -> \1
safe_replace_pattern = regex.sub(r'\$<([^>]+)>', r'\\g<\1>', replace_pattern)
safe_replace_pattern = regex.sub(r'\$(\d+)', r'\\\1', safe_replace_pattern)
logger.debug(f" replace: {replace_pattern}")
logger.debug(f" safe replace: {safe_replace_pattern}")
# Apply the transformation
stream_url = re.sub(search_pattern, safe_replace_pattern, input_url)
# Apply the transformation (regex module accepts JS-style (?<name>...) natively)
stream_url = regex.sub(search_pattern, safe_replace_pattern, input_url)
logger.info(f"Generated stream url: {stream_url}")
return stream_url
@ -211,9 +212,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int]
existing_stream_id = redis_client.get(f"channel_stream:{channel.id}")
if existing_stream_id:
# Decode bytes to string/int for proper Redis key lookup
existing_stream_id = existing_stream_id.decode('utf-8')
existing_stream_id = existing_stream_id
existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}")
if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id:
if existing_profile_id and int(existing_profile_id) == profile.id:
channel_using_profile = True
logger.debug(f"Channel {channel.id} already using profile {profile.id}")
@ -349,9 +350,9 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No
existing_stream_id = redis_client.get(f"channel_stream:{channel.id}")
if existing_stream_id:
# Decode bytes to string/int for proper Redis key lookup
existing_stream_id = existing_stream_id.decode('utf-8')
existing_stream_id = existing_stream_id
existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}")
if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id:
if existing_profile_id and int(existing_profile_id) == profile.id:
channel_using_profile = True
logger.debug(f"Channel {channel.id} already using profile {profile.id}")

4
apps/proxy/ts_proxy/utils.py
View file

@ -83,7 +83,7 @@ def create_ts_packet(packet_type='null', message=None):
# Add message to payload if provided
if message:
msg_bytes = message.encode('utf-8')
msg_bytes = message
packet[4:4+min(len(msg_bytes), 180)] = msg_bytes[:180]
return bytes(packet)
@ -113,4 +113,4 @@ def get_logger(component_name=None):
# Default if detection fails
logger_name = "ts_proxy"
return logging.getLogger(logger_name)
return logging.getLogger(logger_name)

48
apps/proxy/ts_proxy/views.py
View file

@ -19,6 +19,7 @@ from apps.m3u.models import M3UAccount, M3UAccountProfile
from apps.accounts.models import User
from core.models import UserAgent, CoreSettings, PROXY_PROFILE_NAME
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from apps.accounts.permissions import (
IsAdmin,
@ -40,16 +41,21 @@ from .utils import get_logger
from uuid import UUID
import gevent
from dispatcharr.utils import network_access_allowed
from apps.proxy.utils import check_user_stream_limits
logger = get_logger()
@api_view(["GET"])
def stream_ts(request, channel_id):
@permission_classes([AllowAny])
def stream_ts(request, channel_id, user=None):
if not network_access_allowed(request, "STREAMS"):
return JsonResponse({"error": "Forbidden"}, status=403)
"""Stream TS data to client with immediate response and keep-alive packets during initialization"""
if user is None and hasattr(request, 'user') and request.user.is_authenticated:
user = request.user
channel = get_stream_object(channel_id)
client_user_agent = None
@ -71,6 +77,13 @@ def stream_ts(request, channel_id):
)
break
if user:
if not check_user_stream_limits(user, client_id, media_id=channel_id):
return JsonResponse(
{"error": f"Stream limit exceeded ({user.stream_limit} concurrent streams allowed)"},
status=429
)
# Check if we need to reinitialize the channel
needs_initialization = True
channel_state = None
@ -81,9 +94,9 @@ def stream_ts(request, channel_id):
metadata_key = RedisKeys.channel_metadata(channel_id)
if proxy_server.redis_client.exists(metadata_key):
metadata = proxy_server.redis_client.hgetall(metadata_key)
state_field = ChannelMetadataField.STATE.encode("utf-8")
state_field = ChannelMetadataField.STATE
if state_field in metadata:
channel_state = metadata[state_field].decode("utf-8")
channel_state = metadata[state_field]
# Active/running states - channel is operational, don't reinitialize
if channel_state in [
@ -119,9 +132,9 @@ def stream_ts(request, channel_id):
)
# Unknown/empty state - check if owner is alive
else:
owner_field = ChannelMetadataField.OWNER.encode("utf-8")
owner_field = ChannelMetadataField.OWNER
if owner_field in metadata:
owner = metadata[owner_field].decode("utf-8")
owner = metadata[owner_field]
owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat"
if proxy_server.redis_client.exists(owner_heartbeat_key):
# Owner is still active with unknown state - don't reinitialize
@ -399,7 +412,7 @@ def stream_ts(request, channel_id):
metadata_key, ChannelMetadataField.STATE
)
if state_bytes:
current_state = state_bytes.decode("utf-8")
current_state = state_bytes
logger.debug(
f"[{client_id}] Current state of channel {channel_id}: {current_state}"
)
@ -475,12 +488,12 @@ def stream_ts(request, channel_id):
)
if url_bytes:
url = url_bytes.decode("utf-8")
url = url_bytes
if ua_bytes:
stream_user_agent = ua_bytes.decode("utf-8")
stream_user_agent = ua_bytes
# Extract transcode setting from Redis
if profile_bytes:
profile_str = profile_bytes.decode("utf-8")
profile_str = profile_bytes
use_transcode = (
profile_str == PROXY_PROFILE_NAME or profile_str == "None"
)
@ -516,12 +529,12 @@ def stream_ts(request, channel_id):
# Register client
buffer = proxy_server.stream_buffers[channel_id]
client_manager = proxy_server.client_managers[channel_id]
client_manager.add_client(client_id, client_ip, client_user_agent)
client_manager.add_client(client_id, client_ip, client_user_agent, user)
logger.info(f"[{client_id}] Client registered with channel {channel_id}")
# Create a stream generator for this client
generate = create_stream_generator(
channel_id, client_id, client_ip, client_user_agent, channel_initializing
channel_id, client_id, client_ip, client_user_agent, channel_initializing, user=user
)
# Return the StreamingHttpResponse from the main function
@ -543,6 +556,7 @@ def stream_ts(request, channel_id):
@api_view(["GET"])
@permission_classes([AllowAny])
def stream_xc(request, username, password, channel_id):
user = get_object_or_404(User, username=username)
@ -557,7 +571,6 @@ def stream_xc(request, username, password, channel_id):
if custom_properties["xc_password"] != password:
return Response({"error": "Invalid credentials"}, status=401)
print(f"Fetchin channel with ID: {channel_id}")
if user.user_level < 10:
user_profile_count = user.channel_profiles.count()
@ -585,7 +598,7 @@ def stream_xc(request, username, password, channel_id):
channel = get_object_or_404(Channel, id=channel_id)
# @TODO: we've got the file 'type' via extension, support this when we support multiple outputs
return stream_ts(request._request, str(channel.uuid))
return stream_ts(request._request, str(channel.uuid), user)
@csrf_exempt
@ -713,7 +726,7 @@ def channel_status(request, channel_id=None):
)
for key in keys:
channel_id_match = re.search(
r"ts_proxy:channel:(.*):metadata", key.decode("utf-8")
r"ts_proxy:channel:(.*):metadata", key
)
if channel_id_match:
ch_id = channel_id_match.group(1)
@ -834,7 +847,7 @@ def next_stream(request, channel_id):
metadata_key, ChannelMetadataField.STREAM_ID
)
if stream_id_bytes:
current_stream_id = int(stream_id_bytes.decode("utf-8"))
current_stream_id = int(stream_id_bytes)
logger.info(
f"Found current stream ID {current_stream_id} in Redis for channel {channel_id}"
)
@ -844,7 +857,7 @@ def next_stream(request, channel_id):
metadata_key, ChannelMetadataField.M3U_PROFILE
)
if profile_id_bytes:
profile_id = int(profile_id_bytes.decode("utf-8"))
profile_id = int(profile_id_bytes)
logger.info(
f"Found M3U profile ID {profile_id} in Redis for channel {channel_id}"
)
@ -916,7 +929,8 @@ def next_stream(request, channel_id):
channel_id,
stream_info["url"],
stream_info["user_agent"],
next_stream_id, # Pass the stream_id to be stored in Redis
next_stream_id,
stream_info.get("m3u_profile_id"),
)
if result.get("status") == "error":

2
apps/proxy/urls.py
View file

@ -6,4 +6,4 @@ urlpatterns = [
path('ts/', include('apps.proxy.ts_proxy.urls')),
path('hls/', include('apps.proxy.hls_proxy.urls')),
path('vod/', include('apps.proxy.vod_proxy.urls')),
]
]

185
apps/proxy/utils.py Normal file
View file

@ -0,0 +1,185 @@
import logging
from core.utils import RedisClient
from apps.proxy.vod_proxy.multi_worker_connection_manager import MultiWorkerVODConnectionManager, get_vod_client_stop_key
from core.models import CoreSettings
from apps.proxy.ts_proxy.services.channel_service import ChannelService
logger = logging.getLogger("proxy")
def attempt_stream_termination(user_id, requesting_client_id, active_connections):
try:
logger.info("[stream limits]" f"[{requesting_client_id}] User {user_id} has {len(active_connections)} active connections, checking termination candidates")
user_limit_settings = CoreSettings.get_user_limits_settings()
terminate_oldest = user_limit_settings.get("terminate_oldest", True)
prioritize_single = user_limit_settings.get("prioritize_single_client_channels", True)
ignore_same_channel = user_limit_settings.get("ignore_same_channel_connections", False)
channel_counts = {}
for connection in active_connections:
media_id = connection['media_id']
channel_counts[media_id] = channel_counts.get(media_id, 0) + 1
def prioritize(connection):
is_multi = channel_counts[connection['media_id']] > 1
# if we're ignoring same-channel connections, put them at the end
same_ch_key = 1 if (ignore_same_channel and is_multi) else 0
# key for prioritizing single-client channels
single_key = 0 if (prioritize_single and not is_multi) else 1
# sort by age setting
time_key = connection['connected_at'] if terminate_oldest else -connection['connected_at']
return (same_ch_key, single_key, time_key)
termination_candidates = sorted(active_connections, key=prioritize)
if not termination_candidates:
logger.warning("[stream limits]" f"[{requesting_client_id}] No termination candidates found for user {user_id}")
return False
target = termination_candidates[0]
logger.info("[stream limits]"
f"[{requesting_client_id}] Terminating client {target['client_id']} "
f"on media {target['media_id']} (connected_at={target['connected_at']})"
)
# When counting by unique channel, freeing one connection from a multi-connection
# channel doesn't free a slot — terminate all connections to that channel so the
# unique-channel count actually drops by one.
targets = (
[c for c in active_connections if c['media_id'] == target['media_id']]
if ignore_same_channel
else [target]
)
for t in targets:
if t['type'] == 'live':
result = ChannelService.stop_client(t['media_id'], t['client_id'])
if result.get("status") == "error":
logger.warning(f"[stream limits][{requesting_client_id}] Failed to stop client {t['client_id']} on channel {t['media_id']}")
else:
connection_manager = MultiWorkerVODConnectionManager.get_instance()
redis_client = connection_manager.redis_client
if not redis_client:
return False
connection_key = f"vod_persistent_connection:{t['client_id']}"
connection_data = redis_client.hgetall(connection_key)
if not connection_data:
logger.warning(f"VOD connection not found: {t['client_id']}")
continue
stop_key = get_vod_client_stop_key(t['client_id'])
redis_client.setex(stop_key, 60, "true") # 60 second TTL
return True
except Exception as e:
logger.error("[stream limits]" f"[{requesting_client_id}] Error during stream termination for user {user_id}: {e}")
return False
def get_user_active_connections(user_id):
redis_client = RedisClient.get_client()
connections = []
try:
# Grab live streams
for key in redis_client.scan_iter(match="ts_proxy:channel:*:clients:*", count=1000):
parts = key.split(':')
if len(parts) >= 5:
channel_id = parts[2]
client_id = parts[4]
client_user_id = redis_client.hget(key, 'user_id')
connected_at = redis_client.hget(key, 'connected_at')
logger.debug(f"[stream limits] user_id = {user_id}")
logger.debug(f"[stream limits] channel_id = {channel_id}")
logger.debug(f"[stream limits] client_id = {client_id}")
if client_user_id and int(client_user_id) == user_id:
try:
logger.debug(f"[stream limits] Found LIVE connection for user {user_id} on channel {channel_id} with client ID {client_id}")
connected_at = float(connected_at) if connected_at else 0
connections.append({
'media_id': channel_id,
'client_id': client_id,
'connected_at': connected_at,
'type': 'live',
})
except (ValueError, TypeError):
pass
# Grab VOD
for key in redis_client.scan_iter(match="vod_persistent_connection:*", count=1000):
parts = key.split(':')
if len(parts) >= 2:
client_id = parts[1]
client_user_id = redis_client.hget(key, 'user_id')
connected_at = redis_client.hget(key, 'created_at')
content_uuid = redis_client.hget(key, 'content_uuid')
logger.debug(f"[stream limits] user_id = {user_id}")
logger.debug(f"[stream limits] client_id = {client_id}")
if client_user_id and int(client_user_id) == user_id:
try:
logger.debug(f"[stream limits] Found VOD connection for user {user_id} on content {content_uuid} with client ID {client_id}")
connected_at = float(connected_at) if connected_at else 0
connections.append({
'media_id': content_uuid or client_id,
'client_id': client_id,
'connected_at': connected_at,
'type': 'vod',
})
except (ValueError, TypeError):
pass
return connections
except Exception as e:
logger.warning(f"Error getting active channel details for user {user_id}: {e}")
return []
def check_user_stream_limits(user, client_id, media_id=None):
# Check user stream limits
if user and user.stream_limit > 0:
logger.debug("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) is requesting a stream (stream_limit: {user.stream_limit})")
user_limit_settings = CoreSettings.get_user_limits_settings()
ignore_same_channel = user_limit_settings.get("ignore_same_channel_connections", False)
active_connections = get_user_active_connections(user.id)
unique_channel_count = set([conn['media_id'] for conn in active_connections])
user_stream_count = len(unique_channel_count) if ignore_same_channel else len(active_connections)
logger.debug(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if ignore_same_channel else 'total connections'})")
# If ignore_same_channel is enabled and this request is for a live channel the user
# is already watching, allow it through without counting against the limit.
# VOD is excluded: connections aren't shared so multiple VOD connections to the
# same content would mean multiple upstream connections.
live_channel_ids = {str(conn['media_id']) for conn in active_connections if conn['type'] == 'live'}
if ignore_same_channel and media_id and str(media_id) in live_channel_ids:
logger.debug(f"[stream limits][{client_id}] Same-channel reconnect for {media_id} allowed (ignore_same_channel=True)")
return True
if user_stream_count >= user.stream_limit:
if user_limit_settings.get("terminate_on_limit_exceeded", True) == False:
return False
if user_stream_count >= user.stream_limit:
logger.warning("[stream limits]"
f"[{client_id}] User {user.username} (ID: {user.id}) has reached stream limit "
f"({user_stream_count}/{user.stream_limit} streams), attempting to free up slot"
)
if not attempt_stream_termination(user.id, client_id, active_connections):
return False
return True

1505
apps/proxy/vod_proxy/connection_manager.py
View file

File diff suppressed because it is too large Load diff

51
apps/proxy/vod_proxy/multi_worker_connection_manager.py
View file

@ -93,7 +93,7 @@ class SerializableConnectionState:
content_name: str = None, client_ip: str = None,
client_user_agent: str = None, utc_start: str = None,
utc_end: str = None, offset: str = None,
worker_id: str = None, connection_type: str = "redis_backed"):
worker_id: str = None, connection_type: str = "redis_backed", user_id: str = "unknown"):
self.session_id = session_id
self.stream_url = stream_url
self.headers = headers
@ -104,6 +104,7 @@ class SerializableConnectionState:
self.last_activity = time.time()
self.request_count = 0
self.active_streams = 0
self.user_id = user_id
# Session metadata (consolidated from vod_session key)
self.content_obj_type = content_obj_type
@ -160,7 +161,8 @@ class SerializableConnectionState:
'last_seek_byte': str(self.last_seek_byte),
'last_seek_percentage': str(self.last_seek_percentage),
'total_content_size': str(self.total_content_size),
'last_seek_timestamp': str(self.last_seek_timestamp)
'last_seek_timestamp': str(self.last_seek_timestamp),
'user_id': str(self.user_id),
}
@classmethod
@ -184,7 +186,8 @@ class SerializableConnectionState:
utc_end=data.get('utc_end') or '',
offset=data.get('offset') or '',
worker_id=data.get('worker_id') or None,
connection_type=data.get('connection_type', 'redis_backed')
connection_type=data.get('connection_type', 'redis_backed'),
user_id=data.get('user_id', 'unknown')
)
obj.last_activity = float(data.get('last_activity', time.time()))
obj.request_count = int(data.get('request_count', 0))
@ -224,7 +227,7 @@ class RedisBackedVODConnection:
# Convert bytes keys/values to strings if needed
if isinstance(list(data.keys())[0], bytes):
data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()}
data = {k: v for k, v in data.items()}
return SerializableConnectionState.from_dict(data)
except Exception as e:
@ -281,7 +284,7 @@ class RedisBackedVODConnection:
content_name: str = None, client_ip: str = None,
client_user_agent: str = None, utc_start: str = None,
utc_end: str = None, offset: str = None,
worker_id: str = None) -> bool:
worker_id: str = None, user=None) -> bool:
"""Create a new connection state in Redis with consolidated session metadata"""
if not self._acquire_lock():
logger.warning(f"[{self.session_id}] Could not acquire lock for connection creation")
@ -309,7 +312,8 @@ class RedisBackedVODConnection:
utc_start=utc_start,
utc_end=utc_end,
offset=offset,
worker_id=worker_id
worker_id=worker_id,
user_id=user.id if user else "unknown"
)
success = self._save_connection_state(state)
@ -365,6 +369,24 @@ class RedisBackedVODConnection:
timeout=(10, 10),
allow_redirects=allow_redirects
)
# If the cached final_url returned an error (e.g. an ephemeral dispatcharr session
# that has since expired), clear it and retry from the original stream_url.
if response.status_code >= 400 and state.final_url:
logger.warning(
f"[{self.session_id}] Cached final_url returned {response.status_code}, "
f"clearing and retrying from stream_url"
)
response.close()
state.final_url = None
response = self.local_session.get(
state.stream_url,
headers=headers,
stream=True,
timeout=(10, 10),
allow_redirects=True
)
response.raise_for_status()
# Update state with response info on first request
@ -798,7 +820,7 @@ class MultiWorkerVODConnectionManager:
def stream_content_with_session(self, session_id, content_obj, stream_url, m3u_profile,
client_ip, client_user_agent, request,
utc_start=None, utc_end=None, offset=None, range_header=None):
utc_start=None, utc_end=None, offset=None, range_header=None, user=None):
"""Stream content with Redis-backed persistent connection"""
# Generate client ID
@ -895,7 +917,8 @@ class MultiWorkerVODConnectionManager:
utc_start=utc_start,
utc_end=utc_end,
offset=str(offset) if offset else None,
worker_id=self.worker_id
worker_id=self.worker_id,
user=user
):
logger.error(f"[{client_id}] Worker {self.worker_id} - Failed to create Redis connection")
# Roll back the profile slot reservation since connection failed
@ -1316,14 +1339,14 @@ class MultiWorkerVODConnectionManager:
# Convert bytes to strings if needed
if isinstance(list(data.keys())[0], bytes):
data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()}
data = {k: v for k, v in data.items()}
last_activity = float(data.get('last_activity', 0))
active_streams = int(data.get('active_streams', 0))
# Clean up if stale and no active streams
if (current_time - last_activity > max_age_seconds) and active_streams == 0:
session_id = key.decode('utf-8').replace('vod_persistent_connection:', '')
session_id = key.replace('vod_persistent_connection:', '')
logger.info(f"Cleaning up stale connection: {session_id}")
# Clean up connection and related keys
@ -1420,7 +1443,7 @@ class MultiWorkerVODConnectionManager:
if connection_data:
# Convert bytes to strings if needed
if isinstance(list(connection_data.keys())[0], bytes):
connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()}
connection_data = {k: v for k, v in connection_data.items()}
profile_id = connection_data.get('m3u_profile_id')
if profile_id:
@ -1482,7 +1505,7 @@ class MultiWorkerVODConnectionManager:
# Convert bytes keys/values to strings if needed
if isinstance(list(connection_data.keys())[0], bytes):
connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()}
connection_data = {k: v for k, v in connection_data.items()}
# Check if content matches (using consolidated data)
stored_content_type = connection_data.get('content_obj_type', '')
@ -1492,7 +1515,7 @@ class MultiWorkerVODConnectionManager:
continue
# Extract session ID
session_id = key.decode('utf-8').replace('vod_persistent_connection:', '')
session_id = key.replace('vod_persistent_connection:', '')
# Check if Redis-backed connection exists and has no active streams
redis_connection = RedisBackedVODConnection(session_id, self.redis_client)
@ -1570,4 +1593,4 @@ class MultiWorkerVODConnectionManager:
return redis_connection.get_session_metadata()
except Exception as e:
logger.error(f"Error getting session info for {session_id}: {e}")
return None
return None

18
apps/proxy/vod_proxy/urls.py
View file

@ -1,26 +1,20 @@
from django.urls import path
from . import views
from .views import stream_vod
app_name = 'vod_proxy'
urlpatterns = [
# Generic VOD streaming with session ID in path (for compatibility)
path('<str:content_type>/<uuid:content_id>/<str:session_id>', views.VODStreamView.as_view(), name='vod_stream_with_session'),
path('<str:content_type>/<uuid:content_id>/<str:session_id>/<int:profile_id>/', views.VODStreamView.as_view(), name='vod_stream_with_session_and_profile'),
path('<str:content_type>/<uuid:content_id>/<str:session_id>', stream_vod, name='vod_stream_with_session'),
path('<str:content_type>/<uuid:content_id>/<str:session_id>/<int:profile_id>/', stream_vod, name='vod_stream_with_session_and_profile'),
# Generic VOD streaming (supports movies, episodes, series) - legacy patterns
path('<str:content_type>/<uuid:content_id>', views.VODStreamView.as_view(), name='vod_stream'),
path('<str:content_type>/<uuid:content_id>/<int:profile_id>/', views.VODStreamView.as_view(), name='vod_stream_with_profile'),
# VOD playlist generation
path('playlist/', views.VODPlaylistView.as_view(), name='vod_playlist'),
path('playlist/<int:profile_id>/', views.VODPlaylistView.as_view(), name='vod_playlist_with_profile'),
# Position tracking
path('position/<uuid:content_id>/', views.VODPositionView.as_view(), name='vod_position'),
path('<str:content_type>/<uuid:content_id>', stream_vod, name='vod_stream'),
path('<str:content_type>/<uuid:content_id>/<int:profile_id>/', stream_vod, name='vod_stream_with_profile'),
# VOD Stats
path('stats/', views.VODStatsView.as_view(), name='vod_stats'),
path('stats/', views.vod_stats, name='vod_stats'),
# Stop VOD client connection
path('stop_client/', views.stop_vod_client, name='stop_vod_client'),

1824
apps/proxy/vod_proxy/views.py
View file

File diff suppressed because it is too large Load diff

18
apps/vod/api_views.py
View file

@ -578,13 +578,15 @@ class UnifiedContentViewSet(viewsets.ReadOnlyModelViewSet):
"series.id IN (SELECT DISTINCT series_id FROM vod_m3useriesrelation msr JOIN m3u_m3uaccount ma ON msr.m3u_account_id = ma.id WHERE ma.is_active = true)"
]
params = []
movie_params = []
series_params = []
if search:
where_conditions[0] += " AND LOWER(movies.name) LIKE %s"
where_conditions[1] += " AND LOWER(series.name) LIKE %s"
search_param = f"%{search.lower()}%"
params.extend([search_param, search_param])
movie_params.append(search_param)
series_params.append(search_param)
if category:
if '|' in category:
@ -592,15 +594,20 @@ class UnifiedContentViewSet(viewsets.ReadOnlyModelViewSet):
if cat_type == 'movie':
where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)"
where_conditions[1] = "1=0" # Exclude series
params.append(cat_name)
movie_params.append(cat_name)
series_params = [] # no params needed for "1=0"
elif cat_type == 'series':
where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)"
where_conditions[0] = "1=0" # Exclude movies
params.append(cat_name)
series_params.append(cat_name)
movie_params = [] # no params needed for "1=0"
else:
where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)"
where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)"
params.extend([category, category])
movie_params.append(category)
series_params.append(category)
params = movie_params + series_params
# Use UNION ALL with ORDER BY and LIMIT/OFFSET for true unified pagination
# This is much more efficient than Python sorting
@ -896,4 +903,3 @@ class VODLogoViewSet(viewsets.ModelViewSet):
{"error": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR
)

47
core/api_views.py
View file

@ -3,12 +3,13 @@
import json
import ipaddress
import logging
from django.conf import settings as django_settings
from django.db import models
from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.views import APIView
from django.shortcuts import get_object_or_404
from rest_framework.permissions import IsAuthenticated
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.decorators import api_view, permission_classes, action
from drf_spectacular.utils import extend_schema, OpenApiParameter
from drf_spectacular.types import OpenApiTypes
@ -34,6 +35,9 @@ import os
from core.tasks import rehash_streams
from apps.accounts.permissions import (
Authenticated,
IsAdmin,
IsStandardUser,
permission_classes_by_action,
)
from dispatcharr.utils import get_client_ip
@ -49,6 +53,12 @@ class UserAgentViewSet(viewsets.ModelViewSet):
queryset = UserAgent.objects.all()
serializer_class = UserAgentSerializer
def get_permissions(self):
try:
return [perm() for perm in permission_classes_by_action[self.action]]
except KeyError:
return [Authenticated()]
class StreamProfileViewSet(viewsets.ModelViewSet):
"""
@ -58,6 +68,12 @@ class StreamProfileViewSet(viewsets.ModelViewSet):
queryset = StreamProfile.objects.all()
serializer_class = StreamProfileSerializer
def get_permissions(self):
try:
return [perm() for perm in permission_classes_by_action[self.action]]
except KeyError:
return [Authenticated()]
class CoreSettingsViewSet(viewsets.ModelViewSet):
"""
@ -68,6 +84,12 @@ class CoreSettingsViewSet(viewsets.ModelViewSet):
queryset = CoreSettings.objects.all()
serializer_class = CoreSettingsSerializer
def get_permissions(self):
try:
return [perm() for perm in permission_classes_by_action[self.action]]
except KeyError:
return [Authenticated()]
def update(self, request, *args, **kwargs):
instance = self.get_object()
old_value = instance.value
@ -170,6 +192,11 @@ class ProxySettingsViewSet(viewsets.ViewSet):
"""
serializer_class = ProxySettingsSerializer
def get_permissions(self):
if self.action in ('list', 'retrieve'):
return [IsStandardUser()]
return [IsAdmin()]
def _get_or_create_settings(self):
"""Get or create the proxy settings CoreSettings entry"""
try:
@ -301,7 +328,9 @@ def environment(request):
country_code = None
country_name = None
# 4) Get environment mode from system environment variable
# 4) Get environment mode and TLS status from settings
postgres_ssl = getattr(django_settings, "POSTGRES_SSL", False)
return Response(
{
"authenticated": True,
@ -309,7 +338,17 @@ def environment(request):
"local_ip": local_ip,
"country_code": country_code,
"country_name": country_name,
"env_mode": "dev" if os.getenv("DISPATCHARR_ENV") == "dev" else "prod",
"env_mode": os.getenv("DISPATCHARR_ENV", "aio"),
"redis_tls": {
"enabled": getattr(django_settings, "REDIS_SSL", False),
"verify": getattr(django_settings, "REDIS_SSL_VERIFY", True),
"mtls": bool(getattr(django_settings, "REDIS_SSL_CERT", "") and getattr(django_settings, "REDIS_SSL_KEY", "")),
},
"postgres_tls": {
"enabled": postgres_ssl,
"ssl_mode": getattr(django_settings, "POSTGRES_SSL_MODE", "verify-full") if postgres_ssl else None,
"mtls": bool(getattr(django_settings, "POSTGRES_SSL_CERT", "") and getattr(django_settings, "POSTGRES_SSL_KEY", "")),
},
}
)
@ -317,8 +356,8 @@ def environment(request):
@extend_schema(
description="Get application version information",
)
@api_view(["GET"])
@permission_classes([AllowAny])
def version(request):
# Import version information
from version import __version__, __timestamp__

14
core/management/commands/dropdb.py
View file

@ -1,5 +1,6 @@
import sys
import psycopg2
from psycopg2 import sql
from django.core.management.base import BaseCommand
from django.conf import settings
from django.db import connection
@ -15,6 +16,13 @@ class Command(BaseCommand):
host = db_settings.get('HOST', 'localhost')
port = db_settings.get('PORT', 5432)
# Read TLS parameters from Django OPTIONS (populated when POSTGRES_SSL=true)
db_options = db_settings.get('OPTIONS', {})
ssl_kwargs = {}
for key in ('sslmode', 'sslrootcert', 'sslcert', 'sslkey'):
if key in db_options:
ssl_kwargs[key] = db_options[key]
self.stdout.write(self.style.WARNING(
f"WARNING: This will irreversibly drop the entire database '{db_name}'!"
))
@ -30,13 +38,13 @@ class Command(BaseCommand):
maintenance_db = 'postgres'
try:
self.stdout.write("Connecting to maintenance database...")
conn = psycopg2.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port)
conn = psycopg2.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, **ssl_kwargs)
conn.autocommit = True
cur = conn.cursor()
self.stdout.write(f"Dropping database '{db_name}'...")
cur.execute(f"DROP DATABASE IF EXISTS {db_name};")
cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name)))
self.stdout.write(f"Creating database '{db_name}'...")
cur.execute(f"CREATE DATABASE {db_name};")
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_name)))
cur.close()
conn.close()
self.stdout.write(self.style.SUCCESS(f"Database '{db_name}' has been dropped and recreated."))

24
core/migrations/022_default_user_limit_settings.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-03-01 14:01
from django.db import migrations
from django.utils.text import slugify
def preload_user_limit_settings(apps, schema_editor):
CoreSettings = apps.get_model("core", "CoreSettings")
CoreSettings.objects.create(
key="user_limit_settings",
name="User Limit Settings",
value={},
)
class Migration(migrations.Migration):
dependencies = [
("core", "0021_systemnotification_notificationdismissal"),
]
operations = [
migrations.RunPython(preload_user_limit_settings),
]

33
core/models.py
View file

@ -156,6 +156,7 @@ PROXY_SETTINGS_KEY = "proxy_settings"
NETWORK_ACCESS_KEY = "network_access"
SYSTEM_SETTINGS_KEY = "system_settings"
EPG_SETTINGS_KEY = "epg_settings"
USER_LIMITS_SETTINGS_KEY = "user_limit_settings"
class CoreSettings(models.Model):
@ -239,17 +240,24 @@ class CoreSettings(models.Model):
"epg_match_ignore_custom": [],
})
@classmethod
def _safe_string_list(cls, value):
"""Return a list of strings, filtering out non-list or non-string values."""
if not isinstance(value, list):
return []
return [v for v in value if isinstance(v, str)]
@classmethod
def get_epg_match_ignore_prefixes(cls):
return cls.get_epg_settings().get("epg_match_ignore_prefixes", [])
return cls._safe_string_list(cls.get_epg_settings().get("epg_match_ignore_prefixes", []))
@classmethod
def get_epg_match_ignore_suffixes(cls):
return cls.get_epg_settings().get("epg_match_ignore_suffixes", [])
return cls._safe_string_list(cls.get_epg_settings().get("epg_match_ignore_suffixes", []))
@classmethod
def get_epg_match_ignore_custom(cls):
return cls.get_epg_settings().get("epg_match_ignore_custom", [])
return cls._safe_string_list(cls.get_epg_settings().get("epg_match_ignore_custom", []))
# DVR Settings
@classmethod
@ -312,12 +320,16 @@ class CoreSettings(models.Model):
@classmethod
def get_dvr_series_rules(cls):
return cls.get_dvr_settings().get("series_rules", [])
rules = cls.get_dvr_settings().get("series_rules", [])
if not isinstance(rules, list):
return []
return [r for r in rules if isinstance(r, dict)]
@classmethod
def set_dvr_series_rules(cls, rules):
cls._update_group(DVR_SETTINGS_KEY, "DVR Settings", {"series_rules": rules})
return rules
clean = [r for r in rules if isinstance(r, dict)] if isinstance(rules, list) else []
cls._update_group(DVR_SETTINGS_KEY, "DVR Settings", {"series_rules": clean})
return clean
# Proxy Settings
@classmethod
@ -351,6 +363,15 @@ class CoreSettings(models.Model):
cls._update_group(SYSTEM_SETTINGS_KEY, "System Settings", {"time_zone": value})
return value
@classmethod
def get_user_limits_settings(cls):
return cls._get_group(USER_LIMITS_SETTINGS_KEY, {
"terminate_on_limit_exceeded": True,
"prioritize_single_client_channels": True,
"ignore_same_channel_connections": False,
"terminate_oldest": True,
})
class SystemEvent(models.Model):
"""

4
core/redis_pubsub.py
View file

@ -201,10 +201,6 @@ class RedisPubSubManager:
channel = message.get('channel')
if channel:
# Decode binary channel name if needed
if isinstance(channel, bytes):
channel = channel.decode('utf-8')
# Find and call the appropriate handler
handler = self.message_handlers.get(channel)
if handler:

15
core/serializers.py
View file

@ -3,7 +3,7 @@ import json
import ipaddress
from rest_framework import serializers
from .models import CoreSettings, UserAgent, StreamProfile, NETWORK_ACCESS_KEY
from .models import CoreSettings, UserAgent, StreamProfile, DVR_SETTINGS_KEY, NETWORK_ACCESS_KEY
class UserAgentSerializer(serializers.ModelSerializer):
@ -64,6 +64,19 @@ class CoreSettingsSerializer(serializers.ModelSerializer):
}
)
# Sanitize series_rules when DVR settings are saved through the
# generic settings API (e.g. Settings page round-trip) to prevent
# corrupted non-dict entries from persisting.
if instance.key == DVR_SETTINGS_KEY:
value = validated_data.get("value")
if isinstance(value, dict) and "series_rules" in value:
rules = value["series_rules"]
value["series_rules"] = (
[r for r in rules if isinstance(r, dict)]
if isinstance(rules, list)
else []
)
result = super().update(instance, validated_data)
# Note: Cache invalidation and notification sync is handled by post_save signal

18
core/tasks.py
View file

@ -13,7 +13,7 @@ from apps.epg.models import EPGSource
from apps.m3u.tasks import refresh_single_m3u_account
from apps.epg.tasks import refresh_epg_data
from .models import CoreSettings
from apps.channels.models import Stream, ChannelStream
from apps.channels.models import ChannelStream
from django.db import transaction
logger = logging.getLogger(__name__)
@ -404,7 +404,7 @@ def fetch_channel_stats():
while True:
cursor, keys = redis_client.scan(cursor, match=channel_pattern)
for key in keys:
channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8'))
channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key)
if channel_id_match:
ch_id = channel_id_match.group(1)
channel_info = ChannelStatus.get_basic_channel_info(ch_id)
@ -753,20 +753,6 @@ def _determine_stream_to_keep(stream_a, stream_b):
return (stream_b, stream_a)
@shared_task
def cleanup_vod_persistent_connections():
"""Clean up stale VOD persistent connections"""
try:
from apps.proxy.vod_proxy.connection_manager import VODConnectionManager
# Clean up connections older than 30 minutes
VODConnectionManager.cleanup_stale_persistent_connections(max_age_seconds=1800)
logger.info("VOD persistent connection cleanup completed")
except Exception as e:
logger.error(f"Error during VOD persistent connection cleanup: {e}")
@shared_task
def check_for_version_update():
"""

222
core/tests.py
View file

@ -1,3 +1,223 @@
from unittest.mock import patch, MagicMock
from django.test import TestCase
# Create your tests here.
from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY
class GetDvrSeriesRulesTest(TestCase):
"""Verify get_dvr_series_rules handles corrupted stored data."""
def _set_series_rules_raw(self, raw_value):
"""Write a raw series_rules value into the DB, bypassing set_dvr_series_rules."""
obj, _ = CoreSettings.objects.get_or_create(
key=DVR_SETTINGS_KEY,
defaults={"name": "DVR Settings", "value": {}},
)
current = obj.value if isinstance(obj.value, dict) else {}
current["series_rules"] = raw_value
obj.value = current
obj.save()
def test_valid_rules_returned_as_is(self):
rules = [{"tvg_id": "abc", "mode": "all", "title": "Show"}]
self._set_series_rules_raw(rules)
result = CoreSettings.get_dvr_series_rules()
self.assertEqual(result, rules)
def test_non_dict_elements_filtered(self):
"""Strings in the list cause 'str' has no attribute 'get'."""
self._set_series_rules_raw(["bad_string", {"tvg_id": "abc", "mode": "all", "title": ""}])
result = CoreSettings.get_dvr_series_rules()
self.assertEqual(len(result), 1)
self.assertEqual(result[0]["tvg_id"], "abc")
def test_non_list_value_returns_empty(self):
"""If series_rules is a JSON string instead of a list, return empty."""
self._set_series_rules_raw("[]")
result = CoreSettings.get_dvr_series_rules()
self.assertEqual(result, [])
def test_none_value_returns_empty(self):
self._set_series_rules_raw(None)
result = CoreSettings.get_dvr_series_rules()
self.assertEqual(result, [])
def test_mixed_corrupt_elements(self):
self._set_series_rules_raw([42, None, True, {"tvg_id": "x", "mode": "new", "title": "T"}])
result = CoreSettings.get_dvr_series_rules()
self.assertEqual(len(result), 1)
self.assertEqual(result[0]["tvg_id"], "x")
class SetDvrSeriesRulesTest(TestCase):
"""Verify set_dvr_series_rules sanitizes input before persisting."""
def test_valid_rules_persisted(self):
rules = [{"tvg_id": "abc", "mode": "all", "title": "Show"}]
result = CoreSettings.set_dvr_series_rules(rules)
self.assertEqual(result, rules)
self.assertEqual(CoreSettings.get_dvr_series_rules(), rules)
def test_non_dict_elements_stripped_on_write(self):
dirty = ["bad", 42, {"tvg_id": "abc", "mode": "all", "title": ""}]
result = CoreSettings.set_dvr_series_rules(dirty)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]["tvg_id"], "abc")
self.assertEqual(CoreSettings.get_dvr_series_rules(), result)
def test_non_list_input_stores_empty(self):
result = CoreSettings.set_dvr_series_rules("not a list")
self.assertEqual(result, [])
self.assertEqual(CoreSettings.get_dvr_series_rules(), [])
class CoreSettingsSerializerDvrTest(TestCase):
"""Verify the generic settings API sanitizes series_rules on save."""
def test_serializer_strips_corrupt_series_rules(self):
"""Settings page round-trip must not persist corrupt series_rules."""
from core.serializers import CoreSettingsSerializer
obj, _ = CoreSettings.objects.get_or_create(
key=DVR_SETTINGS_KEY,
defaults={"name": "DVR Settings", "value": {"series_rules": []}},
)
dirty_value = {
**obj.value,
"series_rules": ["bad", {"tvg_id": "ok", "mode": "all", "title": ""}],
}
serializer = CoreSettingsSerializer(obj, data={"value": dirty_value}, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
obj.refresh_from_db()
rules = obj.value.get("series_rules", [])
self.assertEqual(len(rules), 1)
self.assertEqual(rules[0]["tvg_id"], "ok")
def test_serializer_handles_non_list_series_rules(self):
from core.serializers import CoreSettingsSerializer
obj, _ = CoreSettings.objects.get_or_create(
key=DVR_SETTINGS_KEY,
defaults={"name": "DVR Settings", "value": {"series_rules": []}},
)
dirty_value = {**obj.value, "series_rules": "not a list"}
serializer = CoreSettingsSerializer(obj, data={"value": dirty_value}, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
obj.refresh_from_db()
self.assertEqual(obj.value.get("series_rules"), [])
class EpgIgnoreListsTest(TestCase):
"""Verify EPG ignore list getters handle corrupted stored data."""
def _set_epg_field_raw(self, field, raw_value):
obj, _ = CoreSettings.objects.get_or_create(
key=EPG_SETTINGS_KEY,
defaults={"name": "EPG Settings", "value": {}},
)
current = obj.value if isinstance(obj.value, dict) else {}
current[field] = raw_value
obj.value = current
obj.save()
def test_valid_string_lists_returned(self):
for field, getter in [
("epg_match_ignore_prefixes", CoreSettings.get_epg_match_ignore_prefixes),
("epg_match_ignore_suffixes", CoreSettings.get_epg_match_ignore_suffixes),
("epg_match_ignore_custom", CoreSettings.get_epg_match_ignore_custom),
]:
self._set_epg_field_raw(field, ["HD", "SD"])
self.assertEqual(getter(), ["HD", "SD"])
def test_non_string_elements_filtered(self):
for field, getter in [
("epg_match_ignore_prefixes", CoreSettings.get_epg_match_ignore_prefixes),
("epg_match_ignore_suffixes", CoreSettings.get_epg_match_ignore_suffixes),
("epg_match_ignore_custom", CoreSettings.get_epg_match_ignore_custom),
]:
self._set_epg_field_raw(field, [42, None, "HD", True, "SD"])
result = getter()
self.assertEqual(result, ["HD", "SD"])
def test_non_list_value_returns_empty(self):
for field, getter in [
("epg_match_ignore_prefixes", CoreSettings.get_epg_match_ignore_prefixes),
("epg_match_ignore_suffixes", CoreSettings.get_epg_match_ignore_suffixes),
("epg_match_ignore_custom", CoreSettings.get_epg_match_ignore_custom),
]:
self._set_epg_field_raw(field, "not a list")
self.assertEqual(getter(), [])
class DropDBCommandTlsTest(TestCase):
"""Verify dropdb management command passes TLS parameters to psycopg2."""
databases = []
_DB_WITH_TLS = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'testdb',
'USER': 'testuser',
'PASSWORD': 'testpass',
'HOST': 'localhost',
'PORT': 5432,
'OPTIONS': {
'sslmode': 'verify-full',
'sslrootcert': '/certs/ca.crt',
'sslcert': '/certs/client.crt',
'sslkey': '/certs/client.key',
},
}
}
_DB_NO_TLS = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'testdb',
'USER': 'testuser',
'PASSWORD': 'testpass',
'HOST': 'localhost',
'PORT': 5432,
}
}
@patch('core.management.commands.dropdb.psycopg2.connect')
@patch('core.management.commands.dropdb.connection')
@patch('builtins.input', return_value='yes')
def test_dropdb_passes_ssl_kwargs_when_tls_enabled(self, _inp, _conn, mock_connect):
mock_pg = MagicMock()
mock_connect.return_value = mock_pg
mock_pg.cursor.return_value = MagicMock()
with self.settings(DATABASES=self._DB_WITH_TLS):
from django.core.management import call_command
call_command('dropdb')
mock_connect.assert_called_once_with(
dbname='postgres', user='testuser', password='testpass',
host='localhost', port=5432,
sslmode='verify-full',
sslrootcert='/certs/ca.crt',
sslcert='/certs/client.crt',
sslkey='/certs/client.key',
)
@patch('core.management.commands.dropdb.psycopg2.connect')
@patch('core.management.commands.dropdb.connection')
@patch('builtins.input', return_value='yes')
def test_dropdb_no_ssl_kwargs_when_tls_disabled(self, _inp, _conn, mock_connect):
mock_pg = MagicMock()
mock_connect.return_value = mock_pg
mock_pg.cursor.return_value = MagicMock()
with self.settings(DATABASES=self._DB_NO_TLS):
from django.core.management import call_command
call_command('dropdb')
mock_connect.assert_called_once_with(
dbname='postgres', user='testuser', password='testpass',
host='localhost', port=5432,
)

212
core/utils.py
View file

@ -3,6 +3,7 @@ import logging
import time
import os
import threading
from pathlib import Path
import re
from django.conf import settings
from redis.exceptions import ConnectionError, TimeoutError
@ -13,6 +14,8 @@ from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
import gc
_REDIS_TLS_HINT = " (TLS is enabled — verify certificate paths and that Redis is configured for TLS)"
logger = logging.getLogger(__name__)
# Import the command detector
@ -43,103 +46,116 @@ def natural_sort_key(text):
class RedisClient:
_client = None
_buffer = None
_pubsub_client = None
@classmethod
def get_client(cls, max_retries=5, retry_interval=1):
if cls._client is None:
retry_count = 0
while retry_count < max_retries:
def _init_client(cls, decode_responses=True, max_retries=5, retry_interval=1):
retry_count = 0
while retry_count < max_retries:
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
redis_password = os.environ.get("REDIS_PASSWORD", getattr(settings, 'REDIS_PASSWORD', ''))
redis_user = os.environ.get("REDIS_USER", getattr(settings, 'REDIS_USER', ''))
# Use standardized settings
socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5)
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
# TLS params from settings (empty dict when TLS is disabled)
ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {})
# Create Redis client with better defaults
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
password=redis_password if redis_password else None,
username=redis_user if redis_user else None,
socket_timeout=socket_timeout,
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout,
decode_responses=decode_responses,
**ssl_params
)
# Validate connection with ping
client.ping()
# Disable persistence on first connection - improves performance
# Only try to disable if not in a read-only environment
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
redis_password = os.environ.get("REDIS_PASSWORD", getattr(settings, 'REDIS_PASSWORD', ''))
redis_user = os.environ.get("REDIS_USER", getattr(settings, 'REDIS_USER', ''))
client.config_set('save', '') # Disable RDB snapshots
client.config_set('appendonly', 'no') # Disable AOF logging
# Use standardized settings
socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5)
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
# Disable protected mode when in debug mode
if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true':
client.config_set('protected-mode', 'no') # Disable protected mode in debug
logger.warning("Redis protected mode disabled for debug environment")
# Create Redis client with better defaults
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
password=redis_password if redis_password else None,
username=redis_user if redis_user else None,
socket_timeout=socket_timeout,
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
)
# Validate connection with ping
client.ping()
# Disable persistence on first connection - improves performance
# Only try to disable if not in a read-only environment
try:
client.config_set('save', '') # Disable RDB snapshots
client.config_set('appendonly', 'no') # Disable AOF logging
# Set optimal memory settings with environment variable support
# Get max memory from environment or use a larger default (512MB instead of 256MB)
#max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb')
#eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru')
# Apply memory settings
#client.config_set('maxmemory-policy', eviction_policy)
#client.config_set('maxmemory', max_memory)
#logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}")
# Disable protected mode when in debug mode
if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true':
client.config_set('protected-mode', 'no') # Disable protected mode in debug
logger.warning("Redis protected mode disabled for debug environment")
logger.trace("Redis persistence disabled for better performance")
except redis.exceptions.ResponseError as e:
# Improve error handling for Redis configuration errors
if "OOM" in str(e):
logger.error(f"Redis OOM during configuration: {e}")
# Try to increase maxmemory as an emergency measure
try:
client.config_set('maxmemory', '768mb')
logger.warning("Applied emergency Redis memory increase to 768MB")
except:
pass
else:
logger.error(f"Redis configuration error: {e}")
logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}")
cls._client = client
break
except (ConnectionError, TimeoutError) as e:
retry_count += 1
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}")
return None
logger.trace("Redis persistence disabled for better performance")
except redis.exceptions.ResponseError as e:
# Improve error handling for Redis configuration errors
if "OOM" in str(e):
logger.error(f"Redis OOM during configuration: {e}")
# Try to increase maxmemory as an emergency measure
try:
client.config_set('maxmemory', '768mb')
logger.warning("Applied emergency Redis memory increase to 768MB")
except:
pass
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
logger.error(f"Redis configuration error: {e}")
except Exception as e:
logger.error(f"Unexpected error connecting to Redis: {e}")
logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}")
return client
except (ConnectionError, TimeoutError) as e:
retry_count += 1
_tls_hint = _REDIS_TLS_HINT if ssl_params else ""
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}{_tls_hint}")
return None
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
except Exception as e:
_tls_hint = ""
try:
_tls_hint = _REDIS_TLS_HINT if ssl_params else ""
except NameError:
pass
logger.error(f"Unexpected error connecting to Redis: {e}{_tls_hint}")
return None
return None
@classmethod
def get_client(cls, max_retries=5, retry_interval=1):
"""Get Redis client optimized for non-binary data (decoded responses)"""
if cls._client is None:
cls._client = cls._init_client(decode_responses=True, max_retries=max_retries, retry_interval=retry_interval)
return cls._client
@classmethod
def get_buffer(cls, max_retries=5, retry_interval=1):
"""Get Redis client optimized for binary data (no decoding)"""
if cls._buffer is None:
cls._buffer = cls._init_client(decode_responses=False, max_retries=max_retries, retry_interval=retry_interval)
return cls._buffer
@classmethod
def get_pubsub_client(cls, max_retries=5, retry_interval=1):
"""Get Redis client optimized for PubSub operations"""
@ -161,6 +177,8 @@ class RedisClient:
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {})
# Create Redis client with PubSub-optimized settings - no timeout
client = redis.Redis(
host=redis_host,
@ -172,7 +190,9 @@ class RedisClient:
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
retry_on_timeout=retry_on_timeout,
decode_responses=True,
**ssl_params
)
# Validate connection with ping
@ -185,8 +205,9 @@ class RedisClient:
except (ConnectionError, TimeoutError) as e:
retry_count += 1
_tls_hint = _REDIS_TLS_HINT if ssl_params else ""
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}")
logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}{_tls_hint}")
return None
else:
# Use exponential backoff for retries
@ -195,7 +216,8 @@ class RedisClient:
time.sleep(wait_time)
except Exception as e:
logger.error(f"Unexpected error connecting to Redis for PubSub: {e}")
_tls_hint = _REDIS_TLS_HINT if ssl_params else ""
logger.error(f"Unexpected error connecting to Redis for PubSub: {e}{_tls_hint}")
return None
return cls._pubsub_client
@ -410,6 +432,20 @@ def cleanup_memory(log_usage=False, force_collection=True):
pass
logger.trace("Memory cleanup complete for django")
def safe_upload_path(filename: str, base_dir) -> str:
"""Return a safe absolute path for an uploaded file within base_dir.
Strips all directory components from *filename* and verifies the resolved
path stays inside *base_dir*. Raises ValueError on path traversal attempts.
"""
safe_name = Path(filename).name
base = Path(base_dir).resolve()
file_path = (base / safe_name).resolve()
if not file_path.is_relative_to(base):
raise ValueError("Invalid filename.")
return str(file_path)
def is_protected_path(file_path):
"""
Determine if a file path is in a protected directory that shouldn't be deleted.

13
core/views.py
View file

@ -4,7 +4,7 @@ from shlex import split as shlex_split
import sys
import subprocess
import logging
import re
import regex
import redis
from django.conf import settings
@ -42,12 +42,14 @@ def stream_view(request, channel_uuid):
redis_db = int(getattr(settings, "REDIS_DB", "0"))
redis_password = getattr(settings, "REDIS_PASSWORD", "")
redis_user = getattr(settings, "REDIS_USER", "")
ssl_params = getattr(settings, "REDIS_SSL_PARAMS", {})
redis_client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
password=redis_password if redis_password else None,
username=redis_user if redis_user else None
username=redis_user if redis_user else None,
**ssl_params
)
# Retrieve the channel by the provided stream_id.
@ -130,10 +132,13 @@ def stream_view(request, channel_uuid):
# Prepare the pattern replacement.
logger.debug("Executing the following pattern replacement:")
logger.debug(f" search: {active_profile.search_pattern}")
safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', active_profile.replace_pattern)
# Convert JS-style backreferences in replace: $<name> -> \g<name>, $1 -> \1
safe_replace_pattern = regex.sub(r'\$<([^>]+)>', r'\\g<\1>', active_profile.replace_pattern)
safe_replace_pattern = regex.sub(r'\$(\d+)', r'\\\1', safe_replace_pattern)
logger.debug(f" replace: {active_profile.replace_pattern}")
logger.debug(f" safe replace: {safe_replace_pattern}")
stream_url = re.sub(active_profile.search_pattern, safe_replace_pattern, input_url)
# regex module accepts JS-style (?<name>...) named groups natively
stream_url = regex.sub(active_profile.search_pattern, safe_replace_pattern, input_url)
logger.debug(f"Generated stream url: {stream_url}")
# Get the stream profile set on the channel.

2
core/xtream_codes.py
View file

@ -62,7 +62,7 @@ class Client:
url = f"{self.server_url}/{endpoint}"
logger.debug(f"XC API Request: {url} with params: {params}")
response = self.session.get(url, params=params, timeout=30)
response = self.session.get(url, params=params, timeout=60)
response.raise_for_status()
# Check if response is empty

6
dispatcharr/consumers.py
View file

@ -1,6 +1,6 @@
import json
from channels.generic.websocket import AsyncWebsocketConsumer
import re, logging
import regex, logging
logger = logging.getLogger(__name__)
@ -54,9 +54,9 @@ class MyWebSocketConsumer(AsyncWebsocketConsumer):
# Apply the transformation using the replace_with_mark function
try:
search_preview = re.sub(data["search"], replace_with_mark, data["url"])
search_preview = regex.sub(data["search"], replace_with_mark, data["url"])
except Exception as e:
search_preview = data["search"]
search_preview = data["url"]
logger.error(f"Failed to generate replace preview: {e}")
result = transform_url(data["url"], data["search"], data["replace"])

26
dispatcharr/persistent_lock.py
View file

@ -48,7 +48,7 @@ class PersistentLock:
Returns True if the expiration was successfully extended.
"""
current_value = self.redis_client.get(self.lock_key)
if current_value and current_value.decode("utf-8") == self.lock_token:
if current_value and current_value == self.lock_token:
self.redis_client.expire(self.lock_key, self.lock_timeout)
self.has_lock = False
return True
@ -74,18 +74,40 @@ class PersistentLock:
# Example usage (for testing purposes only):
if __name__ == "__main__":
import os
import sys
# Connect to Redis using environment variables; adjust connection parameters as needed.
redis_host = os.environ.get("REDIS_HOST", "localhost")
redis_port = int(os.environ.get("REDIS_PORT", 6379))
redis_db = int(os.environ.get("REDIS_DB", 0))
redis_password = os.environ.get("REDIS_PASSWORD", "")
redis_user = os.environ.get("REDIS_USER", "")
ssl_kwargs = {}
if os.environ.get("REDIS_SSL", "false").lower() == "true":
import ssl as _ssl
ssl_kwargs["ssl"] = True
ssl_kwargs["ssl_cert_reqs"] = (
_ssl.CERT_REQUIRED if os.environ.get("REDIS_SSL_VERIFY", "true").lower() == "true"
else _ssl.CERT_NONE
)
for env_var, key in [
("REDIS_SSL_CA_CERT", "ssl_ca_certs"),
("REDIS_SSL_CERT", "ssl_certfile"),
("REDIS_SSL_KEY", "ssl_keyfile"),
]:
path = os.environ.get(env_var, "")
if path:
if not os.path.isfile(path):
print(f"Redis TLS: {env_var}={path!r} — file not found.")
sys.exit(1)
ssl_kwargs[key] = path
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
password=redis_password if redis_password else None,
username=redis_user if redis_user else None
username=redis_user if redis_user else None,
**ssl_kwargs
)
lock = PersistentLock(client, "lock:example_account", lock_timeout=120)

200
dispatcharr/settings.py
View file

@ -1,7 +1,23 @@
import os
import ssl
from pathlib import Path
from datetime import timedelta
from urllib.parse import quote_plus
from django.core.exceptions import ImproperlyConfigured
def _validate_tls_cert_paths(paths, service_name):
"""Validate that configured TLS certificate file paths exist on disk.
Raises ImproperlyConfigured with a clear message identifying the
service and missing file so operators can fix their environment.
"""
for env_var, file_path in paths:
if file_path and not Path(file_path).is_file():
raise ImproperlyConfigured(
f"{service_name} TLS: {env_var}={file_path!r} — file not found. "
f"Check that the certificate file exists and the volume is mounted correctly."
)
BASE_DIR = Path(__file__).resolve().parent.parent
@ -12,6 +28,37 @@ REDIS_DB = os.environ.get("REDIS_DB", "0")
REDIS_USER = os.environ.get("REDIS_USER", "")
REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "")
# Redis TLS configuration
REDIS_SSL = os.environ.get("REDIS_SSL", "false").lower() == "true"
REDIS_SSL_VERIFY = os.environ.get("REDIS_SSL_VERIFY", "true").lower() == "true"
REDIS_SSL_CA_CERT = os.environ.get("REDIS_SSL_CA_CERT", "")
REDIS_SSL_CERT = os.environ.get("REDIS_SSL_CERT", "")
REDIS_SSL_KEY = os.environ.get("REDIS_SSL_KEY", "")
# Reusable dict of SSL kwargs for redis.Redis() constructors
REDIS_SSL_PARAMS = {}
if REDIS_SSL:
_validate_tls_cert_paths([
("REDIS_SSL_CA_CERT", REDIS_SSL_CA_CERT),
("REDIS_SSL_CERT", REDIS_SSL_CERT),
("REDIS_SSL_KEY", REDIS_SSL_KEY),
], "Redis")
REDIS_SSL_PARAMS["ssl"] = True
REDIS_SSL_PARAMS["ssl_cert_reqs"] = ssl.CERT_REQUIRED if REDIS_SSL_VERIFY else ssl.CERT_NONE
if REDIS_SSL_CA_CERT:
REDIS_SSL_PARAMS["ssl_ca_certs"] = REDIS_SSL_CA_CERT
if REDIS_SSL_CERT:
REDIS_SSL_PARAMS["ssl_certfile"] = REDIS_SSL_CERT
if REDIS_SSL_KEY:
REDIS_SSL_PARAMS["ssl_keyfile"] = REDIS_SSL_KEY
_mtls = "enabled" if REDIS_SSL_CERT and REDIS_SSL_KEY else "disabled"
_verify = "on" if REDIS_SSL_VERIFY else "off"
print(f"Redis TLS: enabled (verify={_verify}, mTLS={_mtls})")
else:
print("Redis TLS: disabled")
# Set DEBUG to True for development, False for production
if os.environ.get("DISPATCHARR_DEBUG", "False").lower() == "true":
DEBUG = True
@ -120,20 +167,46 @@ TEMPLATES = [
WSGI_APPLICATION = "dispatcharr.wsgi.application"
ASGI_APPLICATION = "dispatcharr.asgi.application"
_redis_scheme = "rediss" if REDIS_SSL else "redis"
# URL-encoded auth string shared by CHANNEL_LAYERS and Celery broker URLs
if REDIS_PASSWORD:
_encoded_password = quote_plus(REDIS_PASSWORD)
if REDIS_USER:
_redis_auth = f"{quote_plus(REDIS_USER)}:{_encoded_password}@"
else:
_redis_auth = f":{_encoded_password}@"
else:
_redis_auth = ""
_channels_redis_url = f"{_redis_scheme}://{_redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
# channels_redis accepts either a URL string or a dict with "address" + kwargs.
# When TLS is enabled, pass SSL params alongside the URL so the connection pool
# uses the correct CA cert and verification settings.
if REDIS_SSL:
# Filter out "ssl" key — the rediss:// scheme already enables SSL.
# Passing ssl=True as a kwarg to aioredis from_url causes an error.
_channels_ssl = {k: v for k, v in REDIS_SSL_PARAMS.items() if k != "ssl"}
_channels_host = {"address": _channels_redis_url, **_channels_ssl}
else:
_channels_host = _channels_redis_url
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": ["redis://{redis_auth}{host}:{port}/{db}".format(
redis_auth=f"{quote_plus(REDIS_USER)}:{quote_plus(REDIS_PASSWORD)}@" if REDIS_PASSWORD and REDIS_USER else f":{quote_plus(REDIS_PASSWORD)}@" if REDIS_PASSWORD else "",
host=REDIS_HOST,
port=REDIS_PORT,
db=REDIS_DB
)], # URL format supports authentication
"hosts": [_channels_host],
},
},
}
# PostgreSQL TLS configuration (defined before DATABASES for module-level access)
POSTGRES_SSL = os.environ.get("POSTGRES_SSL", "false").lower() == "true"
POSTGRES_SSL_MODE = os.environ.get("POSTGRES_SSL_MODE", "verify-full")
POSTGRES_SSL_CA_CERT = os.environ.get("POSTGRES_SSL_CA_CERT", "")
POSTGRES_SSL_CERT = os.environ.get("POSTGRES_SSL_CERT", "")
POSTGRES_SSL_KEY = os.environ.get("POSTGRES_SSL_KEY", "")
if os.getenv("DB_ENGINE", None) == "sqlite":
DATABASES = {
"default": {
@ -154,6 +227,28 @@ else:
}
}
if POSTGRES_SSL:
_validate_tls_cert_paths([
("POSTGRES_SSL_CA_CERT", POSTGRES_SSL_CA_CERT),
("POSTGRES_SSL_CERT", POSTGRES_SSL_CERT),
("POSTGRES_SSL_KEY", POSTGRES_SSL_KEY),
], "PostgreSQL")
DATABASES["default"]["OPTIONS"] = {
"sslmode": POSTGRES_SSL_MODE,
}
if POSTGRES_SSL_CA_CERT:
DATABASES["default"]["OPTIONS"]["sslrootcert"] = POSTGRES_SSL_CA_CERT
if POSTGRES_SSL_CERT:
DATABASES["default"]["OPTIONS"]["sslcert"] = POSTGRES_SSL_CERT
if POSTGRES_SSL_KEY:
DATABASES["default"]["OPTIONS"]["sslkey"] = POSTGRES_SSL_KEY
_mtls = "enabled" if POSTGRES_SSL_CERT and POSTGRES_SSL_KEY else "disabled"
print(f"PostgreSQL TLS: enabled (sslmode={POSTGRES_SSL_MODE}, mTLS={_mtls})")
else:
print("PostgreSQL TLS: disabled")
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
@ -170,7 +265,14 @@ REST_FRAMEWORK = {
"rest_framework_simplejwt.authentication.JWTAuthentication",
"apps.accounts.authentication.ApiKeyAuthentication",
],
"DEFAULT_PERMISSION_CLASSES": [
"apps.accounts.permissions.IsAdmin",
],
"DEFAULT_FILTER_BACKENDS": ["django_filters.rest_framework.DjangoFilterBackend"],
"DEFAULT_THROTTLE_CLASSES": [],
"DEFAULT_THROTTLE_RATES": {
"login": "3/minute",
},
}
SPECTACULAR_SETTINGS = {
@ -178,17 +280,6 @@ SPECTACULAR_SETTINGS = {
"DESCRIPTION": "API documentation for Dispatcharr",
"VERSION": "1.0.0",
"SERVE_INCLUDE_SCHEMA": False,
"SECURITY": [{"BearerAuth": []}],
"COMPONENTS": {
"securitySchemes": {
"BearerAuth": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT",
"description": "Enter your JWT access token. The 'Bearer ' prefix is added automatically.",
}
}
},
}
LANGUAGE_CODE = "en-us"
@ -208,22 +299,52 @@ STATICFILES_DIRS = [
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
AUTH_USER_MODEL = "accounts.User"
# Build default Redis URL from components for Celery with optional authentication
# Build auth string conditionally with URL encoding for special characters
if REDIS_PASSWORD:
encoded_password = quote_plus(REDIS_PASSWORD)
if REDIS_USER:
encoded_user = quote_plus(REDIS_USER)
redis_auth = f"{encoded_user}:{encoded_password}@"
else:
redis_auth = f":{encoded_password}@"
_default_redis_url = f"{_redis_scheme}://{_redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
# Celery/Kombu require SSL parameters in the URL query string because
# internal URL parsing can overwrite the CELERY_BROKER_USE_SSL dict.
if REDIS_SSL:
_celery_ssl_params = [
f"ssl_cert_reqs={'CERT_REQUIRED' if REDIS_SSL_VERIFY else 'CERT_NONE'}",
]
if REDIS_SSL_CA_CERT:
_celery_ssl_params.append(f"ssl_ca_certs={REDIS_SSL_CA_CERT}")
if REDIS_SSL_CERT:
_celery_ssl_params.append(f"ssl_certfile={REDIS_SSL_CERT}")
if REDIS_SSL_KEY:
_celery_ssl_params.append(f"ssl_keyfile={REDIS_SSL_KEY}")
_default_celery_url = f"{_default_redis_url}?{'&'.join(_celery_ssl_params)}"
else:
redis_auth = ""
_default_redis_url = f"redis://{redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}"
CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", _default_redis_url)
_default_celery_url = _default_redis_url
CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", _default_celery_url)
CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", CELERY_BROKER_URL)
# Validate that URL overrides don't conflict with TLS settings
for _url_var, _url_val in [
("CELERY_BROKER_URL", CELERY_BROKER_URL),
("CELERY_RESULT_BACKEND", CELERY_RESULT_BACKEND),
]:
_is_override = os.environ.get(_url_var) is not None
if not _is_override:
continue
_url_is_ssl = _url_val.startswith("rediss://")
if REDIS_SSL and not _url_is_ssl:
raise ImproperlyConfigured(
f"REDIS_SSL is enabled but {_url_var} uses redis:// (plaintext). "
f"Change the URL scheme to rediss:// or remove the {_url_var} override."
)
if not REDIS_SSL and _url_is_ssl:
raise ImproperlyConfigured(
f"{_url_var} uses rediss:// (TLS) but REDIS_SSL is not enabled. "
f"Set REDIS_SSL=true and configure the TLS certificate settings."
)
# Celery TLS configuration — required in addition to the rediss:// URL scheme.
# Uses the same cert params as REDIS_SSL_PARAMS, minus the "ssl" key that
# redis-py needs but Celery/Kombu does not.
if REDIS_SSL:
CELERY_BROKER_USE_SSL = {k: v for k, v in REDIS_SSL_PARAMS.items() if k != "ssl"}
CELERY_RESULT_BACKEND_USE_SSL = CELERY_BROKER_USE_SSL
# Configure Redis key prefix
CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = {
"global_keyprefix": "celery-tasks:", # Set the Redis key prefix for Celery
@ -267,6 +388,11 @@ CELERY_BEAT_SCHEDULE = {
"task": "core.tasks.check_for_version_update",
"schedule": 86400.0, # Once every 24 hours
},
# Check for account expirations daily
"check-account-expirations": {
"task": "apps.m3u.tasks.check_account_expirations",
"schedule": 86400.0, # Once every 24 hours
},
}
MEDIA_ROOT = BASE_DIR / "media"
@ -283,7 +409,6 @@ BACKUP_DATA_DIRS = [
SERVER_IP = "127.0.0.1"
CORS_ALLOW_ALL_ORIGINS = True
CORS_ALLOW_CREDENTIALS = True
CSRF_TRUSTED_ORIGINS = ["http://*", "https://*"]
APPEND_SLASH = True
@ -294,8 +419,19 @@ SIMPLE_JWT = {
"BLACKLIST_AFTER_ROTATION": True, # Optional: Whether to blacklist refresh tokens
}
# Redis connection settings
# Redis connection settings — _default_redis_url uses rediss:// when REDIS_SSL is enabled
REDIS_URL = os.environ.get("REDIS_URL", _default_redis_url)
if os.environ.get("REDIS_URL") is not None:
if REDIS_SSL and not REDIS_URL.startswith("rediss://"):
raise ImproperlyConfigured(
"REDIS_SSL is enabled but REDIS_URL uses redis:// (plaintext). "
"Change the URL scheme to rediss:// or remove the REDIS_URL override."
)
if not REDIS_SSL and REDIS_URL.startswith("rediss://"):
raise ImproperlyConfigured(
"REDIS_URL uses rediss:// (TLS) but REDIS_SSL is not enabled. "
"Set REDIS_SSL=true and configure the TLS certificate settings."
)
REDIS_SOCKET_TIMEOUT = 60 # Socket timeout in seconds
REDIS_SOCKET_CONNECT_TIMEOUT = 5 # Connection timeout in seconds
REDIS_HEALTH_CHECK_INTERVAL = 15 # Health check every 15 seconds

10
dispatcharr/urls.py
View file

@ -6,7 +6,7 @@ from django.views.generic import TemplateView, RedirectView
from .routing import websocket_urlpatterns
from apps.output.views import xc_player_api, xc_panel_api, xc_get, xc_xmltv
from apps.proxy.ts_proxy.views import stream_xc
from apps.output.views import xc_movie_stream, xc_series_stream
from apps.proxy.vod_proxy.views import stream_xc_movie, stream_xc_episode
urlpatterns = [
# API Routes
@ -44,13 +44,13 @@ urlpatterns = [
# XC VOD endpoints
path(
"movie/<str:username>/<str:password>/<str:stream_id>.<str:extension>",
xc_movie_stream,
name="xc_movie_stream",
stream_xc_movie,
name="stream_xc_movie",
),
path(
"series/<str:username>/<str:password>/<str:stream_id>.<str:extension>",
xc_series_stream,
name="xc_series_stream",
stream_xc_episode,
name="stream_xc_episode",
),
# Admin
path("admin", RedirectView.as_view(url="/admin/", permanent=True)),

3
docker/DispatcharrBase
View file

@ -15,9 +15,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
&& apt-get update \
&& apt-get install --no-install-recommends -y \
python3.13 python3.13-dev python3.13-venv libpython3.13 \
python-is-python3 python3-pip \
libpcre3 libpcre3-dev libpq-dev procps pciutils \
nginx streamlink comskip \
nginx comskip \
vlc-bin vlc-plugin-base \
build-essential gcc g++ gfortran libopenblas-dev libopenblas0 ninja-build

8
docker/Dockerfile
View file

@ -12,7 +12,7 @@ COPY ./frontend /app/frontend
# remove any node_modules that may have been copied from the host (x86)
RUN rm -rf node_modules || true; \
npm install --no-audit --progress=false;
RUN npm run build; \
RUN npm run build && \
rm -rf node_modules .cache
# --- Redeclare build arguments for the next stage ---
@ -32,9 +32,9 @@ COPY . /app
COPY ./docker/nginx.conf /etc/nginx/sites-enabled/default
# Fix line endings and make entrypoint scripts executable
RUN for f in /app/docker/entrypoint*.sh; do \
if [ -f "$f" ]; then \
sed -i 's/\r$//' "$f" && chmod +x "$f"; \
fi; \
if [ -f "$f" ]; then \
sed -i 's/\r$//' "$f" && chmod +x "$f"; \
fi; \
done
# Clean out existing frontend folder
RUN rm -rf /app/frontend

36
docker/docker-compose.yml
View file

@ -14,6 +14,7 @@ services:
- 9191:9191
volumes:
- ./data:/data
#- ./certs:/certs:ro # TLS certificates (optional)
depends_on:
db:
condition: service_healthy
@ -33,15 +34,26 @@ services:
- POSTGRES_DB=dispatcharr
- POSTGRES_USER=dispatch
- POSTGRES_PASSWORD=secret
# PostgreSQL TLS (optional) — mount certs via the volume above
#- POSTGRES_SSL=true # required to enable TLS
#- POSTGRES_SSL_MODE=verify-full # optional: verify-full (default) | verify-ca | require
#- POSTGRES_SSL_CA_CERT=/certs/postgres/ca.crt # optional: CA cert to verify the server
#- POSTGRES_SSL_CERT=/certs/postgres/client.crt # optional: client cert (only if server requires client auth)
#- POSTGRES_SSL_KEY=/certs/postgres/client.key # optional: client key (only if server requires client auth)
# Redis Connection
- REDIS_HOST=redis
- REDIS_PORT=6379
# Redis Authentication (Optional)
# Uncomment and set if your Redis requires authentication:
#- REDIS_PASSWORD=your_strong_redis_password
#- REDIS_USER=your_redis_username # For Redis 6+ ACL - see Redis service below
# Redis TLS (optional) — mount certs via the volume above
#- REDIS_SSL=true # required to enable TLS
#- REDIS_SSL_VERIFY=true # optional: set false for self-signed certs without a CA
#- REDIS_SSL_CA_CERT=/certs/redis/ca.crt # optional: CA cert to verify the server
#- REDIS_SSL_CERT=/certs/redis/client.crt # optional: client cert (only if server requires client auth)
#- REDIS_SSL_KEY=/certs/redis/client.key # optional: client key (only if server requires client auth)
# Logging
- DISPATCHARR_LOG_LEVEL=info
@ -95,6 +107,7 @@ services:
condition: service_started
volumes:
- ./data:/data
#- ./certs:/certs:ro # TLS certificates (optional)
extra_hosts:
- "host.docker.internal:host-gateway"
entrypoint: ["/app/docker/entrypoint.celery.sh"]
@ -108,21 +121,30 @@ services:
# Must match the web service port for DVR recording and internal API calls
- DISPATCHARR_PORT=9191
# PostgreSQL Connection
# PostgreSQL — must match web service settings
- POSTGRES_HOST=db
- POSTGRES_PORT=5432
- POSTGRES_DB=dispatcharr
- POSTGRES_USER=dispatch
- POSTGRES_PASSWORD=secret
# PostgreSQL TLS — must match web service
#- POSTGRES_SSL=true
#- POSTGRES_SSL_MODE=verify-full
#- POSTGRES_SSL_CA_CERT=/certs/postgres/ca.crt
#- POSTGRES_SSL_CERT=/certs/postgres/client.crt
#- POSTGRES_SSL_KEY=/certs/postgres/client.key
# Redis Connection
# Redis — must match web service settings
- REDIS_HOST=redis
- REDIS_PORT=6379
# Redis Authentication (Optional)
# Uncomment and set if your Redis requires authentication:
#- REDIS_PASSWORD=your_strong_redis_password
#- REDIS_USER=your_redis_username # For Redis 6+ ACL - see Redis service below
#- REDIS_USER=your_redis_username
# Redis TLS — must match web service
#- REDIS_SSL=true
#- REDIS_SSL_VERIFY=true
#- REDIS_SSL_CA_CERT=/certs/redis/ca.crt
#- REDIS_SSL_CERT=/certs/redis/client.crt
#- REDIS_SSL_KEY=/certs/redis/client.key
# Logging
- DISPATCHARR_LOG_LEVEL=info

4
docker/entrypoint.celery.sh
View file

@ -36,6 +36,10 @@ if [ "$USE_LEGACY_NUMPY" = "true" ]; then
fi
fi
# Fix TLS client key permissions/ownership for PostgreSQL.
FIXED_KEY_PATH="/data/.pg-client-celery.key"
. /app/docker/init/00-fix-pg-ssl-key.sh
# Wait for migrations to complete
# Uses 'migrate --check' which exits 0 only when all migrations are applied,
# and exits 1 on unapplied migrations OR connection errors (safe either way)

47
docker/entrypoint.sh
View file

@ -30,7 +30,14 @@ echo_with_timestamp() {
# Set PostgreSQL environment variables
export POSTGRES_DB=${POSTGRES_DB:-dispatcharr}
export POSTGRES_USER=${POSTGRES_USER:-dispatch}
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-secret}
# AIO mode: default to 'secret' for internal DB.
# Modular mode + TLS: no default — cert-only auth (mTLS) uses no password.
# Modular mode + no TLS: preserve 'secret' default for backward compatibility.
if [[ "${DISPATCHARR_ENV:-}" == "modular" && "${POSTGRES_SSL:-}" == "true" ]]; then
export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-}"
else
export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-secret}"
fi
export POSTGRES_HOST=${POSTGRES_HOST:-localhost}
export POSTGRES_PORT=${POSTGRES_PORT:-5432}
export PG_VERSION=$(ls /usr/lib/postgresql/ | sort -V | tail -n 1)
@ -96,6 +103,20 @@ echo "Environment DISPATCHARR_LOG_LEVEL set to: '${DISPATCHARR_LOG_LEVEL}'"
# Also make the log level available in /etc/environment for all login shells
#grep -q "DISPATCHARR_LOG_LEVEL" /etc/environment || echo "DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL}" >> /etc/environment
# Translate Dispatcharr POSTGRES_SSL_* env vars into libpq-recognized PGSSL*
# env vars. Called once before any external PostgreSQL connection; all child
# processes (psql, pg_dump, pg_isready, createdb, dropdb) inherit these
# automatically. No-op when POSTGRES_SSL is not "true".
setup_pg_ssl_env() {
if [ "${POSTGRES_SSL:-false}" != "true" ]; then
return 0
fi
export PGSSLMODE="${POSTGRES_SSL_MODE:-verify-full}"
if [ -n "${POSTGRES_SSL_CA_CERT:-}" ]; then export PGSSLROOTCERT="$POSTGRES_SSL_CA_CERT"; fi
if [ -n "${POSTGRES_SSL_CERT:-}" ]; then export PGSSLCERT="$POSTGRES_SSL_CERT"; fi
if [ -n "${POSTGRES_SSL_KEY:-}" ]; then export PGSSLKEY="$POSTGRES_SSL_KEY"; fi
}
# READ-ONLY - don't let users change these
export POSTGRES_DIR=/data/db
@ -112,6 +133,14 @@ variables=(
CELERY_NICE_LEVEL UWSGI_NICE_LEVEL DJANGO_SECRET_KEY
)
# TLS variables are optional — only propagate when set to avoid noisy warnings
for _tls_var in POSTGRES_SSL POSTGRES_SSL_MODE POSTGRES_SSL_CA_CERT POSTGRES_SSL_CERT POSTGRES_SSL_KEY \
REDIS_SSL REDIS_SSL_VERIFY REDIS_SSL_CA_CERT REDIS_SSL_CERT REDIS_SSL_KEY; do
if [ -n "${!_tls_var+x}" ]; then
variables+=("$_tls_var")
fi
done
# Truncate files before rewriting
> /etc/profile.d/dispatcharr.sh
@ -146,6 +175,22 @@ fi
echo "Starting user setup..."
. /app/docker/init/01-user-setup.sh
# Fix TLS client key permissions/ownership BEFORE any external PG connections.
# Must run after 01-user-setup.sh (user exists for chown) and before
# 02-postgres.sh / pg_isready (which make the first external PG connections).
FIXED_KEY_PATH="/data/.pg-client.key"
. /app/docker/init/00-fix-pg-ssl-key.sh
# Propagate the fixed path to login shells (su - strips env vars)
if [ "${POSTGRES_SSL_KEY:-}" = "$FIXED_KEY_PATH" ]; then
sed -i "/^POSTGRES_SSL_KEY=/d" /etc/environment
echo "POSTGRES_SSL_KEY='$FIXED_KEY_PATH'" >> /etc/environment
sed -i "s|export POSTGRES_SSL_KEY=.*|export POSTGRES_SSL_KEY='$FIXED_KEY_PATH'|" /etc/profile.d/dispatcharr.sh
fi
# Export libpq TLS env vars so all subsequent psql/pg_dump/pg_isready calls
# (in 02-postgres.sh, modular-mode checks, etc.) use TLS automatically.
setup_pg_ssl_env
# Initialize PostgreSQL (script handles modular vs internal mode internally)
echo "Setting up PostgreSQL..."
. /app/docker/init/02-postgres.sh

44
docker/init/00-fix-pg-ssl-key.sh Normal file
View file

@ -0,0 +1,44 @@
#!/bin/bash
#
# Fix TLS client key permissions and ownership for PostgreSQL.
# libpq requires the client key to be 0600 or stricter.
#
# Triggers on:
# - Permissions too open (Docker Desktop mounts files as 0777)
# - Wrong ownership (Kubernetes secrets / Docker volumes mount as root;
# the application user can't read a root-owned 0600 key)
# - Read-only source (volume mounted :ro — can't chmod in place)
#
# Usage: source this script with FIXED_KEY_PATH set to the destination.
# FIXED_KEY_PATH="/data/.pg-client.key"
# . /app/docker/init/00-fix-pg-ssl-key.sh
#
# After sourcing, POSTGRES_SSL_KEY is updated to the fixed path if a copy
# was needed. The caller is responsible for propagating the new value to
# /etc/environment or profile.d if required.
: "${FIXED_KEY_PATH:?FIXED_KEY_PATH must be set before sourcing fix-pg-ssl-key.sh}"
if [ -n "${POSTGRES_SSL_KEY:-}" ] && [ -f "$POSTGRES_SSL_KEY" ]; then
_key_perms=$(stat -c '%a' "$POSTGRES_SSL_KEY" 2>/dev/null)
_key_owner=$(stat -c '%u' "$POSTGRES_SSL_KEY" 2>/dev/null)
_needs_fix=false
if [ "$_key_perms" != "600" ] && [ "$_key_perms" != "640" ]; then
_needs_fix=true
elif [ "$(id -u)" = "0" ] && [ -n "${PUID:-}" ] && [ "$_key_owner" != "$PUID" ]; then
_needs_fix=true
fi
if [ "$_needs_fix" = true ]; then
cp "$POSTGRES_SSL_KEY" "$FIXED_KEY_PATH"
chmod 600 "$FIXED_KEY_PATH"
if [ "$(id -u)" = "0" ] && [ -n "${PUID:-}" ]; then
chown "${PUID}:${PGID:-$PUID}" "$FIXED_KEY_PATH"
fi
export POSTGRES_SSL_KEY="$FIXED_KEY_PATH"
echo "Fixed PostgreSQL client key (perms: ${_key_perms}, owner: ${_key_owner} → ${PUID:-root}:600)"
fi
unset _key_perms _key_owner _needs_fix
fi

23
docker/init/01-user-setup.sh
View file

@ -6,24 +6,11 @@
# runtime behavior since all postgres operations run as $POSTGRES_USER
# rather than the postgres system user.
# Auto-detect PUID/PGID from existing data when not explicitly set.
# Avoids a cross-UID chown on upgrade, which would fail on restricted
# filesystems (NFS root_squash, CIFS). UID/GID 0 is excluded — PostgreSQL
# refuses to run as root. Falls through to default 1000 for new installs.
if [ -z "${PUID+x}" ] && [ -f "${POSTGRES_DIR}/PG_VERSION" ]; then
_data_uid=$(stat -c '%u' "${POSTGRES_DIR}/PG_VERSION")
if [ "$_data_uid" -ne 0 ] 2>/dev/null; then
export PUID=$_data_uid
echo "PUID not set — defaulting to existing data owner UID: $PUID"
fi
fi
if [ -z "${PGID+x}" ] && [ -f "${POSTGRES_DIR}/PG_VERSION" ]; then
_data_gid=$(stat -c '%g' "${POSTGRES_DIR}/PG_VERSION")
if [ "$_data_gid" -ne 0 ] 2>/dev/null; then
export PGID=$_data_gid
echo "PGID not set — defaulting to existing data owner GID: $PGID"
fi
fi
# Default PUID/PGID to 1000 when not explicitly set.
# The old image ran Django as UID 1000 and PostgreSQL as UID 102. Since
# DATA_DIRS and host-side files are owned by 1000, defaulting to 1000
# preserves access for upgrading users without requiring configuration.
# The DB ownership migration (102 → 1000) is handled by 02-postgres.sh.
export PUID=${PUID:-1000}
export PGID=${PGID:-1000}

75
docker/init/03-init-dispatcharr.sh
View file

@ -1,7 +1,11 @@
#!/bin/bash
# Define directories that need to exist and be owned by PUID:PGID
# Define directories that need to exist and be owned by PUID:PGID.
# DATA_DIRS may reside on external mounts (NFS, SMB/CIFS, FUSE) where
# mkdir and chown can fail. Failures are collected and reported as a
# single consolidated warning so the container still starts.
DATA_DIRS=(
"/data/backups"
"/data/logos"
"/data/recordings"
"/data/uploads/m3us"
@ -13,17 +17,25 @@ DATA_DIRS=(
"/data/scripts"
)
# APP_DIRS live on the image layer and are always locally writable.
APP_DIRS=(
"/app/logo_cache"
"/app/media"
"/app/static"
)
# Create all directories
for dir in "${DATA_DIRS[@]}" "${APP_DIRS[@]}"; do
# Create app directories (image layer — always writable)
for dir in "${APP_DIRS[@]}"; do
mkdir -p "$dir"
done
# Create data directories, tolerating failures on external mounts
_failed_mkdir=()
_failed_chown=()
for dir in "${DATA_DIRS[@]}"; do
_mkdir_err=$(mkdir -p "$dir" 2>&1) || _failed_mkdir+=("$dir ($_mkdir_err)")
done
# Ensure /app itself is owned by PUID:PGID (needed for uwsgi socket creation)
if [ "$(id -u)" = "0" ] && [ -d "/app" ]; then
if [ "$(stat -c '%u:%g' /app)" != "$PUID:$PGID" ]; then
@ -49,11 +61,15 @@ fi
# NOTE: mac doesn't run as root, so only manage permissions
# if this script is running as root
if [ "$(id -u)" = "0" ]; then
# Fix data directories (non-recursive to avoid touching user files)
# Fix data directories (non-recursive to avoid touching user files).
# Failures are collected rather than fatal — directories may be on
# external mounts (NFS, SMB/CIFS, FUSE) that reject chown.
for dir in "${DATA_DIRS[@]}"; do
if [ -d "$dir" ] && [ "$(stat -c '%u:%g' "$dir")" != "$PUID:$PGID" ]; then
echo "Fixing ownership for $dir"
chown "$PUID:$PGID" "$dir"
if [ -d "$dir" ] && [ "$(stat -c '%u:%g' "$dir" 2>/dev/null)" != "$PUID:$PGID" ]; then
_chown_err=$(chown "$PUID:$PGID" "$dir" 2>&1) || {
_current_owner=$(stat -c '%u:%g' "$dir" 2>/dev/null || echo "unknown")
_failed_chown+=("$dir (current: $_current_owner, error: $_chown_err)")
}
fi
done
@ -69,11 +85,46 @@ if [ "$(id -u)" = "0" ]; then
# No secondary check needed here — duplicating it could chown without updating
# the sentinel, creating inconsistent state.
# Fix /data directory ownership (non-recursive)
if [ -d "/data" ] && [ "$(stat -c '%u:%g' /data)" != "$PUID:$PGID" ]; then
echo "Fixing ownership for /data (non-recursive)"
chown "$PUID:$PGID" /data
# Fix /data directory ownership (non-recursive).
# Tolerates failure for the same external-mount reasons as DATA_DIRS.
if [ -d "/data" ] && [ "$(stat -c '%u:%g' /data 2>/dev/null)" != "$PUID:$PGID" ]; then
_chown_err=$(chown "$PUID:$PGID" /data 2>&1) || {
_current_owner=$(stat -c '%u:%g' /data 2>/dev/null || echo "unknown")
_failed_chown+=("/data (current: $_current_owner, error: $_chown_err)")
}
fi
chmod +x /data
chmod +x /data 2>/dev/null || true
fi
# Consolidated warning for all mkdir/chown failures.
# Emitted outside the root guard so non-root mkdir failures are also reported.
if [ ${#_failed_mkdir[@]} -gt 0 ] || [ ${#_failed_chown[@]} -gt 0 ]; then
echo ""
echo "================================================================"
echo "WARNING: Some data directories could not be created or updated."
echo " This typically occurs with NFS, SMB/CIFS, or other external"
echo " mounts that restrict ownership changes."
echo ""
if [ ${#_failed_mkdir[@]} -gt 0 ]; then
echo " Could not create:"
for entry in "${_failed_mkdir[@]}"; do
echo " - $entry"
done
echo ""
fi
if [ ${#_failed_chown[@]} -gt 0 ]; then
echo " Could not set ownership to $PUID:$PGID:"
for entry in "${_failed_chown[@]}"; do
echo " - $entry"
done
echo ""
fi
echo " To fix, either:"
echo " 1. Set PUID/PGID to match your mount's owner"
echo " 2. Fix ownership on the host/NAS:"
echo " sudo chown $PUID:$PGID <path>"
echo " 3. For SMB/CIFS: set uid=$PUID,gid=$PGID in mount options"
echo "================================================================"
echo ""
fi

50
docker/init/99-init-dev.sh
View file

@ -1,25 +1,33 @@
#!/bin/bash
echo "🚀 Development Mode - Setting up Frontend..."
if [ ! -e "/tmp/init" ]; then
echo "🚀 Development Mode - Setting up Frontend..."
# Install Node.js
if ! command -v node 2>&1 >/dev/null
then
echo "=== setting up nodejs ==="
curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh
bash /tmp/nodesource_setup.sh
apt-get update
apt-get install -y --no-install-recommends \
nodejs
fi
# Install frontend dependencies
cd /app/frontend && npm install
# Install Python dependencies using UV
cd /app && uv sync --python $UV_PROJECT_ENVIRONMENT/bin/python --no-install-project --no-dev
# Install debugpy for remote debugging
if [ "$DISPATCHARR_DEBUG" = "true" ]; then
echo "=== setting up debugpy ==="
uv pip install --python $UV_PROJECT_ENVIRONMENT/bin/python debugpy
# Install Node.js
if ! command -v node 2>&1 >/dev/null
then
echo "=== setting up nodejs ==="
curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh
bash /tmp/nodesource_setup.sh
apt-get update
apt-get install -y --no-install-recommends \
nodejs
fi
# Install frontend dependencies
cd /app/frontend && npm install
# Install Python dependencies using UV
cd /app && uv sync --python $UV_PROJECT_ENVIRONMENT/bin/python --no-install-project --no-dev
# Install debugpy for remote debugging
if [ "$DISPATCHARR_DEBUG" = "true" ]; then
echo "=== setting up debugpy ==="
uv pip install --python $UV_PROJECT_ENVIRONMENT/bin/python debugpy
fi
if [[ "$DISPATCHARR_ENV" = "dev" ]]; then
touch /tmp/init
fi
else
echo "Development mode initialization already done. Skipping dev setup."
fi

892
docker/tests/test-tls-postgres.sh Normal file
View file

@ -0,0 +1,892 @@
#!/bin/bash
#
# Integration test suite for TLS/mTLS in modular mode.
# Validates that Dispatcharr connects correctly to external PostgreSQL and
# Redis services using various TLS configurations.
#
# Prerequisites:
# - Docker Desktop (or Docker Engine) running
# - Internet access (pulls postgres:17, redis:latest)
# - ~10-15 minutes for a full run
#
# Usage:
# cd <repo_root>
# bash docker/tests/test-tls-postgres.sh [--skip-build] [--keep-on-fail] [scenario_name]
#
# Options:
# --skip-build Skip Docker image build (use existing dispatcharr:tls-test image)
# --keep-on-fail Don't clean up containers/volumes on failure (for debugging)
# scenario_name Run only the named scenario
#
# Scenarios:
# modular_mtls_no_password PG mTLS cert-only auth, no password
# modular_mtls_with_password PG mTLS + password auth combined
# modular_tls_server_only PG server-side TLS only (no client cert)
# modular_tls_key_permission PG mTLS with 0777 client key (Docker Desktop scenario)
# modular_no_tls_regression Non-TLS modular mode still works
# modular_pg_verify_full PG mTLS with verify-full (CN must match hostname)
# modular_redis_tls Redis with TLS (server-side verification)
# modular_full_tls_celery PG mTLS + Redis TLS with separate Celery container
#
# Exit codes:
# 0 All tests passed
# 1 One or more tests failed (or build failed)
set -uo pipefail
# Prevent Git Bash (MINGW) from converting Unix paths
export MSYS_NO_PATHCONV=1
###############################################################################
# Configuration
###############################################################################
IMAGE_NAME="dispatcharr:tls-test"
TEST_PREFIX="tls_test"
STARTUP_TIMEOUT=120
SKIP_BUILD=false
KEEP_ON_FAIL=false
SINGLE_SCENARIO=""
PASS=0
FAIL=0
SKIP=0
ERRORS=()
CERT_DIR=""
# Colors (disabled if not a terminal)
if [ -t 1 ]; then
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
CYAN='\033[0;36m'; BOLD='\033[1m'; NC='\033[0m'
else
RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; NC=''
fi
###############################################################################
# Parse arguments
###############################################################################
for arg in "$@"; do
case "$arg" in
--skip-build) SKIP_BUILD=true ;;
--keep-on-fail) KEEP_ON_FAIL=true ;;
-*) echo "Unknown option: $arg"; exit 1 ;;
*) SINGLE_SCENARIO="$arg" ;;
esac
done
###############################################################################
# Helpers
###############################################################################
CURRENT_SCENARIO=""
CLEANUP_ITEMS=()
log_pass() { echo -e " ${GREEN}✅ $1${NC}"; PASS=$((PASS + 1)); }
log_fail() { echo -e " ${RED}❌ $1${NC}"; FAIL=$((FAIL + 1)); ERRORS+=("[$CURRENT_SCENARIO] $1"); }
log_skip() { echo -e " ${YELLOW}⏭️ $1${NC}"; SKIP=$((SKIP + 1)); }
log_info() { echo -e " ${CYAN}ℹ️ $1${NC}"; }
section() { echo -e "\n${BOLD}━━━ $1 ━━━${NC}"; SCENARIO_FAIL_BEFORE=$FAIL; }
track_container() { CLEANUP_ITEMS+=("container:$1"); }
track_volume() { CLEANUP_ITEMS+=("volume:$1"); }
track_network() { CLEANUP_ITEMS+=("network:$1"); }
fresh_volume() {
local vol="$1"
docker rm -f $(docker ps -aq --filter "volume=${vol}") 2>/dev/null || true
docker volume rm "$vol" 2>/dev/null || true
docker volume create "$vol" >/dev/null
track_volume "$vol"
}
cleanup_scenario() {
if [ "$KEEP_ON_FAIL" = true ] && [ "$FAIL" -gt "${SCENARIO_FAIL_BEFORE:-0}" ]; then
log_info "Keeping resources for debugging (--keep-on-fail)"
CLEANUP_ITEMS=()
return
fi
for item in "${CLEANUP_ITEMS[@]}"; do
local type="${item%%:*}"
local name="${item#*:}"
case "$type" in
container) docker stop "$name" 2>/dev/null; docker rm -f "$name" 2>/dev/null ;;
volume) docker volume rm "$name" 2>/dev/null ;;
network) docker network rm "$name" 2>/dev/null ;;
esac
done
CLEANUP_ITEMS=()
}
trap 'cleanup_scenario; [ -n "$CERT_DIR" ] && rm -rf "$CERT_DIR"' EXIT
wait_for_ready() {
local name="$1"
local timeout="${2:-$STARTUP_TIMEOUT}"
local elapsed=0
while [ $elapsed -lt $timeout ]; do
if ! docker ps -q -f "name=^${name}$" 2>/dev/null | grep -q .; then
echo " Container $name exited unexpectedly"
return 1
fi
if docker logs "$name" 2>&1 | grep -q "uwsgi started with PID"; then
return 0
fi
sleep 3
((elapsed+=3))
done
echo " Timeout (${timeout}s) waiting for $name"
return 1
}
_capture_logs() {
local container="$1" logfile="$2"
docker logs "$container" > "$logfile" 2>&1
}
check_log_contains() {
local container="$1" pattern="$2" description="$3"
local tmplog; tmplog=$(mktemp)
_capture_logs "$container" "$tmplog"
if grep -q "$pattern" "$tmplog"; then
log_pass "$description"
else
log_fail "$description (pattern not found: $pattern)"
fi
rm -f "$tmplog"
}
check_log_absent() {
local container="$1" pattern="$2" description="$3"
local tmplog; tmplog=$(mktemp)
_capture_logs "$container" "$tmplog"
if grep -q "$pattern" "$tmplog"; then
log_fail "$description (unexpected pattern found: $pattern)"
else
log_pass "$description"
fi
rm -f "$tmplog"
}
check_migrations_done() {
local container="$1"
local tmplog; tmplog=$(mktemp)
_capture_logs "$container" "$tmplog"
if grep -qE "Running migrations|No migrations to apply|Operations to perform|Applying .+\.\.\. OK" "$tmplog"; then
log_pass "Django migrations completed"
elif grep -q "uwsgi started with PID" "$tmplog"; then
log_pass "Django migrations completed (confirmed via uwsgi startup)"
else
log_fail "Django migrations did not complete"
fi
rm -f "$tmplog"
}
check_no_permission_errors() {
local container="$1"
local tmplog; tmplog=$(mktemp)
_capture_logs "$container" "$tmplog"
local errors
errors=$(grep -iE "permission denied|operation not permitted" "$tmplog" \
| grep -v "GPU acceleration" | grep -v "Warning:" | head -5)
rm -f "$tmplog"
if [ -n "$errors" ]; then
log_fail "Permission errors in logs:"
echo "$errors" | while read -r line; do echo " $line"; done
else
log_pass "No permission errors in logs"
fi
}
dump_logs_on_fail() {
local container="$1"
if [ $FAIL -gt ${SCENARIO_FAIL_BEFORE:-0} ]; then
echo -e " ${YELLOW}--- Container logs ($container) ---${NC}"
docker logs "$container" 2>&1 | tail -30 | sed 's/^/ /'
echo -e " ${YELLOW}--- End logs ---${NC}"
fi
}
###############################################################################
# Certificate generation
###############################################################################
generate_test_certs() {
CERT_DIR=$(mktemp -d)
log_info "Generating test certificates in $CERT_DIR"
# Generate certs inside a container for cross-platform compatibility.
# Shared CA for both PG and Redis. CN of server certs must match their
# Docker container hostnames for verify-full mode.
docker run --rm --entrypoint sh \
-v "$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR"):/certs" \
-w /certs alpine/openssl -c '
# Shared CA
openssl req -new -x509 -days 1 -nodes \
-keyout ca.key -out ca.crt -subj "/CN=Test CA" 2>/dev/null &&
# PostgreSQL server cert (CN = PG container hostname)
openssl req -new -nodes \
-keyout pg-server.key -out pg-server.csr -subj "/CN='"${TEST_PREFIX}"'_pg" 2>/dev/null &&
openssl x509 -req -days 1 -in pg-server.csr \
-CA ca.crt -CAkey ca.key -CAcreateserial -out pg-server.crt 2>/dev/null &&
# PostgreSQL client cert (CN = POSTGRES_USER)
openssl req -new -nodes \
-keyout pg-client.key -out pg-client.csr -subj "/CN=dispatch" 2>/dev/null &&
openssl x509 -req -days 1 -in pg-client.csr \
-CA ca.crt -CAkey ca.key -CAcreateserial -out pg-client.crt 2>/dev/null &&
# Redis server cert (CN = Redis container hostname)
openssl req -new -nodes \
-keyout redis-server.key -out redis-server.csr -subj "/CN='"${TEST_PREFIX}"'_redis" 2>/dev/null &&
openssl x509 -req -days 1 -in redis-server.csr \
-CA ca.crt -CAkey ca.key -CAcreateserial -out redis-server.crt 2>/dev/null &&
# Backwards-compat aliases (existing PG-only scenarios use these names)
cp pg-server.crt server.crt && cp pg-server.key server.key &&
cp pg-client.crt client.crt && cp pg-client.key client.key &&
chmod 600 pg-server.key pg-client.key redis-server.key server.key client.key
' || { log_fail "Certificate generation failed"; return 1; }
log_pass "Test certificates generated"
}
###############################################################################
# Start a TLS-enabled Redis container
###############################################################################
start_tls_redis() {
local name="$1" net="$2"
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
# Redis needs certs owned by redis user (uid 999 in the official image).
# Mount certs, copy to a writable location, fix ownership, then start
# with TLS flags.
docker run -d --name "$name" --network "$net" \
-v "${cert_mount}:/certs:ro" \
redis:latest \
sh -c '
cp /certs/redis-server.crt /certs/redis-server.key /certs/ca.crt /tmp/ &&
chmod 600 /tmp/redis-server.key &&
chown redis:redis /tmp/redis-server.crt /tmp/redis-server.key /tmp/ca.crt &&
exec redis-server \
--tls-port 6379 --port 0 \
--tls-cert-file /tmp/redis-server.crt \
--tls-key-file /tmp/redis-server.key \
--tls-ca-cert-file /tmp/ca.crt \
--tls-auth-clients no
' >/dev/null
# Wait for Redis TLS to be ready
local elapsed=0
while [ $elapsed -lt 20 ]; do
if docker exec "$name" redis-cli --tls \
--cert /certs/redis-server.crt --key /certs/redis-server.key --cacert /certs/ca.crt \
ping 2>/dev/null | grep -q "PONG"; then
break
fi
sleep 2; elapsed=$((elapsed + 2))
done
}
###############################################################################
# Start a TLS-enabled PostgreSQL container
###############################################################################
start_tls_postgres() {
local name="$1" net="$2" hba_auth="$3"
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
docker run -d --name "$name" --network "$net" \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=tempsetup \
-e POSTGRES_DB=dispatcharr \
-v "${cert_mount}:/certs:ro" \
postgres:17 >/dev/null
# Wait for PG to initialize
local elapsed=0
while [ $elapsed -lt 30 ]; do
if docker exec "$name" su postgres -c "/usr/lib/postgresql/17/bin/pg_isready" 2>/dev/null | grep -q "accepting"; then
break
fi
sleep 2; ((elapsed+=2))
done
# Configure SSL and pg_hba.conf
docker exec "$name" bash -c "
cp /certs/server.crt /certs/server.key /certs/ca.crt /var/lib/postgresql/
chown postgres:postgres /var/lib/postgresql/server.crt /var/lib/postgresql/server.key /var/lib/postgresql/ca.crt
chmod 600 /var/lib/postgresql/server.key
echo \"ssl = on\" >> /var/lib/postgresql/data/postgresql.conf
echo \"ssl_cert_file = '/var/lib/postgresql/server.crt'\" >> /var/lib/postgresql/data/postgresql.conf
echo \"ssl_key_file = '/var/lib/postgresql/server.key'\" >> /var/lib/postgresql/data/postgresql.conf
echo \"ssl_ca_file = '/var/lib/postgresql/ca.crt'\" >> /var/lib/postgresql/data/postgresql.conf
cat > /var/lib/postgresql/data/pg_hba.conf << HBA
local all all trust
hostssl all all 0.0.0.0/0 ${hba_auth}
hostssl all all ::0/0 ${hba_auth}
HBA
su postgres -c '/usr/lib/postgresql/17/bin/pg_ctl reload -D /var/lib/postgresql/data'
" >/dev/null 2>&1
sleep 1
}
###############################################################################
# Test scenarios
###############################################################################
test_modular_mtls_no_password() {
CURRENT_SCENARIO="modular_mtls_no_password"
section "Modular mode — mTLS cert-only auth (no password)"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
start_tls_postgres "$pg_name" "$net" "cert"
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
# No POSTGRES_PASSWORD — cert-only auth
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e POSTGRES_SSL=true \
-e POSTGRES_SSL_MODE=verify-ca \
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt \
-e POSTGRES_SSL_CERT=/certs/client.crt \
-e POSTGRES_SSL_KEY=/certs/client.key \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with mTLS cert-only auth"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed with mTLS"
check_log_contains "$name" "PostgreSQL TLS: enabled" \
"Django sees TLS enabled"
check_migrations_done "$name"
check_no_permission_errors "$name"
else
log_fail "Container failed to start with mTLS cert-only auth"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_mtls_with_password() {
CURRENT_SCENARIO="modular_mtls_with_password"
section "Modular mode — mTLS + password auth"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
# cert + md5 password
start_tls_postgres "$pg_name" "$net" "cert"
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=tempsetup \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e POSTGRES_SSL=true \
-e POSTGRES_SSL_MODE=verify-ca \
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt \
-e POSTGRES_SSL_CERT=/certs/client.crt \
-e POSTGRES_SSL_KEY=/certs/client.key \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with mTLS + password"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed with mTLS + password"
check_migrations_done "$name"
else
log_fail "Container failed to start with mTLS + password"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_tls_server_only() {
CURRENT_SCENARIO="modular_tls_server_only"
section "Modular mode — server-only TLS (no client cert)"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
# md5 auth over TLS (no client cert required)
start_tls_postgres "$pg_name" "$net" "md5"
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=tempsetup \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e POSTGRES_SSL=true \
-e POSTGRES_SSL_MODE=verify-ca \
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with server-only TLS"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed with server-only TLS"
check_migrations_done "$name"
else
log_fail "Container failed to start with server-only TLS"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_tls_key_permission() {
CURRENT_SCENARIO="modular_tls_key_permission"
section "Modular mode — mTLS with 0777 client key (Docker Desktop scenario)"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
start_tls_postgres "$pg_name" "$net" "cert"
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
# Create a copy of certs with 0777 key permissions
local bad_perms_dir
bad_perms_dir=$(mktemp -d)
cp "$CERT_DIR"/ca.crt "$CERT_DIR"/client.crt "$CERT_DIR"/client.key "$bad_perms_dir/"
chmod 777 "$bad_perms_dir/client.key"
local cert_mount
cert_mount="$(cygpath -w "$bad_perms_dir" 2>/dev/null || echo "$bad_perms_dir")"
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e POSTGRES_SSL=true \
-e POSTGRES_SSL_MODE=verify-ca \
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt \
-e POSTGRES_SSL_CERT=/certs/client.crt \
-e POSTGRES_SSL_KEY=/certs/client.key \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with 0777 client key"
check_log_contains "$name" "Fixed PostgreSQL client key" \
"Key permission fix triggered"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed after key fix"
check_migrations_done "$name"
else
log_fail "Container failed to start with 0777 client key"
fi
dump_logs_on_fail "$name"
rm -rf "$bad_perms_dir"
cleanup_scenario
}
test_modular_no_tls_regression() {
CURRENT_SCENARIO="modular_no_tls_regression"
section "Modular mode — no TLS (regression check)"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
# Plain PostgreSQL — no TLS
docker run -d --name "$pg_name" --network "$net" \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=secret \
-e POSTGRES_DB=dispatcharr \
postgres:17 >/dev/null
local elapsed=0
while [ $elapsed -lt 30 ]; do
if docker exec "$pg_name" su postgres -c "/usr/lib/postgresql/17/bin/pg_isready" 2>/dev/null | grep -q "accepting"; then
break
fi
sleep 2; ((elapsed+=2))
done
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=secret \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started without TLS (regression check)"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed without TLS"
check_log_absent "$name" "Fixed PostgreSQL client key" \
"No key fix when TLS disabled"
check_migrations_done "$name"
else
log_fail "Container failed to start without TLS"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_pg_verify_full() {
CURRENT_SCENARIO="modular_pg_verify_full"
section "Modular mode — PG mTLS with verify-full (CN must match hostname)"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
start_tls_postgres "$pg_name" "$net" "cert"
docker run -d --name "$redis_name" --network "$net" redis:latest >/dev/null
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
# verify-full requires server cert CN to match the hostname used to connect.
# Our PG server cert CN is "${TEST_PREFIX}_pg", which matches the container name
# used in POSTGRES_HOST.
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e POSTGRES_SSL=true \
-e POSTGRES_SSL_MODE=verify-full \
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt \
-e POSTGRES_SSL_CERT=/certs/client.crt \
-e POSTGRES_SSL_KEY=/certs/client.key \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with verify-full"
check_log_contains "$name" "PostgreSQL version check passed" \
"Version check passed with verify-full"
check_log_contains "$name" "sslmode=verify-full" \
"Django reports verify-full mode"
check_migrations_done "$name"
else
log_fail "Container failed to start with verify-full"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_redis_tls() {
CURRENT_SCENARIO="modular_redis_tls"
section "Modular mode — Redis with TLS"
local name="${TEST_PREFIX}_app"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"; track_container "$name"
# Plain PG (no TLS) — isolate Redis TLS testing
docker run -d --name "$pg_name" --network "$net" \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=secret \
-e POSTGRES_DB=dispatcharr \
postgres:17 >/dev/null
local elapsed=0
while [ $elapsed -lt 30 ]; do
if docker exec "$pg_name" su postgres -c "/usr/lib/postgresql/17/bin/pg_isready" 2>/dev/null | grep -q "accepting"; then
break
fi
sleep 2; elapsed=$((elapsed + 2))
done
start_tls_redis "$redis_name" "$net"
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
docker run -d --name "$name" --network "$net" \
-e DISPATCHARR_ENV=modular \
-e POSTGRES_HOST="$pg_name" \
-e POSTGRES_PORT=5432 \
-e POSTGRES_USER=dispatch \
-e POSTGRES_PASSWORD=secret \
-e POSTGRES_DB=dispatcharr \
-e REDIS_HOST="$redis_name" \
-e REDIS_SSL=true \
-e REDIS_SSL_VERIFY=false \
-e REDIS_SSL_CA_CERT=/certs/ca.crt \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if wait_for_ready "$name"; then
log_pass "Container started with Redis TLS"
check_log_contains "$name" "Redis TLS: enabled" \
"Django reports Redis TLS enabled"
check_log_contains "$name" "Redis at ${redis_name}" \
"Redis connected via TLS"
check_migrations_done "$name"
else
log_fail "Container failed to start with Redis TLS"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
test_modular_full_tls_celery() {
CURRENT_SCENARIO="modular_full_tls_celery"
section "Modular mode — PG mTLS + Redis TLS with Celery container"
local name="${TEST_PREFIX}_app"
local celery_name="${TEST_PREFIX}_celery"
local pg_name="${TEST_PREFIX}_pg"
local redis_name="${TEST_PREFIX}_redis"
local net="${TEST_PREFIX}_net"
local vol="${name}_data"
cleanup_scenario
docker network create "$net" >/dev/null 2>&1
fresh_volume "$vol"
track_network "$net"
track_container "$pg_name"; track_container "$redis_name"
track_container "$name"; track_container "$celery_name"
start_tls_postgres "$pg_name" "$net" "cert"
start_tls_redis "$redis_name" "$net"
local cert_mount
cert_mount="$(cygpath -w "$CERT_DIR" 2>/dev/null || echo "$CERT_DIR")"
# Shared env vars for both web and celery containers
local -a tls_env=(
-e DISPATCHARR_ENV=modular
-e POSTGRES_HOST="$pg_name"
-e POSTGRES_PORT=5432
-e POSTGRES_USER=dispatch
-e POSTGRES_DB=dispatcharr
-e REDIS_HOST="$redis_name"
-e POSTGRES_SSL=true
-e POSTGRES_SSL_MODE=verify-ca
-e POSTGRES_SSL_CA_CERT=/certs/ca.crt
-e POSTGRES_SSL_CERT=/certs/client.crt
-e POSTGRES_SSL_KEY=/certs/client.key
-e REDIS_SSL=true
-e REDIS_SSL_VERIFY=false
-e REDIS_SSL_CA_CERT=/certs/ca.crt
)
# Start web container first (generates JWT, runs migrations)
docker run -d --name "$name" --network "$net" \
"${tls_env[@]}" \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
"$IMAGE_NAME" >/dev/null
if ! wait_for_ready "$name"; then
log_fail "Web container failed to start with full TLS"
dump_logs_on_fail "$name"
cleanup_scenario
return
fi
log_pass "Web container started with PG mTLS + Redis TLS"
# Start Celery container (shares /data volume for JWT, waits for migrations)
docker run -d --name "$celery_name" --network "$net" \
"${tls_env[@]}" \
-e DJANGO_SETTINGS_MODULE=dispatcharr.settings \
-e PYTHONUNBUFFERED=1 \
-v "${cert_mount}:/certs:ro" \
-v "${vol}:/data" \
--entrypoint /app/docker/entrypoint.celery.sh \
"$IMAGE_NAME" >/dev/null
# Wait for Celery to start (look for "starting Celery" message)
local elapsed=0
local celery_ok=false
while [ $elapsed -lt 90 ]; do
if ! docker ps -q -f "name=^${celery_name}$" 2>/dev/null | grep -q .; then
echo " Celery container exited unexpectedly"
break
fi
if docker logs "$celery_name" 2>&1 | grep -q "starting Celery"; then
celery_ok=true
break
fi
sleep 3; elapsed=$((elapsed + 3))
done
if [ "$celery_ok" = true ]; then
log_pass "Celery container started with PG mTLS + Redis TLS"
check_log_contains "$celery_name" "Migrations complete" \
"Celery confirmed migrations complete via TLS"
check_log_contains "$celery_name" "PostgreSQL TLS: enabled" \
"Celery sees PostgreSQL TLS enabled"
check_log_contains "$celery_name" "Redis TLS: enabled" \
"Celery sees Redis TLS enabled"
else
log_fail "Celery container failed to start with full TLS"
echo -e " ${YELLOW}--- Celery logs ---${NC}"
docker logs "$celery_name" 2>&1 | tail -20 | sed 's/^/ /'
echo -e " ${YELLOW}--- End logs ---${NC}"
fi
dump_logs_on_fail "$name"
cleanup_scenario
}
###############################################################################
# Main
###############################################################################
echo -e "${BOLD}╔═══════════════════════════════════════════════════════════╗${NC}"
echo -e "${BOLD}║ Dispatcharr — TLS Integration Tests ║${NC}"
echo -e "${BOLD}╚═══════════════════════════════════════════════════════════╝${NC}"
# Build image
if [ "$SKIP_BUILD" = false ]; then
echo -e "\n${BOLD}Building test image...${NC}"
if ! docker build -t "$IMAGE_NAME" -f docker/Dockerfile . 2>&1 | tail -5; then
echo -e "${RED}Build failed${NC}"
exit 1
fi
echo -e "${GREEN}Build complete${NC}"
else
echo -e "\n${YELLOW}Skipping build (--skip-build)${NC}"
fi
# Generate certificates
generate_test_certs || exit 1
# Run scenarios
SCENARIOS=(
modular_mtls_no_password
modular_mtls_with_password
modular_tls_server_only
modular_tls_key_permission
modular_no_tls_regression
modular_pg_verify_full
modular_redis_tls
modular_full_tls_celery
)
for scenario in "${SCENARIOS[@]}"; do
if [ -n "$SINGLE_SCENARIO" ] && [ "$scenario" != "$SINGLE_SCENARIO" ]; then
continue
fi
"test_${scenario}"
done
# Clean up certs
rm -rf "$CERT_DIR"
# Summary
echo -e "\n${BOLD}═══════════════════════════════════════════════════════════${NC}"
echo -e " ${GREEN}Passed: $PASS${NC} ${RED}Failed: $FAIL${NC} ${YELLOW}Skipped: $SKIP${NC}"
if [ ${#ERRORS[@]} -gt 0 ]; then
echo -e "\n ${RED}Failures:${NC}"
for err in "${ERRORS[@]}"; do
echo -e " ${RED}• $err${NC}"
done
fi
echo -e "${BOLD}═══════════════════════════════════════════════════════════${NC}"
[ $FAIL -eq 0 ]

2
docker/uwsgi.debug.ini
View file

@ -28,8 +28,6 @@ static-map = /static=/app/static
# Worker configuration
workers = 1
threads = 8
enable-threads = true
lazy-apps = true
# HTTP server

8
docker/uwsgi.dev.ini
View file

@ -8,7 +8,7 @@
exec-pre = python /app/scripts/wait_for_redis.py
; Start Redis first
attach-daemon = redis-server
attach-daemon = redis-server --protected-mode no
; Then start other services with configurable nice level (default: 5 for low priority)
; Users can override via CELERY_NICE_LEVEL environment variable in docker-compose
attach-daemon = nice -n $(CELERY_NICE_LEVEL) celery -A dispatcharr worker --autoscale=6,1
@ -28,10 +28,8 @@ vacuum = true
die-on-term = true
static-map = /static=/app/static
# Worker management (Optimize for I/O bound tasks)
# Worker management
workers = 4
threads = 2
enable-threads = true
# Optimize for streaming
http = 0.0.0.0:5656
@ -59,4 +57,4 @@ logformat-strftime = true
log-date = %%Y-%%m-%%d %%H:%%M:%%S,000
# Use formatted time with environment variable for log level
log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms
log-buffering = 1024 # Add buffer size limit for logging
log-buffering = 1024 # Add buffer size limit for logging

62
frontend/package-lock.json generated
View file

@ -2518,9 +2518,9 @@
}
},
"node_modules/@xmldom/xmldom": {
"version": "0.8.11",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
"integrity": "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==",
"version": "0.8.12",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.12.tgz",
"integrity": "sha512-9k/gHF6n/pAi/9tqr3m3aqkuiNosYTurLLUtc7xQ9sxB/wm7WPygCv8GYa6mS0fLJEHhqMC1ATYhz++U/lRHqg==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
@ -2704,16 +2704,16 @@
}
},
"node_modules/brace-expansion": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz",
"integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==",
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz",
"integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^4.0.2"
},
"engines": {
"node": "20 || >=22"
"node": "18 || 20 || >=22"
}
},
"node_modules/cac": {
@ -2850,9 +2850,9 @@
}
},
"node_modules/cosmiconfig/node_modules/yaml": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz",
"integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==",
"version": "1.10.3",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz",
"integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==",
"license": "ISC",
"engines": {
"node": ">= 6"
@ -3554,9 +3554,9 @@
}
},
"node_modules/flatted": {
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz",
"integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==",
"version": "3.4.2",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
"integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
"dev": true,
"license": "ISC"
},
@ -3986,9 +3986,9 @@
}
},
"node_modules/lodash": {
"version": "4.17.23",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
"integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
"version": "4.18.1",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
"integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
"license": "MIT"
},
"node_modules/lodash.clamp": {
@ -4362,9 +4362,9 @@
"license": "ISC"
},
"node_modules/picomatch": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
"integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
"dev": true,
"license": "MIT",
"engines": {
@ -5490,9 +5490,9 @@
}
},
"node_modules/vite": {
"version": "7.3.1",
"resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz",
"integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==",
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz",
"integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -5794,6 +5794,24 @@
"dev": true,
"license": "MIT"
},
"node_modules/yaml": {
"version": "2.8.3",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
"integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
"dev": true,
"license": "ISC",
"optional": true,
"peer": true,
"bin": {
"yaml": "bin.mjs"
},
"engines": {
"node": ">= 14.6"
},
"funding": {
"url": "https://github.com/sponsors/eemeli"
}
},
"node_modules/yocto-queue": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

5
frontend/src/App.jsx
View file

@ -14,6 +14,7 @@ import Stats from './pages/Stats';
import DVR from './pages/DVR';
import Settings from './pages/Settings';
import PluginsPage from './pages/Plugins';
import PluginBrowsePage from './pages/PluginBrowse';
import ConnectPage from './pages/Connect';
import ConnectLogsPage from './pages/ConnectLogs';
import Users from './pages/Users';
@ -153,6 +154,10 @@ const App = () => {
<Route path="/guide" element={<Guide />} />
<Route path="/dvr" element={<DVR />} />
<Route path="/stats" element={<Stats />} />
<Route
path="/plugins/browse"
element={<PluginBrowsePage />}
/>
<Route path="/plugins" element={<PluginsPage />} />
<Route path="/connect" element={<ConnectPage />} />
<Route

182
frontend/src/api.js
View file

@ -1193,6 +1193,7 @@ export default class API {
if (values.file) {
body = new FormData();
for (const prop in values) {
if (values[prop] === null || values[prop] === undefined) continue;
body.append(prop, values[prop]);
}
} else {
@ -1286,6 +1287,7 @@ export default class API {
body = new FormData();
for (const prop in values) {
if (values[prop] === null || values[prop] === undefined) continue;
body.append(prop, values[prop]);
}
} else {
@ -1600,9 +1602,7 @@ export default class API {
});
const playlist = await API.getPlaylist(accountId);
usePlaylistsStore
.getState()
.updateProfiles(playlist.id, playlist.profiles);
usePlaylistsStore.getState().updatePlaylist(playlist);
} catch (e) {
errorNotification(`Failed to update profile for account ${accountId}`, e);
}
@ -1909,26 +1909,28 @@ export default class API {
}
}
static async importPlugin(file) {
static async importPlugin(file, overwrite = false, silent = false) {
try {
const form = new FormData();
form.append('file', file);
if (overwrite) form.append('overwrite', 'true');
const response = await request(`${host}/api/plugins/plugins/import/`, {
method: 'POST',
body: form,
});
return response;
} catch (e) {
// Show only the concise error message for plugin import
const msg =
(e?.body && (e.body.error || e.body.detail)) ||
e?.message ||
'Failed to import plugin';
notifications.show({
title: 'Import failed',
message: msg,
color: 'red',
});
if (!silent) {
const msg =
(e?.body && (e.body.error || e.body.detail)) ||
e?.message ||
'Failed to import plugin';
notifications.show({
title: 'Import failed',
message: msg,
color: 'red',
});
}
throw e;
}
}
@ -1993,6 +1995,130 @@ export default class API {
}
}
// Plugin Repos API
static async getPluginRepos() {
try {
return await request(`${host}/api/plugins/repos/`);
} catch (e) {
errorNotification('Failed to retrieve plugin repos', e);
return [];
}
}
static async addPluginRepo(data) {
try {
return await request(`${host}/api/plugins/repos/`, {
method: 'POST',
body: data,
});
} catch (e) {
errorNotification('Failed to add plugin repo', e);
throw e;
}
}
static async deletePluginRepo(id) {
try {
return await request(`${host}/api/plugins/repos/${id}/`, {
method: 'DELETE',
});
} catch (e) {
errorNotification('Failed to delete plugin repo', e);
throw e;
}
}
static async updatePluginRepo(id, data) {
try {
return await request(`${host}/api/plugins/repos/${id}/`, {
method: 'PUT',
body: data,
});
} catch (e) {
errorNotification('Failed to update plugin repo', e);
}
}
static async refreshPluginRepo(id) {
try {
return await request(`${host}/api/plugins/repos/${id}/refresh/`, {
method: 'POST',
});
} catch (e) {
errorNotification('Failed to refresh plugin repo', e);
}
}
static async getAvailablePlugins() {
try {
const response = await request(`${host}/api/plugins/repos/available/`);
return response.plugins || [];
} catch (e) {
errorNotification('Failed to retrieve available plugins', e);
return [];
}
}
static async getPluginDetailManifest(repoId, manifestUrl) {
try {
const response = await request(
`${host}/api/plugins/repos/plugin-detail/`,
{
method: 'POST',
body: { repo_id: repoId, manifest_url: manifestUrl },
}
);
return response;
} catch (e) {
errorNotification('Failed to retrieve plugin details', e);
return null;
}
}
static async getPluginRepoSettings() {
try {
return await request(`${host}/api/plugins/repos/settings/`);
} catch (e) {
errorNotification('Failed to retrieve repo settings', e);
return null;
}
}
static async updatePluginRepoSettings(data) {
try {
return await request(`${host}/api/plugins/repos/settings/`, {
method: 'PUT',
body: data,
});
} catch (e) {
errorNotification('Failed to update repo settings', e);
return null;
}
}
static async installPluginFromRepo(data) {
try {
return await request(`${host}/api/plugins/repos/install/`, {
method: 'POST',
body: data,
});
} catch (e) {
errorNotification('Failed to install plugin', e);
return null;
}
}
static async previewPluginRepo(url, publicKey) {
try {
return await request(`${host}/api/plugins/repos/preview/`, {
method: 'POST',
body: { url, public_key: publicKey || '' },
});
} catch {
return null;
}
}
static async checkSetting(values) {
const { id, ...payload } = values;
@ -2999,12 +3125,15 @@ export default class API {
}
}
static async updateUser(id, body) {
static async updateUser(id, body, self = false) {
try {
const response = await request(`${host}/api/accounts/users/${id}/`, {
method: 'PATCH',
body,
});
const response = await request(
`${host}/api/accounts/users/${self ? 'me' : id}/`,
{
method: 'PATCH',
body,
}
);
useUsersStore.getState().updateUser(response);
@ -3210,21 +3339,6 @@ export default class API {
}
}
static async updateVODPosition(vodUuid, clientId, position) {
try {
const response = await request(
`${host}/proxy/vod/stream/${vodUuid}/position/`,
{
method: 'POST',
body: { client_id: clientId, position },
}
);
return response;
} catch (e) {
errorNotification('Failed to update playback position', e);
}
}
static async getSystemEvents(limit = 100, offset = 0, eventType = null) {
try {
const params = new URLSearchParams();

Some files were not shown because too many files have changed in this diff Show more