From d824cd164653bc6fb3e79e9b9ee757cdec076ea3 Mon Sep 17 00:00:00 2001 From: Nick Sandstrom <32273437+nick4810@users.noreply.github.com> Date: Wed, 6 May 2026 23:54:47 -0700 Subject: [PATCH] Merge remote-tracking branch 'upstream/dev' into test/component-cleanup --- CHANGELOG.md | 74 + apps/accounts/api_views.py | 4 +- apps/accounts/permissions.py | 3 +- apps/accounts/serializers.py | 2 +- apps/channels/api_urls.py | 4 +- apps/channels/api_views.py | 937 +++++++- apps/channels/compact_numbering.py | 363 +++ apps/channels/managers.py | 55 + .../migrations/0037_auto_sync_overhaul.py | 156 ++ apps/channels/models.py | 149 +- apps/channels/serializers.py | 424 +++- apps/channels/signals.py | 133 +- apps/channels/tasks.py | 106 +- apps/channels/tests/test_channel_api.py | 297 ++- apps/channels/tests/test_series_rule_dedup.py | 118 + apps/channels/utils.py | 12 + apps/epg/api_views.py | 228 +- apps/epg/query_utils.py | 137 ++ apps/epg/serializers.py | 71 +- apps/epg/tests/test_epg_search_api.py | 339 +++ apps/hdhr/api_views.py | 32 +- apps/hdhr/views.py | 30 +- apps/m3u/api_views.py | 261 ++- apps/m3u/models.py | 1 - apps/m3u/serializers.py | 35 +- apps/m3u/tasks.py | 1132 ++++++--- apps/m3u/tests/test_account_destroy.py | 143 ++ apps/m3u/tests/test_sync_compound.py | 240 ++ apps/m3u/tests/test_sync_correctness.py | 2051 +++++++++++++++++ apps/output/views.py | 355 +-- apps/proxy/ts_proxy/views.py | 3 + apps/proxy/vod_proxy/views.py | 6 + .../management/commands/reset_user_network.py | 22 + dispatcharr/utils.py | 19 +- frontend/src/api.js | 112 +- .../src/components/M3URefreshNotification.jsx | 156 +- .../__tests__/M3URefreshNotification.test.jsx | 137 ++ frontend/src/components/forms/Channel.jsx | 281 ++- .../src/components/forms/ChannelBatch.jsx | 113 +- .../components/forms/GroupConfigureModal.jsx | 50 + .../forms/ProgramRecordingModal.jsx | 39 +- .../components/forms/RecurringRuleModal.jsx | 35 +- .../components/forms/SeriesRecordingModal.jsx | 177 +- .../forms/SeriesRuleEditorModal.jsx | 423 ++++ frontend/src/components/forms/User.jsx | 48 +- .../forms/__tests__/Channel.test.jsx | 135 +- .../forms/__tests__/ChannelBatch.test.jsx | 121 +- .../__tests__/GroupConfigureModal.test.jsx | 95 + .../forms/__tests__/LiveGroupFilter.test.jsx | 2 +- .../forms/settings/NetworkAccessForm.jsx | 69 +- .../__tests__/NetworkAccessForm.test.jsx | 34 +- .../src/components/tables/ChannelsTable.jsx | 117 +- .../ChannelsTable/ChannelTableHeader.jsx | 56 +- .../tables/ChannelsTable/EditableCell.jsx | 117 +- .../__tests__/EditableCell.test.jsx | 82 + frontend/src/components/tables/M3UsTable.jsx | 75 +- frontend/src/pages/DVR.jsx | 6 +- frontend/src/pages/Guide.jsx | 34 +- .../src/pages/__tests__/guideUtils.test.js | 16 +- .../src/utils/cards/RecordingCardUtils.js | 28 +- .../__tests__/RecordingCardUtils.test.js | 22 +- frontend/src/utils/forms/ChannelBatchUtils.js | 50 + frontend/src/utils/forms/ChannelUtils.js | 215 +- frontend/src/utils/forms/GroupSyncUtils.js | 70 + .../utils/forms/RecurringRuleModalUtils.js | 5 +- ...nnelBatchUtils.bulkOverrideRouting.test.js | 105 + .../forms/__tests__/ChannelUtils.test.js | 340 +++ .../forms/__tests__/GroupSyncUtils.test.js | 178 ++ .../forms/settings/NetworkAccessFormUtils.js | 56 +- .../__tests__/NetworkAccessFormUtils.test.js | 45 +- frontend/src/utils/guideUtils.js | 4 +- version.py | 2 +- 72 files changed, 10546 insertions(+), 1046 deletions(-) create mode 100644 apps/channels/compact_numbering.py create mode 100644 apps/channels/managers.py create mode 100644 apps/channels/migrations/0037_auto_sync_overhaul.py create mode 100644 apps/epg/query_utils.py create mode 100644 apps/epg/tests/test_epg_search_api.py create mode 100644 apps/m3u/tests/test_account_destroy.py create mode 100644 apps/m3u/tests/test_sync_compound.py create mode 100644 apps/m3u/tests/test_sync_correctness.py create mode 100644 core/management/commands/reset_user_network.py create mode 100644 frontend/src/components/forms/GroupConfigureModal.jsx create mode 100644 frontend/src/components/forms/SeriesRuleEditorModal.jsx create mode 100644 frontend/src/components/forms/__tests__/GroupConfigureModal.test.jsx create mode 100644 frontend/src/components/tables/ChannelsTable/__tests__/EditableCell.test.jsx create mode 100644 frontend/src/utils/forms/GroupSyncUtils.js create mode 100644 frontend/src/utils/forms/__tests__/ChannelBatchUtils.bulkOverrideRouting.test.js create mode 100644 frontend/src/utils/forms/__tests__/GroupSyncUtils.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 66bb7297..3dc31e9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,80 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **DVR series rules: EPG channel is now optional.** Series recording rules no longer require a `tvg_id`. Rules with only a title or description filter search across all EPG channels in the 7-day horizon. The evaluator pre-loads one channel per EPG source (lowest channel number) in a single query and resolves the recording channel per-program, so cross-channel searches remain efficient. Saves and the preview endpoint now require at least one of `title`, or `description` instead of requiring `tvg_id`. The preview response includes a `warn: true` flag when more than 50 programs match, and the editor shows an orange alert when this flag is set. The upsert key for rules changed from `tvg_id` alone to `(tvg_id, title)` so multiple rules can target the same channel. +- **DVR series rules: rich title and description matching.** Series recording rules now accept the same boolean / quoted-phrase / regex / whole-word semantics as the EPG Program Search API on both the title and description fields, plus an optional pinned channel. Existing rules continue to work unchanged (default `title_mode=exact`, no description filter, no pinned channel). + - New rule fields: `title_mode` (`exact` / `contains` / `search` / `regex`), `description`, `description_mode` (`contains` / `search` / `regex`), and `channel_id` (optional integer to pin recordings to a specific channel; defaults to the lowest-numbered channel for the EPG as before). + - The boolean/quoted/regex/whole-word parser was extracted from `apps/epg/api_views.py` into a shared `apps/epg/query_utils.py` so the rule evaluator and the search endpoint use the same code path. No behavioral changes to the existing `/api/epg/programs/search/` endpoint. + - The evaluator builds a single `ProgramData` queryset with `.distinct()` and reuses the existing `(tvg_id, start_time, end_time)` dedup key, so dropping in a description filter or a fuzzy title still preserves the dedup, episode collapsing, and offset-adjustment behavior. No N+1 introduced. +- **`POST /api/channels/series-rules/preview/`** returns up to 25 (configurable, max 100) upcoming programs that a candidate rule would match within the standard 7-day evaluation horizon, without persisting anything. Used by the new rule editor to give live feedback as the user types. +- **`GET /api/epg/programs/search/?tvg_id=`** filter parameter for exact `epg.tvg_id` matches. +- **Series rule editor modal** with form (title + match mode, description + match mode, episodes mode, pinned channel) and a debounced (500ms) preview pane backed by an `AbortController` so per-keystroke calls don't pile up. A "Customize rule..." link in the program record-choice modal opens the editor pre-filled with the program's `tvg_id` and title; the series rules modal gains an "Add rule" button and an "Edit" button per existing rule. + +### Changed + +- **Series rules modal redesign.** Rules now display a one-line summary with the title-match mode, description filter (when present), and a "Pinned channel" badge when a `channel_id` is configured. The previous list rendering remains for legacy rules. +- **EPG Program Search API** (`GET /api/epg/programs/search/`): a new endpoint for querying EPG program data with rich filtering and query support. - Thanks [@northernpowerhouse](https://github.com/northernpowerhouse) + - **Text search** on title and description with AND/OR boolean operators (case-insensitive), quoted phrase matching (`"Law and Order"` treats _and_ as literal text), parenthetical grouping (`(Newcastle OR NEW) AND (Villa OR AST)`), whole-word mode (`title_whole_words=true`), and regex mode (`title_regex=true`). + - **Time filters**: `airing_at` (programs live at a specific instant), `start_after`, `start_before`, `end_after`, `end_before`. + - **Relational filters**: `channel`, `channel_id`, `stream`, `group`, `epg_source`. + - **Field selection**: `fields=title,start_time,channels` returns only the requested keys; channel and stream data is skipped server-side (no wasted serialization) when not requested. + - **Pagination**: default 50 results per page, configurable up to 500 via `page_size`. + - **Access control**: results are scoped to channels the requesting user can access. `user_level` and the per-user adult-content filter are both enforced, matching the access model used by the M3U playlist, XC API, and stream proxy. Admin users receive unfiltered results. + - Full OpenAPI/Swagger documentation available. + - Requires `IsStandardUser` permission (user level ≥ 1). +- **Per-user IP/CIDR network allowlists**. Admins can now assign IP address and CIDR range restrictions to individual user accounts via the API & XC tab on the user edit form. When a user has one or more allowed ranges configured, requests from IPs outside that list are rejected with `403 Forbidden` regardless of the global network access policy; if no ranges are configured, the user inherits global settings unchanged. The existing `network_access_allowed()` utility is extended with an optional `user` argument so the per-user check is enforced at all access-controlled entry points (M3U/EPG, Streams, XC API, UI) without duplicating IP-matching logic. Per-user restrictions are stored in `custom_properties['allowed_networks']`; no model changes or migrations are required. — Thanks [@sethwv](https://github.com/sethwv) +- **`reset_user_network` management command**: `manage.py reset_user_network ` clears the per-user `allowed_networks` restriction for the specified account, restoring it to global-policy inheritance. Useful for recovering a user locked out by a misconfigured allowlist. — Thanks [@sethwv](https://github.com/sethwv) +- **Auto-sync overhaul**: comprehensive rebuild of the M3U auto-channel-sync flow. Introduces a per-field override system, hide-from-output flag, range-bounded auto-numbering with a re-pack helper, multi-stream channel safety, multi-provider shared-range merging, and an across-the-board move from per-row writes to bulk operations. (Closes #1196) — Thanks [@CodeBormen](https://github.com/CodeBormen) + - **Per-field channel overrides for auto-synced channels.** A new `ChannelOverride` table (one-to-one with `Channel`) holds user-specified values for `name`, `channel_number`, `channel_group`, `logo`, `tvg_id`, `tvc_guide_stationid`, `epg_data`, and `stream_profile`. Sync writes only to `Channel.*` columns; the override row takes precedence at read time via a new `with_effective_values()` queryset helper that coalesces both sources at the SQL layer. Every output surface that consumes channel data reads through this helper so overrides surface consistently: HDHR (`/hdhr/lineup.json` and per-profile variants, `/hdhr/discover.json`), M3U (`/output/m3u`, per-profile variants), EPG (`/output/epg`, per-profile variants), XC API (`get_live_streams`, `xmltv.php`), the channels list/detail endpoints, and the TV Guide summary endpoint (`GET /api/channels/channels/summary/`). Channel API responses now include both the raw `name`/`channel_number`/etc. fields AND new `effective_*` annotations plus the `override` object, so frontend consumers can show both "what the user picked" and "what the provider sent" simultaneously. + - **Per-field reset-to-provider icon.** FK pickers (channel group, logo, EPG, stream profile) and scalar inputs (name, channel_number, tvg_id, tvc_guide_stationid) display a small reset icon ("Provider: " subtext + Undo2 button) when the field has an active override on an auto-synced channel. Clicking it sets the form value back to the provider value; the override for that field is cleared on save while other overrides remain intact. + - **Auto-created channel source attribution.** The channel edit form shows an "Auto-created from: / " label at the top of auto-synced channels so the user can see which M3U account and which stream produced the row. + - **Hide-from-output flag (`hidden_from_output`).** A user-toggleable boolean that excludes a channel from HDHR, M3U, EPG, and XC output without deleting it. Hidden channels are preserved across auto-sync refreshes and excluded from auto-cleanup. The channels table shows an EyeOff icon on hidden rows. + - **Channels table visibility filter and inline indicators.** The table header has a new visibility selector ("Active Only" / "Hidden Only" / "Show All") plus a "Has Overrides" filter that narrows to channels with an active override row. Each row's name cell renders a yellow Pencil icon when overrides are active (tooltip lists the overridden field names) and an EyeOff icon when the channel is hidden. + - **Bulk hide / unhide in the bulk edit modal.** Selecting any number of channels and toggling `hidden_from_output` in bulk edit applies the change in a single PATCH; auto-routes through the override-aware bulk endpoint so it works equivalently for auto-synced and manual channels. + - **Auto-sync channel ranges per group.** `ChannelGroupM3UAccount` accepts `auto_sync_channel_start` (lower bound, inclusive) and `auto_sync_channel_end` (upper bound, inclusive; nullable for unbounded fill). Channels created by auto-sync are assigned numbers within the configured range; streams that do not fit are surfaced in the failure detail modal with a typed "RANGE_EXHAUSTED" reason. + - **Per-group inline configuration in the M3U Live group filter.** Each group row now exposes a Sync toggle, a Numbering Mode selector (Provider / Next Available / Fixed), and Start/End channel-number inputs alongside the existing fields. The numbering modes control how each stream's channel number is chosen during sync: Provider tries the M3U-supplied number first then falls back to next-available; Next Available always picks the lowest free number from 1 upward; Fixed picks sequentially from the configured Start. + - **Per-group "Configure" modal.** A gear-icon button on each group row opens a `GroupConfigureModal` containing the full set of advanced options. Fields surfaced in the modal: Channel Sort Order (provider order / name / tvg_id / updated_at, with reverse toggle), Compact Numbering toggle and "Re-pack now" button, Group Override (re-route this group's auto-created channels into a different `ChannelGroup` so multiple provider groups merge into one logical group), Name Regex Find/Replace pattern, Exclude Regex pattern, Force Dummy EPG, and a live regex preview pane. + - **Live regex preview pane.** A new `GET /api/channels/streams/regex-preview/` endpoint scans the group's streams (capped at 5000) for the find / match / exclude patterns the user is editing and returns up to 10 sample matches per pattern plus accurate total counts. The frontend debounces the call (500ms) and runs it inside the configure modal so the user sees live evidence of what their patterns will match before saving. + - **Live overlap warning across rows.** As the user edits Start/End values across multiple groups in the same provider, an `AbortController`-debounced scan calls a new `GET /api/channels/channels/numbers-in-range/` endpoint to detect (a) cross-row range overlaps (in-memory, all group pairs) and (b) channels already occupying the configured range that are not the expected occupants for that group. An informational warning surfaces inline; configurations are not blocked. + - **Compact numbering with re-pack helper.** When `compact_numbering=True` on a group's account relation, sync packs visible auto-sync channels sequentially into the configured range. A "Re-pack now" button in the group config modal runs the pack manually via a new `POST /api/m3u/accounts/{id}/repack-group/` endpoint. Hidden channels do not occupy slots; un-hiding shifts visible numbers down to fill the gap. Override-pinned channel numbers are respected as fixed anchors. Single-channel hide / unhide via the post-save signal triggers an incremental `assign_compact_numbers_for_channels` pass without requiring a full sync. + - **Multi-stream channel safety in sync.** A user-attached second stream on an auto-created channel no longer causes the channel to be deleted when one of those streams disappears from the provider. The deletion path filters out channels where at least one current stream remains alive, and per-stream iteration mutates the same `Channel` instance so `bulk_update` writes the merged final state. + - **Orphan channel cleanup mode (3-state, account-scoped).** Stored under `M3UAccount.custom_properties.orphan_channel_cleanup` and surfaced as a SegmentedControl at the top of the M3U account's group-settings page. Three positions: `always` (default; removes every auto-channel whose source stream has disappeared), `preserve_customized` (removes orphans without a `ChannelOverride` row, preserves those with one), and `never` (preserves all orphans, intended for users with manual redundancy/failover stream setups). Hidden channels are universally preserved regardless of mode. + - **Failure modal grouped by reason.** The auto-sync completion notification's failure detail modal groups entries by typed reason (`RANGE_EXHAUSTED`, `INTEGRITY_ERROR`, `OTHER`) with collapsible sections and a per-group cap. The in-memory cap was raised from 50 to 1000 so realistic multi-provider failure sets are not truncated. The notification's auto-close timeout extends from 4s to 12s when failures are present so users have time to see and click into the modal. + - **Multi-provider shared-range merging.** Two M3U accounts can target the same group with overlapping channel-number ranges. Sync seeds `used_numbers` globally so the second provider's channels pick up where the first left off without colliding. The overlap warning in the group-settings UI surfaces this configuration as informational; the merge is intentional, not a hard error. + - **Selection summary in the bulk edit modal.** The bulk edit form shows `Selection: X auto-synced, Y manual` at the top so users can see how a single set of changes will route between override rows and direct writes. + - **Delete-Playlist preview.** A new `GET /api/m3u/accounts/{id}/auto-created-channels-count/` endpoint returns the count and up to 5 sample names of auto-created channels owned by the account. The Delete Playlist confirmation dialog uses this to show the user exactly how many channels will cascade away before they confirm. + - **Custom `Channel.objects` manager.** A thin manager wraps the existing `with_effective_values()` helper so new code can call `Channel.objects.with_effective_values(...)` directly. The module-level helper remains the canonical implementation; existing call sites are unchanged. + +### Changed + +- **Global Network Access settings use tag-style inputs**: the IP/CIDR range fields in the Network Access settings panel now use tag-style chip inputs instead of a plain text field, making it easier to add, review, and remove individual addresses or ranges. — Thanks [@sethwv](https://github.com/sethwv) +- **Auto-sync overhaul** sync and channel behavior changes for the new override system. — Thanks [@CodeBormen](https://github.com/CodeBormen) + - **`Channel.channel_number` is now nullable.** Auto-sync may produce channels without an assigned number (for example, a sync run where the configured range was exhausted before this stream could be slotted, or a hidden channel in compact mode whose slot was released for visible channels). Existing channels are unchanged. HDHR / M3U / EPG / XC output endpoints filter out channels with `null` channel_number, so a number-less row is invisible to clients but visible in the channels page where the user can assign one. The 0037 auto-sync overhaul migration includes a reverse-direction backfill on the `channel_number` AlterField step so a rollback that re-imposes NOT NULL still succeeds even if NULLs are present. + - **M3U account delete always cascades auto-created channels.** The delete dialog is a single confirmation showing the count of affected channels (fetched from the new auto-created-channels-count endpoint). Auto-created channels are removed regardless of `hidden_from_output` or override state: override rows cascade via the one-to-one FK; hidden status does not preserve the channel because there is no provider left to populate it on the next sync. Manual channels are unaffected; their non-provider streams remain intact, and only streams owned by the deleted account are removed. The destroy response now returns `{"deleted_channels": N}` (HTTP 200) instead of an empty 204 so the confirmation toast can show the actual count. + - **Bulk channel edit auto-routes auto-created channels through the override path.** When a bulk edit includes auto-synced channels, the changed fields are written to each channel's `ChannelOverride` row instead of the raw `Channel.*` columns, so the edit survives the next sync. Manual channels in the same selection write directly to `Channel.*`. A "Clear all overrides" affordance pre-clears overrides on the selection before applying new edits in the same submit (clear runs before the routing PATCH so a same-submit clear cannot wipe just-written overrides). + - **Inline edits on the channels table route through the override row for auto-synced channels.** Editing a cell directly in the table (number, name, group, EPG, logo) now writes to `ChannelOverride.` for auto-synced rows so the edit survives the next refresh; manual rows continue to write directly. The save mechanic and validation are unchanged from the user's perspective. + - **Channel-number duplicates are explicitly allowed.** Sync's `used_numbers` seed still avoids assigning the same number to two newly-created auto channels in the same run, so accidental duplication during sync remains impossible. Manual or override-driven duplication is permitted; downstream client behavior on duplicates varies by client. + +### Fixed + +- **Bulk channel edit silently failed on API rejection.** The bulk-edit submit handler in `ChannelBatch.jsx` only wrote `console.error` when the PATCH was rejected; the user saw the spinner stop but received no notification, leading them to assume the save succeeded. The catch block now surfaces a red toast with the server-provided detail (or a generic fallback), and the form stays open with the in-progress selection intact so the user can correct and retry. — Thanks [@CodeBormen](https://github.com/CodeBormen) +- **Recurring rule edit modal showed a blank Channel field.** The `RecurringRuleModal` read channel data from a Zustand store slice (`channels`) that nothing populates - the channels page and all other consumers switched to on-demand summary fetches in a prior release. Opening the edit modal from the DVR page always produced an empty Channel select. The modal now fetches channels via `API.getChannelsSummary()` when it opens, matching the lightweight approach used by the one-time recording form and other modals. +- **DVR section count badges appeared at the far right of the screen.** The "Currently Recording", "Upcoming Recordings", and "Previously Recorded" section headers wrapped the title and badge in a `Group justify="space-between"`, which pushed the badge to the opposite edge of the full-width container. Changed to `Group gap="xs" align="center"` so each badge sits inline with its heading. + +### Performance + +- **Auto-sync at scale**: the new override-aware sync flow is more capable than the prior path but the implementation choices below keep it viable on libraries with thousands of channels. — Thanks [@CodeBormen](https://github.com/CodeBormen) + - **Bulk writes throughout `sync_auto_channels`.** Per-row `Channel.objects.create()` and `.save()` calls were replaced with `bulk_create()` and `bulk_update()` paths that batch the entire group's create + update sets into single round-trips. The renumber pass collects all dirty channels into one list and flushes with a single `bulk_update` at the end of the loop. `ChannelStream.order` writes were similarly consolidated into a single `bulk_update`. + - **Single-pass collision detection via a global `used_numbers` set.** Choosing a free channel number was previously an `O(N)` DB query per stream. The new path seeds a `set()` of all reserved numbers (existing channels + override pins) once per run; `_next_available_number()` is `O(cluster size)` against that set. The seed deliberately excludes only this account's visible auto-created channels (the rows about to be reassigned), so cross-account and hidden-channel reservations are honored without extra queries. + - **SQL-level effective-value coalescing via `with_effective_values()`.** All channel-data consumers (HDHR, M3U, EPG, XC, channel list / detail, TV Guide summary) read effective values directly from a single annotated queryset that left-joins the override row at the database layer. Prevents N+1 `ChannelOverride` lookups across the channel scan and lets the same query serve the join. + - **EPG dispatch deduplication.** `bulk_create` and `bulk_update` bypass `post_save`, so the post-loop step explicitly dispatches `parse_programs_for_tvg_id.delay` once per unique `epg_data_id` actually changed in the run, not per channel. Avoids fan-out where one EPG source change otherwise queued thousands of redundant parse tasks. + - **Logo and EPGData lookup caching during sync.** `sync_auto_channels` now caches Logo and EPGData lookups in a per-run dict so repeated provider URLs / IDs across many streams do not produce duplicate `Logo.objects.get_or_create` or `EPGData.objects.filter` calls. + - **Override-aware bulk PATCH endpoint.** `update_channels_with_override_routing` partitions the bulk-edit selection into auto-created (override write) vs manual (direct write) once, then issues two bulk PATCHes instead of N single-row updates. + +## [0.24.0] - 2026-05-03 + ### Security - **HDHomeRun discovery endpoints now respect the `M3U_EPG` network access policy**. `DiscoverAPIView`, `LineupAPIView`, `LineupStatusAPIView`, and `HDHRDeviceXMLAPIView` were marked `AllowAny` so HDHR clients (Plex, Emby, Jellyfin, Channels DVR, etc.) can discover the tuner without authenticating, but they were not gated by any network allowlist. The lineup enumerates every channel name and per-channel UUID stream URL, so any client that could reach the server could full-enumerate the lineup. All four views now call `network_access_allowed(request, "M3U_EPG")` and return `403 Forbidden` for clients outside the allowlist, matching the gating already applied to the M3U and EPG endpoints (and matching what the Network Access settings UI already advertised: "Limit access to M3U, EPG, and HDHR URLs"). Operators with a restrictive `M3U_EPG` policy will see HDHR discovery start being blocked for off-LAN clients on upgrade; loosen the policy if remote HDHR access is required. diff --git a/apps/accounts/api_views.py b/apps/accounts/api_views.py index 2ac410f2..b4cdc8bc 100644 --- a/apps/accounts/api_views.py +++ b/apps/accounts/api_views.py @@ -252,8 +252,8 @@ class UserViewSet(viewsets.ModelViewSet): request.data.pop(key, None) # Strip admin-managed keys from custom_properties so users cannot - # set their own XC credentials via this endpoint. - ADMIN_ONLY_PROPS = {"xc_password"} + # set their own XC credentials or network rules via this endpoint. + ADMIN_ONLY_PROPS = {"xc_password", "allowed_networks"} cp = request.data.get("custom_properties") if isinstance(cp, dict): for key in ADMIN_ONLY_PROPS: diff --git a/apps/accounts/permissions.py b/apps/accounts/permissions.py index 62673038..f564f68f 100644 --- a/apps/accounts/permissions.py +++ b/apps/accounts/permissions.py @@ -6,7 +6,8 @@ from dispatcharr.utils import network_access_allowed class Authenticated(IsAuthenticated): def has_permission(self, request, view): is_authenticated = super().has_permission(request, view) - network_allowed = network_access_allowed(request, "UI") + user = request.user if hasattr(request, 'user') and request.user.is_authenticated else None + network_allowed = network_access_allowed(request, "UI", user) return is_authenticated and network_allowed diff --git a/apps/accounts/serializers.py b/apps/accounts/serializers.py index 8a9f609e..a5ce9075 100644 --- a/apps/accounts/serializers.py +++ b/apps/accounts/serializers.py @@ -11,7 +11,7 @@ VALID_NAV_ITEM_IDS = { 'channels', 'vods', 'sources', 'guide', 'dvr', 'stats', 'plugins', 'integrations', 'system', 'settings' } -MAX_CUSTOM_PROPS_SIZE = 10240 # 10KB limit +MAX_CUSTOM_PROPS_SIZE = 102400 # 100KB limit def validate_nav_array(value, field_name): diff --git a/apps/channels/api_urls.py b/apps/channels/api_urls.py index 756e2341..ff00e0f7 100644 --- a/apps/channels/api_urls.py +++ b/apps/channels/api_urls.py @@ -17,7 +17,7 @@ from .api_views import ( GetChannelStreamsAPIView, GetChannelStreamStatsAPIView, SeriesRulesAPIView, - DeleteSeriesRuleAPIView, + SeriesRulePreviewAPIView, EvaluateSeriesRulesAPIView, BulkRemoveSeriesRecordingsAPIView, BulkDeleteUpcomingRecordingsAPIView, @@ -47,9 +47,9 @@ urlpatterns = [ path('profiles//channels/bulk-update/', BulkUpdateChannelMembershipAPIView.as_view(), name='bulk_update_channel_membership'), # DVR series rules (order matters: specific routes before catch-all slug) path('series-rules/', SeriesRulesAPIView.as_view(), name='series_rules'), + path('series-rules/preview/', SeriesRulePreviewAPIView.as_view(), name='series_rules_preview'), path('series-rules/evaluate/', EvaluateSeriesRulesAPIView.as_view(), name='evaluate_series_rules'), path('series-rules/bulk-remove/', BulkRemoveSeriesRecordingsAPIView.as_view(), name='bulk_remove_series_recordings'), - path('series-rules//', DeleteSeriesRuleAPIView.as_view(), name='delete_series_rule'), path('recordings/bulk-delete-upcoming/', BulkDeleteUpcomingRecordingsAPIView.as_view(), name='bulk_delete_upcoming_recordings'), path( 'recordings//hls/', diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 5ef575a0..882e90b9 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -8,13 +8,12 @@ from drf_spectacular.utils import extend_schema, OpenApiParameter, inline_serial from drf_spectacular.types import OpenApiTypes from rest_framework import serializers from django.shortcuts import get_object_or_404, get_list_or_404 -from django.db import transaction -from django.db.models import Count, F +from django.db import connection, transaction +from django.db.models import Count, F, Prefetch from django.db.models import Q import os, json, requests, logging, mimetypes, threading, time from datetime import timedelta from django.utils.http import http_date -from urllib.parse import unquote from apps.accounts.permissions import ( Authenticated, IsAdmin, @@ -200,6 +199,249 @@ class StreamViewSet(viewsets.ModelViewSet): # Return the response with the list of IDs return Response(list(stream_ids)) + @extend_schema( + parameters=[ + OpenApiParameter( + name="channel_group", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + required=True, + description="Channel group name to scope the preview to.", + ), + OpenApiParameter( + name="find", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Find regex for the rename preview. When supplied, " + "the response includes find_matches and find_match_count." + ), + ), + OpenApiParameter( + name="replace", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Replacement string used with the find pattern. " + "Defaults to empty string when omitted." + ), + ), + OpenApiParameter( + name="match", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Filter regex for the include preview. When supplied, " + "the response includes filter_matches and filter_match_count." + ), + ), + OpenApiParameter( + name="exclude", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Filter regex for the exclude preview. When supplied, " + "the response includes exclude_matches and " + "exclude_match_count." + ), + ), + OpenApiParameter( + name="limit", + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY, + required=False, + description="Max preview entries per match list (default 10, capped at 50).", + ), + ], + responses={ + 200: inline_serializer( + name="StreamRegexPreviewResponse", + fields={ + "total_in_group": serializers.IntegerField(), + "total_scanned": serializers.IntegerField(), + "scan_limit_hit": serializers.BooleanField(), + "find_matches": serializers.ListField( + child=serializers.DictField(), required=False + ), + "find_match_count": serializers.IntegerField(required=False), + "filter_matches": serializers.ListField( + child=serializers.DictField(), required=False + ), + "filter_match_count": serializers.IntegerField(required=False), + "exclude_matches": serializers.ListField( + child=serializers.DictField(), required=False + ), + "exclude_match_count": serializers.IntegerField(required=False), + "find_error": serializers.CharField(required=False), + "match_error": serializers.CharField(required=False), + "exclude_error": serializers.CharField(required=False), + }, + ) + }, + description=( + "Returns regex preview info for a group's streams. Used by the " + "auto-sync gear modal so users can see how their find/replace " + "or filter pattern affects real stream names before saving. " + "Caps in-memory iteration at SCAN_CAP streams per call so the " + "endpoint stays bounded even on groups with tens of thousands " + "of streams; the caller surfaces total_in_group and " + "scan_limit_hit so users know whether the preview is complete." + ), + ) + @action(detail=False, methods=["get"], url_path="regex-preview") + def regex_preview(self, request, *args, **kwargs): + # `regex` (third-party) supports a per-call timeout that bounds + # catastrophic backtracking; paired with PATTERN_MAX_LEN to keep + # the endpoint safe under adversarial input. + import regex as re + + SCAN_CAP = 5000 + PATTERN_MAX_LEN = 512 + REGEX_TIMEOUT = 0.1 + + group_name = request.query_params.get("channel_group") + if not group_name: + return Response( + {"detail": "channel_group is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Group names are not unique across M3U accounts (two providers + # can both publish a "Sports" group). Scope to the calling + # account so the sample reflects only the user's edits. + m3u_account_id = request.query_params.get("m3u_account_id") + if m3u_account_id is not None: + try: + m3u_account_id = int(m3u_account_id) + except (TypeError, ValueError): + return Response( + {"detail": "m3u_account_id must be an integer"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + find_pat = request.query_params.get("find") or None + replace_pat = request.query_params.get("replace") or "" + match_pat = request.query_params.get("match") or None + exclude_pat = request.query_params.get("exclude") or None + for label, value in ( + ("find", find_pat), + ("replace", replace_pat), + ("match", match_pat), + ("exclude", exclude_pat), + ): + if value is not None and len(value) > PATTERN_MAX_LEN: + return Response( + {"detail": f"{label} exceeds {PATTERN_MAX_LEN} characters"}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + limit = int(request.query_params.get("limit", 10)) + except (TypeError, ValueError): + limit = 10 + limit = max(1, min(limit, 50)) + + find_re = None + match_re = None + exclude_re = None + find_error = None + match_error = None + exclude_error = None + if find_pat: + try: + find_re = re.compile(find_pat) + except re.error as e: + find_error = str(e) + if match_pat: + try: + match_re = re.compile(match_pat) + except re.error as e: + match_error = str(e) + if exclude_pat: + try: + exclude_re = re.compile(exclude_pat) + except re.error as e: + exclude_error = str(e) + + # Capped at SCAN_CAP to bound memory on huge groups; the + # separate COUNT lets the client surface scan_limit_hit when + # the preview covers only a sample. + base_qs = Stream.objects.filter(channel_group__name=group_name) + if m3u_account_id is not None: + base_qs = base_qs.filter(m3u_account_id=m3u_account_id) + names_iter = base_qs.values_list("name", flat=True)[:SCAN_CAP] + total_in_group = base_qs.count() + + find_matches = [] + filter_matches = [] + exclude_matches = [] + find_match_count = 0 + filter_match_count = 0 + exclude_match_count = 0 + total_scanned = 0 + # Abort a pattern on timeout to bound CPU; partial counts and + # an `*_error` field still flow back to the client. + for name in names_iter: + total_scanned += 1 + if find_re is not None: + try: + new_name = find_re.sub(replace_pat, name, timeout=REGEX_TIMEOUT) + except (TimeoutError, re.error) as e: + find_error = find_error or f"Pattern timed out: {e}" + find_re = None + continue + if new_name != name: + find_match_count += 1 + if len(find_matches) < limit: + find_matches.append({"before": name, "after": new_name}) + if match_re is not None: + try: + matched = match_re.search(name, timeout=REGEX_TIMEOUT) + except (TimeoutError, re.error) as e: + match_error = match_error or f"Pattern timed out: {e}" + match_re = None + continue + if matched: + filter_match_count += 1 + if len(filter_matches) < limit: + filter_matches.append({"name": name, "matches": True}) + if exclude_re is not None: + try: + matched = exclude_re.search(name, timeout=REGEX_TIMEOUT) + except (TimeoutError, re.error) as e: + exclude_error = exclude_error or f"Pattern timed out: {e}" + exclude_re = None + continue + if matched: + exclude_match_count += 1 + if len(exclude_matches) < limit: + exclude_matches.append({"name": name, "matches": True}) + + response_payload = { + "total_in_group": total_in_group, + "total_scanned": total_scanned, + "scan_limit_hit": total_in_group > SCAN_CAP, + } + if find_pat: + response_payload["find_matches"] = find_matches + response_payload["find_match_count"] = find_match_count + if find_error: + response_payload["find_error"] = find_error + if match_pat: + response_payload["filter_matches"] = filter_matches + response_payload["filter_match_count"] = filter_match_count + if match_error: + response_payload["match_error"] = match_error + if exclude_pat: + response_payload["exclude_matches"] = exclude_matches + response_payload["exclude_match_count"] = exclude_match_count + if exclude_error: + response_payload["exclude_error"] = exclude_error + return Response(response_payload) + @action(detail=False, methods=["get"], url_path="groups") def get_groups(self, request, *args, **kwargs): # Get unique ChannelGroup names that are linked to streams @@ -322,8 +564,56 @@ class ChannelGroupViewSet(viewsets.ModelViewSet): return [Authenticated()] def get_queryset(self): - """Return channel groups with prefetched relations for efficient counting""" - return ChannelGroup.objects.prefetch_related('channels', 'm3u_accounts').all() + # Annotate both counts at the SQL level so the serializer methods + # can read them from the object rather than issuing a COUNT per row. + # `distinct=True` is required when multiple reverse-FK annotations + # share the same queryset to avoid row-multiplication artifacts. + # m3u_accounts is still prefetched for the nested serializer data. + return ( + ChannelGroup.objects + .annotate( + channel_count=Count('channels', distinct=True), + m3u_account_count=Count('m3u_accounts', distinct=True), + ) + .prefetch_related('m3u_accounts') + .all() + ) + + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + # Evaluate the queryset once so the annotations and prefetch cache are + # populated together, then extract IDs from the in-memory objects. + # A second .values_list() call would fire a separate SQL query. + groups = list(queryset) + group_ids = [g.id for g in groups] + + # Pre-aggregate stream counts for all (account, group) pairs in a + # single query so the nested ChannelGroupM3UAccountSerializer never + # fires a COUNT per row. + counts_qs = ( + Stream.objects.filter(channel_group_id__in=group_ids) + .values('m3u_account_id', 'channel_group_id') + .annotate(c=Count('id')) + ) + stream_counts = { + (row['m3u_account_id'], row['channel_group_id']): row['c'] + for row in counts_qs + } + + page = self.paginate_queryset(groups) + if page is not None: + serializer = self.get_serializer( + page, many=True, + context={**self.get_serializer_context(), 'stream_counts': stream_counts}, + ) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer( + groups, many=True, + context={**self.get_serializer_context(), 'stream_counts': stream_counts}, + ) + return Response(serializer.data) def update(self, request, *args, **kwargs): """Override update to check M3U associations""" @@ -563,17 +853,31 @@ class ChannelViewSet(viewsets.ModelViewSet): return [Authenticated()] def get_queryset(self): - qs = ( - super() - .get_queryset() - .select_related( + # get_ids and summary only need the filter conditions, not the full + # object graph. Skipping the 5 select_related joins and 2 prefetch + # queries for those actions cuts their DB cost significantly. + action = getattr(self, "action", None) + qs = super().get_queryset() + + if action not in ("get_ids", "summary"): + qs = qs.select_related( "channel_group", "logo", "epg_data", "stream_profile", + "override", + "auto_created_by", + ).prefetch_related( + "streams", + # Default-attr prefetch shares the cache with M2M writes; + # a named `to_attr` would isolate it and trigger N+1. + Prefetch( + "channelstream_set", + queryset=ChannelStream.objects.select_related( + "stream__m3u_account" + ).order_by("order"), + ), ) - .prefetch_related("streams") - ) channel_group = self.request.query_params.get("channel_group") if channel_group: @@ -587,6 +891,8 @@ class ChannelViewSet(viewsets.ModelViewSet): show_disabled_param = self.request.query_params.get("show_disabled", None) only_streamless = self.request.query_params.get("only_streamless", None) only_stale = self.request.query_params.get("only_stale", None) + only_has_overrides = self.request.query_params.get("only_has_overrides", None) + visibility_filter = self.request.query_params.get("visibility_filter", "active") if channel_profile_id: try: @@ -609,6 +915,18 @@ class ChannelViewSet(viewsets.ModelViewSet): if only_stale: # Filter channels that have at least one related stream marked as stale q_filters &= Q(streams__is_stale=True) + if only_has_overrides: + q_filters &= Q(override__isnull=False) + + # Visibility filter applies to list-style reads only; retrieve / + # update / delete must still reach a hidden channel by id so the + # frontend can unhide. Summary powers the TV Guide and follows + # the same hidden semantic as downstream clients. + if self.action in ("list", "get_ids", "summary"): + if visibility_filter == "hidden": + q_filters &= Q(hidden_from_output=True) + elif visibility_filter != "all": + q_filters &= Q(hidden_from_output=False) if self.request.user.user_level < 10: filters["user_level__lte"] = self.request.user.user_level @@ -630,6 +948,14 @@ class ChannelViewSet(viewsets.ModelViewSet): self.request.query_params.get("include_streams", "false") == "true" ) context["include_streams"] = include_streams + # source_stream is only needed by the channel edit form. For get_ids + # and summary the channelstream_set prefetch is skipped entirely, so + # source_stream cannot be computed without hitting the DB per channel. + # For list and write/retrieve paths the prefetch is present, so we + # can populate it from memory without extra queries. + context["include_source_stream"] = action not in ( + "get_ids", "summary" + ) if (action := getattr(self, "action", None)) else False return context @extend_schema( @@ -756,14 +1082,44 @@ class ChannelViewSet(viewsets.ModelViewSet): status=status.HTTP_400_BAD_REQUEST, ) + # Capture override intent from the raw payload: presence of the + # key distinguishes "no change" from explicit null ("clear"), + # which `validated_data` would collapse. + override_intents = {} + for channel_data in data: + if "override" in channel_data: + cid = channel_data.get("id") + override_intents[cid] = channel_data["override"] + + # Capture hide / unhide transitions before setattr overwrites + # the in-memory channels. bulk_update skips post_save, so the + # compact-mode assign / release runs explicitly below. + unhide_transition_ids = [ + channel.id + for channel, validated_data in validated_updates + if validated_data.get("hidden_from_output") is False + and channel.hidden_from_output is True + ] + hide_transition_candidates = [ + channel + for channel, validated_data in validated_updates + if validated_data.get("hidden_from_output") is True + and channel.hidden_from_output is False + and channel.channel_number is not None + and channel.auto_created + and channel.auto_created_by_id + ] + # Apply all updates in a transaction with transaction.atomic(): streams_updates = [] for channel, validated_data in validated_updates: - # Pop streams before setattr loop — M2M fields can't be set via setattr + # Streams (M2M) and override (reverse OneToOne) cannot + # ride the setattr loop; handle each separately below. streams = validated_data.pop("streams", None) if streams is not None: streams_updates.append((channel, streams)) + validated_data.pop("override", None) for key, value in validated_data.items(): setattr(channel, key, value) @@ -783,6 +1139,179 @@ class ChannelViewSet(viewsets.ModelViewSet): batch_size=100 ) + # On unhide under compact mode, assign a number so the + # channel is immediately addressable by clients. + if unhide_transition_ids: + from .compact_numbering import ( + assign_compact_numbers_for_channels, + ) + assign_compact_numbers_for_channels(unhide_transition_ids) + + # On hide under compact mode, release the channel_number + # so the slot is reused. The bulk path bypasses the + # post_save signal that handles single-row hides. + if hide_transition_candidates: + from .compact_numbering import ( + get_group_relation_for_channel, + is_compact_group, + ) + ids_to_release = [] + for ch in hide_transition_candidates: + relation = get_group_relation_for_channel(ch) + if relation and is_compact_group(relation): + ids_to_release.append(ch.id) + if ids_to_release: + Channel.objects.filter(id__in=ids_to_release).update( + channel_number=None + ) + # Refresh in-memory copies so the response shape + # reflects the cleared numbers. + for ch in channels_to_update: + if ch.id in ids_to_release: + ch.channel_number = None + + # Override (reverse OneToOne) needs a separate write path. + if override_intents: + from apps.channels.models import ChannelOverride + from apps.channels.managers import OVERRIDABLE_FIELDS + override_fields = OVERRIDABLE_FIELDS + # Block override mutations on manual channels. Mixed + # selections with override:null are tolerated because + # clearing a non-existent row is a no-op. + manual_with_override = [] + for channel, _ in validated_updates: + if channel.id not in override_intents: + continue + intent = override_intents[channel.id] + if ( + intent is not None + and intent != {} + and not channel.auto_created + ): + manual_with_override.append(channel.id) + if manual_with_override: + return Response( + { + "errors": [ + { + "channel_id": cid, + "error": ( + "Cannot set override on a manual channel; " + "overrides only apply to auto-created channels." + ), + } + for cid in manual_with_override + ] + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + channels_to_clear = [] + overrides_to_upsert = [] + for channel, _ in validated_updates: + if channel.id not in override_intents: + continue + intent = override_intents[channel.id] + if intent is None: + channels_to_clear.append(channel.id) + elif intent == {}: + # Empty dict means no override intent; treat as no-op. + continue + else: + defaults = { + f: intent.get(f) + for f in override_fields + if f in intent + } + # Coerce FK aliases (logo, channel_group, ...) to + # the *_id columns ChannelOverride actually stores. + for raw, mapped in ( + ("logo", "logo_id"), + ("channel_group", "channel_group_id"), + ("epg_data", "epg_data_id"), + ("stream_profile", "stream_profile_id"), + ): + if raw in intent and mapped not in defaults: + val = intent[raw] + defaults[mapped] = ( + val.id if hasattr(val, "id") else val + ) + overrides_to_upsert.append((channel.id, defaults)) + + if channels_to_clear: + ChannelOverride.objects.filter( + channel_id__in=channels_to_clear + ).delete() + + # Bulk upsert keeps a 1000-channel batch to two + # statements (one INSERT, one UPDATE) instead of the + # per-row SELECT + INSERT-or-UPDATE that update_or_create + # would generate. + if overrides_to_upsert: + existing_overrides = { + o.channel_id: o + for o in ChannelOverride.objects.filter( + channel_id__in=[ + cid for cid, _ in overrides_to_upsert + ] + ) + } + to_create = [] + to_update = [] + update_field_set = set() + for channel_id, defaults in overrides_to_upsert: + existing = existing_overrides.get(channel_id) + if existing: + for f, v in defaults.items(): + setattr(existing, f, v) + update_field_set.add(f) + to_update.append(existing) + else: + to_create.append( + ChannelOverride( + channel_id=channel_id, **defaults + ) + ) + if to_update: + ChannelOverride.objects.bulk_update( + to_update, + fields=list(update_field_set), + batch_size=200, + ) + if to_create: + ChannelOverride.objects.bulk_create( + to_create, batch_size=200 + ) + + # Drop override rows that ended up all-null; an empty + # override would falsely surface as active in the UI. + touched_ids = [cid for cid, _ in overrides_to_upsert] + empty_overrides = [ + o for o in ChannelOverride.objects.filter( + channel_id__in=touched_ids + ) + if not o.has_any_override() + ] + if empty_overrides: + ChannelOverride.objects.filter( + id__in=[o.id for o in empty_overrides] + ).delete() + + # Queryset writes leave the reverse-OneToOne cache stale; + # clear it so the serializer reads the new override state. + touched_channel_ids = {cid for cid, _ in overrides_to_upsert} + touched_channel_ids.update(channels_to_clear) + if touched_channel_ids: + for channel, _ in validated_updates: + if channel.id not in touched_channel_ids: + continue + try: + channel._state.fields_cache.pop("override", None) + except AttributeError: + pass + if hasattr(channel, "_channel_override_cache"): + delattr(channel, "_channel_override_cache") + # Handle streams M2M updates separately for channel, streams in streams_updates: normalized_ids = [ @@ -1032,21 +1561,158 @@ class ChannelViewSet(viewsets.ModelViewSet): @action(detail=False, methods=["get"], url_path="summary") def summary(self, request, *args, **kwargs): - """Return a lightweight list of channels with only the fields needed by the TV Guide.""" - queryset = self.filter_queryset(self.get_queryset()) - data = list( - queryset.values( - "id", - "name", - "logo_id", - "channel_number", - "uuid", - "epg_data_id", - "channel_group_id", - ) + """Return a lightweight list of channels with only the fields needed by the TV Guide. + + The TV Guide is a downstream output surface like HDHR / M3U / EPG / + XC and must reflect the user's overrides. Effective values are + coalesced at the SQL layer; the annotated columns are renamed + back to the raw field names on the way out so the response + shape stays unchanged for the frontend. + """ + from .managers import with_effective_values + + queryset = with_effective_values( + self.filter_queryset(self.get_queryset()) ) + data = [ + { + "id": row["id"], + "uuid": row["uuid"], + "name": row["effective_name"], + "logo_id": row["effective_logo_id"], + "channel_number": row["effective_channel_number"], + "epg_data_id": row["effective_epg_data_id"], + "channel_group_id": row["effective_channel_group_id"], + } + for row in queryset.values( + "id", + "uuid", + "effective_name", + "effective_logo_id", + "effective_channel_number", + "effective_epg_data_id", + "effective_channel_group_id", + ) + ] return Response(data) + @extend_schema( + parameters=[ + OpenApiParameter( + name="start", + type=OpenApiTypes.NUMBER, + location=OpenApiParameter.QUERY, + required=True, + description="Inclusive lower bound of the range to scan.", + ), + OpenApiParameter( + name="end", + type=OpenApiTypes.NUMBER, + location=OpenApiParameter.QUERY, + required=False, + description=( + "Inclusive upper bound. If omitted or equal to start, " + "behaves as a single-number lookup." + ), + ), + ], + responses={ + 200: inline_serializer( + name="ChannelsInRangeResponse", + fields={ + "occupants": serializers.ListField( + child=serializers.DictField() + ) + }, + ) + }, + description=( + "Returns the channels (including those whose effective number is " + "set via override) currently occupying numbers within the given " + "range. Used by the group settings form to surface inline range " + "conflict warnings. Capped at 50 entries to bound the response " + "payload; the frontend only needs to know whether any conflicts " + "exist after filtering, not the entire list." + ), + ) + @action(detail=False, methods=["get"], url_path="numbers-in-range") + def numbers_in_range(self, request, *args, **kwargs): + from .managers import with_effective_values + + raw_start = request.query_params.get("start") + raw_end = request.query_params.get("end") + if raw_start is None or raw_start == "": + return Response({"occupants": []}) + try: + start = float(raw_start) + except (TypeError, ValueError): + return Response( + {"detail": "Invalid start value"}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + end = float(raw_end) if raw_end not in (None, "") else start + except (TypeError, ValueError): + return Response( + {"detail": "Invalid end value"}, + status=status.HTTP_400_BAD_REQUEST, + ) + if end < start: + start, end = end, start + + queryset = ( + with_effective_values( + Channel.objects.all(), select_related_fks=True + ) + .filter( + effective_channel_number__gte=start, + effective_channel_number__lte=end, + ) + .order_by("effective_channel_number")[:50] + ) + + occupants = [] + for occupant in queryset: + effective_group = getattr( + occupant, "effective_channel_group_obj", None + ) + group_name = ( + getattr(effective_group, "name", None) + if effective_group is not None + else None + ) + effective_group_id = getattr( + occupant, "effective_channel_group_id", None + ) + override = getattr(occupant, "override", None) + override_sets_number = bool( + override is not None and override.channel_number is not None + ) + occupants.append( + { + "id": occupant.id, + "name": getattr( + occupant, "effective_name", occupant.name + ), + "channel_number": getattr( + occupant, + "effective_channel_number", + occupant.channel_number, + ), + "channel_group": group_name, + "channel_group_id": effective_group_id, + "auto_created": bool(occupant.auto_created), + "auto_created_by_account_id": ( + occupant.auto_created_by_id + if occupant.auto_created_by_id + else None + ), + "has_channel_number_override": override_sets_number, + } + ) + + return Response({"occupants": occupants}) + @extend_schema( methods=["POST"], description="Retrieve channels by a list of UUIDs using POST to avoid URL length limitations", @@ -3252,9 +3918,13 @@ class SeriesRulesAPIView(APIView): request=inline_serializer( name="SeriesRuleRequest", fields={ - 'tvg_id': serializers.CharField(help_text='Channel TVG ID'), + 'tvg_id': serializers.CharField(required=False, allow_blank=True, help_text='Optional channel TVG ID. Omit to match across all channels.'), 'mode': serializers.ChoiceField(choices=['all', 'new'], default='all', help_text='all: record all episodes, new: record only new episodes'), 'title': serializers.CharField(help_text='Series title', required=False), + 'title_mode': serializers.ChoiceField(choices=['exact', 'contains', 'search', 'regex'], default='exact', required=False, help_text='How to match the title field'), + 'description': serializers.CharField(required=False, help_text='Optional description match expression'), + 'description_mode': serializers.ChoiceField(choices=['contains', 'search', 'regex'], default='contains', required=False, help_text='How to match the description field'), + 'channel_id': serializers.IntegerField(required=False, help_text='Optional channel to pin recordings to (defaults to lowest-numbered channel for the EPG)'), }, ), ) @@ -3263,62 +3933,96 @@ class SeriesRulesAPIView(APIView): tvg_id = str(data.get("tvg_id") or "").strip() mode = (data.get("mode") or "all").lower() title = data.get("title") or "" + title_mode = (data.get("title_mode") or "exact").lower() + description = data.get("description") or "" + description_mode = (data.get("description_mode") or "contains").lower() + channel_id = data.get("channel_id") if mode not in ("all", "new"): return Response({"error": "mode must be 'all' or 'new'"}, status=status.HTTP_400_BAD_REQUEST) - if not tvg_id: - return Response({"error": "tvg_id is required"}, status=status.HTTP_400_BAD_REQUEST) + if title_mode not in ("exact", "contains", "search", "regex"): + return Response({"error": "title_mode must be one of exact, contains, search, regex"}, status=status.HTTP_400_BAD_REQUEST) + if description_mode not in ("contains", "search", "regex"): + return Response({"error": "description_mode must be one of contains, search, regex"}, status=status.HTTP_400_BAD_REQUEST) + if not title.strip() and not description.strip(): + return Response({"error": "A title or description is required"}, status=status.HTTP_400_BAD_REQUEST) + + # Coerce / validate optional pinned channel + pinned_channel_id = None + if channel_id not in (None, ""): + try: + pinned_channel_id = int(channel_id) + except (TypeError, ValueError): + return Response({"error": "channel_id must be an integer"}, status=status.HTTP_400_BAD_REQUEST) + from .models import Channel + if not Channel.objects.filter(id=pinned_channel_id).exists(): + return Response({"error": "channel_id does not exist"}, status=status.HTTP_400_BAD_REQUEST) + + rule_record = { + "tvg_id": tvg_id, + "mode": mode, + "title": title, + "title_mode": title_mode, + "description": description, + "description_mode": description_mode, + } + if pinned_channel_id is not None: + rule_record["channel_id"] = pinned_channel_id + rules = CoreSettings.get_dvr_series_rules() - # Upsert by tvg_id - existing = next((r for r in rules if str(r.get("tvg_id")) == tvg_id), None) + # Upsert by tvg_id + title so multiple rules can target the same channel + existing = next( + (r for r in rules if + str(r.get("tvg_id") or "") == tvg_id and + str(r.get("title") or "") == title), + None + ) if existing: - existing.update({"mode": mode, "title": title}) + existing.clear() + existing.update(rule_record) else: - rules.append({"tvg_id": tvg_id, "mode": mode, "title": title}) + rules.append(rule_record) CoreSettings.set_dvr_series_rules(rules) # Note: frontend calls the evaluate endpoint explicitly after creating # the rule, so do NOT fire evaluate_series_rules.delay() here to # avoid a race that creates duplicate recordings. return Response({"success": True, "rules": rules}) - -class DeleteSeriesRuleAPIView(APIView): - def get_permissions(self): - try: - return [perm() for perm in permission_classes_by_method[self.request.method]] - except KeyError: - return [Authenticated()] - @extend_schema( summary="Delete a series rule", - description="Remove a series recording rule by TVG ID and clean up future scheduled recordings.", + description="Remove a series recording rule by tvg_id + title and clean up future scheduled recordings.", parameters=[ - OpenApiParameter('tvg_id', str, OpenApiParameter.PATH, required=True, description='Channel TVG ID'), + OpenApiParameter('tvg_id', str, OpenApiParameter.QUERY, required=False, description='Channel TVG ID (may be blank for title-only rules)'), + OpenApiParameter('title', str, OpenApiParameter.QUERY, required=False, description='Series title'), ], ) - def delete(self, request, tvg_id): - tvg_id = unquote(str(tvg_id or "")) + def delete(self, request): + tvg_id = str(request.query_params.get("tvg_id") or "").strip() + title = request.query_params.get("title") - # Find the rule before removing to retain the title for cleanup rules = CoreSettings.get_dvr_series_rules() - deleted_rule = next((r for r in rules if str(r.get("tvg_id")) == tvg_id), None) - remaining = [r for r in rules if str(r.get("tvg_id")) != tvg_id] + + def _matches(r): + tvg_match = str(r.get("tvg_id") or "") == tvg_id + title_match = title is None or str(r.get("title") or "") == title + return tvg_match and title_match + + deleted_rule = next((r for r in rules if _matches(r)), None) + remaining = [r for r in rules if not _matches(r)] CoreSettings.set_dvr_series_rules(remaining) - # Delete only FUTURE recordings — preserve previously recorded episodes removed = 0 if deleted_rule: from .models import Recording - qs = Recording.objects.filter( - start_time__gte=timezone.now(), - custom_properties__program__tvg_id=tvg_id, - ) - title = deleted_rule.get("title") - if title: - qs = qs.filter(custom_properties__program__title=title) + qs = Recording.objects.filter(start_time__gte=timezone.now()) + rule_tvg_id = deleted_rule.get("tvg_id") or "" + if rule_tvg_id: + qs = qs.filter(custom_properties__program__tvg_id=rule_tvg_id) + rule_title = deleted_rule.get("title") or "" + if rule_title: + qs = qs.filter(custom_properties__program__title=rule_title) removed = qs.count() qs.delete() - # Notify frontend to refresh recordings list try: from core.utils import send_websocket_update send_websocket_update('updates', 'update', { @@ -3330,6 +4034,118 @@ class DeleteSeriesRuleAPIView(APIView): return Response({"success": True, "rules": remaining, "removed": removed}) +class SeriesRulePreviewAPIView(APIView): + """Preview which upcoming programs a series rule would match. + + Accepts the same payload as SeriesRulesAPIView.post but does not persist + anything. Returns up to `limit` upcoming programs (default 25, max 100) + within the standard 7-day evaluation horizon. + """ + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_method[self.request.method]] + except KeyError: + return [Authenticated()] + + @extend_schema( + summary="Preview series rule matches", + description="Return upcoming programs that the given rule would match without persisting the rule.", + request=inline_serializer( + name="SeriesRulePreviewRequest", + fields={ + 'tvg_id': serializers.CharField(required=False, allow_blank=True, help_text='Optional channel TVG ID. Omit to search across all channels.'), + 'mode': serializers.ChoiceField(choices=['all', 'new'], default='all', required=False), + 'title': serializers.CharField(required=False), + 'title_mode': serializers.ChoiceField(choices=['exact', 'contains', 'search', 'regex'], default='exact', required=False), + 'description': serializers.CharField(required=False), + 'description_mode': serializers.ChoiceField(choices=['contains', 'search', 'regex'], default='contains', required=False), + 'limit': serializers.IntegerField(required=False, help_text='Max programs to return (default 25, max 100)'), + }, + ), + ) + def post(self, request): + from apps.epg.models import EPGData, ProgramData + from apps.epg.query_utils import parse_text_query + + data = request.data or {} + tvg_id = str(data.get("tvg_id") or "").strip() + mode = (data.get("mode") or "all").lower() + title = (data.get("title") or "").strip() + title_mode = (data.get("title_mode") or "exact").lower() + description = (data.get("description") or "").strip() + description_mode = (data.get("description_mode") or "contains").lower() + try: + limit = int(data.get("limit") or 25) + except (TypeError, ValueError): + limit = 25 + limit = max(1, min(limit, 100)) + + if not title and not description: + return Response({"error": "A title or description is required"}, status=status.HTTP_400_BAD_REQUEST) + + now = timezone.now() + horizon = now + timedelta(days=7) + + if tvg_id: + epg = EPGData.objects.filter(tvg_id=tvg_id).first() + if not epg: + return Response({"matches": [], "total": 0, "epg_found": False}) + qs = ProgramData.objects.filter(epg=epg, end_time__gt=now, start_time__lte=horizon) + else: + qs = ProgramData.objects.filter(end_time__gt=now, start_time__lte=horizon) + + if title: + if title_mode == "exact": + qs = qs.filter(title__iexact=title) + else: + qs = qs.filter(parse_text_query( + "title", title, + use_regex=(title_mode == "regex"), + whole_words=(title_mode == "search"), + )) + if description: + qs = qs.filter(parse_text_query( + "description", description, + use_regex=(description_mode == "regex"), + whole_words=(description_mode == "search"), + )) + + qs = qs.distinct().order_by("start_time") + + # Apply "new" filter in Python (custom_properties JSON lookup), but only + # over the bounded result set we already filtered down to. + candidates = list(qs[:limit * 4]) # small overshoot to allow new-only filtering + if mode == "new": + candidates = [p for p in candidates if (p.custom_properties or {}).get("new")] + + total = len(candidates) + candidates = candidates[:limit] + + matches = [] + for p in candidates: + cp = p.custom_properties or {} + matches.append({ + "id": p.id, + "tvg_id": p.tvg_id, + "title": p.title, + "sub_title": p.sub_title, + "description": p.description, + "start_time": p.start_time.isoformat(), + "end_time": p.end_time.isoformat(), + "season": cp.get("season"), + "episode": cp.get("episode"), + "is_new": bool(cp.get("new")), + }) + + return Response({ + "matches": matches, + "total": total, + "limit": limit, + "epg_found": True, + "warn": total > 50, + }) + + class EvaluateSeriesRulesAPIView(APIView): def get_permissions(self): try: @@ -3385,13 +4201,12 @@ class BulkRemoveSeriesRecordingsAPIView(APIView): tvg_id = str(request.data.get("tvg_id") or "").strip() title = request.data.get("title") scope = (request.data.get("scope") or "title").lower() - if not tvg_id: - return Response({"error": "tvg_id is required"}, status=status.HTTP_400_BAD_REQUEST) + if not tvg_id and not title: + return Response({"error": "tvg_id or title is required"}, status=status.HTTP_400_BAD_REQUEST) - qs = Recording.objects.filter( - start_time__gte=timezone.now(), - custom_properties__program__tvg_id=tvg_id, - ) + qs = Recording.objects.filter(start_time__gte=timezone.now()) + if tvg_id: + qs = qs.filter(custom_properties__program__tvg_id=tvg_id) if scope == "title" and title: qs = qs.filter(custom_properties__program__title=title) diff --git a/apps/channels/compact_numbering.py b/apps/channels/compact_numbering.py new file mode 100644 index 00000000..f3d7183c --- /dev/null +++ b/apps/channels/compact_numbering.py @@ -0,0 +1,363 @@ +"""Compact channel numbering helpers. + +The per-group `compact_numbering` custom_property is opt-in. When enabled +on a ChannelGroupM3UAccount, the group's auto-created channels get packed +contiguously into the group's [start, end] range: + + * Visible without channel_number override: assigned sequentially + * Hidden without channel_number override: channel_number set to NULL + (released; the slot becomes available for visible channels) + * Override-pinned (any visibility): untouched; the override's + channel_number is treated as a global reservation that other channels + skip when packing + +Used by sync_auto_channels (full-pack pass), the post_save Channel signal +(single-channel unhide), the bulk-edit endpoint (bulk unhide), and a +manual per-group re-pack endpoint. + +Trade-off the user opts into: channel numbers may shift when hide / unhide +state changes. To pin a number through hide/unhide cycles, set a +channel_number override - the override is honored as a reservation. +""" + +import logging + +from django.db import transaction + +from .models import Channel, ChannelOverride, ChannelGroupM3UAccount +from apps.m3u.tasks import _custom_properties_as_dict, _next_available_number +from core.utils import ( + acquire_task_lock, + natural_sort_key, + release_task_lock, +) + +logger = logging.getLogger(__name__) + + +def is_compact_group(group_relation): + """Return True if the given ChannelGroupM3UAccount is in compact mode.""" + cp = _custom_properties_as_dict(group_relation.custom_properties) + return bool(cp.get("compact_numbering")) + + +def get_group_relation_for_channel(channel): + """Resolve the ChannelGroupM3UAccount that owns this auto-created + channel. Returns None for manual channels, or when the relation has + been deleted.""" + if ( + not channel.auto_created + or not channel.auto_created_by_id + or not channel.channel_group_id + ): + return None + try: + return ChannelGroupM3UAccount.objects.get( + m3u_account_id=channel.auto_created_by_id, + channel_group_id=channel.channel_group_id, + ) + except ChannelGroupM3UAccount.DoesNotExist: + return None + + +def build_reserved_set(exclude_channel_ids=None, range_start=None, range_end=None): + """Return the set of channel numbers currently 'claimed' system-wide + for the purposes of a compact pack: + + * Every ChannelOverride.channel_number value (overrides reserve + their effective number; duplicates across overrides are allowed + and collapse to a single reservation via set semantics) + * Every Channel.channel_number value EXCEPT for channels in the + passed-in exclude set (those are about to be reassigned) + + When the caller knows the target group has a bounded range, pass + `range_start` and `range_end` to scope the scan to numbers that + could possibly collide. Without scoping, a single signal-driven + unhide reads every channel_number in the database; with scoping it + reads at most the values within [start, end] which for typical + cable-style ranges is hundreds rather than tens of thousands. + + Float vs int normalization is unnecessary because Python treats + 50 and 50.0 as equal and produces the same hash, so set membership + works directly across both types. + """ + exclude_channel_ids = set(exclude_channel_ids or []) + override_qs = ChannelOverride.objects.filter(channel_number__isnull=False) + other_qs = Channel.objects.exclude(channel_number__isnull=True).exclude( + id__in=exclude_channel_ids + ) + if range_start is not None: + override_qs = override_qs.filter(channel_number__gte=range_start) + other_qs = other_qs.filter(channel_number__gte=range_start) + if range_end is not None: + override_qs = override_qs.filter(channel_number__lte=range_end) + other_qs = other_qs.filter(channel_number__lte=range_end) + reserved = set(override_qs.values_list("channel_number", flat=True)) + reserved.update(other_qs.values_list("channel_number", flat=True)) + reserved.discard(None) + return reserved + + +def _channel_has_number_override(channel): + try: + ov = channel.override + except ChannelOverride.DoesNotExist: + return False + return ov is not None and ov.channel_number is not None + + +def assign_compact_numbers_for_channels(channel_ids): + """For each channel ID in the input that became eligible for a number + (visible, auto-created, no number override, in a compact-mode group), + assign the next available channel number in the group's [start, end] + range. Channels whose group is not in compact mode are skipped silently + so callers can pass mixed batches without filtering up front. + + Used by both the post_save signal (single-channel unhide) and the bulk + edit endpoint (bulk unhide). Returns dict {channel_id: number_or_None}. + """ + if not channel_ids: + return {} + channels = list( + Channel.objects.filter( + id__in=channel_ids, + auto_created=True, + auto_created_by__isnull=False, + hidden_from_output=False, + channel_number__isnull=True, + ).select_related("override", "channel_group", "auto_created_by") + ) + if not channels: + return {} + + # Group channels by (account_id, group_id) so each unique pair's + # ChannelGroupM3UAccount is resolved with a single SELECT rather than + # one per channel. + by_pair = {} + for ch in channels: + if _channel_has_number_override(ch): + continue + key = (ch.auto_created_by_id, ch.channel_group_id) + by_pair.setdefault(key, []).append(ch) + if not by_pair: + return {} + + pair_keys = list(by_pair.keys()) + relations_by_pair = {} + relations_qs = ChannelGroupM3UAccount.objects.filter( + m3u_account_id__in={k[0] for k in pair_keys}, + channel_group_id__in={k[1] for k in pair_keys}, + ) + for rel in relations_qs: + relations_by_pair[(rel.m3u_account_id, rel.channel_group_id)] = rel + + by_relation = {} + for key, group_channels in by_pair.items(): + rel = relations_by_pair.get(key) + if rel is None or not is_compact_group(rel): + continue + by_relation[rel.id] = (rel, group_channels) + + # Group by account so writes share the `refresh_single_m3u_account` + # lock used by sync_auto_channels and the manual repack endpoint. + # If sync is in flight for an account, defer to it. + by_account = {} + for rel, group_channels in by_relation.values(): + by_account.setdefault(rel.m3u_account_id, []).append( + (rel, group_channels) + ) + + results = {} + for account_id, account_pairs in by_account.items(): + if not acquire_task_lock( + "refresh_single_m3u_account", account_id + ): + logger.info( + "Compact unhide deferred for account %s: refresh in progress; " + "next sync will assign numbers.", + account_id, + ) + for _, group_channels in account_pairs: + for ch in group_channels: + results[ch.id] = None + continue + + # try/finally release: the Redis lock is not transactional, so a + # transaction.on_commit release would leak the lock when an + # outer atomic rolls back, blocking subsequent syncs until TTL. + try: + with transaction.atomic(): + for rel, group_channels in account_pairs: + start = int(rel.auto_sync_channel_start or 1) + end = ( + int(rel.auto_sync_channel_end) + if rel.auto_sync_channel_end + else None + ) + reserved = build_reserved_set( + exclude_channel_ids=[c.id for c in group_channels], + range_start=start, + range_end=end, + ) + to_update = [] + for ch in group_channels: + next_num = _next_available_number( + reserved, start, end=end + ) + if next_num is None: + results[ch.id] = None + continue + ch.channel_number = next_num + reserved.add(next_num) + results[ch.id] = next_num + to_update.append(ch) + if to_update: + Channel.objects.bulk_update( + to_update, ["channel_number"], batch_size=100 + ) + finally: + try: + release_task_lock( + "refresh_single_m3u_account", account_id + ) + except Exception as e: + logger.warning( + "Failed to release compact-unhide lock for account " + "%s: %s", + account_id, + e, + ) + return results + + +def repack_group(group_relation): + """Renumber every auto-created channel in the given group+account. + + Visible non-override channels are assigned sequentially in + [start, end] using the group's configured channel_sort_order. + Hidden non-override channels have their channel_number set to + None (slot released). Override-pinned channels are untouched. + + Returns a dict with assigned/released/failed counts. ``failed`` + counts visible channels that could not fit because the range was + exhausted; their channel_number is set to None so the state is + unambiguous instead of stuck at a stale number. + + All writes run inside a single transaction so concurrent readers + (HDHR/M3U/EPG output paths) never observe a half-packed state. + """ + with transaction.atomic(): + return _repack_inner(group_relation) + + +def _repack_inner(group_relation): + account_id = group_relation.m3u_account_id + group_id = group_relation.channel_group_id + + cp = _custom_properties_as_dict(group_relation.custom_properties) + sort_order = cp.get("channel_sort_order") or "" + sort_reverse = bool(cp.get("channel_sort_reverse")) + + start = int(group_relation.auto_sync_channel_start or 1) + end = ( + int(group_relation.auto_sync_channel_end) + if group_relation.auto_sync_channel_end + else None + ) + + channels = list( + Channel.objects.filter( + auto_created=True, + auto_created_by_id=account_id, + channel_group_id=group_id, + ).select_related("override") + ) + + visible = [] + hidden = [] + pinned = [] + for ch in channels: + if _channel_has_number_override(ch): + pinned.append(ch) + elif ch.hidden_from_output: + hidden.append(ch) + else: + visible.append(ch) + + # Sort the visible set by the group's configured channel_sort_order. + # Provider order (the default) preserves DB-iteration order which is + # roughly creation order; treat unrecognized values the same way. + if sort_order == "name": + visible.sort( + key=lambda c: natural_sort_key(c.name or ""), + reverse=sort_reverse, + ) + elif sort_order == "tvg_id": + visible.sort( + key=lambda c: c.tvg_id or "", + reverse=sort_reverse, + ) + elif sort_order == "updated_at": + visible.sort( + key=lambda c: c.updated_at, + reverse=sort_reverse, + ) + + # Exclude every channel in this group: the pinned channel's raw value + # is irrelevant (the override is reserved globally and cleared below). + # Scope to the group's range when bounded to keep the set small. + affected_ids = [c.id for c in (visible + hidden + pinned)] + reserved = build_reserved_set( + exclude_channel_ids=affected_ids, + range_start=start, + range_end=end, + ) + + assigned_count = 0 + failed_count = 0 + visible_to_update = [] + for ch in visible: + next_num = _next_available_number(reserved, start, end=end) + if next_num is None: + failed_count += 1 + if ch.channel_number is not None: + ch.channel_number = None + visible_to_update.append(ch) + continue + if ch.channel_number != next_num: + ch.channel_number = next_num + visible_to_update.append(ch) + reserved.add(next_num) + assigned_count += 1 + + if visible_to_update: + Channel.objects.bulk_update( + visible_to_update, ["channel_number"], batch_size=100 + ) + + released_count = 0 + hidden_with_num = [c for c in hidden if c.channel_number is not None] + if hidden_with_num: + for c in hidden_with_num: + c.channel_number = None + Channel.objects.bulk_update( + hidden_with_num, ["channel_number"], batch_size=100 + ) + released_count = len(hidden_with_num) + + # Pinned channels: clear raw channel_number. The override controls + # their effective number; leaving a stale raw value would pollute + # uniqueness checks and could resurrect on override clear. + pinned_with_num = [c for c in pinned if c.channel_number is not None] + if pinned_with_num: + for c in pinned_with_num: + c.channel_number = None + Channel.objects.bulk_update( + pinned_with_num, ["channel_number"], batch_size=100 + ) + + return { + "assigned": assigned_count, + "released": released_count, + "failed": failed_count, + } diff --git a/apps/channels/managers.py b/apps/channels/managers.py new file mode 100644 index 00000000..ac5c8115 --- /dev/null +++ b/apps/channels/managers.py @@ -0,0 +1,55 @@ +""" +Queryset helpers that resolve effective Channel field values. + +Each Channel can optionally have a related ChannelOverride row carrying user +edits to any subset of its user-facing fields. Sync never touches the override +row; provider metadata flows directly into Channel.* and the override table +sits alongside with a nullable value per field. The helpers here coalesce the +two sources into `effective_*` annotations so output querysets can sort, +filter, and emit values correctly at SQL level (avoiding 23+ Python-side +resolutions across the codebase). +""" + +from django.db.models.functions import Coalesce + + +OVERRIDABLE_FIELDS = ( + "name", + "channel_number", + "channel_group_id", + "logo_id", + "tvg_id", + "tvc_guide_stationid", + "epg_data_id", + "stream_profile_id", +) + + +def with_effective_values(queryset, select_related_fks=False): + """ + Annotate the channels queryset with `effective_*` columns that resolve to + the override value when set, otherwise fall back to the channel's own + value. Always eagerly loads the override one-to-one to avoid N+1 when the + caller reads annotated attributes and then the related override. + + Pass `select_related_fks=True` when the output path will access FK objects + through the `effective_*_obj` Channel properties; this pulls the override's + logo, channel_group, epg_data, and stream_profile in the same query so + those accessors do not trigger per-row lookups. + """ + annotations = { + f"effective_{field}": Coalesce( + f"override__{field}", + field, + ) + for field in OVERRIDABLE_FIELDS + } + qs = queryset.select_related("override").annotate(**annotations) + if select_related_fks: + qs = qs.select_related( + "override__logo", + "override__channel_group", + "override__epg_data", + "override__stream_profile", + ) + return qs diff --git a/apps/channels/migrations/0037_auto_sync_overhaul.py b/apps/channels/migrations/0037_auto_sync_overhaul.py new file mode 100644 index 00000000..d93a1789 --- /dev/null +++ b/apps/channels/migrations/0037_auto_sync_overhaul.py @@ -0,0 +1,156 @@ +""" +Auto-sync overhaul (FR #1196): per-field channel overrides, hide-from-output +flag, configurable auto-sync number range, and nullable channel_number for +compact-numbering slot release. + +Bundled operations (in forward order; reversed on rollback): + 1. AddField Channel.hidden_from_output + 2. CreateModel ChannelOverride (one-to-one Channel) + 3. AddField ChannelGroupM3UAccount.auto_sync_channel_end + 4. RunPython backfill_auto_created_by_null (orphan re-attribution / demotion) + 5. AlterField Channel.channel_number (nullable) + 6. RunPython noop / reverse_backfill_channel_number_nulls + (rollback-safety hook; runs FIRST on un-apply, fills NULLs + so step 5 reverse can re-impose NOT NULL) +""" + +import django.db.models.deletion +from django.db import migrations, models +from django.db.models import Max + + +def backfill_auto_created_by_null(apps, schema_editor): + """ + Re-attribute or demote `auto_created=True, auto_created_by=NULL` rows. + + Sync only touches rows where `auto_created_by=account`, so orphans + accumulate indefinitely. Best-effort re-attribute via the channel's + streams' single owning account; otherwise demote to manual by clearing + `auto_created`. The channel and any user customization survive; sync + will not touch a demoted row again. + """ + Channel = apps.get_model("dispatcharr_channels", "Channel") + ChannelStream = apps.get_model("dispatcharr_channels", "ChannelStream") + + orphans = Channel.objects.filter(auto_created=True, auto_created_by__isnull=True) + total = orphans.count() + if total == 0: + return + + print(f"\n Found {total} auto_created channels with NULL auto_created_by") + reattributed = 0 + demoted = 0 + + for channel in orphans.iterator(chunk_size=200): + account_ids = set( + ChannelStream.objects.filter(channel=channel) + .values_list("stream__m3u_account_id", flat=True) + ) + account_ids.discard(None) + + if len(account_ids) == 1: + channel.auto_created_by_id = next(iter(account_ids)) + channel.save(update_fields=["auto_created_by"]) + reattributed += 1 + else: + channel.auto_created = False + channel.save(update_fields=["auto_created"]) + demoted += 1 + + print( + f" Re-attributed: {reattributed}, demoted to manual " + f"(ambiguous/no streams): {demoted}" + ) + + +def reverse_auto_created_by_null(apps, schema_editor): + # Forward decisions cannot be cleanly reverted (no record of the + # original NULL state). Leaving the re-attributions and demotions in + # place is safer than restoring NULLs the schema may not accept. + pass + + +def noop(apps, schema_editor): + pass + + +def reverse_backfill_channel_number_nulls(apps, schema_editor): + """ + Rollback-safety hook for the channel_number nullable AlterField. + + Runs FIRST on un-apply (operations reverse in list order) and assigns + sequential channel numbers above the current max to any NULL rows so + the AlterField reverse (nullable to NOT NULL) succeeds without a + constraint violation. The user can re-hide or re-number these + channels after they have rolled back. + """ + Channel = apps.get_model("dispatcharr_channels", "Channel") + null_qs = Channel.objects.filter(channel_number__isnull=True) + null_count = null_qs.count() + if null_count == 0: + return + + max_num = Channel.objects.aggregate(m=Max("channel_number"))["m"] or 0.0 + next_num = float(max_num) + 1.0 + print( + f"\n Backfilling channel_number on {null_count} NULL row(s) " + f"starting at {int(next_num)} so rollback can re-impose NOT NULL" + ) + for ch in null_qs.order_by("id"): + ch.channel_number = next_num + ch.save(update_fields=["channel_number"]) + next_num += 1.0 + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '022_default_user_limit_settings'), + ('dispatcharr_channels', '0036_alter_stream_name'), + ('epg', '0022_alter_epgdata_name'), + ] + + operations = [ + migrations.AddField( + model_name='channel', + name='hidden_from_output', + field=models.BooleanField( + db_index=True, + default=False, + help_text='Exclude this channel from downstream client output (HDHR, M3U, EPG, XC). Auto-sync still updates provider metadata.', + ), + ), + migrations.CreateModel( + name='ChannelOverride', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=512, null=True)), + ('channel_number', models.FloatField(blank=True, null=True)), + ('tvg_id', models.CharField(blank=True, max_length=255, null=True)), + ('tvc_guide_stationid', models.CharField(blank=True, max_length=255, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('channel', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='override', to='dispatcharr_channels.channel')), + ('channel_group', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.channelgroup')), + ('epg_data', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='epg.epgdata')), + ('logo', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.logo')), + ('stream_profile', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.streamprofile')), + ], + ), + migrations.AddField( + model_name='channelgroupm3uaccount', + name='auto_sync_channel_end', + field=models.FloatField( + blank=True, + help_text='Optional upper bound for auto-created channel numbers in this group. Leave blank for unlimited fill. Overflow streams are skipped and reported in the completion notification.', + null=True, + ), + ), + migrations.RunPython(backfill_auto_created_by_null, reverse_auto_created_by_null), + migrations.AlterField( + model_name='channel', + name='channel_number', + field=models.FloatField(blank=True, db_index=True, null=True), + ), + migrations.RunPython(noop, reverse_backfill_channel_number_nulls), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index c0de0fe1..7bf53b73 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -291,9 +291,23 @@ class ChannelManager(models.Manager): def active(self): return self.all() + def with_effective_values(self, select_related_fks=False): + """ + Chainable shortcut for the override-aware queryset annotations, + delegating to the canonical helper in ``apps.channels.managers`` + so the function form (``with_effective_values(qs)``) and the + manager form (``Channel.objects.with_effective_values()``) are + both valid entry points and stay in sync. + """ + from apps.channels.managers import with_effective_values + + return with_effective_values( + self.get_queryset(), select_related_fks=select_related_fks + ) + class Channel(models.Model): - channel_number = models.FloatField(db_index=True) + channel_number = models.FloatField(db_index=True, null=True, blank=True) name = models.CharField(max_length=512) logo = models.ForeignKey( "Logo", @@ -360,6 +374,16 @@ class Channel(models.Model): help_text="The M3U account that auto-created this channel" ) + # Hidden channels are excluded from HDHR, M3U, EPG, and XC output queries. + # Auto-sync still recognizes them so they are not recreated when their + # underlying provider stream persists; this is an output-layer concern, not + # a sync-time flag. + hidden_from_output = models.BooleanField( + default=False, + db_index=True, + help_text="Exclude this channel from downstream client output (HDHR, M3U, EPG, XC). Auto-sync still updates provider metadata." + ) + created_at = models.DateTimeField( auto_now_add=True, help_text="Timestamp when this channel was created" @@ -369,6 +393,8 @@ class Channel(models.Model): help_text="Timestamp when this channel was last updated" ) + objects = ChannelManager() + def clean(self): # Enforce unique channel_number within a given group existing = Channel.objects.filter( @@ -384,11 +410,46 @@ class Channel(models.Model): @classmethod def get_next_available_channel_number(cls, starting_from=1): - used_numbers = set(cls.objects.all().values_list("channel_number", flat=True)) - n = starting_from - while n in used_numbers: - n += 1 - return n + # Both raw and override channel numbers are reserved. Handing out a + # raw number currently masked by an override would create a deferred + # collision once the override is cleared. + from apps.channels.compact_numbering import build_reserved_set + from apps.m3u.tasks import _next_available_number + + reserved = build_reserved_set() + return _next_available_number(reserved, starting_from) + + def _resolved_override(self): + """Return the related ChannelOverride or None, tolerant of no row.""" + try: + return self.override + except ChannelOverride.DoesNotExist: + return None + + def _resolve_effective_fk(self, field_name): + """Pick the override's FK object if set, otherwise the channel's own.""" + override = self._resolved_override() + if override is not None: + override_val = getattr(override, field_name, None) + if override_val is not None: + return override_val + return getattr(self, field_name) + + @property + def effective_logo_obj(self): + return self._resolve_effective_fk("logo") + + @property + def effective_channel_group_obj(self): + return self._resolve_effective_fk("channel_group") + + @property + def effective_epg_data_obj(self): + return self._resolve_effective_fk("epg_data") + + @property + def effective_stream_profile_obj(self): + return self._resolve_effective_fk("stream_profile") # @TODO: honor stream's stream profile def get_stream_profile(self): @@ -840,6 +901,76 @@ class Channel(models.Model): return True +class ChannelOverride(models.Model): + """ + Per-field user overrides for auto-synced channels. + + Each nullable column represents a user-provided value that takes precedence + over the matching field on the related Channel. Sync writes only to + Channel.* fields and never to this table, so provider metadata keeps + flowing while user customizations persist across refreshes. Output + querysets resolve the effective value via + `apps.channels.managers.with_effective_values`. + """ + channel = models.OneToOneField( + Channel, + on_delete=models.CASCADE, + related_name="override", + ) + name = models.CharField(max_length=512, null=True, blank=True) + channel_number = models.FloatField(null=True, blank=True) + channel_group = models.ForeignKey( + ChannelGroup, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="+", + ) + logo = models.ForeignKey( + "Logo", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="+", + ) + tvg_id = models.CharField(max_length=255, null=True, blank=True) + tvc_guide_stationid = models.CharField(max_length=255, null=True, blank=True) + epg_data = models.ForeignKey( + EPGData, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="+", + ) + stream_profile = models.ForeignKey( + StreamProfile, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="+", + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return f"Override for channel {self.channel_id}" + + def has_any_override(self) -> bool: + return any( + getattr(self, field) is not None + for field in ( + "name", + "channel_number", + "channel_group_id", + "logo_id", + "tvg_id", + "tvc_guide_stationid", + "epg_data_id", + "stream_profile_id", + ) + ) + + class ChannelProfile(models.Model): name = models.CharField(max_length=100, unique=True) @@ -887,6 +1018,12 @@ class ChannelGroupM3UAccount(models.Model): blank=True, help_text='Starting channel number for auto-created channels in this group' ) + # Optional upper bound; out-of-range streams fail. NULL = unlimited. + auto_sync_channel_end = models.FloatField( + null=True, + blank=True, + help_text='Optional upper bound for auto-created channel numbers in this group. Leave blank for unlimited fill. Overflow streams are skipped and reported in the completion notification.' + ) last_seen = models.DateTimeField( default=timezone.now, db_index=True, diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 039386a3..2c7fee3e 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -6,6 +6,7 @@ from .models import ( Stream, Channel, ChannelGroup, + ChannelOverride, ChannelStream, ChannelGroupM3UAccount, Logo, @@ -17,6 +18,7 @@ from .models import ( from apps.epg.serializers import EPGDataSerializer from core.models import StreamProfile from apps.epg.models import EPGData +from django.db import connection, transaction from django.urls import reverse from rest_framework import serializers from django.utils import timezone @@ -154,12 +156,52 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): m3u_accounts = serializers.IntegerField(source="m3u_accounts.id", read_only=True) enabled = serializers.BooleanField() auto_channel_sync = serializers.BooleanField(default=False) - auto_sync_channel_start = serializers.FloatField(allow_null=True, required=False) + auto_sync_channel_start = serializers.FloatField( + allow_null=True, required=False, min_value=1 + ) + auto_sync_channel_end = serializers.FloatField( + allow_null=True, required=False, min_value=1 + ) custom_properties = serializers.JSONField(required=False) + # Provider stream count for this group+account. Lets users size an + # optional end-range without first running a blind sync. + stream_count = serializers.SerializerMethodField() class Meta: model = ChannelGroupM3UAccount - fields = ["m3u_accounts", "channel_group", "enabled", "auto_channel_sync", "auto_sync_channel_start", "custom_properties", "is_stale", "last_seen"] + fields = [ + "m3u_accounts", + "channel_group", + "enabled", + "auto_channel_sync", + "auto_sync_channel_start", + "auto_sync_channel_end", + "custom_properties", + "is_stale", + "last_seen", + "stream_count", + ] + + def get_stream_count(self, obj): + """ + Return the number of streams for this (m3u_account, channel_group) + pair. A parent serializer (e.g. M3UAccountSerializer) may seed + ``context["stream_counts"]`` with a pre-aggregated dict keyed by + ``(m3u_account_id, channel_group_id)`` to avoid one COUNT per row; + when present, it is used as the source of truth. The per-row + COUNT fallback is correct for stand-alone serialization (rare, + low-volume) and exists so direct ChannelGroupM3UAccount queries + do not require callers to know the seeding pattern. + """ + counts = self.context.get("stream_counts") + if counts is not None: + return counts.get((obj.m3u_account_id, obj.channel_group_id), 0) + from apps.channels.models import Stream + + return Stream.objects.filter( + m3u_account_id=obj.m3u_account_id, + channel_group_id=obj.channel_group_id, + ).count() def to_representation(self, instance): data = super().to_representation(instance) @@ -179,6 +221,26 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): return super().to_internal_value(data) + def validate(self, attrs): + # Partial PATCHes only carry submitted fields; fill missing + # start/end from the instance so the validator catches a PATCH + # that lowers end past the existing start. + start = attrs.get("auto_sync_channel_start") + end = attrs.get("auto_sync_channel_end") + if start is None and self.instance is not None: + start = self.instance.auto_sync_channel_start + if end is None and self.instance is not None: + end = self.instance.auto_sync_channel_end + if start is not None and end is not None and end < start: + raise serializers.ValidationError( + { + "auto_sync_channel_end": ( + "End must be greater than or equal to start." + ) + } + ) + return super().validate(attrs) + # # Channel Group # @@ -195,12 +257,14 @@ class ChannelGroupSerializer(serializers.ModelSerializer): fields = ["id", "name", "channel_count", "m3u_account_count", "m3u_accounts"] def get_channel_count(self, obj): - """Get count of channels in this group""" - return obj.channels.count() + # Use the queryset annotation when available (list path); fall back + # to a live query for retrieve/create/update where it isn't set. + v = getattr(obj, 'channel_count', None) + return v if v is not None else obj.channels.count() def get_m3u_account_count(self, obj): - """Get count of M3U accounts associated with this group""" - return obj.m3u_accounts.count() + v = getattr(obj, 'm3u_account_count', None) + return v if v is not None else obj.m3u_accounts.count() class ChannelProfileSerializer(serializers.ModelSerializer): @@ -240,6 +304,62 @@ class BulkChannelProfileMembershipSerializer(serializers.Serializer): return value +# +# Channel override +# +# Nullable per-field overrides resolved over the parent Channel in read +# paths. Embedded in ChannelSerializer so clients can upsert/clear in the +# same PATCH that targets direct channel fields. +class ChannelOverrideSerializer(serializers.ModelSerializer): + # HDHR clients reject negative GuideNumber and zero is not a real + # provider value, so reject both at the API boundary. + channel_number = serializers.FloatField( + allow_null=True, required=False, min_value=0.0001 + ) + channel_group_id = serializers.PrimaryKeyRelatedField( + queryset=ChannelGroup.objects.all(), + source="channel_group", + allow_null=True, + required=False, + ) + logo_id = serializers.PrimaryKeyRelatedField( + queryset=Logo.objects.all(), + source="logo", + allow_null=True, + required=False, + ) + epg_data_id = serializers.PrimaryKeyRelatedField( + queryset=EPGData.objects.all(), + source="epg_data", + allow_null=True, + required=False, + ) + stream_profile_id = serializers.PrimaryKeyRelatedField( + queryset=StreamProfile.objects.all(), + source="stream_profile", + allow_null=True, + required=False, + ) + + class Meta: + model = ChannelOverride + fields = [ + "name", + "channel_number", + "channel_group_id", + "logo_id", + "tvg_id", + "tvc_guide_stationid", + "epg_data_id", + "stream_profile_id", + ] + extra_kwargs = { + "name": {"allow_null": True, "required": False}, + "tvg_id": {"allow_null": True, "required": False}, + "tvc_guide_stationid": {"allow_null": True, "required": False}, + } + + # # Channel # @@ -280,6 +400,32 @@ class ChannelSerializer(serializers.ModelSerializer): ) auto_created_by_name = serializers.SerializerMethodField() + override = ChannelOverrideSerializer( + required=False, + allow_null=True, + help_text=( + "Per-field overrides for an auto-created channel. " + 'Send {"override": {"name": "ESPN"}} to upsert the listed ' + 'fields, {"override": {"name": null}} to clear specific fields ' + 'while leaving others, or {"override": null} to delete the ' + "override row entirely. Omitting the key leaves any existing " + "override unchanged. Only valid for auto_created=True channels. " + "Duplicate channel_number values across channels are permitted; " + "downstream client behavior on duplicates varies by client." + ), + ) + source_stream = serializers.SerializerMethodField() + # Effective fields coalesce override over channel column. Consumers + # display these; raw fields remain in the response so the edit form + # can show them as "Provider: X" subtext. + effective_name = serializers.SerializerMethodField() + effective_channel_number = serializers.SerializerMethodField() + effective_channel_group_id = serializers.SerializerMethodField() + effective_logo_id = serializers.SerializerMethodField() + effective_tvg_id = serializers.SerializerMethodField() + effective_tvc_guide_stationid = serializers.SerializerMethodField() + effective_epg_data_id = serializers.SerializerMethodField() + effective_stream_profile_id = serializers.SerializerMethodField() class Meta: model = Channel @@ -297,11 +443,88 @@ class ChannelSerializer(serializers.ModelSerializer): "logo_id", "user_level", "is_adult", + "hidden_from_output", "auto_created", "auto_created_by", "auto_created_by_name", + "override", + "source_stream", + "effective_name", + "effective_channel_number", + "effective_channel_group_id", + "effective_logo_id", + "effective_tvg_id", + "effective_tvc_guide_stationid", + "effective_epg_data_id", + "effective_stream_profile_id", ] + def _effective_value(self, obj, field_name): + override = getattr(obj, "_channel_override_cache", None) + if override is None: + try: + override = obj.override + except ChannelOverride.DoesNotExist: + override = None + obj._channel_override_cache = override + if override is not None: + value = getattr(override, field_name, None) + if value is not None: + return value + return getattr(obj, field_name, None) + + def get_effective_name(self, obj): + return self._effective_value(obj, "name") + + def get_effective_channel_number(self, obj): + return self._effective_value(obj, "channel_number") + + def get_effective_channel_group_id(self, obj): + return self._effective_value(obj, "channel_group_id") + + def get_effective_logo_id(self, obj): + return self._effective_value(obj, "logo_id") + + def get_effective_tvg_id(self, obj): + return self._effective_value(obj, "tvg_id") + + def get_effective_tvc_guide_stationid(self, obj): + return self._effective_value(obj, "tvc_guide_stationid") + + def get_effective_epg_data_id(self, obj): + return self._effective_value(obj, "epg_data_id") + + def get_effective_stream_profile_id(self, obj): + return self._effective_value(obj, "stream_profile_id") + + def get_source_stream(self, obj): + """ + Return the originating provider stream for an auto-created channel. + + Surfaces the provider stream's name and owning M3U account so the + frontend can render "Auto-created from: / " + in the channel edit form. Returns None for manual channels. + """ + if not self.context.get("include_source_stream", False): + return None + if not obj.auto_created: + return None + # Viewset prefetches `channelstream_set` ordered by `order`, so + # `.all()[0]` reuses the cache and returns the lowest-order entry. + prefetched_list = list(obj.channelstream_set.all()) + if not prefetched_list: + return None + cs = prefetched_list[0] + if not cs.stream: + return None + stream = cs.stream + return { + "id": stream.id, + "name": stream.name, + "account_id": stream.m3u_account_id, + "account_name": getattr(stream.m3u_account, "name", None), + } + def to_representation(self, instance): include_streams = self.context.get("include_streams", False) @@ -309,14 +532,14 @@ class ChannelSerializer(serializers.ModelSerializer): self.fields["streams"] = serializers.SerializerMethodField() return super().to_representation(instance) else: - # Fix: For PATCH/PUT responses, ensure streams are ordered + # Read from the prefetched channelstream_set (ordered by the + # viewset's Prefetch); chaining .order_by() rebuilds the + # queryset and fires one SELECT per row in list responses. representation = super().to_representation(instance) if "streams" in representation: - representation["streams"] = list( - instance.streams.all() - .order_by("channelstream__order") - .values_list("id", flat=True) - ) + representation["streams"] = [ + cs.stream_id for cs in instance.channelstream_set.all() + ] return representation def get_logo(self, obj): @@ -330,6 +553,7 @@ class ChannelSerializer(serializers.ModelSerializer): def create(self, validated_data): streams = validated_data.pop("streams", []) + override_data = validated_data.pop("override", None) channel_number = validated_data.pop( "channel_number", Channel.get_next_available_channel_number() ) @@ -341,60 +565,146 @@ class ChannelSerializer(serializers.ModelSerializer): default_group, _ = ChannelGroup.objects.get_or_create(name="Default Group") validated_data["channel_group"] = default_group - channel = Channel.objects.create(**validated_data) + # Atomic wrapper keeps the channel insert and its override row + # in the same transaction so a failure on either rolls both back. + with transaction.atomic(): + channel = Channel.objects.create(**validated_data) - # Add streams in the specified order - for index, stream in enumerate(streams): - ChannelStream.objects.create( - channel=channel, stream_id=stream.id, order=index - ) + # Add streams in the specified order + for index, stream in enumerate(streams): + ChannelStream.objects.create( + channel=channel, stream_id=stream.id, order=index + ) + + if override_data: + # Manual channels (auto_created=False) have no provider + # value to override; reject the override payload here so a + # programmatic client can't write a semantically meaningless + # row that the frontend would then surface as "Overrides + # active". + if not channel.auto_created: + raise serializers.ValidationError( + { + "override": ( + "Cannot set override on a manual channel; " + "overrides only apply to auto-created channels." + ) + } + ) + obj = ChannelOverride.objects.create(channel=channel, **override_data) + # Drop an all-null override row; an empty override would + # falsely surface as active in the UI. + if not obj.has_any_override(): + obj.delete() return channel def update(self, instance, validated_data): + """ + PATCH handler for Channel rows. The ``override`` key carries + per-field user overrides for auto-created channels and follows + these rules: + + * key absent from payload: no change to existing overrides + * ``{"override": {"field": value}}``: upsert those fields + * ``{"override": {"field": null}}``: clear those specific fields + * ``{"override": null}``: delete the override row entirely + + Key presence is what distinguishes "no change" from "delete"; + an explicit null means delete. Override mutations are rejected + on manual channels (auto_created=False) since there is no + provider value to override. + """ streams = validated_data.pop("streams", None) + has_override_key = "override" in self.initial_data + override_data = validated_data.pop("override", None) - # Update standard fields - for attr, value in validated_data.items(): - setattr(instance, attr, value) - - instance.save() - - if streams is not None: - # Normalize stream IDs - normalized_ids = [ - stream.id if hasattr(stream, "id") else stream for stream in streams - ] - - # Get current mapping of stream_id -> ChannelStream - current_links = { - cs.stream_id: cs for cs in instance.channelstream_set.all() - } - - # Track existing stream IDs - existing_ids = set(current_links.keys()) - new_ids = set(normalized_ids) - - # Delete any links not in the new list - to_remove = existing_ids - new_ids - if to_remove: - instance.channelstream_set.filter(stream_id__in=to_remove).delete() - - # Update or create with new order - to_update = [] - for order, stream_id in enumerate(normalized_ids): - if stream_id in current_links: - cs = current_links[stream_id] - if cs.order != order: - cs.order = order - to_update.append(cs) - else: - ChannelStream.objects.create( - channel=instance, stream_id=stream_id, order=order + # Block override mutations on manual channels (no provider + # value to override). Clearing is a tolerated no-op. + if ( + has_override_key + and override_data is not None + and override_data != {} + and not instance.auto_created + ): + raise serializers.ValidationError( + { + "override": ( + "Cannot set override on a manual channel; " + "overrides only apply to auto-created channels." ) + } + ) - if to_update: - ChannelStream.objects.bulk_update(to_update, ["order"]) + # Atomic so a failure on the override row rolls back the + # channel update too. + with transaction.atomic(): + # Skip save() when only override keys were submitted; a + # no-op UPDATE would bump updated_at and bust caches. + if validated_data: + for attr, value in validated_data.items(): + setattr(instance, attr, value) + instance.save() + + if has_override_key: + if override_data is None: + # Explicit null: remove the override row. + ChannelOverride.objects.filter(channel=instance).delete() + elif override_data == {}: + # Empty dict has no field intent; no-op. + pass + else: + obj, _ = ChannelOverride.objects.update_or_create( + channel=instance, defaults=override_data + ) + # Drop an all-null override; would falsely surface + # as active in the UI. + if not obj.has_any_override(): + obj.delete() + # Queryset writes leave the reverse-OneToOne cache stale; + # clear it so to_representation reads the new state. + try: + instance._state.fields_cache.pop("override", None) + except AttributeError: + pass + if hasattr(instance, "_channel_override_cache"): + delattr(instance, "_channel_override_cache") + + if streams is not None: + # Normalize stream IDs + normalized_ids = [ + stream.id if hasattr(stream, "id") else stream for stream in streams + ] + + # Get current mapping of stream_id -> ChannelStream + current_links = { + cs.stream_id: cs for cs in instance.channelstream_set.all() + } + + # Track existing stream IDs + existing_ids = set(current_links.keys()) + new_ids = set(normalized_ids) + + # Delete any links not in the new list + to_remove = existing_ids - new_ids + if to_remove: + instance.channelstream_set.filter(stream_id__in=to_remove).delete() + + # Update or create with new order + to_update = [] + for order, stream_id in enumerate(normalized_ids): + if stream_id in current_links: + cs = current_links[stream_id] + if cs.order != order: + cs.order = order + to_update.append(cs) + else: + ChannelStream.objects.create( + channel=instance, stream_id=stream_id, order=order + ) + + if to_update: + ChannelStream.objects.bulk_update(to_update, ["order"]) return instance diff --git a/apps/channels/signals.py b/apps/channels/signals.py index dd2ac158..2eecbebf 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -1,6 +1,6 @@ # apps/channels/signals.py -from django.db.models.signals import m2m_changed, pre_save, post_save, post_delete +from django.db.models.signals import m2m_changed, pre_save, post_save, post_delete, pre_delete from django.dispatch import receiver from django.utils.timezone import now, is_aware, make_aware from celery.result import AsyncResult @@ -60,6 +60,137 @@ def generate_custom_stream_hash(sender, instance, created, **kwargs): # Use update to avoid triggering signals again Stream.objects.filter(id=instance.id).update(stream_hash=instance.stream_hash) +@receiver(pre_delete, sender=Channel) +def stop_proxy_session_before_channel_delete(sender, instance, **kwargs): + """ + When a Channel is deleted, stop any active TS proxy session for it first. + + Without this, the proxy's Redis state (ts_proxy:channel:{uuid}:*) survives + the Channel row and the connected clients' "Stop" button hits + `ChannelService.stop_channel(uuid)`, which calls `Channel.objects.get(uuid=...)` + and crashes with DoesNotExist (reported as 'Channel not found' in the UI). + Users then cannot close the stream; source-side connection limits stay + consumed. Covers manual deletes, bulk deletes, and sync-driven deletes + via the same signal path. + """ + try: + from apps.proxy.ts_proxy.services.channel_service import ChannelService + + channel_uuid = str(instance.uuid) if instance.uuid else None + if not channel_uuid: + return + # Best-effort: if the channel has no active session the service + # returns a benign 'Channel not found' result, which is ignored. + ChannelService.stop_channel(channel_uuid) + except Exception as e: + # Never block a channel delete on proxy cleanup failure. Log and + # continue so at least the DB row is removed. + logger.warning( + "Failed to stop proxy session before deleting channel %s: %s", + getattr(instance, "id", ""), + e, + ) + + +@receiver(post_save, sender=Channel) +def assign_compact_number_on_unhide(sender, instance, created, **kwargs): + """When a channel transitions from hidden to visible under compact + numbering, immediately assign the next available number from its + group's range. Without this, an unhide would leave the channel at + NULL until the next M3U refresh, which is too long a delay for what + is meant to feel like an instant action. + + Bails out for the common cases where assignment is not appropriate: + a fresh channel (newly created), a channel that already has a + number, a hidden channel, a manual channel, or a channel whose group + is not in compact mode. The assignment helper does the same checks + defensively, but bailing here keeps the signal out of the helper + code path on every channel save. + """ + if created: + return + # Skip the signal when update_fields proves hidden_from_output was not + # touched. Sync sets update_fields on every save, so this keeps the + # signal off the sync hot path. + update_fields = kwargs.get("update_fields") + if update_fields is not None and "hidden_from_output" not in update_fields: + return + if instance.hidden_from_output: + return + if instance.channel_number is not None: + return + if not instance.auto_created or not instance.auto_created_by_id: + return + try: + from .compact_numbering import assign_compact_numbers_for_channels + + assign_compact_numbers_for_channels([instance.id]) + # The helper writes via queryset .update() which skips + # in-memory state. Reload so a same-request serializer + # response carries the assigned number, not stale None. + try: + instance.refresh_from_db(fields=["channel_number"]) + except Exception: + # Refresh failure (race with delete) is non-fatal. + pass + except Exception as e: + # Do not propagate. The save succeeded; the assignment is + # recoverable on next sync or manual repack. + logger.warning( + "Compact unhide assignment failed for channel %s: %s", + instance.id, + e, + ) + + +@receiver(post_save, sender=Channel) +def release_compact_number_on_hide(sender, instance, created, **kwargs): + """When a channel transitions from visible to hidden under compact + numbering, immediately release its channel_number slot. Without + this, the slot stays occupied until the next sync's repack pass, + so the user sees their hidden channels still consuming numbers in + the table for an indeterminate window. Mirror image of + `assign_compact_number_on_unhide` above. + + Bails out for the common cases where release is not appropriate: + a fresh channel (newly created), a non-hidden channel, a channel + that has no number to release, a manual channel, a channel without + a known auto_created_by, or a channel whose group is not in + compact mode. + """ + if created: + return + update_fields = kwargs.get("update_fields") + if update_fields is not None and "hidden_from_output" not in update_fields: + return + if not instance.hidden_from_output: + return + if instance.channel_number is None: + return + if not instance.auto_created or not instance.auto_created_by_id: + return + try: + from .compact_numbering import ( + get_group_relation_for_channel, + is_compact_group, + ) + + relation = get_group_relation_for_channel(instance) + if not relation or not is_compact_group(relation): + return + except Exception: + return + # Release the slot. Queryset .update() bypasses the post_save signal + # chain that this handler is itself running inside. + Channel.objects.filter(id=instance.id).update(channel_number=None) + # Refresh in-memory instance so the same-request serializer + # response surfaces channel_number=None instead of stale value. + try: + instance.refresh_from_db(fields=["channel_number"]) + except Exception: + pass + + @receiver(post_save, sender=Channel) def refresh_epg_programs(sender, instance, created, **kwargs): """ diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 08c7737b..b039d478 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1136,6 +1136,15 @@ def _evaluate_series_rules_locked(tvg_id, result): now = timezone.now() horizon = now + timedelta(days=7) + try: + pre_min = int(CoreSettings.get_dvr_pre_offset_minutes()) + except Exception: + pre_min = 0 + try: + post_min = int(CoreSettings.get_dvr_post_offset_minutes()) + except Exception: + post_min = 0 + # Preload existing recordings keyed by stable program attributes that # survive EPG refreshes (tvg_id + original start/end times stored in # custom_properties). ProgramData.id changes on every EPG refresh so @@ -1160,37 +1169,74 @@ def _evaluate_series_rules_locked(tvg_id, result): rv_tvg = str(rule.get("tvg_id") or "").strip() mode = (rule.get("mode") or "all").lower() series_title = (rule.get("title") or "").strip() - norm_series = normalize_name(series_title) if series_title else None - if not rv_tvg: + title_mode = (rule.get("title_mode") or "exact").lower() + description = (rule.get("description") or "").strip() + description_mode = (rule.get("description_mode") or "contains").lower() + try: + pinned_channel_id = int(rule["channel_id"]) if rule.get("channel_id") not in (None, "") else None + except (TypeError, ValueError): + pinned_channel_id = None + if not series_title and not description: result["details"].append({"tvg_id": rv_tvg, "status": "invalid_rule"}) continue - epg = EPGData.objects.filter(tvg_id=rv_tvg).first() - if not epg: - result["details"].append({"tvg_id": rv_tvg, "status": "no_epg_match"}) - continue - - programs_qs = ProgramData.objects.filter( + if rv_tvg: + epg = EPGData.objects.filter(tvg_id=rv_tvg).first() + if not epg: + result["details"].append({"tvg_id": rv_tvg, "status": "no_epg_match"}) + continue + programs_qs = ProgramData.objects.filter( epg=epg, end_time__gt=now, start_time__lte=horizon, ) - if series_title: - programs_qs = programs_qs.filter(title__iexact=series_title) - programs = list(programs_qs.order_by("start_time")) - # Fallback: if no direct matches and we have a title, try normalized comparison in Python - if series_title and not programs: - all_progs = ProgramData.objects.filter( - epg=epg, + else: + epg = None + programs_qs = ProgramData.objects.select_related("epg").filter( end_time__gt=now, start_time__lte=horizon, - ).only("id", "title", "start_time", "end_time", "custom_properties", "tvg_id") - programs = [p for p in all_progs if normalize_name(p.title) == norm_series] + ) - channel = Channel.objects.filter(epg_data=epg).order_by("channel_number").first() - if not channel: - result["details"].append({"tvg_id": rv_tvg, "status": "no_channel_for_epg"}) - continue + from apps.epg.query_utils import parse_text_query + + if series_title: + if title_mode == "exact": + programs_qs = programs_qs.filter(title__iexact=series_title) + else: + use_regex = title_mode == "regex" + whole_words = title_mode == "search" + programs_qs = programs_qs.filter( + parse_text_query("title", series_title, use_regex=use_regex, whole_words=whole_words) + ) + + if description: + use_regex = description_mode == "regex" + whole_words = description_mode == "search" + programs_qs = programs_qs.filter( + parse_text_query("description", description, use_regex=use_regex, whole_words=whole_words) + ) + + programs = list(programs_qs.distinct().order_by("start_time")) + + if pinned_channel_id is not None: + pinned_channel = Channel.objects.filter(id=pinned_channel_id).first() + if pinned_channel is None: + result["details"].append({"tvg_id": rv_tvg, "status": "pinned_channel_missing", "channel_id": pinned_channel_id}) + continue + channels_by_epg_id = None + elif rv_tvg: + pinned_channel = Channel.objects.filter(epg_data=epg).order_by("channel_number").first() + if not pinned_channel: + result["details"].append({"tvg_id": rv_tvg, "status": "no_channel_for_epg"}) + continue + channels_by_epg_id = None + else: + pinned_channel = None + epg_ids = {p.epg_id for p in programs} + channels_by_epg_id = {} + for ch in Channel.objects.filter(epg_data_id__in=epg_ids).order_by("channel_number"): + if ch.epg_data_id not in channels_by_epg_id: + channels_by_epg_id[ch.epg_data_id] = ch # # Many providers list multiple future airings of the same episode @@ -1247,6 +1293,12 @@ def _evaluate_series_rules_locked(tvg_id, result): created_here = 0 for prog in unique_programs: try: + if pinned_channel is not None: + rec_channel = pinned_channel + else: + rec_channel = channels_by_epg_id.get(prog.epg_id) + if rec_channel is None: + continue # Skip if a recording already exists for this exact airing # (keyed by tvg_id + original program times, which are stable # across EPG refreshes unlike ProgramData.id). @@ -1266,16 +1318,6 @@ def _evaluate_series_rules_locked(tvg_id, result): except Exception: continue # already scheduled/recorded - # Apply global DVR pre/post offsets (in minutes) - try: - pre_min = int(CoreSettings.get_dvr_pre_offset_minutes()) - except Exception: - pre_min = 0 - try: - post_min = int(CoreSettings.get_dvr_post_offset_minutes()) - except Exception: - post_min = 0 - adj_start = prog.start_time adj_end = prog.end_time try: @@ -1290,7 +1332,7 @@ def _evaluate_series_rules_locked(tvg_id, result): pass rec = Recording.objects.create( - channel=channel, + channel=rec_channel, start_time=adj_start, end_time=adj_end, custom_properties={ diff --git a/apps/channels/tests/test_channel_api.py b/apps/channels/tests/test_channel_api.py index bb245da1..363f5925 100644 --- a/apps/channels/tests/test_channel_api.py +++ b/apps/channels/tests/test_channel_api.py @@ -1,9 +1,11 @@ from django.test import TestCase from django.contrib.auth import get_user_model +from django.utils import timezone +from datetime import timedelta from rest_framework.test import APIClient from rest_framework import status -from apps.channels.models import Channel, ChannelGroup +from apps.channels.models import Channel, ChannelGroup, ChannelOverride User = get_user_model() @@ -209,3 +211,296 @@ class ChannelBulkEditAPITests(TestCase): self.assertEqual(self.channel1.name, "Only Name Changed") self.assertEqual(self.channel1.channel_number, original_channel_number) self.assertEqual(self.channel1.tvg_id, original_tvg_id) + + def test_bulk_swap_clear_and_assign_same_number(self): + # User clears channel A's override (which currently pins #10) and + # in the same bulk request sets channel B's override.channel_number + # to #10. Both halves of the swap must succeed; the resulting + # state has A unpinned and B pinned at #10. + auto_a = Channel.objects.create( + channel_number=1.0, + name="Auto A", + tvg_id="auto_a", + channel_group=self.group1, + auto_created=True, + ) + ChannelOverride.objects.create(channel=auto_a, channel_number=10.0) + auto_b = Channel.objects.create( + channel_number=2.0, + name="Auto B", + tvg_id="auto_b", + channel_group=self.group1, + auto_created=True, + ) + + data = [ + {"id": auto_a.id, "override": None}, + {"id": auto_b.id, "override": {"channel_number": 10.0}}, + ] + response = self.client.patch(self.bulk_edit_url, data, format="json") + + self.assertEqual( + response.status_code, + status.HTTP_200_OK, + f"Expected 200; got {response.status_code} body={response.data}", + ) + self.assertFalse( + ChannelOverride.objects.filter(channel=auto_a).exists() + ) + b_override = ChannelOverride.objects.get(channel=auto_b) + self.assertEqual(b_override.channel_number, 10.0) + + +class ChannelSummaryEffectiveValuesTests(TestCase): + """ + The /api/channels/channels/summary/ endpoint feeds the TV Guide. + Like every downstream output surface, it must reflect the user's + overrides (name, channel_number, logo_id, epg_data_id, + channel_group_id) instead of the raw provider values, otherwise + the in-app guide would silently disagree with HDHR / M3U / EPG / + XC clients on the same channel set. + """ + + def setUp(self): + from django.contrib.auth import get_user_model + from rest_framework.test import APIClient + from apps.channels.models import ChannelOverride + + User = get_user_model() + self.user = User.objects.create_user( + username="summary_admin", password="x" + ) + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + + self.group = ChannelGroup.objects.create(name="Summary Group") + self.other_group = ChannelGroup.objects.create(name="Other") + self.channel = Channel.objects.create( + channel_number=10.0, + name="Provider Name", + channel_group=self.group, + auto_created=True, + ) + ChannelOverride.objects.create( + channel=self.channel, + name="Override Name", + channel_number=99.0, + channel_group=self.other_group, + ) + + def test_summary_returns_effective_values(self): + response = self.client.get("/api/channels/channels/summary/") + self.assertEqual(response.status_code, 200) + row = next(r for r in response.data if r["id"] == self.channel.id) + self.assertEqual(row["name"], "Override Name") + self.assertEqual(row["channel_number"], 99.0) + self.assertEqual(row["channel_group_id"], self.other_group.id) + + +class ChannelManagerEffectiveValuesTests(TestCase): + """ + The chainable ``Channel.objects.with_effective_values()`` shortcut + must return rows with the same ``effective_*`` annotations the + module-level helper produces, since both forms are documented + entry points and a divergence would silently change output for + one set of callers. + """ + + def test_manager_shortcut_matches_module_helper(self): + from apps.channels.managers import with_effective_values + + group = ChannelGroup.objects.create(name="Manager Test") + channel = Channel.objects.create( + channel_number=42.0, + name="Original Name", + channel_group=group, + auto_created=True, + ) + ChannelOverride.objects.create( + channel=channel, + name="Renamed", + channel_number=99.0, + ) + + helper_row = with_effective_values( + Channel.objects.filter(id=channel.id) + ).get() + shortcut_row = ( + Channel.objects.with_effective_values() + .filter(id=channel.id) + .get() + ) + + self.assertEqual(helper_row.effective_name, "Renamed") + self.assertEqual(shortcut_row.effective_name, "Renamed") + self.assertEqual(helper_row.effective_channel_number, 99.0) + self.assertEqual(shortcut_row.effective_channel_number, 99.0) + self.assertEqual( + helper_row.effective_channel_group_id, + shortcut_row.effective_channel_group_id, + ) + + +class SeriesRuleAPITests(TestCase): + """API tests for series rule CRUD and bulk-remove endpoints.""" + + def setUp(self): + User = get_user_model() + self.admin = User.objects.create_user(username="admin_sr", password="pass") + self.admin.user_level = 10 + self.admin.save() + self.client = APIClient() + self.client.force_authenticate(user=self.admin) + + from core.models import CoreSettings + CoreSettings.set_dvr_series_rules([]) + + self.rules_url = "/api/channels/series-rules/" + self.bulk_remove_url = "/api/channels/series-rules/bulk-remove/" + + # --- POST (create/upsert) --- + + def test_create_rule_with_tvg_id(self): + resp = self.client.post(self.rules_url, { + "tvg_id": "some.channel", "title": "My Show", "mode": "all", + }, format="json") + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(len(resp.data["rules"]), 1) + self.assertEqual(resp.data["rules"][0]["tvg_id"], "some.channel") + + def test_create_title_only_rule_no_tvg_id(self): + """A rule with no tvg_id (title-only) is accepted when title is provided.""" + resp = self.client.post(self.rules_url, { + "tvg_id": "", "title": "Untethered Show", "mode": "all", + }, format="json") + self.assertEqual(resp.status_code, status.HTTP_200_OK) + rule = resp.data["rules"][0] + self.assertEqual(rule["tvg_id"], "") + self.assertEqual(rule["title"], "Untethered Show") + + def test_create_rule_requires_title_or_description(self): + resp = self.client.post(self.rules_url, { + "tvg_id": "some.channel", "title": "", "description": "", + }, format="json") + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + + def test_upsert_key_is_tvg_id_and_title(self): + """Two POST requests with same tvg_id but different titles create two rules.""" + self.client.post(self.rules_url, { + "tvg_id": "ch.1", "title": "Show A", "mode": "all", + }, format="json") + self.client.post(self.rules_url, { + "tvg_id": "ch.1", "title": "Show B", "mode": "all", + }, format="json") + resp = self.client.get(self.rules_url) + self.assertEqual(len(resp.data["rules"]), 2) + + def test_upsert_updates_existing_rule(self): + """POSTing with an existing (tvg_id, title) pair updates in place.""" + self.client.post(self.rules_url, { + "tvg_id": "ch.1", "title": "Show A", "mode": "all", + }, format="json") + self.client.post(self.rules_url, { + "tvg_id": "ch.1", "title": "Show A", "mode": "new", + }, format="json") + resp = self.client.get(self.rules_url) + self.assertEqual(len(resp.data["rules"]), 1) + self.assertEqual(resp.data["rules"][0]["mode"], "new") + + # --- DELETE (query params) --- + + def test_delete_rule_by_tvg_id_and_title(self): + self.client.post(self.rules_url, { + "tvg_id": "ch.1", "title": "Show A", "mode": "all", + }, format="json") + resp = self.client.delete( + self.rules_url + "?tvg_id=ch.1&title=Show+A" + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(resp.data["rules"], []) + + def test_delete_title_only_rule(self): + """Title-only rules (tvg_id='') are deleted via empty tvg_id query param.""" + self.client.post(self.rules_url, { + "tvg_id": "", "title": "Untethered Show", "mode": "all", + }, format="json") + resp = self.client.delete( + self.rules_url + "?tvg_id=&title=Untethered+Show" + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(resp.data["rules"], []) + + def test_delete_only_removes_matching_rule(self): + """Delete by (tvg_id, title) leaves other rules intact.""" + self.client.post(self.rules_url, {"tvg_id": "ch.1", "title": "Show A", "mode": "all"}, format="json") + self.client.post(self.rules_url, {"tvg_id": "ch.1", "title": "Show B", "mode": "all"}, format="json") + self.client.delete(self.rules_url + "?tvg_id=ch.1&title=Show+A") + resp = self.client.get(self.rules_url) + self.assertEqual(len(resp.data["rules"]), 1) + self.assertEqual(resp.data["rules"][0]["title"], "Show B") + + def test_delete_removes_future_recordings(self): + """DELETE cleans up future recordings that matched the rule.""" + from apps.channels.models import Recording + + group = ChannelGroup.objects.create(name="G") + channel = Channel.objects.create(channel_number=1, name="Ch", channel_group=group) + now = timezone.now() + Recording.objects.create( + channel=channel, + start_time=now + timedelta(hours=1), + end_time=now + timedelta(hours=2), + custom_properties={"program": {"tvg_id": "ch.1", "title": "Show A"}}, + ) + + self.client.post(self.rules_url, {"tvg_id": "ch.1", "title": "Show A", "mode": "all"}, format="json") + self.client.delete(self.rules_url + "?tvg_id=ch.1&title=Show+A") + self.assertEqual(Recording.objects.count(), 0) + + # --- POST bulk-remove --- + + def test_bulk_remove_with_tvg_id(self): + from apps.channels.models import Recording + + group = ChannelGroup.objects.create(name="G2") + channel = Channel.objects.create(channel_number=2, name="Ch2", channel_group=group) + now = timezone.now() + Recording.objects.create( + channel=channel, + start_time=now + timedelta(hours=1), + end_time=now + timedelta(hours=2), + custom_properties={"program": {"tvg_id": "ch.x", "title": "Show X"}}, + ) + resp = self.client.post(self.bulk_remove_url, {"tvg_id": "ch.x"}, format="json") + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(resp.data["removed"], 1) + self.assertEqual(Recording.objects.count(), 0) + + def test_bulk_remove_title_only_no_tvg_id(self): + """Bulk-remove accepts title alone (no tvg_id) for title-only rules.""" + from apps.channels.models import Recording + + group = ChannelGroup.objects.create(name="G3") + channel = Channel.objects.create(channel_number=3, name="Ch3", channel_group=group) + now = timezone.now() + Recording.objects.create( + channel=channel, + start_time=now + timedelta(hours=1), + end_time=now + timedelta(hours=2), + custom_properties={"program": {"tvg_id": "ch.a", "title": "Cross Show"}}, + ) + Recording.objects.create( + channel=channel, + start_time=now + timedelta(hours=3), + end_time=now + timedelta(hours=4), + custom_properties={"program": {"tvg_id": "ch.b", "title": "Cross Show"}}, + ) + resp = self.client.post(self.bulk_remove_url, {"title": "Cross Show"}, format="json") + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(resp.data["removed"], 2) + + def test_bulk_remove_requires_tvg_id_or_title(self): + resp = self.client.post(self.bulk_remove_url, {}, format="json") + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/apps/channels/tests/test_series_rule_dedup.py b/apps/channels/tests/test_series_rule_dedup.py index 33aff695..9c498f99 100644 --- a/apps/channels/tests/test_series_rule_dedup.py +++ b/apps/channels/tests/test_series_rule_dedup.py @@ -716,3 +716,121 @@ class NonSeriesRecordingTests(SeriesRuleDedupBaseTestCase): prog = self._create_program(hours_from_now=2) result = evaluate_series_rules_impl() self.assertEqual(result["scheduled"], 1) + + +# --------------------------------------------------------------------------- +# Title-only rule tests: tvg_id is empty / omitted (cross-EPG matching) +# --------------------------------------------------------------------------- + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class TitleOnlyRuleTests(SeriesRuleDedupBaseTestCase): + """Tests for series rules where tvg_id is omitted (searches all EPG channels).""" + + def setUp(self): + super().setUp() + _set_series_rules([{ + "tvg_id": "", + "mode": "all", + "title": "Test Show", + }]) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_title_only_rule_schedules_recording(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """A rule with no tvg_id matches programs on any EPG channel by title.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2) + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 1) + self.assertEqual(Recording.objects.first().channel, self.channel) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_title_only_rule_matches_across_multiple_epg_channels( + self, mock_release, mock_lock, mock_schedule, mock_artwork + ): + """Title-only rule creates a recording per matching EPG channel (distinct programs).""" + from apps.channels.tasks import evaluate_series_rules_impl + + epg2 = EPGData.objects.create( + tvg_id="test.channel.2", name="Channel 2 EPG", + epg_source=self.epg_source, + ) + Channel.objects.create( + channel_number=2, name="Test Channel 2", epg_data=epg2 + ) + + start1 = self.now + timedelta(hours=2) + ProgramData.objects.create( + epg=self.epg, tvg_id="test.channel.1", + start_time=start1, end_time=start1 + timedelta(hours=1), + title="Test Show", sub_title="Episode 1", + ) + start2 = self.now + timedelta(hours=3) + ProgramData.objects.create( + epg=epg2, tvg_id="test.channel.2", + start_time=start2, end_time=start2 + timedelta(hours=1), + title="Test Show", sub_title="Episode 2", + ) + + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 2) + self.assertEqual(Recording.objects.count(), 2) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_title_only_rule_dedup_after_epg_refresh(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Dedup works for title-only rules after EPG refresh reassigns program IDs.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + result = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + self.assertEqual(result["scheduled"], 0) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_invalid_rule_no_title_no_description_skipped( + self, mock_release, mock_lock, mock_schedule, mock_artwork + ): + """Rules with neither title nor description are skipped and flagged invalid.""" + from apps.channels.tasks import evaluate_series_rules_impl + + _set_series_rules([{"tvg_id": "", "mode": "all", "title": "", "description": ""}]) + self._create_program(hours_from_now=2) + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 0) + statuses = [d.get("status") for d in result["details"]] + self.assertIn("invalid_rule", statuses) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_program_on_epg_with_no_channel_skipped( + self, mock_release, mock_lock, mock_schedule, mock_artwork + ): + """Programs on an EPG source that has no Channel assigned are skipped gracefully.""" + from apps.channels.tasks import evaluate_series_rules_impl + + epg_orphan = EPGData.objects.create( + tvg_id="orphan.channel", name="Orphan EPG", + epg_source=self.epg_source, + ) + start = self.now + timedelta(hours=2) + ProgramData.objects.create( + epg=epg_orphan, tvg_id="orphan.channel", + start_time=start, end_time=start + timedelta(hours=1), + title="Test Show", sub_title="Episode 1", + ) + + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 0) + self.assertEqual(Recording.objects.count(), 0) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index 70358cee..f41c2b72 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -4,6 +4,18 @@ lock = threading.Lock() # Dictionary to track usage: {account_id: current_usage} active_streams_map = {} + +def format_channel_number(value, empty=""): + """Display formatting for an effective channel_number. Returns int for + whole-valued floats (so ``123.0`` renders as ``123``), the float as-is + for fractional values, or ``empty`` when the value is ``None``. + """ + if value is None: + return empty + if value == int(value): + return int(value) + return value + def increment_stream_count(account): with lock: current_usage = active_streams_map.get(account.id, 0) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 33483e60..296f91b8 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -1,11 +1,14 @@ -import logging, os +import logging, os, re from rest_framework import viewsets, status, serializers +from rest_framework.pagination import PageNumberPagination from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.decorators import action from drf_spectacular.utils import extend_schema, OpenApiParameter, inline_serializer from drf_spectacular.types import OpenApiTypes +from django.db.models import Q from django.utils import timezone +from django.utils.dateparse import parse_datetime from datetime import timedelta from .models import EPGSource, ProgramData, EPGData from .serializers import ( @@ -13,10 +16,13 @@ from .serializers import ( ProgramDetailSerializer, EPGSourceSerializer, EPGDataSerializer, + ProgramSearchResultSerializer, ) from .tasks import refresh_epg_data +from .query_utils import parse_text_query from apps.accounts.permissions import ( Authenticated, + IsStandardUser, permission_classes_by_action, permission_classes_by_method, ) @@ -105,6 +111,12 @@ class EPGSourceViewSet(viewsets.ModelViewSet): # ───────────────────────────── # 2) Program API (CRUD) # ───────────────────────────── +class ProgramSearchPagination(PageNumberPagination): + page_size = 50 + page_size_query_param = 'page_size' + max_page_size = 500 + + class ProgramViewSet(viewsets.ModelViewSet): """Handles CRUD operations for EPG programs""" @@ -131,6 +143,219 @@ class ProgramViewSet(viewsets.ModelViewSet): logger.debug("Listing all EPG programs.") return super().list(request, *args, **kwargs) + @extend_schema( + summary="Search EPG programs", + description=""" +**Advanced EPG program search with multiple filter types and complex query support.** + +### Text Search Features + +**Title and Description Search**: +- Supports AND/OR logical operators (case-insensitive: `and`/`AND` both work) +- Wrap phrases in double quotes to match them literally: `"Law and Order"` +- Parenthetical grouping for complex queries: `(Newcastle OR NEW) AND (Villa OR AST)` +- Regex pattern matching with `title_regex=true` (evaluated by the database engine) +- Whole word matching with `title_whole_words=true` to avoid partial matches + +**Examples**: +- Simple: `title=football` +- AND operator: `title=premier AND league` +- OR operator: `title=Newcastle OR Villa` +- Quoted phrase: `title="Law and Order"` (matches the exact phrase; 'and' is literal) +- Mixed: `title="Law and Order" AND crime` +- Nested groups: `title=(Newcastle OR NEW) AND (Villa OR AST)` +- Regex: `title=^Premier&title_regex=true` (programs starting with "Premier") +- Whole words: `title=NEW&title_whole_words=true` (matches "NEW" but not "News") + +### Time Filtering + +**airing_at**: Find programs airing at a specific moment (start_time ≤ airing_at < end_time) + +**Time ranges**: Use combinations of start_after, start_before, end_after, end_before + +### Response Customization + +**fields**: Comma-separated list to include only specific fields in response +- Available: id, title, sub_title, description, start_time, end_time, tvg_id, custom_properties, epg_source, epg_name, epg_icon_url, channels, streams + +### Pagination + +- Default: 50 results per page +- Maximum: 500 results per page +- Use `page` and `page_size` parameters to navigate results + """, + parameters=[ + OpenApiParameter( + 'title', + OpenApiTypes.STR, + description='Title search query. Supports AND/OR operators (case-insensitive), quoted phrases, and parentheses. Double-quote a phrase to match it literally: `"Law and Order"`. Unquoted space-separated terms are matched as a phrase; use AND/OR to combine separate terms.', + ), + OpenApiParameter('title_regex', OpenApiTypes.BOOL, description='Enable regex matching for title (case-insensitive, default: false). e.g. `^The` matches titles starting with "The".'), + OpenApiParameter('title_whole_words', OpenApiTypes.BOOL, description='Match whole words only in title (default: false). e.g. `new` matches "Newcastle" normally but not with whole words enabled.'), + OpenApiParameter( + 'description', + OpenApiTypes.STR, + description='Description search query. Same syntax and features as title search.' + ), + OpenApiParameter('description_regex', OpenApiTypes.BOOL, description='Enable regex matching for description (case-insensitive, default: false).'), + OpenApiParameter('description_whole_words', OpenApiTypes.BOOL, description='Match whole words only in description (default: false). Same behaviour as title_whole_words.'), + OpenApiParameter('start_after', OpenApiTypes.DATETIME, description='Filter programs starting at or after this time. ISO 8601 format, e.g. `2026-02-14T18:00:00Z`.'), + OpenApiParameter('start_before', OpenApiTypes.DATETIME, description='Filter programs starting at or before this time. ISO 8601 format.'), + OpenApiParameter('end_after', OpenApiTypes.DATETIME, description='Filter programs ending at or after this time. ISO 8601 format.'), + OpenApiParameter('end_before', OpenApiTypes.DATETIME, description='Filter programs ending at or before this time. ISO 8601 format.'), + OpenApiParameter('airing_at', OpenApiTypes.DATETIME, description='Find programs airing at this exact moment (start_time ≤ airing_at < end_time). ISO 8601 format, e.g. `2026-02-14T20:00:00Z`.'), + OpenApiParameter('channel', OpenApiTypes.STR, description='Filter by channel name (case-insensitive substring match). e.g. `BBC One`, `Sky Sports`.'), + OpenApiParameter('channel_id', OpenApiTypes.INT, description='Filter by exact channel ID.'), + OpenApiParameter('tvg_id', OpenApiTypes.STR, description='Filter by EPG tvg_id (exact match). e.g. `bbcone.uk`.'), + OpenApiParameter('stream', OpenApiTypes.STR, description='Filter by stream name (case-insensitive substring match).'), + OpenApiParameter('group', OpenApiTypes.STR, description='Filter by channel group or stream group name (case-insensitive substring match). e.g. `Sports`, `UK Channels`.'), + OpenApiParameter('epg_source', OpenApiTypes.INT, description='Filter by EPG source ID.'), + OpenApiParameter('fields', OpenApiTypes.STR, description='Comma-separated list of fields to include. Omit to return all fields. e.g. `title,start_time,end_time`.'), + OpenApiParameter('page', OpenApiTypes.INT, description='Page number for pagination (default: 1).'), + OpenApiParameter('page_size', OpenApiTypes.INT, description='Results per page (default: 50, max: 500).'), + ], + responses={200: ProgramSearchResultSerializer(many=True)}, + tags=['EPG'], + ) + @action(detail=False, methods=['get'], url_path='search', permission_classes=[IsStandardUser]) + def search(self, request): + params = request.query_params + + # Build base queryset with prefetching + queryset = ProgramData.objects.select_related( + 'epg', 'epg__epg_source' + ).prefetch_related( + 'epg__channels', 'epg__channels__channel_group', + 'epg__channels__streams', 'epg__channels__streams__channel_group', + 'epg__channels__streams__m3u_account', + ) + + filters = Q() + + # Text filters + title = params.get('title') + if title: + title_regex = params.get('title_regex', '').lower() in ('true', '1', 'yes') + title_whole_words = params.get('title_whole_words', '').lower() in ('true', '1', 'yes') + filters &= parse_text_query('title', title, use_regex=title_regex, whole_words=title_whole_words) + + description = params.get('description') + if description: + desc_regex = params.get('description_regex', '').lower() in ('true', '1', 'yes') + desc_whole_words = params.get('description_whole_words', '').lower() in ('true', '1', 'yes') + filters &= parse_text_query('description', description, use_regex=desc_regex, whole_words=desc_whole_words) + + # Time filters with validation + start_after = params.get('start_after') + if start_after: + dt = parse_datetime(start_after) + if dt is None: + return Response( + {"error": f"Invalid datetime format for start_after: {start_after}. Use ISO 8601 format (e.g., 2026-02-14T18:00:00Z)"}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__gte=dt) + + start_before = params.get('start_before') + if start_before: + dt = parse_datetime(start_before) + if dt is None: + return Response( + {"error": f"Invalid datetime format for start_before: {start_before}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__lte=dt) + + end_after = params.get('end_after') + if end_after: + dt = parse_datetime(end_after) + if dt is None: + return Response( + {"error": f"Invalid datetime format for end_after: {end_after}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(end_time__gte=dt) + + end_before = params.get('end_before') + if end_before: + dt = parse_datetime(end_before) + if dt is None: + return Response( + {"error": f"Invalid datetime format for end_before: {end_before}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(end_time__lte=dt) + + airing_at = params.get('airing_at') + if airing_at: + dt = parse_datetime(airing_at) + if dt is None: + return Response( + {"error": f"Invalid datetime format for airing_at: {airing_at}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__lte=dt, end_time__gt=dt) + + # Channel/stream filters + channel = params.get('channel') + if channel: + filters &= Q(epg__channels__name__icontains=channel) + + channel_id = params.get('channel_id') + if channel_id: + try: + filters &= Q(epg__channels__id=int(channel_id)) + except (ValueError, TypeError): + pass + + tvg_id = params.get('tvg_id') + if tvg_id: + filters &= Q(epg__tvg_id=tvg_id) + + stream = params.get('stream') + if stream: + filters &= Q(epg__channels__streams__name__icontains=stream) + + group = params.get('group') + if group: + filters &= ( + Q(epg__channels__channel_group__name__icontains=group) + | Q(epg__channels__streams__channel_group__name__icontains=group) + ) + + epg_source = params.get('epg_source') + if epg_source: + try: + filters &= Q(epg__epg_source__id=int(epg_source)) + except (ValueError, TypeError): + pass + + queryset = queryset.filter(filters).distinct().order_by('start_time') + + # Restrict results to programs on channels the user can access + user = request.user + if user.user_level < 10: + access_filter = Q(epg__channels__user_level__lte=user.user_level) + custom_props = user.custom_properties or {} + if custom_props.get('hide_adult_content', False): + access_filter &= Q(epg__channels__is_adult=False) + queryset = queryset.filter(access_filter).distinct() + + # Resolve field selection before serialization so expensive methods can short-circuit + requested_fields = params.get('fields') + allowed = set(f.strip() for f in requested_fields.split(',')) if requested_fields else None + + # Paginate + paginator = ProgramSearchPagination() + page = paginator.paginate_queryset(queryset, request) + serializer = ProgramSearchResultSerializer(page, many=True, context={'fields': allowed, 'user': request.user}) + data = serializer.data + + if allowed: + data = [{k: v for k, v in item.items() if k in allowed} for item in data] + + return paginator.get_paginated_response(data) + # ───────────────────────────── # 3) EPG Grid View @@ -549,5 +774,6 @@ class CurrentProgramsAPIView(APIView): program_data['channel_uuid'] = str(channel.uuid) current_programs.append(program_data) + return Response(current_programs, status=status.HTTP_200_OK) diff --git a/apps/epg/query_utils.py b/apps/epg/query_utils.py new file mode 100644 index 00000000..f2f06b9a --- /dev/null +++ b/apps/epg/query_utils.py @@ -0,0 +1,137 @@ +"""Shared query parsing helpers for EPG program text search. + +Used by the EPG search API and the DVR series rule evaluator to apply +the same boolean / quoted-phrase / regex / whole-word matching semantics. + +Public API: + parse_text_query(field, raw_value, use_regex=False, whole_words=False) -> Q +""" + +import re + +from django.db.models import Q + + +def build_q_object(field_name, term, use_regex=False, whole_words=False): + """Build a single Q object for one search term. + + `whole_words` uses `\\y` on PostgreSQL and `\\b` everywhere else so the + pattern actually anchors on a word boundary regardless of backend. + """ + term = term.strip() + if not term: + return Q() + + if use_regex: + return Q(**{f'{field_name}__iregex': term}) + if whole_words: + from django.db import connection + boundary = r'\y' if connection.vendor == 'postgresql' else r'\b' + pattern = boundary + re.escape(term) + boundary + return Q(**{f'{field_name}__iregex': pattern}) + return Q(**{f'{field_name}__icontains': term}) + + +def parse_text_query(field_name, raw_value, use_regex=False, whole_words=False): + """Parse a search expression into a Q object. + + Supports: + - AND / OR (case-insensitive) between bare terms. + - Double-quoted phrases (atomic; never split on operators). + - Parenthetical grouping with arbitrary nesting. + - Regex mode (entire raw value is treated as a single regex). + - Whole-word mode (each bare term anchored with word boundaries). + + A bare value with no operators is matched as a single phrase via + icontains (or regex / whole-word as configured). + """ + phrases = {} + + def extract_quoted(text): + def replacer(m): + key = f'\x00P{len(phrases)}\x00' + phrases[key] = m.group(1) + return key + return re.sub(r'"([^"]*)"', replacer, text) + + processed = extract_quoted(raw_value) + + def build_q(token): + return build_q_object(field_name, phrases.get(token, token), use_regex, whole_words) + + def parse_expression(expr): + expr = expr.strip() + + if '(' in expr: + paren_start = expr.rfind('(') + paren_end = expr.find(')', paren_start) + if paren_end == -1: + return Q() + + group_q = parse_expression(expr[paren_start + 1:paren_end]) + + before_str = expr[:paren_start].rstrip() + after_str = expr[paren_end + 1:].lstrip() + + before_op = '&' + if before_str.upper().endswith(' AND'): + before_str = before_str[:-4].rstrip() + elif before_str.upper().endswith(' OR'): + before_str = before_str[:-3].rstrip() + before_op = '|' + + after_op = '&' + after_upper = after_str.upper() + if after_upper.startswith('AND '): + after_str = after_str[4:].lstrip() + elif after_upper.startswith('OR '): + after_str = after_str[3:].lstrip() + after_op = '|' + + result = group_q + if before_str: + before_q = parse_expression(before_str) + result = (before_q | result) if before_op == '|' else (before_q & result) + if after_str: + after_q = parse_expression(after_str) + result = (result | after_q) if after_op == '|' else (result & after_q) + return result + + tokens = [] + operators = [] + remaining = expr + + while remaining: + upper = remaining.upper() + and_pos = upper.find(' AND ') + or_pos = upper.find(' OR ') + + if and_pos == -1 and or_pos == -1: + tokens.append(remaining.strip()) + break + + if and_pos == -1: + pos, op, op_len = or_pos, '|', 4 + elif or_pos == -1: + pos, op, op_len = and_pos, '&', 5 + elif and_pos < or_pos: + pos, op, op_len = and_pos, '&', 5 + else: + pos, op, op_len = or_pos, '|', 4 + + token = remaining[:pos].strip() + if token: + tokens.append(token) + operators.append(op) + remaining = remaining[pos + op_len:] + + if not tokens: + return Q() + + result = build_q(tokens[0]) + for i, op in enumerate(operators): + next_q = build_q(tokens[i + 1]) + result = (result & next_q) if op == '&' else (result | next_q) + return result + + return parse_expression(processed) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index efbdcb69..4ab89a9b 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -1,7 +1,7 @@ from core.utils import validate_flexible_url from rest_framework import serializers from .models import EPGSource, EPGData, ProgramData -from apps.channels.models import Channel +from apps.channels.models import Channel, Stream class EPGSourceSerializer(serializers.ModelSerializer): epg_data_count = serializers.SerializerMethodField() @@ -170,3 +170,72 @@ class EPGDataSerializer(serializers.ModelSerializer): 'icon_url', 'epg_source', ] + + +class ProgramSearchChannelSerializer(serializers.ModelSerializer): + """Lightweight channel info for search results.""" + channel_group = serializers.CharField(source='channel_group.name', default=None) + + class Meta: + model = Channel + fields = ['id', 'name', 'channel_number', 'channel_group', 'tvg_id'] + + +class ProgramSearchStreamSerializer(serializers.ModelSerializer): + """Lightweight stream info for search results.""" + channel_group = serializers.CharField(source='channel_group.name', default=None) + m3u_account = serializers.CharField(source='m3u_account.name', default=None) + + class Meta: + model = Stream + fields = ['id', 'name', 'channel_group', 'tvg_id', 'm3u_account'] + + +class ProgramSearchResultSerializer(serializers.ModelSerializer): + """Full program data with associated channels and streams for search results.""" + epg_source = serializers.CharField(source='epg.epg_source.name', default=None) + epg_name = serializers.CharField(source='epg.name', default=None) + epg_icon_url = serializers.URLField(source='epg.icon_url', default=None) + channels = serializers.SerializerMethodField() + streams = serializers.SerializerMethodField() + + class Meta: + model = ProgramData + fields = [ + 'id', 'title', 'sub_title', 'description', + 'start_time', 'end_time', 'tvg_id', 'custom_properties', + 'epg_source', 'epg_name', 'epg_icon_url', + 'channels', 'streams', + ] + + def _accessible_channels(self, obj): + """Return prefetched channels filtered to those the requesting user can access.""" + channels = list(obj.epg.channels.all()) if obj.epg else [] + user = self.context.get('user') + if user is None or user.user_level >= 10: + return channels + custom_props = user.custom_properties or {} + hide_adult = custom_props.get('hide_adult_content', False) + return [ + ch for ch in channels + if ch.user_level <= user.user_level and (not hide_adult or not ch.is_adult) + ] + + def get_channels(self, obj): + fields = self.context.get('fields') + if fields is not None and 'channels' not in fields: + return [] + return ProgramSearchChannelSerializer(self._accessible_channels(obj), many=True).data + + def get_streams(self, obj): + fields = self.context.get('fields') + if fields is not None and 'streams' not in fields: + return [] + stream_ids = set() + streams = [] + for ch in self._accessible_channels(obj): + for s in ch.streams.all(): + if s.id not in stream_ids: + stream_ids.add(s.id) + streams.append(s) + return ProgramSearchStreamSerializer(streams, many=True).data diff --git a/apps/epg/tests/test_epg_search_api.py b/apps/epg/tests/test_epg_search_api.py new file mode 100644 index 00000000..fd44c463 --- /dev/null +++ b/apps/epg/tests/test_epg_search_api.py @@ -0,0 +1,339 @@ +from datetime import timedelta + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APIClient + +from apps.epg.models import EPGData, EPGSource, ProgramData + +User = get_user_model() + +SEARCH_URL = "/api/epg/programs/search/" + + +class ProgramSearchAPIViewTests(TestCase): + """Tests for the /api/epg/programs/search/ endpoint.""" + + @classmethod + def setUpTestData(cls): + cls.epg_source = EPGSource.objects.create(name="Test Source", source_type="xmltv") + cls.epg = EPGData.objects.create( + tvg_id="test-tvg", name="Test EPG", epg_source=cls.epg_source + ) + + now = timezone.now().replace(microsecond=0) + + # Premier League Football — airing now + cls.prog_football = ProgramData.objects.create( + epg=cls.epg, + title="Premier League Football", + description="Live coverage of the Premier League match.", + start_time=now - timedelta(minutes=30), + end_time=now + timedelta(hours=1), + ) + + # Newcastle vs Villa — also airing now + cls.prog_newcastle = ProgramData.objects.create( + epg=cls.epg, + title="Newcastle vs Villa", + description="Match highlights.", + start_time=now - timedelta(minutes=15), + end_time=now + timedelta(hours=2), + ) + + # BBC News — starts in 3 hours + cls.prog_news = ProgramData.objects.create( + epg=cls.epg, + title="BBC News at Ten", + description="The latest news from around the world.", + start_time=now + timedelta(hours=3), + end_time=now + timedelta(hours=4), + ) + + # Nature Documentary — starts in 5 hours + cls.prog_doc = ProgramData.objects.create( + epg=cls.epg, + title="Nature Documentary", + description="Exploring wildlife in the Amazon.", + start_time=now + timedelta(hours=5), + end_time=now + timedelta(hours=6), + ) + + cls.now = now + cls.user = User.objects.create_user(username="testuser", password="pass", user_level=1) + + def setUp(self): + self.client = APIClient(REMOTE_ADDR="127.0.0.1") + self.client.force_authenticate(user=self.user) + + # ------------------------------------------------------------------ + # Response structure + # ------------------------------------------------------------------ + + def test_response_structure(self): + """Response includes pagination envelope and all expected program fields.""" + response = self.client.get(SEARCH_URL, {"page_size": 1}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + data = response.json() + self.assertIn("count", data) + self.assertIn("results", data) + self.assertIn("next", data) + self.assertIn("previous", data) + + program = data["results"][0] + for field in ("id", "title", "start_time", "end_time", "tvg_id", "channels", "streams"): + self.assertIn(field, program) + + # ------------------------------------------------------------------ + # No filter — returns all programs + # ------------------------------------------------------------------ + + def test_no_filters_returns_all(self): + """Omitting filters returns all seeded programs.""" + response = self.client.get(SEARCH_URL) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 4) + + # ------------------------------------------------------------------ + # Title search + # ------------------------------------------------------------------ + + def test_title_simple_match(self): + """Simple title search returns matching programs.""" + response = self.client.get(SEARCH_URL, {"title": "football"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(data["count"], 1) + self.assertEqual(data["results"][0]["title"], "Premier League Football") + + def test_title_multi_word_is_phrase_not_implicit_and(self): + """Space-separated words without AND/OR are matched as a phrase, not as implicit AND. + + 'Premier Football' contains both words from 'Premier League Football' but + not as a consecutive phrase — so it should return 0 results. + Use 'Premier AND Football' to match both words independently. + """ + phrase = self.client.get(SEARCH_URL, {"title": "Premier Football"}).json() + explicit_and = self.client.get(SEARCH_URL, {"title": "Premier AND Football"}).json() + # Phrase match: "Premier Football" is not a substring of "Premier League Football" + self.assertEqual(phrase["count"], 0) + # Explicit AND: both words present → matches + self.assertEqual(explicit_and["count"], 1) + + def test_title_quoted_phrase(self): + """Double-quoted phrases are matched literally; 'and'/'or' inside quotes are not operators. + + This is the standard way to search for program titles that contain conjunctions, + e.g. "Law and Order". Without quotes, lowercase 'and'/'or' are still treated as + case-insensitive boolean operators — so quoting is the reliable way to do a phrase match. + """ + prog = ProgramData.objects.create( + epg=self.epg, + title="Law and Order", + description="Crime drama.", + start_time=self.now + timedelta(hours=10), + end_time=self.now + timedelta(hours=11), + ) + try: + # Quoted phrase → exact substring match → finds the program + quoted = self.client.get(SEARCH_URL, {"title": '"Law and Order"'}).json() + self.assertEqual(quoted["count"], 1) + self.assertEqual(quoted["results"][0]["title"], "Law and Order") + + # Quoted phrase that is not a substring → no match + non_phrase = self.client.get(SEARCH_URL, {"title": '"Law Order"'}).json() + self.assertEqual(non_phrase["count"], 0) + + # Mix: quoted phrase AND bare term present in the title → matches + mixed_match = self.client.get(SEARCH_URL, {"title": '"Law and Order" AND order'}).json() + self.assertEqual(mixed_match["count"], 1) + + # Mix: quoted phrase AND bare term NOT in the title → no match + mixed_no_match = self.client.get(SEARCH_URL, {"title": '"Law and Order" AND crime'}).json() + self.assertEqual(mixed_no_match["count"], 0) + finally: + prog.delete() + def test_title_case_insensitive(self): + """Title search is case-insensitive.""" + lower = self.client.get(SEARCH_URL, {"title": "football"}).json() + upper = self.client.get(SEARCH_URL, {"title": "FOOTBALL"}).json() + self.assertEqual(lower["count"], 1) + self.assertEqual(upper["count"], 1) + self.assertEqual(lower["results"][0]["title"], upper["results"][0]["title"]) + + def test_title_and_operator(self): + """AND operator (case-insensitive) requires both terms to be present in the title.""" + upper = self.client.get(SEARCH_URL, {"title": "Premier AND League"}).json() + lower = self.client.get(SEARCH_URL, {"title": "Premier and League"}).json() + self.assertEqual(upper["count"], 1) + self.assertEqual(lower["count"], 1) + self.assertIn("Premier", upper["results"][0]["title"]) + + def test_title_or_operator(self): + """OR operator (case-insensitive) returns programs matching either term.""" + upper = self.client.get(SEARCH_URL, {"title": "Newcastle OR Football"}) + lower = self.client.get(SEARCH_URL, {"title": "Newcastle or Football"}) + self.assertEqual(upper.status_code, status.HTTP_200_OK) + for response in (upper, lower): + titles = [r["title"] for r in response.json()["results"]] + self.assertIn("Premier League Football", titles) + self.assertIn("Newcastle vs Villa", titles) + + def test_title_no_match_returns_empty(self): + """Search with no matching title returns empty results.""" + response = self.client.get(SEARCH_URL, {"title": "XYZNONEXISTENT999"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(data["count"], 0) + self.assertEqual(data["results"], []) + + def test_title_whole_word_matching(self): + """title_whole_words=true matches complete words but not partial words.""" + # 'new' as substring matches 'Newcastle vs Villa' and 'BBC News at Ten' + partial = self.client.get(SEARCH_URL, {"title": "new"}).json() + # Whole-word \bnew\b matches neither 'Newcastle' nor 'News' (partial matches) + whole_no_match = self.client.get(SEARCH_URL, {"title": "new", "title_whole_words": "true"}).json() + self.assertEqual(partial["count"], 2) + self.assertEqual(whole_no_match["count"], 0) + + # 'football' is a complete word in 'Premier League Football' — must still match + whole_match = self.client.get(SEARCH_URL, {"title": "football", "title_whole_words": "true"}).json() + self.assertEqual(whole_match["count"], 1) + self.assertEqual(whole_match["results"][0]["title"], "Premier League Football") + + # 'league' is also a complete word — and 'Premier AND league' with whole_words works + both_words = self.client.get(SEARCH_URL, {"title": "Premier AND league", "title_whole_words": "true"}).json() + self.assertEqual(both_words["count"], 1) + + def test_title_regex(self): + """title_regex=true applies the query as a regex pattern.""" + response = self.client.get( + SEARCH_URL, {"title": "^Premier", "title_regex": "true"} + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + for program in response.json()["results"]: + self.assertTrue(program["title"].startswith("Premier")) + + def test_title_parenthetical_grouping(self): + """Parenthetical groups with AND/OR are evaluated correctly.""" + # (Newcastle OR Football) AND (Villa OR League) should match both seeded programs: + # "Premier League Football" matches Football AND League + # "Newcastle vs Villa" matches Newcastle AND Villa + response = self.client.get( + SEARCH_URL, {"title": "(Newcastle OR Football) AND (Villa OR League)"} + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + titles = {r["title"] for r in response.json()["results"]} + self.assertIn("Premier League Football", titles) + self.assertIn("Newcastle vs Villa", titles) + self.assertNotIn("BBC News at Ten", titles) + self.assertNotIn("Nature Documentary", titles) + + # ------------------------------------------------------------------ + # Description search + # ------------------------------------------------------------------ + + def test_description_simple_match(self): + """Description search returns programs whose description contains the term.""" + response = self.client.get(SEARCH_URL, {"description": "Premier League"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(data["count"], 1) + self.assertEqual(data["results"][0]["title"], "Premier League Football") + + def test_description_and_operator(self): + """AND operator in description requires both terms.""" + response = self.client.get(SEARCH_URL, {"description": "latest AND news"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(data["count"], 1) + self.assertEqual(data["results"][0]["title"], "BBC News at Ten") + + # ------------------------------------------------------------------ + # Time filters + # ------------------------------------------------------------------ + + def test_airing_at_returns_current_programs(self): + """airing_at returns programs where start_time <= t < end_time.""" + ts = self.now.isoformat() + response = self.client.get(SEARCH_URL, {"airing_at": ts}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + titles = [r["title"] for r in response.json()["results"]] + self.assertIn("Premier League Football", titles) + self.assertIn("Newcastle vs Villa", titles) + self.assertNotIn("BBC News at Ten", titles) + + def test_start_after_filter(self): + """start_after excludes programs that start before the given time.""" + cutoff = (self.now + timedelta(hours=4)).isoformat() + response = self.client.get(SEARCH_URL, {"start_after": cutoff}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + titles = [r["title"] for r in response.json()["results"]] + self.assertIn("Nature Documentary", titles) + self.assertNotIn("Premier League Football", titles) + self.assertNotIn("BBC News at Ten", titles) + + def test_start_before_filter(self): + """start_before excludes programs that start after the given time.""" + cutoff = (self.now + timedelta(hours=1)).isoformat() + response = self.client.get(SEARCH_URL, {"start_before": cutoff}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + titles = [r["title"] for r in response.json()["results"]] + self.assertIn("Premier League Football", titles) + self.assertIn("Newcastle vs Villa", titles) + self.assertNotIn("Nature Documentary", titles) + + def test_invalid_datetime_returns_400(self): + """An unparseable datetime value returns a 400 error.""" + response = self.client.get(SEARCH_URL, {"airing_at": "not-a-date"}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.json()) + + # ------------------------------------------------------------------ + # Field selection + # ------------------------------------------------------------------ + + def test_field_selection_limits_response_keys(self): + """fields param restricts the keys present in each result.""" + response = self.client.get(SEARCH_URL, {"fields": "title,start_time,end_time"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + for program in response.json()["results"]: + self.assertIn("title", program) + self.assertIn("start_time", program) + self.assertIn("end_time", program) + self.assertNotIn("description", program) + self.assertNotIn("channels", program) + self.assertNotIn("streams", program) + + # ------------------------------------------------------------------ + # Pagination + # ------------------------------------------------------------------ + + def test_pagination_page_size(self): + """page_size limits the number of results returned.""" + response = self.client.get(SEARCH_URL, {"page_size": 2}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(len(data["results"]), 2) + self.assertEqual(data["count"], 4) + self.assertIsNotNone(data["next"]) + + def test_pagination_second_page(self): + """Page 2 returns different results from page 1.""" + page1 = self.client.get(SEARCH_URL, {"page": 1, "page_size": 2}).json() + page2 = self.client.get(SEARCH_URL, {"page": 2, "page_size": 2}).json() + ids_p1 = {r["id"] for r in page1["results"]} + ids_p2 = {r["id"] for r in page2["results"]} + self.assertTrue(ids_p1.isdisjoint(ids_p2)) + + def test_page_size_capped_at_maximum(self): + """page_size beyond the 500 maximum is clamped, not rejected.""" + response = self.client.get(SEARCH_URL, {"page_size": 10000}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + # All 4 seeded programs are returned — request was accepted and clamped + self.assertEqual(data["count"], 4) + self.assertEqual(len(data["results"]), 4) diff --git a/apps/hdhr/api_views.py b/apps/hdhr/api_views.py index b1d0e852..41c1a3d9 100644 --- a/apps/hdhr/api_views.py +++ b/apps/hdhr/api_views.py @@ -125,30 +125,38 @@ class LineupAPIView(APIView): if blocked is not None: return blocked + from apps.channels.managers import with_effective_values + from apps.channels.utils import format_channel_number + if profile is not None: channel_profile = ChannelProfile.objects.get(name=profile) - channels = Channel.objects.filter( + base_qs = Channel.objects.filter( channelprofilemembership__channel_profile=channel_profile, channelprofilemembership__enabled=True, - ).order_by("channel_number") + ) else: - channels = Channel.objects.all().order_by("channel_number") + base_qs = Channel.objects.all() + + channels = ( + with_effective_values(base_qs) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) lineup = [] for ch in channels: - # Format channel number as integer if it has no decimal component - if ch.channel_number is not None: - if ch.channel_number == int(ch.channel_number): - formatted_channel_number = str(int(ch.channel_number)) - else: - formatted_channel_number = str(ch.channel_number) - else: - formatted_channel_number = "" + # HDHR clients reject lineup entries with empty/non-numeric + # GuideNumber and may drop the whole lineup. With nullable + # channel_number, skip rows that have no usable number. + formatted = format_channel_number(ch.effective_channel_number, empty=None) + if formatted is None: + continue + formatted_channel_number = str(formatted) lineup.append( { "GuideNumber": formatted_channel_number, - "GuideName": ch.name, + "GuideName": ch.effective_name, "URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}"), "Guide_ID": formatted_channel_number, "Station": formatted_channel_number, diff --git a/apps/hdhr/views.py b/apps/hdhr/views.py index f9dd42d2..6777bdbf 100644 --- a/apps/hdhr/views.py +++ b/apps/hdhr/views.py @@ -84,15 +84,27 @@ class LineupAPIView(APIView): description="Retrieve the available channel lineup", ) def get(self, request): - channels = Channel.objects.all().order_by("channel_number") - lineup = [ - { - "GuideNumber": str(ch.channel_number), - "GuideName": ch.name, - "URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}"), - } - for ch in channels - ] + from apps.channels.managers import with_effective_values + from apps.channels.utils import format_channel_number + + channels = ( + with_effective_values(Channel.objects.all()) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) + lineup = [] + for ch in channels: + formatted = format_channel_number(ch.effective_channel_number, empty=None) + if formatted is None: + continue + formatted_channel_number = str(formatted) + lineup.append( + { + "GuideNumber": formatted_channel_number, + "GuideName": ch.effective_name, + "URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}"), + } + ) return JsonResponse(lineup, safe=False) diff --git a/apps/m3u/api_views.py b/apps/m3u/api_views.py index b8cb099a..c09b668f 100644 --- a/apps/m3u/api_views.py +++ b/apps/m3u/api_views.py @@ -17,6 +17,9 @@ from rest_framework.decorators import action from django.conf import settings from .tasks import refresh_m3u_groups import json +import logging + +logger = logging.getLogger(__name__) from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile from core.models import UserAgent @@ -41,7 +44,7 @@ class M3UAccountViewSet(viewsets.ModelViewSet): queryset = M3UAccount.objects.select_related( "refresh_task__crontab", "refresh_task__interval" - ).prefetch_related("channel_group", "profiles") + ).prefetch_related("channel_group", "profiles", "filters") serializer_class = M3UAccountSerializer def get_permissions(self): @@ -50,6 +53,41 @@ class M3UAccountViewSet(viewsets.ModelViewSet): except KeyError: return [Authenticated()] + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + # Pre-aggregate stream counts for all accounts in one query so the + # nested ChannelGroupM3UAccountSerializer never issues a COUNT per + # group row. The serializer checks for this key and skips its own + # per-instance query when it is present. + from apps.channels.models import Stream + from django.db.models import Count + + account_ids = list(queryset.values_list("id", flat=True)) + counts_qs = ( + Stream.objects.filter(m3u_account_id__in=account_ids) + .values("m3u_account_id", "channel_group_id") + .annotate(c=Count("id")) + ) + stream_counts = { + (row["m3u_account_id"], row["channel_group_id"]): row["c"] + for row in counts_qs + } + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer( + page, many=True, + context={**self.get_serializer_context(), "stream_counts": stream_counts}, + ) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer( + queryset, many=True, + context={**self.get_serializer_context(), "stream_counts": stream_counts}, + ) + return Response(serializer.data) + def create(self, request, *args, **kwargs): # Handle file upload first, if any file_path = None @@ -225,6 +263,198 @@ class M3UAccountViewSet(viewsets.ModelViewSet): # Continue with regular partial update return super().partial_update(request, *args, **kwargs) + def destroy(self, request, *args, **kwargs): + """ + Delete an M3U account and all auto-created channels attributed + to it. Auto-created channels with no surviving provider have no + useful state (they cannot sync, their streams are about to + cascade away), so the delete is unconditional: the only + question for the user is whether to confirm. Manual channels + are untouched, even if they include streams from this account; + those streams cascade away independently and the channels + survive with their other streams. The legacy + ``?cleanup_channels`` query parameter is accepted for backward + compatibility but ignored. + """ + instance = self.get_object() + from apps.channels.models import Channel + from apps.proxy.ts_proxy.services.channel_service import ( + ChannelService, + ) + + # Snapshot channels so proxy sessions can be stopped outside + # the DB transaction. The pre_delete signal would otherwise + # fire ChannelService.stop_channel (Redis pub / hgetall / + # setex) per channel inside the atomic, holding the DB + # connection across thousands of blocking RPCs and gumming up + # the connection pool. + channels_to_delete = list( + Channel.objects.filter( + auto_created=True, + auto_created_by=instance, + ).values_list("id", "uuid") + ) + for _, channel_uuid in channels_to_delete: + if not channel_uuid: + continue + try: + ChannelService.stop_channel(str(channel_uuid)) + except Exception as e: + logger.warning( + "Failed to stop proxy session for channel %s " + "during account cleanup: %s", + channel_uuid, + e, + ) + + channel_ids = [cid for cid, _ in channels_to_delete] + # Channel + account writes share an atomic so an account + # delete failure rolls back the channel deletes too. The + # pre_delete signal will fire again here but its proxy stop + # is fast on already-stopped channels (a single Redis check + # returns "not found" immediately). + with transaction.atomic(): + if channel_ids: + _, per_model = Channel.objects.filter( + id__in=channel_ids + ).delete() + deleted_channels = per_model.get( + "dispatcharr_channels.Channel", 0 + ) + else: + deleted_channels = 0 + response = super().destroy(request, *args, **kwargs) + + # Surface the channel count alongside the standard 204; the + # confirmation toast renders the number to acknowledge what + # the cascade actually removed. + if response.status_code == status.HTTP_204_NO_CONTENT: + return Response( + {"deleted_channels": deleted_channels}, + status=status.HTTP_200_OK, + ) + return response + + @extend_schema( + responses={ + 200: { + "type": "object", + "properties": { + "count": {"type": "integer"}, + "sample_names": {"type": "array", "items": {"type": "string"}}, + }, + }, + }, + ) + @action(detail=True, methods=["get"], url_path="auto-created-channels-count") + def auto_created_channels_count(self, request, pk=None): + """ + Preview how many auto-created channels would be removed if the account + were deleted with cleanup_channels=true. The frontend calls this when + the user clicks Delete, to render a truthful confirmation dialog + ("Also delete N channels auto-created by this provider?"). + """ + account = self.get_object() + from apps.channels.models import Channel + + qs = Channel.objects.filter( + auto_created=True, auto_created_by=account + ) + count = qs.count() + sample_names = list(qs.values_list("name", flat=True)[:5]) + return Response({"count": count, "sample_names": sample_names}) + + @extend_schema( + parameters=[ + OpenApiParameter( + name="channel_group_id", + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY, + required=True, + description=( + "ID of the ChannelGroup whose auto-created channels " + "should be repacked." + ), + ), + ], + responses={ + 200: { + "type": "object", + "properties": { + "assigned": {"type": "integer"}, + "released": {"type": "integer"}, + "failed": {"type": "integer"}, + }, + }, + }, + ) + @action(detail=True, methods=["post"], url_path="repack-group") + def repack_group(self, request, pk=None): + """ + Manually re-pack visible channels in one of this account's + groups into the group's [start, end] range. Override-pinned + numbers are treated as reservations and skipped. Hidden channels + without overrides have their channel_number set to NULL. + + Useful when the user has just finished customizing channels + (setting overrides as pins, hiding unwanted streams) and wants + the result reflected immediately rather than on the next M3U + refresh. Also acts as a one-shot cleanup for groups that aren't + running in compact mode but have accumulated gaps. + """ + account = self.get_object() + group_id_raw = request.query_params.get("channel_group_id") + if not group_id_raw: + return Response( + {"detail": "channel_group_id is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + group_id = int(group_id_raw) + except (TypeError, ValueError): + return Response( + {"detail": "channel_group_id must be an integer"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + from apps.channels.models import ChannelGroupM3UAccount + from apps.channels.compact_numbering import repack_group as _repack + from core.utils import acquire_task_lock, release_task_lock + + # Share the lock that wraps the entire refresh-plus-sync pipeline + # (`refresh_single_m3u_account`). The narrower + # `refresh_m3u_account_groups` lock is released before + # `sync_auto_channels` runs, so it would not protect this writer + # from racing against the channel_number writes inside sync. + if not acquire_task_lock("refresh_single_m3u_account", account.id): + return Response( + {"detail": "An M3U refresh is in progress for this account."}, + status=status.HTTP_409_CONFLICT, + ) + try: + # Re-fetch under the lock so a sync that just released its lock + # cannot leave the cached group_relation reflecting pre-sync + # custom_properties (auto_sync_channel_start/end, etc.). + try: + group_relation = ChannelGroupM3UAccount.objects.get( + m3u_account=account, channel_group_id=group_id + ) + except ChannelGroupM3UAccount.DoesNotExist: + return Response( + {"detail": "Group is not associated with this account"}, + status=status.HTTP_404_NOT_FOUND, + ) + result = _repack(group_relation) + finally: + try: + release_task_lock("refresh_single_m3u_account", account.id) + except Exception as e: + logger.warning( + f"Failed to release repack lock for account " + f"{account.id}: {e}" + ) + return Response(result) + @action(detail=True, methods=["post"], url_path="refresh-vod") def refresh_vod(self, request, pk=None): """Trigger VOD content refresh for XtreamCodes accounts""" @@ -270,6 +500,33 @@ class M3UAccountViewSet(viewsets.ModelViewSet): category_settings = request.data.get("category_settings", []) try: + for setting in group_settings: + start = setting.get("auto_sync_channel_start") + end = setting.get("auto_sync_channel_end") + if (start is not None and start < 1) or ( + end is not None and end < 1 + ): + return Response( + { + "error": ( + f"Channel group {setting.get('channel_group')}: " + f"channel range must be >= 1." + ) + }, + status=status.HTTP_400_BAD_REQUEST, + ) + if start is not None and end is not None and end < start: + return Response( + { + "error": ( + f"Channel group {setting.get('channel_group')}: " + f"auto_sync_channel_end must be >= " + f"auto_sync_channel_start." + ) + }, + status=status.HTTP_400_BAD_REQUEST, + ) + with transaction.atomic(): group_objects = [ ChannelGroupM3UAccount( @@ -278,6 +535,7 @@ class M3UAccountViewSet(viewsets.ModelViewSet): enabled=setting.get("enabled", True), auto_channel_sync=setting.get("auto_channel_sync", False), auto_sync_channel_start=setting.get("auto_sync_channel_start"), + auto_sync_channel_end=setting.get("auto_sync_channel_end"), custom_properties=setting.get("custom_properties", {}), ) for setting in group_settings @@ -293,6 +551,7 @@ class M3UAccountViewSet(viewsets.ModelViewSet): "enabled", "auto_channel_sync", "auto_sync_channel_start", + "auto_sync_channel_end", "custom_properties", ], ) diff --git a/apps/m3u/models.py b/apps/m3u/models.py index c1cb5a99..6fc48bec 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -100,7 +100,6 @@ class M3UAccount(models.Model): default=0, help_text="Priority for VOD provider selection (higher numbers = higher priority). Used when multiple providers offer the same content.", ) - def __str__(self): return self.name diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index f72566cf..485a3687 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -193,6 +193,24 @@ class M3UAccountSerializer(serializers.ModelSerializer): } def to_representation(self, instance): + # When the list() view pre-aggregates stream counts for all accounts + # in a single query, it seeds "stream_counts" into the context before + # serialization. Avoid issuing a redundant per-instance COUNT in that + # case. The per-instance fallback handles direct serialization (e.g. + # retrieve, create) where only one account is in scope. + if "stream_counts" not in self.context: + from django.db.models import Count + from apps.channels.models import Stream + + counts_qs = ( + Stream.objects.filter(m3u_account_id=instance.id) + .values("channel_group_id") + .annotate(c=Count("id")) + ) + self.context["stream_counts"] = { + (instance.id, row["channel_group_id"]): row["c"] for row in counts_qs + } + data = super().to_representation(instance) # Parse custom_properties to get VOD preference and auto_enable_new_groups settings @@ -247,10 +265,17 @@ class M3UAccountSerializer(serializers.ModelSerializer): auto_enable_new_groups_vod = validated_data.pop("auto_enable_new_groups_vod", None) auto_enable_new_groups_series = validated_data.pop("auto_enable_new_groups_series", None) - # Get existing custom_properties - custom_props = instance.custom_properties or {} + # Merge client-supplied custom_properties over the existing blob + # so unrelated keys persist. The dedicated preference fields below + # overwrite their corresponding keys; clients should set those via + # the typed top-level fields rather than the custom_properties + # payload. + incoming_custom = validated_data.get("custom_properties") or {} + custom_props = { + **(instance.custom_properties or {}), + **incoming_custom, + } - # Update preferences if enable_vod is not None: custom_props["enable_vod"] = enable_vod if auto_enable_new_groups_live is not None: @@ -345,7 +370,9 @@ class M3UAccountSerializer(serializers.ModelSerializer): return instance def get_filters(self, obj): - filters = obj.filters.order_by("order") + # Sort over the prefetch cache; .order_by() would fire one SELECT + # per account (viewset prefetches "filters"). + filters = sorted(obj.filters.all(), key=lambda f: f.order) return M3UFilterSerializer(filters, many=True).data def get_earliest_expiration(self, obj): diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 0a441e18..4599d688 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1686,6 +1686,113 @@ def delete_m3u_refresh_task_by_id(account_id): return False +def _next_available_number(used_numbers, start, end=None): + """ + Return the smallest integer >= start that is not present in `used_numbers`. + + When `end` is provided (inclusive upper bound for range-constrained groups), + returns None if the search exceeds that bound instead of running + indefinitely. The search is O(cluster size) per call against the set; + maintaining the cursor monotonically across calls keeps the + "next_available" numbering mode from becoming O(N^2) on large groups. + """ + n = start + while n in used_numbers: + n += 1 + if end is not None and n > end: + return None + if end is not None and n > end: + return None + return n + + +def _pick_target_number( + mode, + stream, + used_numbers, + fixed_cursor, + fallback_start, + end_number=None, + range_start=None, +): + """ + Return the channel number a given stream should claim under the group's + numbering mode, or None if the configured range is exhausted. + + Shared by the existing-channel renumber pass and the new-channel create + pass so both honor identical mode semantics: provider-supplied number + when available and free, otherwise fall back; or always next-available; + or fixed-cursor sequential. + + `range_start`, when provided, is the inclusive lower bound for the + group's configured numbering range. Provider-supplied numbers below + this bound fall back to the next-available picker so freshly-created + channels never land outside the configured range. + """ + if mode == "provider": + chno = stream.stream_chno + if ( + chno is not None + and chno not in used_numbers + and (range_start is None or chno >= range_start) + and (end_number is None or chno <= end_number) + ): + return chno + # No usable provider number: walk from fallback_start, bumped up + # to range_start when set so the fallback never lands below the + # configured range. + effective_start = ( + max(fallback_start, range_start) + if range_start is not None + else fallback_start + ) + return _next_available_number(used_numbers, effective_start, end=end_number) + if mode == "next_available": + return _next_available_number(used_numbers, 1, end=end_number) + return _next_available_number(used_numbers, fixed_cursor, end=end_number) + + +def _custom_properties_as_dict(value): + """ + Normalize a JSONField-backed custom_properties value into a dict. + + Historical data has rows where the field holds a JSON-encoded string + instead of a dict. Django's JSONField serializes whatever it gets, so + `.get()` on one of those rows raises AttributeError and aborts the + entire sync. Treat string values as JSON to parse, and fall back to an + empty dict for anything that isn't a dict after parsing. + """ + import json + + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except (ValueError, TypeError): + logger.warning( + "custom_properties stored as non-JSON string; ignoring: %r", + value[:100], + ) + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + +def _classify_sync_failure(exc): + """ + Map an exception raised during per-stream sync to a coarse typed + reason used by the completion notification's grouped failure list. + Keeps the bucket count small so the modal stays readable; the + underlying exception text is preserved verbatim in ``error``. + """ + from django.db import IntegrityError + + if isinstance(exc, IntegrityError): + return "INTEGRITY_ERROR" + return "OTHER" + + @shared_task def sync_auto_channels(account_id, scan_start_time=None): """ @@ -1722,17 +1829,47 @@ def sync_auto_channels(account_id, scan_start_time=None): channels_created = 0 channels_updated = 0 channels_deleted = 0 + channels_failed = 0 + # Per-failure context for the completion notification. Each entry + # carries a typed ``reason`` so the modal can group counts by + # cause; the cap keeps the WebSocket payload bounded but is sized + # generously to cover realistic multi-provider failure sets. + # Full per-stream detail still goes to ``logger.warning`` for + # power-user diagnostics regardless of the cap. + failed_stream_details = [] + FAILURE_LOG_LIMIT = 1000 - # Get all channel numbers that are already in use by other channels (not auto-created by this account) + # Group range reservations (start+end) are advisory and NOT seeded + # here: two groups with overlapping ranges must cooperate, so only + # actually-occupied numbers constrain assignment. + # Hidden auto-created channels stay in the seed because the renumber + # loop iterates current provider streams (which excludes hidden + # ones); excluding them here would let sync reclaim their numbers. used_numbers = set( Channel.objects.exclude( - auto_created=True, auto_created_by=account + auto_created=True, + auto_created_by=account, + hidden_from_output=False, ).values_list("channel_number", flat=True) ) + # Override pins are global reservations: effective_channel_number + # uses the override, so the picker must treat those numbers as + # taken or sync can produce duplicate effective channel numbers. + from apps.channels.models import ChannelOverride + + used_numbers.update( + ChannelOverride.objects.filter( + channel_number__isnull=False + ).values_list("channel_number", flat=True) + ) + used_numbers.discard(None) for group_relation in auto_sync_groups: channel_group = group_relation.channel_group start_number = group_relation.auto_sync_channel_start or 1.0 + # Optional upper bound; _next_available_number returns None when + # exhausted, which the per-stream loop converts to a failure. + end_number = group_relation.auto_sync_channel_end # Get force_dummy_epg, group_override, and regex patterns from group custom_properties group_custom_props = {} @@ -1741,6 +1878,7 @@ def sync_auto_channels(account_id, scan_start_time=None): name_regex_pattern = None name_replace_pattern = None name_match_regex = None + name_match_exclude_regex = None channel_profile_ids = None channel_sort_order = None channel_sort_reverse = False @@ -1749,8 +1887,10 @@ def sync_auto_channels(account_id, scan_start_time=None): custom_epg_id = None # New option: select specific EPG source (takes priority over force_dummy_epg) channel_numbering_mode = "fixed" # Default mode channel_numbering_fallback = 1 # Default fallback for provider mode - if group_relation.custom_properties: - group_custom_props = group_relation.custom_properties + group_custom_props = _custom_properties_as_dict( + group_relation.custom_properties + ) + if group_custom_props: force_dummy_epg = group_custom_props.get("force_dummy_epg", False) override_group_id = group_custom_props.get("group_override") name_regex_pattern = group_custom_props.get("name_regex_pattern") @@ -1758,6 +1898,9 @@ def sync_auto_channels(account_id, scan_start_time=None): "name_replace_pattern" ) name_match_regex = group_custom_props.get("name_match_regex") + name_match_exclude_regex = group_custom_props.get( + "name_match_exclude_regex" + ) channel_profile_ids = group_custom_props.get("channel_profile_ids") custom_epg_id = group_custom_props.get("custom_epg_id") channel_sort_order = group_custom_props.get("channel_sort_order") @@ -1793,9 +1936,11 @@ def sync_auto_channels(account_id, scan_start_time=None): last_seen__gte=scan_start_time, ) - # --- FILTER STREAMS BY NAME MATCH REGEX IF SPECIFIED --- + # Pre-compile with Python re so an invalid pattern fails here + # rather than as a Postgres exception inside the per-stream loop. if name_match_regex: try: + re.compile(name_match_regex) current_streams = current_streams.filter( name__iregex=name_match_regex ) @@ -1804,6 +1949,19 @@ def sync_auto_channels(account_id, scan_start_time=None): f"Invalid name_match_regex '{name_match_regex}' for group '{channel_group.name}': {e}. Skipping name filter." ) + # Exclude regex runs after the include filter so the two + # compose: include narrows, exclude removes from what's left. + if name_match_exclude_regex: + try: + re.compile(name_match_exclude_regex) + current_streams = current_streams.exclude( + name__iregex=name_match_exclude_regex + ) + except re.error as e: + logger.warning( + f"Invalid name_match_exclude_regex '{name_match_exclude_regex}' for group '{channel_group.name}': {e}. Skipping exclude filter." + ) + # --- APPLY CHANNEL SORT ORDER --- streams_is_list = False # Track if we converted to list if channel_sort_order and channel_sort_order != "": @@ -1834,27 +1992,27 @@ def sync_auto_channels(account_id, scan_start_time=None): order_prefix = "-" if channel_sort_reverse else "" current_streams = current_streams.order_by(f"{order_prefix}id") - # Get existing auto-created channels for this account (regardless of current group) - # We'll find them by their stream associations instead of just group location - existing_channels = Channel.objects.filter( - auto_created=True, auto_created_by=account - ).select_related("logo", "epg_data") - - # Create mapping of existing channels by their associated stream - # This approach finds channels even if they've been moved to different groups + # Scoped to this group so the loop below runs in O(group size). + # Multi-stream channels are deduped by channel_id so every + # stream_id maps to the same in-memory Channel instance and + # post-loop bulk_update writes the merged state. existing_channel_map = {} - for channel in existing_channels: - # Get streams associated with this channel that belong to our M3U account and original group - channel_streams = ChannelStream.objects.filter( - channel=channel, + existing_channels_by_id = {} + existing_channel_streams = ( + ChannelStream.objects.filter( + channel__auto_created=True, + channel__auto_created_by=account, stream__m3u_account=account, - stream__channel_group=channel_group, # Match streams from the original group - ).select_related("stream") - - # Map each of our M3U account's streams to this channel - for channel_stream in channel_streams: - if channel_stream.stream: - existing_channel_map[channel_stream.stream.id] = channel + stream__channel_group=channel_group, + ) + .select_related("channel") + ) + for cs in existing_channel_streams: + if cs.stream_id and cs.channel_id: + canonical = existing_channels_by_id.setdefault( + cs.channel_id, cs.channel + ) + existing_channel_map[cs.stream_id] = canonical # Track which streams we've processed processed_stream_ids = set() @@ -1866,10 +2024,142 @@ def sync_auto_channels(account_id, scan_start_time=None): else current_streams.exists() ) + # Bulk pre-fetch collapses N+1 Logo/EPGData lookups into a + # pair of in_bulk() calls. + from apps.channels.models import Logo + from apps.epg.models import EPGSource + + # Resolve the group's custom EPG source once. + custom_epg_source = None + custom_dummy_epg_data = None + if custom_epg_id: + try: + custom_epg_source = EPGSource.objects.get(id=custom_epg_id) + if custom_epg_source.source_type == "dummy": + custom_dummy_epg_data = ( + EPGData.objects.filter( + epg_source=custom_epg_source + ).first() + ) + if not custom_dummy_epg_data: + logger.warning( + f"No EPGData found for dummy EPG source " + f"{custom_epg_source.name} (ID: {custom_epg_id})" + ) + except EPGSource.DoesNotExist: + logger.warning( + f"Custom EPG source with ID {custom_epg_id} not found " + f"for group '{channel_group.name}', falling back to " + f"auto-match" + ) + + # Resolve the group's custom logo once. + custom_logo = None + if custom_logo_id: + try: + custom_logo = Logo.objects.get(id=custom_logo_id) + except Logo.DoesNotExist: + logger.warning( + f"Custom logo with ID {custom_logo_id} not found for " + f"group '{channel_group.name}', falling back to stream " + f"logos" + ) + + logo_cache_by_url = {} + epg_cache_by_tvg_id = {} + if has_streams: + # Collect unique URLs / tvg_ids in one DB call each. + stream_iter = ( + current_streams + if streams_is_list + else list(current_streams.values("logo_url", "tvg_id")) + ) + unique_logo_urls = { + s.get("logo_url") if isinstance(s, dict) else getattr(s, "logo_url", None) + for s in stream_iter + } + unique_logo_urls.discard(None) + unique_logo_urls.discard("") + if unique_logo_urls: + logo_cache_by_url = { + lg.url: lg + for lg in Logo.objects.filter(url__in=unique_logo_urls) + } + + unique_tvg_ids = { + s.get("tvg_id") if isinstance(s, dict) else getattr(s, "tvg_id", None) + for s in stream_iter + } + unique_tvg_ids.discard(None) + unique_tvg_ids.discard("") + # Skip the EPG cache when force_dummy_epg with no + # custom source override; the resolver always returns None. + want_epg_cache = unique_tvg_ids and ( + not force_dummy_epg or custom_epg_id + ) + if want_epg_cache: + # Scope to the group's pinned source so foreign-source + # tvg_id matches do not leak in. + epg_q = EPGData.objects.filter(tvg_id__in=unique_tvg_ids) + if ( + custom_epg_source is not None + and custom_epg_source.source_type != "dummy" + ): + epg_q = epg_q.filter(epg_source=custom_epg_source) + epg_cache_by_tvg_id = {d.tvg_id: d for d in epg_q} + + def _resolve_logo_for_stream(stream): + """Return a Logo for stream.logo_url, creating it once if needed.""" + url = getattr(stream, "logo_url", None) + if not url: + return None + cached = logo_cache_by_url.get(url) + if cached is not None: + return cached + created, _ = Logo.objects.get_or_create( + url=url, + defaults={"name": stream.name or stream.tvg_id or "Unknown"}, + ) + logo_cache_by_url[url] = created + return created + + def _resolve_epg_for_stream(stream): + """Return the EPGData row that should be assigned to this + stream's channel. Encodes all four group-level EPG modes: + + 1. custom dummy source: single shared EPGData + 2. custom non-dummy source: cache lookup, scoped to + that source + 3. force_dummy_epg with no custom: None (clear EPG) + 4. default auto-match: cache lookup, any source + + The cache (epg_cache_by_tvg_id) is built once above with the + correct scope so the per-stream lookup is a dict access. + """ + if custom_epg_source is not None: + if custom_epg_source.source_type == "dummy": + return custom_dummy_epg_data + tvg_id = getattr(stream, "tvg_id", None) + if not tvg_id: + return None + return epg_cache_by_tvg_id.get(tvg_id) + if force_dummy_epg: + return None + tvg_id = getattr(stream, "tvg_id", None) + if not tvg_id: + return None + return epg_cache_by_tvg_id.get(tvg_id) + if not has_streams: logger.debug(f"No streams found in group {channel_group.name}") - # Delete all existing auto channels if no streams - channels_to_delete = [ch for ch in existing_channel_map.values()] + # No streams left in the group: drop the visible auto + # channels. Hidden channels are preserved so the hide + # flag survives temporary provider drops (event/PPV). + channels_to_delete = [ + ch + for ch in existing_channel_map.values() + if not ch.hidden_from_output + ] if channels_to_delete: deleted_count = len(channels_to_delete) Channel.objects.filter( @@ -1915,43 +2205,37 @@ def sync_auto_channels(account_id, scan_start_time=None): ) stream_profile_to_assign = None - # Process each current stream current_channel_number = start_number - # Always renumber all existing channels to match current sort order - # This ensures channels are always in the correct sequence + # Renumber existing channels to match sort order. Compact + # mode skips this; the end-of-iteration pack is the source + # of truth and would overwrite the renumber. + compact_mode = bool(group_custom_props.get("compact_numbering")) channels_to_renumber = [] temp_channel_number = start_number - for stream in current_streams: + for stream in current_streams if not compact_mode else []: if stream.id in existing_channel_map: channel = existing_channel_map[stream.id] - # Determine target number based on numbering mode - if channel_numbering_mode == "provider": - # Use provider number if available, otherwise use fallback with next available logic - if stream.stream_chno is not None: - target_number = stream.stream_chno - # If provider number is already used, find next available - if target_number in used_numbers: - target_number = channel_numbering_fallback - while target_number in used_numbers: - target_number += 1 - else: - # No provider number, use fallback and find next available - target_number = channel_numbering_fallback - while target_number in used_numbers: - target_number += 1 - elif channel_numbering_mode == "next_available": - # Find next available starting from 1 - target_number = 1 - while target_number in used_numbers: - target_number += 1 - else: # fixed mode (default) - # Find next available number starting from temp_channel_number - target_number = temp_channel_number - while target_number in used_numbers: - target_number += 1 + target_number = _pick_target_number( + channel_numbering_mode, + stream, + used_numbers, + temp_channel_number, + channel_numbering_fallback, + end_number=end_number, + range_start=start_number, + ) + + # Range exhausted: leave the channel at its existing + # number. The renumber pass is sort-optimization only; + # no failure record needed. + if target_number is None: + # Preserve the channel's current number in used_numbers + if channel.channel_number is not None: + used_numbers.add(channel.channel_number) + continue # Add this number to used_numbers so we don't reuse it in this batch used_numbers.add(target_number) @@ -1971,14 +2255,73 @@ def sync_auto_channels(account_id, scan_start_time=None): # Bulk update channel numbers if any need renumbering if channels_to_renumber: - Channel.objects.bulk_update(channels_to_renumber, ["channel_number"]) + Channel.objects.bulk_update( + channels_to_renumber, ["channel_number"], batch_size=500 + ) logger.info( f"Renumbered {len(channels_to_renumber)} channels to maintain sort order" ) + # When the group's configured range is narrower than its existing + # channels span, any non-hidden auto-created channel whose number + # falls outside [start, end] gets deleted. The new-channel + # creation loop below picks up the freed stream and re-creates + # the channel at a slot inside the new range, so the net user + # outcome is a renumber, not a failure. Counted in + # channels_deleted; the replacement counts in channels_created. + # Hidden channels are preserved. Runs BEFORE new-channel creation + # so slots freed by the deletions are available to incoming + # streams. + if end_number is not None: + overflow_delete_ids = [] + for stream_id, ch in list(existing_channel_map.items()): + if ch.hidden_from_output: + continue + num = ch.channel_number + if num is None: + continue + if num < start_number or num > end_number: + overflow_delete_ids.append(ch.id) + existing_channel_map.pop(stream_id, None) + used_numbers.discard(num) + if overflow_delete_ids: + deleted = Channel.objects.filter( + id__in=overflow_delete_ids + ).delete() + removed_count = ( + deleted[1].get("dispatcharr_channels.Channel", 0) + if isinstance(deleted, tuple) and len(deleted) > 1 + else len(overflow_delete_ids) + ) + channels_deleted += removed_count + logger.info( + f"Deleted {removed_count} channels outside the " + f"range {int(start_number)}-{int(end_number)} for " + f"group '{channel_group.name}'" + ) + # Reset channel number counter for processing new channels current_channel_number = start_number + # Per-channel changes are buffered and bulk_update'd once after the + # loop. update_fields is set explicitly so post_save signals only + # fire for receivers whose tracked field actually changed. + existing_dirty_channels = [] + existing_dirty_ids = set() + existing_dirty_field_set = set() + # Subset of channels whose epg_data actually changed in this + # pass. Used by the dispatcher below to fire the EPG parse + # task only for those, not for every channel in + # existing_dirty_channels. + epg_dirty_channel_ids = set() + + # New channels are buffered and bulk_create'd after the loop. + # bulk_create skips post_save, so the EPG parse task is dispatched + # once per unique epg_data_id below rather than per channel. + # Pairs are (Channel(), Stream) so the post-loop step can attach + # ChannelStream rows using the IDs Postgres returns. + new_channels_pending = [] + for stream in current_streams: processed_stream_ids.add(stream.id) try: @@ -2012,305 +2355,132 @@ def sync_auto_channels(account_id, scan_start_time=None): existing_channel = existing_channel_map.get(stream.id) if existing_channel: - # Update existing channel if needed (channel number already handled above) - channel_updated = False + # Track only the fields that actually changed, so the + # eventual UPDATE writes one column per change instead + # of every column on every channel. The dirty list is + # accumulated and bulk_update'd after the loop - + # which avoids issuing an UPDATE per channel and + # avoids firing the EPG post_save signal on saves + # that didn't touch epg_data. + dirty_fields = [] - # Use new_name instead of stream.name if existing_channel.name != new_name: existing_channel.name = new_name - channel_updated = True + dirty_fields.append("name") if existing_channel.tvg_id != stream.tvg_id: existing_channel.tvg_id = stream.tvg_id - channel_updated = True + dirty_fields.append("tvg_id") if existing_channel.tvc_guide_stationid != tvc_guide_stationid: existing_channel.tvc_guide_stationid = tvc_guide_stationid - channel_updated = True + dirty_fields.append("tvc_guide_stationid") - # Check if channel group needs to be updated (in case override was added/changed) - if existing_channel.channel_group != target_group: + # The group override may direct sync to a different + # ChannelGroup than the one currently on the row. + if existing_channel.channel_group_id != target_group.id: existing_channel.channel_group = target_group - channel_updated = True - logger.info( - f"Moved auto channel '{existing_channel.name}' from '{existing_channel.channel_group.name if existing_channel.channel_group else 'None'}' to '{target_group.name}'" - ) + dirty_fields.append("channel_group") - # Handle logo updates - current_logo = None - if custom_logo_id: - # Use the custom logo specified in group settings - from apps.channels.models import Logo - try: - current_logo = Logo.objects.get(id=custom_logo_id) - except Logo.DoesNotExist: - logger.warning( - f"Custom logo with ID {custom_logo_id} not found for existing channel, falling back to stream logo" - ) - # Fall back to stream logo if custom logo not found - if stream.logo_url: - current_logo, _ = Logo.objects.get_or_create( - url=stream.logo_url, - defaults={ - "name": stream.name or stream.tvg_id or "Unknown" - }, - ) - elif stream.logo_url: - # No custom logo configured, use stream logo - from apps.channels.models import Logo - - current_logo, _ = Logo.objects.get_or_create( - url=stream.logo_url, - defaults={ - "name": stream.name or stream.tvg_id or "Unknown" - }, - ) - - if existing_channel.logo != current_logo: + # Logo: custom group setting wins; otherwise stream logo + current_logo = ( + custom_logo + if custom_logo_id and custom_logo is not None + else _resolve_logo_for_stream(stream) + ) + current_logo_id = current_logo.id if current_logo else None + if existing_channel.logo_id != current_logo_id: existing_channel.logo = current_logo - channel_updated = True + dirty_fields.append("logo") - # Handle EPG data updates - current_epg_data = None - if custom_epg_id: - # Use the custom EPG specified in group settings (e.g., a dummy EPG) - from apps.epg.models import EPGSource - try: - epg_source = EPGSource.objects.get(id=custom_epg_id) - # For dummy EPGs, select the first (and typically only) EPGData entry from this source - if epg_source.source_type == 'dummy': - current_epg_data = EPGData.objects.filter( - epg_source=epg_source - ).first() - if not current_epg_data: - logger.warning( - f"No EPGData found for dummy EPG source {epg_source.name} (ID: {custom_epg_id})" - ) - else: - # For non-dummy sources, try to find existing EPGData by tvg_id - if stream.tvg_id: - current_epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id, - epg_source=epg_source - ).first() - except EPGSource.DoesNotExist: - logger.warning( - f"Custom EPG source with ID {custom_epg_id} not found for existing channel, falling back to auto-match" - ) - # Fall back to auto-match by tvg_id - if stream.tvg_id and not force_dummy_epg: - current_epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id - ).first() - elif stream.tvg_id and not force_dummy_epg: - # Auto-match EPG by tvg_id (original behavior) - current_epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id - ).first() - # If force_dummy_epg is True and no custom_epg_id, current_epg_data stays None - - if existing_channel.epg_data != current_epg_data: + # EPG: handled centrally by _resolve_epg_for_stream + current_epg_data = _resolve_epg_for_stream(stream) + current_epg_id = ( + current_epg_data.id if current_epg_data else None + ) + if existing_channel.epg_data_id != current_epg_id: existing_channel.epg_data = current_epg_data - channel_updated = True + dirty_fields.append("epg_data") + if current_epg_id is not None: + epg_dirty_channel_ids.add(existing_channel.id) - # Handle stream profile updates for the channel - if stream_profile_to_assign and existing_channel.stream_profile != stream_profile_to_assign: + # Stream profile: only set if group has one configured + if ( + stream_profile_to_assign is not None + and existing_channel.stream_profile_id + != stream_profile_to_assign.id + ): existing_channel.stream_profile = stream_profile_to_assign - channel_updated = True + dirty_fields.append("stream_profile") - if channel_updated: - existing_channel.save() - channels_updated += 1 - logger.debug( - f"Updated auto channel: {existing_channel.channel_number} - {existing_channel.name}" - ) - - # Update channel profile memberships for existing channels - current_memberships = set( - ChannelProfileMembership.objects.filter( - channel=existing_channel, enabled=True - ).values_list("channel_profile_id", flat=True) - ) - - target_profile_ids = set( - profile.id for profile in profiles_to_assign - ) - - # Only update if memberships have changed - if current_memberships != target_profile_ids: - # Disable all current memberships - ChannelProfileMembership.objects.filter( - channel=existing_channel - ).update(enabled=False) - - # Enable/create memberships for target profiles - for profile in profiles_to_assign: - membership, created = ( - ChannelProfileMembership.objects.get_or_create( - channel_profile=profile, - channel=existing_channel, - defaults={"enabled": True}, - ) - ) - if not created and not membership.enabled: - membership.enabled = True - membership.save() - - logger.debug( - f"Updated profile memberships for auto channel: {existing_channel.name}" - ) + if dirty_fields: + # Multi-stream channels appear once per stream; + # dedupe by id so bulk_update does not double-fire + # and channels_updated does not double-count. + if existing_channel.id not in existing_dirty_ids: + existing_dirty_channels.append(existing_channel) + existing_dirty_ids.add(existing_channel.id) + channels_updated += 1 + existing_dirty_field_set.update(dirty_fields) else: - # Create new channel - # Determine channel number based on numbering mode - if channel_numbering_mode == "provider": - # Use provider number if available, otherwise use fallback with next available logic - if stream.stream_chno is not None: - target_number = stream.stream_chno - # If provider number is already used, find next available from fallback - if target_number in used_numbers: - target_number = channel_numbering_fallback - while target_number in used_numbers: - target_number += 1 - else: - # No provider number, use fallback and find next available - target_number = channel_numbering_fallback - while target_number in used_numbers: - target_number += 1 - elif channel_numbering_mode == "next_available": - # Find next available starting from 1 - target_number = 1 - while target_number in used_numbers: - target_number += 1 - else: # fixed mode (default) - # Find next available channel number starting from current_channel_number - target_number = current_channel_number - while target_number in used_numbers: - target_number += 1 + # Range exhaustion is surfaced to the user via the + # completion notification, not swallowed. + target_number = _pick_target_number( + channel_numbering_mode, + stream, + used_numbers, + current_channel_number, + channel_numbering_fallback, + end_number=end_number, + range_start=start_number, + ) + + if target_number is None: + channels_failed += 1 + if len(failed_stream_details) < FAILURE_LOG_LIMIT: + failed_stream_details.append({ + "stream_name": stream.name, + "stream_id": stream.id, + "group": channel_group.name, + "reason": "RANGE_EXHAUSTED", + "error": ( + f"Channel number range " + f"{int(start_number)}-{int(end_number)} is full" + ), + }) + processed_stream_ids.add(stream.id) + continue # Add this number to used_numbers used_numbers.add(target_number) - channel = Channel.objects.create( - channel_number=target_number, - name=new_name, - tvg_id=stream.tvg_id, - tvc_guide_stationid=tvc_guide_stationid, - channel_group=target_group, - user_level=0, - auto_created=True, - auto_created_by=account, + # Resolve every FK BEFORE the create call so the + # initial INSERT carries the complete row. + new_logo = ( + custom_logo + if custom_logo_id and custom_logo is not None + else _resolve_logo_for_stream(stream) ) + new_epg_data = _resolve_epg_for_stream(stream) - # Associate the stream with the channel - ChannelStream.objects.create( - channel=channel, stream=stream, order=0 - ) - - # Assign to correct profiles - memberships = [ - ChannelProfileMembership( - channel_profile=profile, channel=channel, enabled=True + new_channels_pending.append( + ( + Channel( + channel_number=target_number, + name=new_name, + tvg_id=stream.tvg_id, + tvc_guide_stationid=tvc_guide_stationid, + channel_group=target_group, + user_level=0, + auto_created=True, + auto_created_by=account, + logo=new_logo, + epg_data=new_epg_data, + stream_profile=stream_profile_to_assign, + ), + stream, ) - for profile in profiles_to_assign - ] - if memberships: - ChannelProfileMembership.objects.bulk_create(memberships) - - # Try to match EPG data - if custom_epg_id: - # Use the custom EPG specified in group settings (e.g., a dummy EPG) - from apps.epg.models import EPGSource - try: - epg_source = EPGSource.objects.get(id=custom_epg_id) - # For dummy EPGs, select the first (and typically only) EPGData entry from this source - if epg_source.source_type == 'dummy': - epg_data = EPGData.objects.filter( - epg_source=epg_source - ).first() - if epg_data: - channel.epg_data = epg_data - channel.save(update_fields=["epg_data"]) - else: - logger.warning( - f"No EPGData found for dummy EPG source {epg_source.name} (ID: {custom_epg_id})" - ) - else: - # For non-dummy sources, try to find existing EPGData by tvg_id - if stream.tvg_id: - epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id, - epg_source=epg_source - ).first() - if epg_data: - channel.epg_data = epg_data - channel.save(update_fields=["epg_data"]) - except EPGSource.DoesNotExist: - logger.warning( - f"Custom EPG source with ID {custom_epg_id} not found, falling back to auto-match" - ) - # Fall back to auto-match by tvg_id - if stream.tvg_id and not force_dummy_epg: - epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id - ).first() - if epg_data: - channel.epg_data = epg_data - channel.save(update_fields=["epg_data"]) - elif stream.tvg_id and not force_dummy_epg: - # Auto-match EPG by tvg_id (original behavior) - epg_data = EPGData.objects.filter( - tvg_id=stream.tvg_id - ).first() - if epg_data: - channel.epg_data = epg_data - channel.save(update_fields=["epg_data"]) - elif force_dummy_epg: - # Force dummy EPG with no custom EPG selected (set to None) - channel.epg_data = None - channel.save(update_fields=["epg_data"]) - - # Handle logo - if custom_logo_id: - # Use the custom logo specified in group settings - from apps.channels.models import Logo - try: - custom_logo = Logo.objects.get(id=custom_logo_id) - channel.logo = custom_logo - channel.save(update_fields=["logo"]) - except Logo.DoesNotExist: - logger.warning( - f"Custom logo with ID {custom_logo_id} not found, falling back to stream logo" - ) - # Fall back to stream logo if custom logo not found - if stream.logo_url: - logo, _ = Logo.objects.get_or_create( - url=stream.logo_url, - defaults={ - "name": stream.name or stream.tvg_id or "Unknown" - }, - ) - channel.logo = logo - channel.save(update_fields=["logo"]) - elif stream.logo_url: - from apps.channels.models import Logo - - logo, _ = Logo.objects.get_or_create( - url=stream.logo_url, - defaults={ - "name": stream.name or stream.tvg_id or "Unknown" - }, - ) - channel.logo = logo - channel.save(update_fields=["logo"]) - - # Handle stream profile assignment - if stream_profile_to_assign: - channel.stream_profile = stream_profile_to_assign - channel.save(update_fields=['stream_profile']) - channels_created += 1 - logger.debug( - f"Created auto channel: {channel.channel_number} - {channel.name}" ) # Increment channel number for next iteration (only in fixed mode) @@ -2323,12 +2493,171 @@ def sync_auto_channels(account_id, scan_start_time=None): logger.error( f"Error processing auto channel for stream {stream.name}: {str(e)}" ) + channels_failed += 1 + if len(failed_stream_details) < FAILURE_LOG_LIMIT: + failed_stream_details.append({ + "stream_name": stream.name, + "stream_id": stream.id, + "group": channel_group.name, + "reason": _classify_sync_failure(e), + "error": str(e), + }) continue - # Delete channels for streams that no longer exist - channels_to_delete = [] + # Bulk-create channels, then dependent rows using the IDs + # Postgres returns. bulk_create skips post_save, so the EPG + # parse task is dispatched explicitly per-epg_data_id below + # to avoid flooding Celery at scale. + if new_channels_pending: + channel_objs = [pair[0] for pair in new_channels_pending] + streams_for_new = [pair[1] for pair in new_channels_pending] + Channel.objects.bulk_create(channel_objs, batch_size=500) + + ChannelStream.objects.bulk_create( + [ + ChannelStream( + channel_id=channel_objs[i].id, + stream_id=streams_for_new[i].id, + order=0, + ) + for i in range(len(channel_objs)) + ], + batch_size=500, + ) + + if profiles_to_assign: + ChannelProfileMembership.objects.bulk_create( + [ + ChannelProfileMembership( + channel_id=ch.id, + channel_profile_id=profile.id, + enabled=True, + ) + for ch in channel_objs + for profile in profiles_to_assign + ], + ignore_conflicts=True, + batch_size=500, + ) + + channels_created += len(channel_objs) + + # One EPG parse task per unique EPGData replaces the + # per-channel post_save dispatch bypassed by bulk_create. + from apps.epg.tasks import parse_programs_for_tvg_id + + unique_epg_ids = { + ch.epg_data_id for ch in channel_objs if ch.epg_data_id + } + for epg_id in unique_epg_ids: + parse_programs_for_tvg_id.delay(epg_id) + + logger.debug( + f"Bulk created {len(channel_objs)} channels in group " + f"'{channel_group.name}'; dispatched " + f"{len(unique_epg_ids)} unique EPG parse task(s)" + ) + + # bulk_update writes only the columns named in `fields` and + # bypasses post_save, so the EPG refresh signal cannot fire here. + # Dispatch one parse task per unique EPGData id when epg_data was + # in the dirty set, mirroring the new-channel path above. + if existing_dirty_channels: + Channel.objects.bulk_update( + existing_dirty_channels, + fields=list(existing_dirty_field_set), + batch_size=500, + ) + if epg_dirty_channel_ids: + from apps.epg.tasks import parse_programs_for_tvg_id + + # Dispatch only for channels whose epg_data_id changed. + # Other dirty channels would queue redundant parses. + unique_epg_ids = { + ch.epg_data_id + for ch in existing_dirty_channels + if ch.id in epg_dirty_channel_ids and ch.epg_data_id + } + for epg_id in unique_epg_ids: + parse_programs_for_tvg_id.delay(epg_id) + logger.debug( + f"Bulk updated {len(existing_dirty_channels)} existing " + f"channels (fields: {sorted(existing_dirty_field_set)})" + ) + + # Reconcile ChannelProfileMembership in two writes: one + # bulk_update for enable-flips, one bulk_create for missing + # rows. Avoids a per-channel save loop. + existing_channel_ids = [ + c.id for c in existing_channel_map.values() + ] + target_profile_ids = {p.id for p in profiles_to_assign} + if existing_channel_ids: + membership_rows = list( + ChannelProfileMembership.objects.filter( + channel_id__in=existing_channel_ids + ).only("id", "channel_id", "channel_profile_id", "enabled") + ) + memberships_by_channel = {} + for m in membership_rows: + memberships_by_channel.setdefault(m.channel_id, []).append(m) + + rows_to_flip = [] + rows_to_create = [] + for ch_id in existing_channel_ids: + rows = memberships_by_channel.get(ch_id, []) + have_for_target = set() + for m in rows: + if m.channel_profile_id in target_profile_ids: + have_for_target.add(m.channel_profile_id) + if not m.enabled: + m.enabled = True + rows_to_flip.append(m) + else: + if m.enabled: + m.enabled = False + rows_to_flip.append(m) + missing = target_profile_ids - have_for_target + for pid in missing: + rows_to_create.append( + ChannelProfileMembership( + channel_id=ch_id, + channel_profile_id=pid, + enabled=True, + ) + ) + + if rows_to_flip: + ChannelProfileMembership.objects.bulk_update( + rows_to_flip, ["enabled"], batch_size=500 + ) + if rows_to_create: + ChannelProfileMembership.objects.bulk_create( + rows_to_create, ignore_conflicts=True, batch_size=500 + ) + if rows_to_flip or rows_to_create: + logger.debug( + f"Reconciled memberships for " + f"{len(existing_channel_ids)} channels " + f"({len(rows_to_flip)} flipped, " + f"{len(rows_to_create)} created)" + ) + + # Delete channels whose streams have all disappeared. + # Hidden channels are preserved so event/PPV holds across + # provider drops. + channel_streams_in_group = {} for stream_id, channel in existing_channel_map.items(): - if stream_id not in processed_stream_ids: + channel_streams_in_group.setdefault(channel.id, []).append( + (stream_id, channel) + ) + channels_to_delete = [] + for ch_id, pairs in channel_streams_in_group.items(): + channel = pairs[0][1] + if channel.hidden_from_output: + continue + stream_ids = {sid for sid, _ in pairs} + if not (stream_ids & processed_stream_ids): channels_to_delete.append(channel) if channels_to_delete: @@ -2341,34 +2670,98 @@ def sync_auto_channels(account_id, scan_start_time=None): f"Deleted {deleted_count} auto channels for removed streams" ) - # Additional cleanup: Remove auto-created channels that no longer have any valid streams - # This handles the case where streams were deleted due to stale retention policy - orphaned_channels = Channel.objects.filter( - auto_created=True, - auto_created_by=account - ).exclude( - # Exclude channels that still have valid stream associations - id__in=ChannelStream.objects.filter( - stream__m3u_account=account, - stream__isnull=False - ).values_list('channel_id', flat=True) - ) + # Compact-mode pack: hidden channels release their number and + # visible channels pack contiguously into [start, end]. Runs + # after create/update/delete so the channel set is stable. + if compact_mode: + from apps.channels.compact_numbering import repack_group - deleted_total, _ = orphaned_channels.delete() - if deleted_total: - channels_deleted += deleted_total - logger.info( - f"Deleted {deleted_total} orphaned auto channels with no valid streams" + pack_result = repack_group(group_relation) + if pack_result["failed"]: + channels_failed += pack_result["failed"] + if ( + len(failed_stream_details) < FAILURE_LOG_LIMIT + ): + failed_stream_details.append( + { + "stream_name": None, + "stream_id": None, + "group": channel_group.name, + "reason": "RANGE_EXHAUSTED", + "error": ( + f"Compact pack: {pack_result['failed']} " + f"visible channel(s) could not fit in range " + f"{int(start_number)}" + + ( + f"-{int(end_number)}" + if end_number + else "+" + ) + ), + } + ) + logger.debug( + f"Compact pack for group '{channel_group.name}': " + f"{pack_result['assigned']} assigned, " + f"{pack_result['released']} released, " + f"{pack_result['failed']} failed" + ) + + # Cleanup mode read from account.custom_properties.orphan_channel_cleanup: + # "always" (default; key absent) removes every orphan auto channel; + # "preserve_customized" keeps those with a ChannelOverride row; + # "never" disables cleanup. Hidden channels are preserved across all + # modes so event/PPV channels that come and go are not silently lost. + cleanup_mode = (account.custom_properties or {}).get( + "orphan_channel_cleanup", "always" + ) + if cleanup_mode != "never": + orphaned_channels = Channel.objects.filter( + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ).exclude( + id__in=ChannelStream.objects.filter( + stream__m3u_account=account, + stream__isnull=False, + ).values_list("channel_id", flat=True) ) + if cleanup_mode == "preserve_customized": + orphaned_channels = orphaned_channels.filter(override__isnull=True) + + _, per_model = orphaned_channels.delete() + deleted_channels = per_model.get("dispatcharr_channels.Channel", 0) + if deleted_channels: + channels_deleted += deleted_channels + logger.info( + f"Deleted {deleted_channels} orphaned auto channels with no valid streams (mode={cleanup_mode})" + ) logger.info( - f"Auto channel sync complete for account {account.name}: {channels_created} created, {channels_updated} updated, {channels_deleted} deleted" + f"Auto channel sync complete for account {account.name}: " + f"{channels_created} created, {channels_updated} updated, " + f"{channels_deleted} deleted, {channels_failed} failed" ) - return f"Auto sync: {channels_created} channels created, {channels_updated} updated, {channels_deleted} deleted" + return { + "status": "ok", + "channels_created": channels_created, + "channels_updated": channels_updated, + "channels_deleted": channels_deleted, + "channels_failed": channels_failed, + "failed_stream_details": failed_stream_details, + } except Exception as e: logger.error(f"Error in auto channel sync for account {account_id}: {str(e)}") - return f"Auto sync error: {str(e)}" + return { + "status": "error", + "error": str(e), + "channels_created": 0, + "channels_updated": 0, + "channels_deleted": 0, + "channels_failed": 0, + "failed_stream_details": [], + } def get_transformed_credentials(account, profile=None): @@ -3074,15 +3467,34 @@ def _refresh_single_m3u_account_impl(account_id): # Run auto channel sync after successful refresh auto_sync_message = "" + auto_sync_result = {} try: - sync_result = sync_auto_channels( + auto_sync_result = sync_auto_channels( account_id, scan_start_time=str(refresh_start_timestamp) - ) + ) or {} logger.info( - f"Auto channel sync result for account {account_id}: {sync_result}" + f"Auto channel sync result for account {account_id}: {auto_sync_result}" ) - if sync_result and "created" in sync_result: - auto_sync_message = f" {sync_result}." + if auto_sync_result.get("status") == "ok": + created = auto_sync_result.get("channels_created", 0) + updated = auto_sync_result.get("channels_updated", 0) + deleted = auto_sync_result.get("channels_deleted", 0) + failed = auto_sync_result.get("channels_failed", 0) + if created or updated or deleted or failed: + parts = [] + if created: + parts.append(f"{created} channel(s) created") + if updated: + parts.append(f"{updated} updated") + if deleted: + parts.append(f"{deleted} deleted") + if failed: + parts.append(f"{failed} failed") + auto_sync_message = f" Auto-sync: {', '.join(parts)}." + elif auto_sync_result.get("status") == "error": + auto_sync_message = ( + f" Auto-sync error: {auto_sync_result.get('error', 'unknown')}." + ) except Exception as e: logger.error( f"Error running auto channel sync for account {account_id}: {str(e)}" @@ -3127,6 +3539,14 @@ def _refresh_single_m3u_account_impl(account_id): streams_created=streams_created, streams_updated=streams_updated, streams_deleted=streams_deleted, + # Structured auto-sync counts so the frontend can render a + # warning card when anything failed, without parsing the + # free-text last_message. + channels_created=auto_sync_result.get("channels_created", 0), + channels_updated=auto_sync_result.get("channels_updated", 0), + channels_deleted=auto_sync_result.get("channels_deleted", 0), + channels_failed=auto_sync_result.get("channels_failed", 0), + failed_stream_details=auto_sync_result.get("failed_stream_details", []), message=account.last_message, ) diff --git a/apps/m3u/tests/test_account_destroy.py b/apps/m3u/tests/test_account_destroy.py new file mode 100644 index 00000000..e3a46aa2 --- /dev/null +++ b/apps/m3u/tests/test_account_destroy.py @@ -0,0 +1,143 @@ +""" +DELETE /api/m3u/accounts/{id}/ behavior. + +The endpoint always cascade-deletes auto-created channels owned by the +account. Manual channels survive even if some of their streams were +owned by the deleted account; only the streams from that account go +away with the account, and the channel keeps any streams sourced from +other accounts. The legacy ``cleanup_channels`` query parameter is +accepted but ignored. +""" +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APIClient + +from apps.channels.models import ( + Channel, + ChannelGroup, + ChannelStream, + Stream, +) +from apps.m3u.models import M3UAccount + +User = get_user_model() + + +class M3UAccountDestroyTests(TestCase): + def setUp(self): + self.user = User.objects.create_user( + username="destroyer", password="testpass123" + ) + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + # Patching the proxy stop here keeps the Redis-backed call out + # of the test path. The endpoint loops over every auto-created + # channel before the DB transaction; the real implementation is + # exercised by integration tests, not unit tests. + self._stop_patch = patch( + "apps.proxy.ts_proxy.services.channel_service." + "ChannelService.stop_channel" + ) + self._stop_patch.start() + self.addCleanup(self._stop_patch.stop) + + self.account = M3UAccount.objects.create( + name="ProviderA", + server_url="http://example.com/a.m3u", + ) + self.other_account = M3UAccount.objects.create( + name="ProviderB", + server_url="http://example.com/b.m3u", + ) + self.group = ChannelGroup.objects.create(name="News") + + def _make_stream(self, account, name="ESPN"): + return Stream.objects.create( + name=name, + url=f"http://example.com/{name.lower()}.m3u8", + m3u_account=account, + channel_group=self.group, + tvg_id=name.lower(), + last_seen=timezone.now(), + ) + + def test_cascade_deletes_auto_created_channels(self): + # Two auto-created channels under the account being deleted. + for n in (101.0, 102.0): + ch = Channel.objects.create( + channel_number=n, + name=f"Auto {n}", + channel_group=self.group, + auto_created=True, + auto_created_by=self.account, + ) + ChannelStream.objects.create( + channel=ch, + stream=self._make_stream(self.account, name=f"Auto{int(n)}"), + order=0, + ) + + response = self.client.delete( + f"/api/m3u/accounts/{self.account.id}/" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["deleted_channels"], 2) + self.assertFalse(M3UAccount.objects.filter(id=self.account.id).exists()) + self.assertFalse( + Channel.objects.filter(auto_created_by_id=self.account.id).exists() + ) + + def test_manual_channel_survives_with_other_provider_streams(self): + # Manual channel with one stream from each account. The provider + # account's stream goes away with the account; the other + # account's stream stays, and the channel survives. + manual = Channel.objects.create( + channel_number=200.0, + name="Manual", + channel_group=self.group, + auto_created=False, + ) + provider_stream = self._make_stream(self.account, name="ProviderA") + other_stream = self._make_stream(self.other_account, name="ProviderB") + ChannelStream.objects.create(channel=manual, stream=provider_stream, order=0) + ChannelStream.objects.create(channel=manual, stream=other_stream, order=1) + + response = self.client.delete( + f"/api/m3u/accounts/{self.account.id}/" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["deleted_channels"], 0) + + manual.refresh_from_db() + remaining = list(manual.channelstream_set.values_list("stream__id", flat=True)) + self.assertEqual(remaining, [other_stream.id]) + + def test_legacy_cleanup_channels_param_is_ignored(self): + # Behavior must be identical with or without the deprecated + # ``cleanup_channels`` query parameter. + ch = Channel.objects.create( + channel_number=300.0, + name="Auto", + channel_group=self.group, + auto_created=True, + auto_created_by=self.account, + ) + response = self.client.delete( + f"/api/m3u/accounts/{self.account.id}/?cleanup_channels=false" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["deleted_channels"], 1) + self.assertFalse(Channel.objects.filter(id=ch.id).exists()) + + def test_no_op_when_account_has_no_auto_created_channels(self): + response = self.client.delete( + f"/api/m3u/accounts/{self.account.id}/" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["deleted_channels"], 0) diff --git a/apps/m3u/tests/test_sync_compound.py b/apps/m3u/tests/test_sync_compound.py new file mode 100644 index 00000000..5f7b4742 --- /dev/null +++ b/apps/m3u/tests/test_sync_compound.py @@ -0,0 +1,240 @@ +""" +Compound-fixture sync tests. + +Where individual sync tests cover one variation at a time (multi-stream, +hidden, override, manual), this module seeds all of them in the same +fixture and asserts the constraints still hold when sync sees the full +mix on a single account. The point is to catch interactions that pass +each isolated test but break when the conditions overlap. +""" + +from datetime import timedelta + +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import ( + Channel, + ChannelGroup, + ChannelGroupM3UAccount, + ChannelOverride, + ChannelStream, + Stream, +) +from apps.m3u.models import M3UAccount +from apps.m3u.tasks import sync_auto_channels + + +def _scan_start_time(): + return (timezone.now() - timedelta(minutes=1)).isoformat() + + +class CompoundFixtureSyncTests(TestCase): + """ + Single fixture covers: multi-stream auto channel, hidden auto channel, + auto channel with channel_number override, manual channel that shares + the group with auto-created rows. After sync runs, every channel must + end up in the state its individual test would have asserted. + """ + + def setUp(self): + self.account = M3UAccount.objects.create( + name="Compound Provider", + server_url="http://example.com/compound.m3u", + ) + self.group = ChannelGroup.objects.create(name="Compound Group") + self.relation = ChannelGroupM3UAccount.objects.create( + m3u_account=self.account, + channel_group=self.group, + enabled=True, + auto_channel_sync=True, + auto_sync_channel_start=100, + auto_sync_channel_end=199, + ) + now = timezone.now() + + # ── Multi-stream auto channel: two streams, one fresh, one stale. + # The channel must survive the stale stream's disappearance because + # the fresh one keeps the channel alive. + self.multi_stream_a = Stream.objects.create( + name="MultiCh HD", + url="http://example.com/multi-a.m3u8", + m3u_account=self.account, + channel_group=self.group, + tvg_id="multi", + last_seen=now, + ) + self.multi_stream_b = Stream.objects.create( + name="MultiCh HD", + url="http://example.com/multi-b.m3u8", + m3u_account=self.account, + channel_group=self.group, + tvg_id="multi", + last_seen=now - timedelta(days=2), + ) + self.multi_channel = Channel.objects.create( + name="MultiCh HD", + channel_number=100, + channel_group=self.group, + auto_created=True, + auto_created_by=self.account, + ) + ChannelStream.objects.create( + channel=self.multi_channel, stream=self.multi_stream_a, order=0 + ) + ChannelStream.objects.create( + channel=self.multi_channel, stream=self.multi_stream_b, order=1 + ) + + # ── Hidden auto channel: visible from the table's perspective only + # to admins with the Hidden filter on. Sync must not reuse its + # channel_number for a different channel just because it's hidden. + self.hidden_stream = Stream.objects.create( + name="HiddenCh", + url="http://example.com/hidden.m3u8", + m3u_account=self.account, + channel_group=self.group, + tvg_id="hidden", + last_seen=now, + ) + self.hidden_channel = Channel.objects.create( + name="HiddenCh", + channel_number=101, + channel_group=self.group, + auto_created=True, + auto_created_by=self.account, + hidden_from_output=True, + ) + ChannelStream.objects.create( + channel=self.hidden_channel, stream=self.hidden_stream, order=0 + ) + + # ── Overridden auto channel: user pinned channel_number to 150 via + # an override row. Sync must not clobber the override; the + # effective channel_number stays 150 even though the raw column + # may evolve. + self.overridden_stream = Stream.objects.create( + name="OverriddenCh", + url="http://example.com/overridden.m3u8", + m3u_account=self.account, + channel_group=self.group, + tvg_id="overridden", + last_seen=now, + ) + self.overridden_channel = Channel.objects.create( + name="OverriddenCh", + channel_number=102, + channel_group=self.group, + auto_created=True, + auto_created_by=self.account, + ) + ChannelStream.objects.create( + channel=self.overridden_channel, + stream=self.overridden_stream, + order=0, + ) + ChannelOverride.objects.create( + channel=self.overridden_channel, + channel_number=150, + name="My Pinned Name", + ) + + # ── Manual channel sharing the group: auto_created=False, user + # picked channel_number 175 themselves. Sync must not touch this + # row at all. + self.manual_channel = Channel.objects.create( + name="ManualCh", + channel_number=175, + channel_group=self.group, + auto_created=False, + ) + + # New stream that has no existing channel; sync should create a + # fresh channel for it within the configured range and skip + # 100-102 (in use), 150 (overridden), 175 (manual). + self.new_stream = Stream.objects.create( + name="FreshCh", + url="http://example.com/fresh.m3u8", + m3u_account=self.account, + channel_group=self.group, + tvg_id="fresh", + last_seen=now, + ) + + def test_compound_fixture_each_invariant_holds_after_sync(self): + sync_auto_channels(self.account.id, scan_start_time=_scan_start_time()) + + # Multi-stream channel survives. + self.assertTrue( + Channel.objects.filter(id=self.multi_channel.id).exists(), + "Multi-stream channel was deleted even though one stream is alive", + ) + + # Hidden channel survives and remains hidden. + self.hidden_channel.refresh_from_db() + self.assertTrue(self.hidden_channel.hidden_from_output) + + # Overridden channel: the override row is intact, not cleared by + # sync. The pinned channel_number persists. + override = ChannelOverride.objects.get(channel=self.overridden_channel) + self.assertEqual(override.channel_number, 150) + self.assertEqual(override.name, "My Pinned Name") + + # Manual channel is untouched. + self.manual_channel.refresh_from_db() + self.assertFalse(self.manual_channel.auto_created) + self.assertEqual(self.manual_channel.channel_number, 175) + + # The fresh stream becomes a new auto channel; its channel_number + # falls inside the configured range and must not collide with any + # existing number (100-102 used by the auto channels, 150 pinned + # by the override, 175 the manual channel). + fresh_stream_id = self.new_stream.id + new_channel_qs = Channel.objects.filter( + channelstream__stream_id=fresh_stream_id, + auto_created=True, + auto_created_by=self.account, + ).distinct() + self.assertEqual( + new_channel_qs.count(), + 1, + "Sync did not create exactly one channel for the new stream", + ) + new_number = new_channel_qs.first().channel_number + self.assertIsNotNone(new_number) + self.assertGreaterEqual(new_number, 100) + self.assertLessEqual(new_number, 199) + self.assertNotIn(new_number, {100, 101, 102, 150, 175}) + + def test_hidden_channel_number_not_reassigned_to_new_stream(self): + # Targeted assertion isolated from the broader fixture invariants + # so the failure mode is unambiguous when this single property + # regresses. + sync_auto_channels(self.account.id, scan_start_time=_scan_start_time()) + + new_channel = ( + Channel.objects.filter( + channelstream__stream_id=self.new_stream.id, + auto_created=True, + auto_created_by=self.account, + ) + .distinct() + .first() + ) + self.assertIsNotNone(new_channel) + self.assertNotEqual( + new_channel.channel_number, + self.hidden_channel.channel_number, + "Hidden channel's number was reassigned; hidden channels must " + "still occupy their slot in the used_numbers set", + ) + + def test_override_channel_number_preserved_through_sync(self): + # If a future regression caused sync to write to ChannelOverride + # (it must not), this test catches it because the override would + # change after refresh. + original_pin = 150 + sync_auto_channels(self.account.id, scan_start_time=_scan_start_time()) + + override = ChannelOverride.objects.get(channel=self.overridden_channel) + self.assertEqual(override.channel_number, original_pin) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py new file mode 100644 index 00000000..e89ac410 --- /dev/null +++ b/apps/m3u/tests/test_sync_correctness.py @@ -0,0 +1,2051 @@ +""" +Correctness reproductions and fixes for sync_auto_channels. + +Each test documents a specific bug in the unpatched code: reproduces the bug +first (fails on HEAD prior to the Tier 2 patch), then is flipped to assert +the correct post-fix behavior. Comments call out the failure mode and the +fix location. +""" +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import ( + Channel, + ChannelGroup, + ChannelGroupM3UAccount, + ChannelStream, + Stream, +) +from apps.m3u.models import M3UAccount +from apps.m3u.tasks import sync_auto_channels + + +def _make_account(name="Test Provider", custom_properties=None): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com/test.m3u", + custom_properties=custom_properties, + ) + + +def _make_group(name="Sports"): + return ChannelGroup.objects.create(name=name) + + +def _attach_group_to_account(account, group, custom_properties=None): + return ChannelGroupM3UAccount.objects.create( + m3u_account=account, + channel_group=group, + enabled=True, + auto_channel_sync=True, + auto_sync_channel_start=100, + custom_properties=custom_properties, + ) + + +def _make_stream(account, group, name="ESPN", tvg_id="espn"): + return Stream.objects.create( + name=name, + url=f"http://example.com/{name.lower()}.m3u8", + m3u_account=account, + channel_group=group, + tvg_id=tvg_id, + last_seen=timezone.now(), + ) + + +def _sync(account): + # Use a scan_start_time prior to the freshly-created streams so + # `last_seen__gte=scan_start_time` includes them. + return sync_auto_channels( + account.id, + scan_start_time=(timezone.now() - timezone.timedelta(minutes=1)).isoformat(), + ) + + +class CustomPropertiesTypeHandlingTests(TestCase): + """ + `custom_properties` is a JSONField but at least one historical + code path stored a JSON string instead of a dict. When + `sync_auto_channels` reads `group_relation.custom_properties` and calls + `.get()` on it, AttributeError is raised because the value is a string. + + Failure mode on unpatched code: + AttributeError: 'str' object has no attribute 'get' + + The outer try/except in sync_auto_channels catches the exception and + returns an error string, aborting the sync for every other well-formed + group in the same account. + """ + + def test_group_custom_properties_as_string_does_not_abort_sync(self): + account = _make_account() + group = _make_group(name="Sports") + # Deliberately stored as a JSON-encoded string (reproducing #432). + _attach_group_to_account( + account, + group, + custom_properties='{"force_dummy_epg": true}', + ) + _make_stream(account, group) + + result = _sync(account) + + # Post-fix: sync does not abort with an error and the stream is + # processed normally. Pre-fix this returned "Auto sync error: ...". + self.assertEqual(result.get("status"), "ok") + + def test_group_custom_properties_as_string_with_streams_still_creates_channels(self): + # Tighter version of the first test: with a real stream attached to + # the group, the pre-fix failure aborts before creating the channel. + # Post-fix, the channel is created normally and the string-typed + # custom_properties is treated as an empty dict (no custom regex, + # no group_override, etc.). + account = _make_account() + group = _make_group(name="Sports") + _attach_group_to_account( + account, + group, + custom_properties='{"name_regex_pattern": "HD"}', + ) + _make_stream(account, group, name="ESPN HD", tvg_id="espn") + + result = _sync(account) + + self.assertEqual(result.get("status"), "ok") + self.assertEqual( + Channel.objects.filter(auto_created=True, auto_created_by=account).count(), + 1, + "Exactly one channel should have been created from the single stream", + ) + + +class NullAutoCreatedByOrphanTests(TestCase): + """ + Channels with `auto_created=True, auto_created_by=NULL` are never touched + by sync (it filters on `auto_created_by=account`). These rows accumulate + indefinitely. + + The fix is a backfill migration that either re-attributes the row by + matching its linked stream to an M3U account, or deletes it if + unattributable. Until that migration runs, sync has no way to clean them + up. This test confirms the current behavior so we can write the migration + with correct expectations. + """ + + def test_null_auto_created_by_rows_are_untouched_by_sync(self): + account = _make_account() + group = _make_group(name="Entertainment") + _attach_group_to_account(account, group) + + orphan = Channel.objects.create( + name="OrphanChannel", + channel_number=999, + channel_group=group, + auto_created=True, + auto_created_by=None, + ) + + # Run sync with no streams; normal orphan cleanup only deletes rows + # scoped to the account. The NULL-owner row should still be there. + _sync(account) + + orphan.refresh_from_db() + self.assertIsNotNone(orphan.id, "Pre-fix: NULL-owner rows are not cleaned") + + +class AccountDeleteCleanupTests(TestCase): + """ + Deleting an M3UAccount unconditionally cascades auto-created + channels owned by the account. An auto-created channel without a + surviving provider has no useful state (it cannot sync, its + streams cascade away), so the destroy endpoint always removes + them. The legacy ``cleanup_channels`` query parameter is accepted + for backward compatibility but ignored. + """ + + def test_destroy_cascades_auto_created_channels_unconditionally(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_delcleanup_a", + password="pw", + user_level=10, + ) + account = _make_account() + group = _make_group(name="Entertainment") + Channel.objects.create( + name="ShouldCascade", + channel_number=700, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.delete(f"/api/m3u/accounts/{account.id}/") + + # Cascade returns 200 with the deleted count body so the UI can + # toast the number of channels removed alongside the account. + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data.get("deleted_channels"), 1) + self.assertEqual( + Channel.objects.filter(auto_created=True).count(), 0 + ) + + def test_destroy_with_cleanup_removes_auto_created_channels(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_delcleanup_b", + password="pw", + user_level=10, + ) + account = _make_account() + group = _make_group(name="News") + Channel.objects.create( + name="Cleanme1", + channel_number=710, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + Channel.objects.create( + name="Cleanme2", + channel_number=711, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.delete( + f"/api/m3u/accounts/{account.id}/?cleanup_channels=true" + ) + + # Cleanup returns 200 with the deleted count body (so the UI can + # toast "Deleted N channels"). Without cleanup it would be 204. + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data.get("deleted_channels"), 2) + self.assertEqual( + Channel.objects.filter(auto_created=True).count(), + 0, + ) + + def test_auto_created_channels_count_endpoint(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_delcleanup_c", + password="pw", + user_level=10, + ) + account = _make_account() + group = _make_group(name="Sports") + for i in range(3): + Channel.objects.create( + name=f"AutoChan{i}", + channel_number=800 + i, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.get( + f"/api/m3u/accounts/{account.id}/auto-created-channels-count/" + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 3) + self.assertEqual(len(response.data["sample_names"]), 3) + + +class ChannelDeleteStopsProxyTests(TestCase): + """ + Issue #870: When an auto-sync refresh deletes a channel that has an + active proxy session, the session's Redis state survives, making the UI + "Stop" button fail with 'Channel not found'. Fix is a pre_delete signal + on Channel that calls ChannelService.stop_channel first, covering + manual, bulk, and sync-triggered deletes uniformly. + """ + + def test_pre_delete_signal_calls_stop_channel(self): + from unittest.mock import patch + + account = _make_account() + group = _make_group(name="Sports") + channel = Channel.objects.create( + name="ESPN", + channel_number=1, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + channel_uuid = str(channel.uuid) + + with patch( + "apps.proxy.ts_proxy.services.channel_service.ChannelService.stop_channel" + ) as mock_stop: + channel.delete() + + mock_stop.assert_called_once_with(channel_uuid) + + def test_pre_delete_signal_swallows_stop_errors(self): + """Proxy failure must not block the DB delete.""" + from unittest.mock import patch + + account = _make_account() + group = _make_group(name="Sports") + channel = Channel.objects.create( + name="ESPN", + channel_number=1, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + with patch( + "apps.proxy.ts_proxy.services.channel_service.ChannelService.stop_channel", + side_effect=Exception("proxy is down"), + ): + channel.delete() + + self.assertFalse(Channel.objects.filter(id=channel.id).exists()) + + +class RangeEnforcementTests(TestCase): + """ + Tier 4 feature: optional per-group `auto_sync_channel_end` caps the + number range a group can use. Streams that don't fit get surfaced in + the completion notification as failures rather than silently spilling + into a neighboring group's range. + """ + + def test_range_allows_streams_within_bounds(self): + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_end = 105 + rel.auto_sync_channel_start = 100 + rel.save() + + for i in range(3): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 3) + self.assertEqual(result["channels_failed"], 0) + + def test_range_exhaustion_surfaces_failures(self): + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + # Range holds 2 slots: 100, 101. + rel.auto_sync_channel_start = 100 + rel.auto_sync_channel_end = 101 + rel.save() + + for i in range(4): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 2) + self.assertEqual(result["channels_failed"], 2) + self.assertEqual(len(result["failed_stream_details"]), 2) + for detail in result["failed_stream_details"]: + self.assertIn("range", detail["error"].lower()) + self.assertEqual(detail["reason"], "RANGE_EXHAUSTED") + + def test_used_numbers_seed_includes_override_pinned_values(self): + # Channel A has override.channel_number=42 pinning effective #42 + # globally. Sync creating a new channel B in fixed mode starting + # at 1 must skip 42; otherwise B's raw channel_number=42 collides + # with A's effective number, producing duplicate output entries + # at the same channel number. + from apps.channels.models import ChannelOverride + + account = _make_account() + group = _make_group(name="News") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 1.0 + rel.save() + + # A: pre-existing manual channel with override pinning #42. + a = Channel.objects.create( + channel_number=999.0, + name="UserPinned", + tvg_id="user_pinned", + channel_group=group, + auto_created=False, + ) + ChannelOverride.objects.create(channel=a, channel_number=42.0) + + # Provide enough new streams to walk past 42 in fixed mode. + for i in range(45): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + self.assertEqual(result["status"], "ok") + + new_numbers = list( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).values_list("channel_number", flat=True) + ) + self.assertNotIn( + 42.0, + new_numbers, + "Sync must skip override-pinned numbers; assigning #42 to a " + "new auto-channel duplicates A's effective channel number.", + ) + + def test_provider_mode_fallback_respects_range_start(self): + # Provider-mode streams without a usable stream_chno fall back to + # the next-available picker. The fallback must honor the group's + # configured `auto_sync_channel_start` so freshly-created channels + # never land below the user's chosen range. Without this, a group + # configured for [100, 200] silently spawned channels at #1 when + # the provider omitted channel-number metadata. + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.auto_sync_channel_end = 200 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + + _make_stream(account, group, name="NoChno", tvg_id="nc") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 1) + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertGreaterEqual(created.channel_number, 100) + self.assertLessEqual(created.channel_number, 200) + + +class ReservationBehaviorTests(TestCase): + """ + Ranges with both `auto_sync_channel_start` and `auto_sync_channel_end` + set are advisory at the UI layer. Sync does not treat another group's + declared range as off-limits; only channels that are actually assigned + a number count toward used_numbers. This lets two groups carrying the + same category (e.g. "Entertainment" from two different providers) + share a number range by cooperative fill, which is the intended UX + for merged-provider setups. + + The UI surfaces overlap between ranges as an advisory heads-up rather + than a blocking error, and the sync task enforces only real occupancy. + """ + + def test_self_reservation_lets_group_use_its_own_range(self): + # A group with a range assigns within that range. This is the + # baseline case the UI advisory is built around. + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 1000 + rel.auto_sync_channel_end = 1010 + rel.save() + + for i in range(3): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 3) + numbers = sorted( + Channel.objects.filter( + channel_group=group, + auto_created=True, + auto_created_by=account, + ).values_list("channel_number", flat=True) + ) + self.assertTrue( + all(1000 <= n <= 1010 for n in numbers), + f"Channels outside own range: {numbers}", + ) + + def test_overlapping_ranges_cooperate_when_no_existing_channels(self): + # Two groups with overlapping ranges but no pre-existing channels + # in the overlap should both be able to fill numbers in the shared + # range. Sync only avoids actual occupancy, not declared intent. + account_a = _make_account(name="Provider A") + account_b = _make_account(name="Provider B") + group_a = _make_group(name="Entertainment A") + group_b = _make_group(name="Entertainment B") + + rel_a = _attach_group_to_account(account_a, group_a) + rel_a.auto_sync_channel_start = 1 + rel_a.auto_sync_channel_end = 10 + rel_a.save() + + rel_b = _attach_group_to_account(account_b, group_b) + rel_b.auto_sync_channel_start = 1 + rel_b.auto_sync_channel_end = 10 + rel_b.save() + + # 3 streams in A, 3 in B. With shared range 1-10 and no prior + # occupancy, A fills some numbers, B fills the remaining free ones. + for i in range(3): + _make_stream(account_a, group_a, name=f"A{i}", tvg_id=f"a{i}") + _make_stream(account_b, group_b, name=f"B{i}", tvg_id=f"b{i}") + + _sync(account_a) + _sync(account_b) + + all_numbers = sorted( + Channel.objects.filter(auto_created=True).values_list( + "channel_number", flat=True + ) + ) + self.assertEqual(len(all_numbers), 6) + # All numbers are unique and all fit inside the shared range. + self.assertEqual(len(set(all_numbers)), 6) + self.assertTrue( + all(1 <= n <= 10 for n in all_numbers), + f"Channels outside the shared range: {all_numbers}", + ) + + +class NumbersInRangeLookupTests(TestCase): + """ + Backend support for the inline range conflict warning on the group + settings form. Returns every channel whose effective channel_number + falls within [start, end], with context fields the frontend uses to + classify each hit (auto-created from this group + account vs anything + else). Matching uses effective values so overrides participate. + """ + + def _client(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_inrange", + password="pw", + user_level=10, + ) + client = APIClient() + client.force_authenticate(user=admin) + return client + + def test_returns_occupants_within_range(self): + group = _make_group(name="Sports") + Channel.objects.create( + name="CNN", channel_number=100, channel_group=group + ) + Channel.objects.create( + name="Local 5", channel_number=105, channel_group=group + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=100&end=110" + ) + + self.assertEqual(response.status_code, 200) + names = sorted(o["name"] for o in response.data["occupants"]) + self.assertEqual(names, ["CNN", "Local 5"]) + + def test_returns_empty_when_no_channels_in_range(self): + client = self._client() + response = client.get( + "/api/channels/channels/numbers-in-range/?start=900&end=999" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["occupants"], []) + + def test_single_number_lookup_when_end_omitted(self): + # Mirrors the case where the user has set Start # but not End #. + # Endpoint should treat omitted end as a single-number lookup. + group = _make_group(name="Sports") + Channel.objects.create( + name="CNN", channel_number=100, channel_group=group + ) + client = self._client() + response = client.get( + "/api/channels/channels/numbers-in-range/?start=100" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data["occupants"]), 1) + + def test_override_channel_number_is_treated_as_effective(self): + from apps.channels.models import ChannelOverride + + group = _make_group(name="News") + ch = Channel.objects.create( + name="Raw", + channel_number=500, + channel_group=group, + auto_created=True, + ) + ChannelOverride.objects.create(channel=ch, channel_number=777) + client = self._client() + + # Raw value 500 must NOT match - overrides are authoritative. + miss = client.get( + "/api/channels/channels/numbers-in-range/?start=500&end=500" + ) + self.assertEqual(miss.data["occupants"], []) + + hit = client.get( + "/api/channels/channels/numbers-in-range/?start=777&end=777" + ) + self.assertEqual(len(hit.data["occupants"]), 1) + self.assertTrue( + hit.data["occupants"][0]["has_channel_number_override"] + ) + + def test_response_includes_group_account_and_override_flags(self): + # The frontend uses these three fields to decide whether a hit is + # a genuine collision or the expected output of the group it's + # currently configuring. The endpoint must carry them through. + account = _make_account() + group = _make_group(name="Sports") + Channel.objects.create( + name="CNN", + channel_number=100, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=100&end=100" + ) + + occupant = response.data["occupants"][0] + self.assertEqual(occupant["channel_group_id"], group.id) + self.assertTrue(occupant["auto_created"]) + self.assertEqual( + occupant["auto_created_by_account_id"], account.id + ) + self.assertFalse(occupant["has_channel_number_override"]) + + def test_manual_channel_exposed_with_auto_created_false(self): + # Manual channels are always a real collision worth surfacing. + # The response must flag them with auto_created=False and a null + # account id so the frontend classifier warns. + group = _make_group(name="Sports") + Channel.objects.create( + name="MyManual", channel_number=100, channel_group=group + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=100&end=100" + ) + + occupant = response.data["occupants"][0] + self.assertFalse(occupant["auto_created"]) + self.assertIsNone(occupant["auto_created_by_account_id"]) + + def test_results_are_capped_at_50_entries(self): + # The endpoint caps at 50 to keep payloads bounded; the frontend + # only needs to know whether any unfiltered occupants remain. + group = _make_group(name="Sports") + for i in range(60): + Channel.objects.create( + name=f"Ch{i}", + channel_number=1000 + i, + channel_group=group, + ) + client = self._client() + response = client.get( + "/api/channels/channels/numbers-in-range/?start=1000&end=1100" + ) + self.assertEqual(len(response.data["occupants"]), 50) + + +class RegexPreviewTests(TestCase): + """ + Backend support for the find/replace and filter regex previews in the + auto-sync gear modal. Returns matched names plus full-group counts so + the UI can show "12 matches across 5,000 streams" without loading the + whole stream list client-side. Hard-caps at SCAN_CAP=5000 names per + call to keep the endpoint bounded on huge groups. + """ + + def _client(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_regex_preview", + password="pw", + user_level=10, + ) + client = APIClient() + client.force_authenticate(user=admin) + return client + + def _make_account(self): + return _make_account(name="Regex Preview Provider") + + def test_find_replace_returns_only_changed_names(self): + account = self._make_account() + group = _make_group(name="Sports") + for name in ["ESPN HD", "Fox Sports HD", "CNN"]: + Stream.objects.create( + name=name, + url=f"http://example.com/{name}.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Sports&find=%20HD%24&replace=" + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["find_match_count"], 2) + names = sorted(m["before"] for m in response.data["find_matches"]) + self.assertEqual(names, ["ESPN HD", "Fox Sports HD"]) + # Replacement is correctly applied in the after field. + self.assertEqual( + sorted(m["after"] for m in response.data["find_matches"]), + ["ESPN", "Fox Sports"], + ) + self.assertEqual(response.data["total_in_group"], 3) + self.assertEqual(response.data["total_scanned"], 3) + self.assertFalse(response.data["scan_limit_hit"]) + + def test_filter_returns_matched_names_with_count(self): + account = self._make_account() + group = _make_group(name="Sports") + for name in ["Sports Central", "News 24", "Sports Live"]: + Stream.objects.create( + name=name, + url=f"http://example.com/{name}.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Sports&match=%5ESports" + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["filter_match_count"], 2) + matched_names = sorted( + m["name"] for m in response.data["filter_matches"] + ) + self.assertEqual(matched_names, ["Sports Central", "Sports Live"]) + + def test_returns_zero_match_counts_when_pattern_matches_nothing(self): + account = self._make_account() + group = _make_group(name="Sports") + for name in ["CNN", "MSNBC"]: + Stream.objects.create( + name=name, + url=f"http://example.com/{name}.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Sports&find=ESPN&replace=Whatever" + ) + + self.assertEqual(response.data["find_match_count"], 0) + self.assertEqual(response.data["find_matches"], []) + + def test_invalid_find_pattern_returns_error_field(self): + # The endpoint reports compile errors via response fields rather + # than 400 so the UI can surface them inline without the request + # being treated as a hard failure. + account = self._make_account() + group = _make_group(name="Sports") + Stream.objects.create( + name="ESPN", + url="http://example.com/espn.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Sports&find=(" + ) + + self.assertEqual(response.status_code, 200) + self.assertIn("find_error", response.data) + + def test_scan_limit_hit_when_group_exceeds_5000_streams(self): + # Build a group with 5050 streams so we cross the SCAN_CAP boundary. + # The endpoint must still return promptly and flag scan_limit_hit + # so the UI can disclose that the preview isn't full coverage. + account = self._make_account() + group = _make_group(name="Bigly") + Stream.objects.bulk_create( + [ + Stream( + name=f"Stream {i}", + url=f"http://example.com/{i}.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + for i in range(5050) + ] + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Bigly&find=Stream" + ) + + self.assertEqual(response.status_code, 200) + self.assertTrue(response.data["scan_limit_hit"]) + self.assertEqual(response.data["total_in_group"], 5050) + self.assertEqual(response.data["total_scanned"], 5000) + + def test_channel_group_required(self): + client = self._client() + response = client.get("/api/channels/streams/regex-preview/") + self.assertEqual(response.status_code, 400) + + def test_exclude_returns_matched_names_with_count(self): + # Exclude pattern returns the streams it would remove, mirroring + # the include preview shape so the UI can render both side-by-side. + account = self._make_account() + group = _make_group(name="Sports") + for name in ["Sports Live", "Sports TEST", "Sports BACKUP"]: + Stream.objects.create( + name=name, + url=f"http://example.com/{name}.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/" + "?channel_group=Sports&exclude=TEST%7CBACKUP" + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["exclude_match_count"], 2) + names = sorted(m["name"] for m in response.data["exclude_matches"]) + self.assertEqual(names, ["Sports BACKUP", "Sports TEST"]) + + +class ExcludeRegexSyncTests(TestCase): + """ + Sync respects the per-group `name_match_exclude_regex` custom property + by dropping any stream whose name matches the pattern, after any + include filter narrows the set. Invalid patterns are logged and + skipped rather than aborting the whole sync. + """ + + def test_exclude_pattern_skips_matching_streams(self): + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account( + account, + group, + custom_properties={"name_match_exclude_regex": "TEST|BACKUP"}, + ) + rel.auto_sync_channel_start = 100 + rel.save() + for name in ["Sports Live", "Sports TEST", "Sports BACKUP", "Sports Pro"]: + _make_stream( + account, group, name=name, tvg_id=name.replace(" ", "_") + ) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + # Only Sports Live and Sports Pro should have been turned into channels. + names = sorted( + Channel.objects.filter( + channel_group=group, + auto_created=True, + auto_created_by=account, + ).values_list("name", flat=True) + ) + self.assertEqual(names, ["Sports Live", "Sports Pro"]) + + def test_invalid_exclude_pattern_does_not_abort_sync(self): + # Bad regex must be logged + ignored, not crash the sync. The + # streams are imported as if no exclude filter was set. + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account( + account, + group, + custom_properties={"name_match_exclude_regex": "("}, + ) + rel.auto_sync_channel_start = 100 + rel.save() + _make_stream(account, group, name="ESPN", tvg_id="espn") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 1) + + +class ChannelOverrideClearResponseTests(TestCase): + """ + Clearing all overrides on a channel via PATCH `override=null` must + return a response where `override` is null and every effective_* + value reflects the post-clear (provider) state. The previous + implementation used a queryset-level delete which left Django's + reverse-OneToOne cache pointing at the just-deleted row; serializing + the response on the same instance returned the stale override data + and kept the frontend's "Clear All Overrides" button stuck visible + after a successful clear. + """ + + def test_clear_response_carries_null_override(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + from apps.channels.models import ChannelOverride + + admin = User.objects.create_superuser( + username="admin_clear_response", + password="pw", + user_level=10, + ) + group = _make_group(name="Sports") + ch = Channel.objects.create( + name="ProviderName", + channel_number=200, + channel_group=group, + auto_created=True, + ) + ChannelOverride.objects.create( + channel=ch, name="UserOverrideName" + ) + client = APIClient() + client.force_authenticate(user=admin) + + response = client.patch( + f"/api/channels/channels/{ch.id}/", + {"override": None}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + self.assertIsNone(response.data["override"]) + # The effective name must come from the provider value now that + # the override is gone, not the cached override that was still + # attached to the in-memory instance. + self.assertEqual(response.data["effective_name"], "ProviderName") + + +class HiddenChannelPreservationTests(TestCase): + """ + Hidden channels (`hidden_from_output=True`) must be preserved across every + sync cleanup path so users can rely on the hide flag as "keep this + channel even if its source stream temporarily disappears". Common + case: event / PPV / seasonal channels whose provider stream comes + and goes between refreshes - if they get deleted, the user loses + the hide state, any overrides, and the channel's identity. + """ + + def test_hidden_channel_survives_when_its_stream_disappears(self): + # Build a channel auto-created from a stream that exists in the + # group, then mark it hidden, then run a sync where the stream is + # absent from the current scan window. The channel must remain. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.save() + + from datetime import timedelta + + old_stream_seen = timezone.now() - timedelta(days=1) + stream = Stream.objects.create( + name="Event A", + url="http://example.com/event-a.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="event-a", + last_seen=old_stream_seen, + ) + ch = Channel.objects.create( + name="Event A", + channel_number=100, + channel_group=group, + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ) + ChannelStream.objects.create(channel=ch, stream=stream, order=0) + + # Sync's scan_start_time is now (stream's last_seen is older, + # so the stream is excluded from current_streams - simulating + # the provider dropping it). Without the hidden_from_output guard, the + # channel would be cleaned up here. + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertTrue( + Channel.objects.filter(id=ch.id).exists(), + "Hidden channel was deleted when its source stream " + "disappeared from the current sync window", + ) + + def test_visible_channel_for_disappeared_stream_is_still_deleted(self): + # The hide-preservation rule must not accidentally apply to + # visible channels - those should still get cleaned up when + # their source stream goes away. Otherwise sync would never + # remove anything. + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.save() + + from datetime import timedelta + + stream = Stream.objects.create( + name="OldStream", + url="http://example.com/old.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="old", + last_seen=timezone.now() - timedelta(days=1), + ) + ch = Channel.objects.create( + name="OldStream", + channel_number=100, + channel_group=group, + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ) + ChannelStream.objects.create(channel=ch, stream=stream, order=0) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertFalse( + Channel.objects.filter(id=ch.id).exists(), + "Visible channel should have been cleaned up when its " + "source stream disappeared", + ) + + +class CompactNumberingTests(TestCase): + """ + Per-group `compact_numbering` custom_property packs visible auto- + created channels into the group's [start, end] range; hidden channels + release their channel numbers; channel_number overrides act as + reservations that survive hide/unhide cycles. Exercised across full + sync, the post_save signal for single-channel unhide, and the + explicit re-pack endpoint. + """ + + def _compact_account_with_group(self, start=100, end=110, name="Sports"): + account = _make_account() + group = _make_group(name=name) + rel = _attach_group_to_account( + account, group, custom_properties={"compact_numbering": True} + ) + rel.auto_sync_channel_start = start + rel.auto_sync_channel_end = end + rel.save() + return account, group, rel + + def test_full_sync_packs_visible_and_releases_hidden(self): + # Five streams, three of which produce visible channels and two of + # which we mark hidden after the first sync. Compact pack should + # leave the three visible channels at 100/101/102 and NULL out the + # hidden ones. + account, group, rel = self._compact_account_with_group( + start=100, end=110 + ) + for i in range(5): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + first = _sync(account) + self.assertEqual(first["status"], "ok") + self.assertEqual(first["channels_created"], 5) + + # Hide two of the resulting channels. + hidden_targets = list( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).order_by("id")[:2] + ) + Channel.objects.filter( + id__in=[c.id for c in hidden_targets] + ).update(hidden_from_output=True) + + # Re-run sync; compact pack runs at the end. + _sync(account) + + nums = sorted( + Channel.objects.filter( + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ).values_list("channel_number", flat=True) + ) + self.assertEqual(nums, [100, 101, 102]) + + hidden_nums = list( + Channel.objects.filter( + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ).values_list("channel_number", flat=True) + ) + self.assertTrue( + all(n is None for n in hidden_nums), + f"Hidden channels still hold numbers: {hidden_nums}", + ) + + def test_override_pinned_number_survives_compact_pass(self): + # Channel B has a channel_number override of 105. After compact + # pack, B's effective number stays at 105, and the other visible + # channels get packed around it (skipping 105 in the pool). + from apps.channels.models import ChannelOverride + + account, group, rel = self._compact_account_with_group( + start=100, end=110 + ) + for i in range(4): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + _sync(account) + all_channels = list( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).order_by("id") + ) + ChannelOverride.objects.create( + channel=all_channels[1], channel_number=105 + ) + + _sync(account) + + # Override-pinned channel: raw cleared to None (override controls + # effective). The other three channels packed into 100/101/102 - + # skipping 105 because the override reserved it. + non_pinned = sorted( + float(n) + for n in Channel.objects.filter( + auto_created=True, + auto_created_by=account, + override__isnull=True, + ).values_list("channel_number", flat=True) + if n is not None + ) + self.assertEqual(non_pinned, [100.0, 101.0, 102.0]) + pinned = Channel.objects.get(id=all_channels[1].id) + self.assertIsNone(pinned.channel_number) + + def test_unhide_signal_assigns_immediately(self): + # Toggle hide → unhide on a single channel; the post_save signal + # under compact mode should give it a number from the range + # without requiring a sync pass to run. + account, group, rel = self._compact_account_with_group( + start=100, end=110 + ) + ch = Channel.objects.create( + name="Test", + channel_number=None, + channel_group=group, + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ) + + ch.hidden_from_output = False + ch.save() + ch.refresh_from_db() + + self.assertEqual(ch.channel_number, 100) + + def test_repack_endpoint_packs_around_overrides(self): + # Simulate a user who set overrides on a couple of channels to + # pin specific numbers and then clicks Re-pack to compact the + # rest. Override-pinned channels survive; everything else fills + # the remaining slots in order. + from rest_framework.test import APIClient + from apps.accounts.models import User + from apps.channels.models import ChannelOverride + + account, group, rel = self._compact_account_with_group( + start=100, end=110 + ) + # Six visible channels, no overrides + channels = [ + Channel.objects.create( + name=f"C{i}", + channel_number=200 + i, # arbitrary starting numbers + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + for i in range(6) + ] + # Pin two of them via override + ChannelOverride.objects.create(channel=channels[0], channel_number=110) + ChannelOverride.objects.create(channel=channels[1], channel_number=105) + + admin = User.objects.create_superuser( + username="admin_repack", password="pw", user_level=10 + ) + client = APIClient() + client.force_authenticate(user=admin) + response = client.post( + f"/api/m3u/accounts/{account.id}/repack-group/?channel_group_id={group.id}" + ) + + self.assertEqual(response.status_code, 200) + # 4 non-pinned channels assigned, no failures + self.assertEqual(response.data["assigned"], 4) + self.assertEqual(response.data["failed"], 0) + # Non-pinned channels get the lowest available numbers in the + # range, skipping 105 and 110 (override reservations). + non_pinned_nums = sorted( + Channel.objects.filter( + id__in=[c.id for c in channels[2:]] + ).values_list("channel_number", flat=True) + ) + self.assertEqual(non_pinned_nums, [100, 101, 102, 103]) + + def test_repack_reports_failed_when_range_too_small(self): + # Range covers 3 slots but visible channels need 5; the extra 2 + # are reported as failed and have their channel_number set to + # None so their state is unambiguous. + account, group, rel = self._compact_account_with_group( + start=100, end=102 + ) + for i in range(5): + Channel.objects.create( + name=f"C{i}", + channel_number=900 + i, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + from apps.channels.compact_numbering import repack_group + + result = repack_group(rel) + self.assertEqual(result["assigned"], 3) + self.assertEqual(result["failed"], 2) + # Those that didn't fit got their channel_number nulled + nulled = Channel.objects.filter( + channel_group=group, + channel_number__isnull=True, + ).count() + self.assertEqual(nulled, 2) + + def test_assign_releases_lock_on_outer_atomic_rollback(self): + # The Redis-backed task lock is not transactional, so a release + # scheduled via `transaction.on_commit` is silently discarded + # when the caller's outer atomic rolls back. This left the lock + # held until its 5-minute TTL expired and silently blocked all + # subsequent syncs for the affected account. The function now + # releases via try/finally so the lock comes back regardless + # of outer transaction state. + from unittest.mock import patch + from django.db import transaction + from apps.channels.compact_numbering import ( + assign_compact_numbers_for_channels, + ) + from core.utils import acquire_task_lock, release_task_lock + + account, group, rel = self._compact_account_with_group( + start=100, end=105 + ) + ch = Channel.objects.create( + name="C1", + channel_group=group, + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ) + + # Make sure the lock starts free. + release_task_lock("refresh_single_m3u_account", account.id) + + try: + with transaction.atomic(): + assign_compact_numbers_for_channels([ch.id]) + # Simulate downstream work in the outer atomic blowing up + # AFTER the inner assignment succeeded. + raise RuntimeError("simulated downstream failure") + except RuntimeError: + pass + + # If the lock is still held, the next acquire returns False and + # subsequent syncs would silently skip auto-sync. + self.assertTrue( + acquire_task_lock("refresh_single_m3u_account", account.id), + "Lock leaked after outer atomic rollback", + ) + # Clean up so other tests in this class are not affected. + release_task_lock("refresh_single_m3u_account", account.id) + + +class OverrideChannelNumberValidationTests(TestCase): + """ + Override PATCH intentionally permits duplicate channel_number values: + users may want two entries at the same number (e.g., one of them + hidden from output). Sync's used_numbers set still avoids collisions + on its own writes via set membership, so cross-provider merge + behavior is unaffected by allowing user-set duplicates. + """ + + def test_override_channel_number_duplicate_is_allowed(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + from apps.channels.models import ChannelOverride + + admin = User.objects.create_superuser( + username="admin_override_a", + password="pw", + user_level=10, + ) + group = _make_group(name="Sports") + Channel.objects.create( + name="Existing", + channel_number=500, + channel_group=group, + ) + target = Channel.objects.create( + name="Target", + channel_number=501, + channel_group=group, + auto_created=True, + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.patch( + f"/api/channels/channels/{target.id}/", + {"override": {"channel_number": 500}}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + ChannelOverride.objects.get(channel=target).channel_number, 500 + ) + + def test_override_channel_number_reusing_own_is_allowed(self): + # Editing an override and re-submitting the same number the channel + # already effectively holds must succeed (no "conflicts with itself"). + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_override_b", + password="pw", + user_level=10, + ) + group = _make_group(name="Sports") + ch = Channel.objects.create( + name="Target", + channel_number=700, + channel_group=group, + auto_created=True, + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.patch( + f"/api/channels/channels/{ch.id}/", + {"override": {"channel_number": 700, "name": "Renamed"}}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + + +class SyncPerformanceRegressionTests(TestCase): + """ + Query-count and throughput guards. The numbers below are ceilings for the + specific scenario, not tight lower bounds. They exist to catch regressions + where a future edit reintroduces N+1 lookups on logos, EPG, or + per-channel ChannelStream joins. If a ceiling needs to be raised, + investigate first (the original audit documented the cost of each pattern). + """ + + def test_sync_of_ten_streams_is_not_n_plus_one_in_logo_lookups(self): + from apps.channels.models import Logo + + account = _make_account() + group = _make_group(name="Sports") + _attach_group_to_account(account, group) + + # Seed 10 streams with 3 distinct logo URLs so the batch cache has + # an interesting ratio of cache hits to misses. + logo_urls = [ + "http://logos.example.com/a.png", + "http://logos.example.com/b.png", + "http://logos.example.com/c.png", + ] + for i in range(10): + Stream.objects.create( + name=f"Chan{i}", + url=f"http://example.com/{i}.m3u8", + m3u_account=account, + channel_group=group, + tvg_id=f"tvg{i}", + logo_url=logo_urls[i % 3], + last_seen=timezone.now(), + ) + + # Count Logo queries during the sync. Without batching, this would + # issue at least one SELECT per stream (10+) plus 10 separate INSERTs. + # With batching, a single SELECT populates the cache for all 3 URLs + # and each new Logo row requires one INSERT (the get_or_create miss + # path). Allow headroom for the initial cache-populate query and + # per-insert overhead, but keep the total well below the N+1 count. + from django.db import connection + from django.test.utils import CaptureQueriesContext + + with CaptureQueriesContext(connection) as ctx: + result = _sync(account) + + logo_queries = [ + q for q in ctx.captured_queries + if 'dispatcharr_channels_logo' in q['sql'].lower() + ] + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 10) + # Target: at most ~6 Logo-related queries for 10 streams / 3 urls. + # Without the batch cache the count would be 10+ SELECTs + 3 INSERTs. + self.assertLessEqual( + len(logo_queries), + 8, + f"Logo queries: {len(logo_queries)} (expected <= 8 after batching)", + ) + + +class OrphanCleanupModeTests(TestCase): + """ + Account-level `custom_properties.orphan_channel_cleanup` is a 3-state + selector that governs how sync handles auto-created channels whose + source streams have disappeared. + + - "always" (default; absent key behaves the same): delete every orphan + auto-created channel. + - "preserve_customized": delete orphans without a ChannelOverride row; + preserve those with one. + - "never": preserve every orphan auto-created channel. + + Hidden channels (`hidden_from_output=True`) are universally preserved + regardless of mode, because the user has explicitly signaled "keep this + around but do not show clients". + """ + + def _set_mode(self, account, mode): + account.custom_properties = {"orphan_channel_cleanup": mode} + account.save() + + def test_default_mode_when_key_absent_is_always(self): + # Account with no custom_properties.orphan_channel_cleanup value + # behaves like "always": orphans get cleaned up. + account = _make_account() + group = _make_group(name="Sports") + _attach_group_to_account(account, group) + Channel.objects.create( + name="OldESPN", + channel_number=500, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_deleted"], 1) + self.assertEqual( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).count(), + 0, + "Default (absent key): orphans must be cleaned up", + ) + + def test_always_mode_removes_orphan_with_override(self): + from apps.channels.models import ChannelOverride + + account = _make_account() + self._set_mode(account, "always") + group = _make_group(name="Sports") + _attach_group_to_account(account, group) + ch = Channel.objects.create( + name="OldESPN", + channel_number=500, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + ChannelOverride.objects.create(channel=ch, name="My Custom ESPN") + + result = _sync(account) + + self.assertEqual(result["channels_deleted"], 1) + self.assertFalse( + Channel.objects.filter(id=ch.id).exists(), + "Always mode: even customized orphans get deleted", + ) + + def test_preserve_customized_mode_spares_overrides(self): + from apps.channels.models import ChannelOverride + + account = _make_account() + self._set_mode(account, "preserve_customized") + group = _make_group(name="Sports") + _attach_group_to_account(account, group) + plain = Channel.objects.create( + name="OldPlain", + channel_number=500, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + customized = Channel.objects.create( + name="OldCustomized", + channel_number=501, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + ChannelOverride.objects.create(channel=customized, name="My Renamed") + + _sync(account) + + self.assertFalse( + Channel.objects.filter(id=plain.id).exists(), + "Preserve-customized mode: orphan without override is removed", + ) + self.assertTrue( + Channel.objects.filter(id=customized.id).exists(), + "Preserve-customized mode: orphan WITH override is preserved", + ) + + def test_never_mode_preserves_all_orphans(self): + account = _make_account() + self._set_mode(account, "never") + group = _make_group(name="Sports") + _attach_group_to_account(account, group) + Channel.objects.create( + name="OldA", + channel_number=500, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + Channel.objects.create( + name="OldB", + channel_number=501, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + result = _sync(account) + + self.assertEqual(result["channels_deleted"], 0) + self.assertEqual( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).count(), + 2, + "Never mode: every orphan survives", + ) + + def test_hidden_channels_universally_preserved(self): + # Hidden orphans must survive in every mode, including the + # default "always" mode. + for mode in ("always", "preserve_customized", "never"): + with self.subTest(mode=mode): + account = _make_account(name=f"Provider-{mode}") + self._set_mode(account, mode) + group = _make_group(name=f"Sports-{mode}") + _attach_group_to_account(account, group) + Channel.objects.create( + name=f"Hidden-{mode}", + channel_number=600, + channel_group=group, + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ) + + _sync(account) + + self.assertEqual( + Channel.objects.filter( + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ).count(), + 1, + f"Hidden channel must survive cleanup in mode={mode}", + ) + + +class MultiStreamChannelTests(TestCase): + """ + A user can manually attach more than one Stream to an auto-created + Channel (typical for backup feeds: same channel, two providers' + versions). Sync must not delete the channel just because ONE of + those streams disappears - other streams may still be valid. + + The deletion path was per-stream: for stream_id, channel in + existing_channel_map.items(): delete the channel if stream_id is not + in processed_stream_ids. With two map entries pointing at the same + channel, the first iteration would queue the channel for delete even + if the second stream is still alive. + + Even more subtle: existing_channel_map gets a fresh Channel instance + per ChannelStream row from the joined query. With two entries, + in-memory mutations during the per-stream iteration would land on + different Python instances and silently drop changes from later + iterations when bulk_update fires on only one of them. + """ + + def test_channel_with_two_streams_survives_when_one_disappears(self): + from datetime import timedelta + + account = _make_account() + group = _make_group(name="Movies") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.save() + + # Two streams in the same group, same account, attached to one + # auto-created channel. Stream A is current, Stream B is stale + # (provider dropped it). + stream_a = Stream.objects.create( + name="StarMovie HD", + url="http://example.com/star-a.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="star", + last_seen=timezone.now(), + ) + stream_b = Stream.objects.create( + name="StarMovie HD", + url="http://example.com/star-b.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="star", + last_seen=timezone.now() - timedelta(days=1), + ) + ch = Channel.objects.create( + name="StarMovie HD", + channel_number=100, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + ChannelStream.objects.create(channel=ch, stream=stream_a, order=0) + ChannelStream.objects.create(channel=ch, stream=stream_b, order=1) + + # Sync sees stream_a as current (live) and stream_b as gone. + # Stream_a means the channel is still wanted; the channel must + # survive. Direct DB check works on both old and new sync return + # shapes (baseline returns a string, overhaul returns a dict). + _sync(account) + + self.assertTrue( + Channel.objects.filter(id=ch.id).exists(), + "Multi-stream channel was deleted even though one of its " + "streams is still alive", + ) + + def test_channel_with_two_streams_metadata_consistent_after_sync(self): + # When two ChannelStream rows resolve to the same Channel, + # in-memory mutations from each iteration must land on the SAME + # Channel instance so the post-loop bulk_update writes the + # merged final state. With per-row Channel instances (the bug), + # later iterations would mutate a different in-memory copy and + # those changes would be lost. + + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.save() + + # Two streams attached to the same channel, both currently + # served by the provider. Each carries different tvg_id so the + # sync code wants to update the channel's tvg_id field. + stream_a = Stream.objects.create( + name="SportsCh HD", + url="http://example.com/sports-a.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="sports-a", + last_seen=timezone.now(), + ) + stream_b = Stream.objects.create( + name="SportsCh HD", + url="http://example.com/sports-b.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="sports-b", + last_seen=timezone.now(), + ) + ch = Channel.objects.create( + name="OldName", + tvg_id="initial", + channel_number=100, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + ChannelStream.objects.create(channel=ch, stream=stream_a, order=0) + ChannelStream.objects.create(channel=ch, stream=stream_b, order=1) + + _sync(account) + + ch.refresh_from_db() + # The channel exists, was updated, and either tvg_id ('sports-a' + # or 'sports-b') is acceptable - the key invariant is that it is + # NOT 'initial' (the pre-sync value), which would mean updates + # were silently dropped because they targeted a different + # in-memory instance than the one bulk_update'd. + self.assertNotEqual( + ch.tvg_id, + "initial", + "tvg_id was not updated; in-memory mutations to the " + "channel were dropped (multi-stream identity bug)", + ) + + +class HiddenChannelNumberCollisionTests(TestCase): + """ + Sync seeds `used_numbers` from existing channels EXCEPT those owned by + the account being synced (those will be re-numbered). The original + pattern excluded ALL of the account's auto-created channels - including + HIDDEN ones, which the renumber loop never visits because they are not + in `current_streams`. Result: the hidden channel's number was free to + be re-assigned to a brand-new visible channel, producing two channels + with the same channel_number on disk. + """ + + def test_hidden_channels_keep_their_number_reserved_during_sync(self): + from datetime import timedelta + + account = _make_account() + group_a = _make_group(name="Hidden Group") + group_b = _make_group(name="New Group") + rel_a = _attach_group_to_account(account, group_a) + rel_a.auto_sync_channel_start = 100 + rel_a.save() + rel_b = _attach_group_to_account(account, group_b) + rel_b.auto_sync_channel_start = 100 + rel_b.save() + + # A hidden auto-created channel pinned at #100 in group_a. + # Its source stream is gone (last_seen old), but hidden_from_output=True + # protects the channel from cleanup. + old_seen = timezone.now() - timedelta(days=1) + Stream.objects.create( + name="HiddenStream", + url="http://example.com/hidden.m3u8", + m3u_account=account, + channel_group=group_a, + tvg_id="hid", + last_seen=old_seen, + ) + hidden_ch = Channel.objects.create( + name="HiddenChannel", + channel_number=100, + channel_group=group_a, + auto_created=True, + auto_created_by=account, + hidden_from_output=True, + ) + + # A NEW stream in group_b, no channel yet. Sync should create a + # channel for it and pick its number. If hidden_ch's number is + # free (the bug), this new channel would also get 100, colliding. + Stream.objects.create( + name="NewStream", + url="http://example.com/new.m3u8", + m3u_account=account, + channel_group=group_b, + tvg_id="new", + last_seen=timezone.now(), + ) + + _sync(account) + + new_ch = Channel.objects.filter( + auto_created=True, + auto_created_by=account, + channel_group=group_b, + ).first() + self.assertIsNotNone(new_ch, "Sync did not create a channel for the new stream") + + # Both rows must coexist with DIFFERENT channel_numbers. + hidden_ch.refresh_from_db() + self.assertNotEqual( + new_ch.channel_number, + hidden_ch.channel_number, + f"New channel #{new_ch.channel_number} collides with hidden " + f"channel #{hidden_ch.channel_number}; sync did not reserve " + f"the hidden channel's number", + ) + + +class HDHRLineupNullChannelNumberTests(TestCase): + """ + With nullable channel_number (migration 0039) it is now valid for a + channel to have effective_channel_number=None - typically a hidden + channel under compact numbering whose number was released. Both HDHR + lineup endpoints (legacy `apps/hdhr/views.py` and new + `apps/hdhr/api_views.py`) must skip those rows rather than emit + `"GuideNumber": "None"` or `"GuideNumber": ""`. HDHR clients reject + such entries and may drop the entire lineup. + """ + + def test_legacy_lineup_skips_channels_with_null_effective_number(self): + # Set up two channels: one with a number, one without. + account = _make_account() + group = _make_group(name="Legacy") + with_number = Channel.objects.create( + name="WithNum", + channel_number=42, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + Channel.objects.create( + name="NoNum", + channel_number=None, + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + + from apps.accounts.models import User + from rest_framework.test import APIClient + + admin = User.objects.create_superuser( + username="hdhr_legacy_admin", password="x", user_level=10 + ) + client = APIClient() + client.force_authenticate(user=admin) + response = client.get("/hdhr/lineup.json") + + self.assertEqual(response.status_code, 200) + body = response.json() if hasattr(response, "json") else response.data + # Body is a list of {GuideNumber, GuideName, URL, ...} + guide_numbers = {entry.get("GuideNumber") for entry in body} + self.assertIn( + "42", guide_numbers, "Numbered channel must appear in lineup" + ) + self.assertNotIn( + "None", guide_numbers, + "Lineup must not contain literal 'None' GuideNumber", + ) + self.assertNotIn( + "", guide_numbers, + "Lineup must not contain empty GuideNumber", + ) + + +class DuplicateOverrideChannelNumberAllowedTests(TestCase): + """ + Override channel_number is intentionally allowed to duplicate an + existing channel's effective number. Downstream clients render two + entries at the same number; users decide whether that is desired + (e.g., one of the duplicates is hidden from output). Sync still + avoids collisions on its own writes via set-membership on + used_numbers, so the merge behavior across providers is unaffected. + """ + + def test_override_channel_number_duplicate_is_accepted(self): + from rest_framework.test import APIClient + from apps.accounts.models import User + from apps.channels.models import Channel as Ch + from apps.channels.models import ChannelOverride + + admin = User.objects.create_superuser( + username="dup_override_admin", password="x", user_level=10 + ) + + # Existing channel claiming #50 by raw channel_number. + Ch.objects.create( + name="Existing", + channel_number=50, + auto_created=False, + ) + editable = Ch.objects.create( + name="Editable", + channel_number=51, + auto_created=True, + auto_created_by=_make_account(), + ) + + client = APIClient() + client.force_authenticate(user=admin) + response = client.patch( + f"/api/channels/channels/{editable.id}/", + {"override": {"channel_number": 50}}, + format="json", + ) + + self.assertEqual( + response.status_code, + 200, + f"Expected 200; got {response.status_code} " + f"body={getattr(response, 'data', None)}", + ) + override = ChannelOverride.objects.get(channel=editable) + self.assertEqual(override.channel_number, 50) + + +class EPGDispatchExistingChannelTests(TestCase): + """ + The new-channel sync path manually dispatches `parse_programs_for_tvg_id` + once per unique epg_data_id (because bulk_create bypasses post_save). + The existing-channel path (bulk_update) ALSO bypasses post_save, so it + needs the same manual dispatch when `epg_data` is in the dirty set; + otherwise EPG re-parse never fires for channels whose epg link was + just changed by sync. + """ + + def test_existing_channel_epg_change_triggers_parse_dispatch(self): + from unittest.mock import patch + + account = _make_account() + group = _make_group(name="EPG Test") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 100 + rel.save() + + # First sync: create channel with one tvg_id. + s1 = Stream.objects.create( + name="EPGCh", + url="http://example.com/epg.m3u8", + m3u_account=account, + channel_group=group, + tvg_id="initial-tvg", + last_seen=timezone.now(), + ) + _sync(account) + + ch = Channel.objects.get( + auto_created=True, auto_created_by=account + ) + # Force epg_data divergence on the next sync by changing the + # stream's tvg_id (the resolver will see a different EPG link). + # Without changing the actual EPG resolution logic here we just + # set epg_data_id to something different in DB and verify the + # next sync detects the change via update_fields="epg_data" in + # the dirty set; the dispatch loop must fire. + ch.epg_data_id = None + ch.save(update_fields=["epg_data"]) + + # Update the stream's tvg_id so the resolver picks up a new + # EPGData on the next sync. + from apps.epg.models import EPGData, EPGSource + + src = EPGSource.objects.create( + name="TestSrc", source_type="manual", url="" + ) + new_epg = EPGData.objects.create( + tvg_id="initial-tvg", name="EPGCh", epg_source=src + ) + # Sync should now associate ch with new_epg via tvg_id match. + + with patch( + "apps.epg.tasks.parse_programs_for_tvg_id.delay" + ) as mock_dispatch: + _sync(account) + + # The exact number of calls is 1 per unique epg_data_id; for one + # changed channel that is at least 1 call. + called_for = {c.args[0] for c in mock_dispatch.call_args_list} + self.assertIn( + new_epg.id, + called_for, + f"parse_programs_for_tvg_id was not dispatched for the EPG " + f"id {new_epg.id} that the existing channel was just " + f"associated with. mock calls: {mock_dispatch.call_args_list}", + ) + + +class Migration0037DemoteOrphansTests(TestCase): + """ + The 0037 auto-sync overhaul migration's `backfill_auto_created_by_null` + step demotes orphaned `auto_created=True, auto_created_by=NULL` channels + to `auto_created=False` instead of deleting them, preserving the + channel and any overrides that may exist on it. + """ + + def test_orphan_with_no_streams_is_demoted_not_deleted(self): + # Create an orphaned auto-created channel with no streams. This + # is the case that a delete-on-orphan strategy would silently lose. + ch = Channel.objects.create( + name="OrphanGhost", + channel_number=999, + auto_created=True, + auto_created_by=None, + ) + + # The migration's backfill function takes (apps, schema_editor) + # where `apps` is normally the historical app registry. For this + # unit test we call with the live registry. The migration file + # name starts with a digit so it must be loaded via importlib. + from importlib import import_module + from django.apps import apps as django_apps + + module = import_module( + "apps.channels.migrations.0037_auto_sync_overhaul" + ) + + # The migration function takes (apps, schema_editor); apps is + # the historical app registry. For this unit test we call with + # the live registry. + module.backfill_auto_created_by_null(django_apps, None) + + ch.refresh_from_db() + self.assertFalse( + ch.auto_created, + "Orphaned auto-created channel with no streams must be " + "demoted to manual (auto_created=False), not left as " + "auto_created=True or deleted", + ) + self.assertIsNone(ch.auto_created_by) diff --git a/apps/output/views.py b/apps/output/views.py index 01c7ee60..16b15236 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -2,6 +2,7 @@ from django.http import HttpResponse, JsonResponse, Http404, HttpResponseForbidd from rest_framework.response import Response from django.urls import reverse from apps.channels.models import Channel, ChannelProfile, ChannelGroup, Stream +from apps.channels.utils import format_channel_number from django.db.models import Prefetch from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods @@ -138,7 +139,7 @@ def generate_m3u(request, profile_name=None, user=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('channel_group', 'logo').order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('channel_group', 'logo') else: # User has specific limited profiles assigned filters = { @@ -149,11 +150,9 @@ def generate_m3u(request, profile_name=None, user=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('channel_group', 'logo').distinct().order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('channel_group', 'logo').distinct() else: - channels = Channel.objects.filter(user_level__lte=user.user_level).select_related('channel_group', 'logo').order_by( - "channel_number" - ) + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('channel_group', 'logo') else: if profile_name is not None: @@ -162,12 +161,22 @@ def generate_m3u(request, profile_name=None, user=None): except ChannelProfile.DoesNotExist: logger.warning("Requested channel profile (%s) during m3u generation does not exist", profile_name) raise Http404(f"Channel profile '{profile_name}' not found") - channels = Channel.objects.filter( + base_qs = Channel.objects.filter( channelprofilemembership__channel_profile=channel_profile, channelprofilemembership__enabled=True - ).select_related('channel_group', 'logo').order_by('channel_number') + ).select_related('channel_group', 'logo') else: - channels = Channel.objects.select_related('channel_group', 'logo').order_by("channel_number") + base_qs = Channel.objects.select_related('channel_group', 'logo') + + # Resolve effective (override | provider) values at SQL level so ordering, + # naming, and logo resolution honor user overrides. `exclude(hidden_from_output=True)` + # is the consumer-facing hide guarantee. + from apps.channels.managers import with_effective_values + channels = ( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) # Check if the request wants to use direct logo URLs instead of cache use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' @@ -212,53 +221,55 @@ def generate_m3u(request, profile_name=None, user=None): m3u_content = f'#EXTM3U x-tvg-url="{epg_url}" url-tvg="{epg_url}"\n' # Start building M3U content + channel_count = 0 for channel in channels: - group_title = channel.channel_group.name if channel.channel_group else "Default" + channel_count += 1 + effective_group = channel.effective_channel_group_obj + effective_logo = channel.effective_logo_obj + effective_name = channel.effective_name + effective_tvg_id_val = channel.effective_tvg_id + effective_tvc_guide = channel.effective_tvc_guide_stationid + effective_number = channel.effective_channel_number - # Format channel number as integer if it has no decimal component - if channel.channel_number is not None: - if channel.channel_number == int(channel.channel_number): - formatted_channel_number = int(channel.channel_number) - else: - formatted_channel_number = channel.channel_number - else: - formatted_channel_number = "" + group_title = effective_group.name if effective_group else "Default" + + formatted_channel_number = format_channel_number(effective_number) # Determine the tvg-id based on the selected source - if tvg_id_source == 'tvg_id' and channel.tvg_id: - tvg_id = channel.tvg_id - elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: - tvg_id = channel.tvc_guide_stationid + if tvg_id_source == 'tvg_id' and effective_tvg_id_val: + tvg_id = effective_tvg_id_val + elif tvg_id_source == 'gracenote' and effective_tvc_guide: + tvg_id = effective_tvc_guide else: # Default to channel number (original behavior) tvg_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - tvg_name = channel.name + tvg_name = effective_name tvg_logo = "" - if channel.logo: + if effective_logo: if use_cached_logos: # Use cached logo as before - tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[channel.logo.id])) + tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[effective_logo.id])) else: # Try to find direct logo URL from channel's streams - direct_logo = channel.logo.url if channel.logo.url.startswith(('http://', 'https://')) else None + direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None # If direct logo found, use it; otherwise fall back to cached version if direct_logo: tvg_logo = direct_logo else: - tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[channel.logo.id])) + tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[effective_logo.id])) # create possible gracenote id insertion tvc_guide_stationid = "" - if channel.tvc_guide_stationid: + if effective_tvc_guide: tvc_guide_stationid = ( - f'tvc-guide-stationid="{channel.tvc_guide_stationid}" ' + f'tvc-guide-stationid="{effective_tvc_guide}" ' ) extinf_line = ( f'#EXTINF:-1 tvg-id="{tvg_id}" tvg-name="{tvg_name}" tvg-logo="{tvg_logo}" ' - f'tvg-chno="{formatted_channel_number}" {tvc_guide_stationid}group-title="{group_title}",{channel.name}\n' + f'tvg-chno="{formatted_channel_number}" {tvc_guide_stationid}group-title="{group_title}",{effective_name}\n' ) # Determine the stream URL based on request type @@ -293,7 +304,7 @@ def generate_m3u(request, profile_name=None, user=None): event_type='m3u_download', profile=profile_name or 'all', user=user.username if user else 'anonymous', - channels=channels.count(), + channels=channel_count, client_ip=client_ip, user_agent=user_agent, ) @@ -1328,7 +1339,7 @@ def generate_epg(request, profile_name=None, user=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') else: # User has specific limited profiles assigned filters = { @@ -1339,11 +1350,9 @@ def generate_epg(request, profile_name=None, user=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct().order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() else: - channels = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source').order_by( - "channel_number" - ) + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') else: if profile_name is not None: try: @@ -1351,12 +1360,21 @@ def generate_epg(request, profile_name=None, user=None): except ChannelProfile.DoesNotExist: logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) raise Http404(f"Channel profile '{profile_name}' not found") - channels = Channel.objects.filter( + base_qs = Channel.objects.filter( channelprofilemembership__channel_profile=channel_profile, channelprofilemembership__enabled=True, - ).select_related('logo', 'epg_data__epg_source').order_by("channel_number") + ).select_related('logo', 'epg_data__epg_source') else: - channels = Channel.objects.all().select_related('logo', 'epg_data__epg_source').order_by("channel_number") + base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') + + # Resolve effective values at SQL level and exclude hidden channels + # so output ordering/display honors user overrides. + from apps.channels.managers import with_effective_values + channels = ( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) # For dummy EPG, use either the specified value or default to 3 days @@ -1376,15 +1394,17 @@ def generate_epg(request, profile_name=None, user=None): # First pass: assign integers for channels that already have integer numbers for channel in channels: - if channel.channel_number == int(channel.channel_number): - num = int(channel.channel_number) + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) channel_num_map[channel.id] = num used_numbers.add(num) # Second pass: assign integers for channels with float numbers for channel in channels: - if channel.channel_number != int(channel.channel_number): - candidate = int(channel.channel_number) + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): + candidate = int(effective_num) while candidate in used_numbers: candidate += 1 channel_num_map[channel.id] = candidate @@ -1392,38 +1412,37 @@ def generate_epg(request, profile_name=None, user=None): # Process channels for the section for channel in channels: + effective_name = channel.effective_name + effective_epg_data = channel.effective_epg_data_obj + effective_logo = channel.effective_logo_obj + effective_number = channel.effective_channel_number + # user is set only for XC clients, which require integer channel numbers if user is not None: formatted_channel_number = channel_num_map[channel.id] else: - if channel.channel_number is not None: - if channel.channel_number == int(channel.channel_number): - formatted_channel_number = int(channel.channel_number) - else: - formatted_channel_number = channel.channel_number - else: - formatted_channel_number = "" + formatted_channel_number = format_channel_number(effective_number) # Determine the channel ID based on the selected source - if tvg_id_source == 'tvg_id' and channel.tvg_id: - channel_id = channel.tvg_id - elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: - channel_id = channel.tvc_guide_stationid + if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: + channel_id = channel.effective_tvg_id + elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: + channel_id = channel.effective_tvc_guide_stationid else: channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) tvg_logo = "" # Check if this is a custom dummy EPG with channel logo URL template - if channel.epg_data and channel.epg_data.epg_source and channel.epg_data.epg_source.source_type == 'dummy': - epg_source = channel.epg_data.epg_source + if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties channel_logo_url_template = custom_props.get('channel_logo_url', '') if channel_logo_url_template: # Determine which name to use for pattern matching (same logic as program generation) - pattern_match_name = channel.name + pattern_match_name = effective_name name_source = custom_props.get('name_source') if name_source == 'stream': @@ -1465,20 +1484,20 @@ def generate_epg(request, profile_name=None, user=None): tvg_logo = channel_logo_url_template logger.debug(f"Built channel logo URL from template: {tvg_logo}") except Exception as e: - logger.warning(f"Failed to build channel logo URL for {channel.name}: {e}") + logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") # If no custom dummy logo, use regular logo logic - if not tvg_logo and channel.logo: + if not tvg_logo and effective_logo: if use_cached_logos: - tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[channel.logo.id])) + tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[effective_logo.id])) else: # Use direct URL if available, otherwise fall back to cached version - direct_logo = channel.logo.url if channel.logo.url.startswith(('http://', 'https://')) else None + direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None if direct_logo: tvg_logo = direct_logo else: - tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[channel.logo.id])) - display_name = channel.name + tvg_logo = build_absolute_uri_with_port(request, reverse('api:channels:logo-cache', args=[effective_logo.id])) + display_name = effective_name xml_lines.append(f' ') xml_lines.append(f' {html.escape(display_name)}') xml_lines.append(f' ') @@ -1495,30 +1514,29 @@ def generate_epg(request, profile_name=None, user=None): dummy_epg_checked = {} # epg_data_id -> bool (has stored programs) for channel in channels: + effective_name = channel.effective_name + effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id + effective_number = channel.effective_channel_number + # Determine channel_id (same logic as channel section) - if tvg_id_source == 'tvg_id' and channel.tvg_id: - channel_id = channel.tvg_id - elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: - channel_id = channel.tvc_guide_stationid + if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: + channel_id = channel.effective_tvg_id + elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: + channel_id = channel.effective_tvc_guide_stationid else: if user is not None: formatted_channel_number = channel_num_map[channel.id] else: - if channel.channel_number is not None: - if channel.channel_number == int(channel.channel_number): - formatted_channel_number = int(channel.channel_number) - else: - formatted_channel_number = channel.channel_number - else: - formatted_channel_number = "" + formatted_channel_number = format_channel_number(effective_number) channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - display_name = channel.epg_data.name if channel.epg_data else channel.name - pattern_match_name = channel.name + display_name = effective_epg_data.name if effective_epg_data else effective_name + pattern_match_name = effective_name # Check if we should use stream name instead of channel name - if channel.epg_data and channel.epg_data.epg_source: - epg_source = channel.epg_data.epg_source + if effective_epg_data and effective_epg_data.epg_source: + epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties name_source = custom_props.get('name_source') @@ -1532,22 +1550,21 @@ def generate_epg(request, profile_name=None, user=None): pattern_match_name = stream.name logger.debug(f"Using stream name for parsing: {pattern_match_name} (stream index: {stream_index})") else: - logger.warning(f"Stream index {stream_index} not found for channel {channel.name}, falling back to channel name") + logger.warning(f"Stream index {stream_index} not found for channel {effective_name}, falling back to channel name") - if not channel.epg_data: + if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) else: - if channel.epg_data.epg_source and channel.epg_data.epg_source.source_type == 'dummy': - epg_data_id = channel.epg_data_id - if epg_data_id not in dummy_epg_checked: - dummy_epg_checked[epg_data_id] = channel.epg_data.programs.exists() - if dummy_epg_checked[epg_data_id]: - real_epg_map.setdefault(epg_data_id, []).append(channel_id) + if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + if effective_epg_data_id not in dummy_epg_checked: + dummy_epg_checked[effective_epg_data_id] = effective_epg_data.programs.exists() + if dummy_epg_checked[effective_epg_data_id]: + real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) else: - dummy_program_list.append((channel_id, pattern_match_name, channel.epg_data.epg_source)) + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) continue - real_epg_map.setdefault(channel.epg_data_id, []).append(channel_id) + real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) # Emit dummy programmes for channel_id, pattern_match_name, epg_source in dummy_program_list: @@ -1880,11 +1897,13 @@ def generate_epg(request, profile_name=None, user=None): client_id, client_ip, user_agent = get_client_identifier(request) event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" if not cache.get(event_cache_key): + # `len()` reuses the queryset's iteration cache populated above; + # `count()` would issue a separate SELECT COUNT(*). log_system_event( event_type='epg_download', profile=profile_name or 'all', user=user.username if user else 'anonymous', - channels=channels.count(), + channels=len(channels), client_ip=client_ip, user_agent=user_agent, ) @@ -1927,13 +1946,13 @@ def xc_get_user(request): if custom_properties["xc_password"] != password: return None + if not network_access_allowed(request, 'XC_API', user): + return None + return user def xc_get_info(request, full=False): - if not network_access_allowed(request, 'XC_API'): - return JsonResponse({'error': 'Forbidden'}, status=403) - user = xc_get_user(request) if user is None: @@ -1990,9 +2009,6 @@ def xc_get_info(request, full=False): def xc_player_api(request, full=False): - if not network_access_allowed(request, 'XC_API'): - return JsonResponse({'error': 'Forbidden'}, status=403) - action = request.GET.get("action") user = xc_get_user(request) @@ -2027,9 +2043,6 @@ def xc_player_api(request, full=False): def xc_panel_api(request): - if not network_access_allowed(request, 'XC_API'): - return JsonResponse({'error': 'Forbidden'}, status=403) - user = xc_get_user(request) if user is None: @@ -2109,8 +2122,18 @@ def xc_xmltv(request): def xc_get_live_categories(user): from django.db.models import Min + from django.db.models.functions import Coalesce + response = [] + # Rank categories by the minimum EFFECTIVE channel number across their + # visible (not hidden_from_output) channels so overridden numbers drive the + # ordering, not the underlying provider values. + effective_min = Min( + Coalesce("channels__override__channel_number", "channels__channel_number") + ) + hidden_exclusion = {"channels__hidden_from_output": False} + if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -2118,20 +2141,25 @@ def xc_get_live_categories(user): if user_profile_count == 0: # No profile filtering - user sees all channel groups channel_groups = ChannelGroup.objects.filter( - channels__isnull=False, channels__user_level__lte=user.user_level - ).distinct().annotate(min_channel_number=Min('channels__channel_number')).order_by('min_channel_number') + channels__isnull=False, + channels__user_level__lte=user.user_level, + **hidden_exclusion, + ).distinct().annotate(min_channel_number=effective_min).order_by('min_channel_number') else: # User has specific limited profiles assigned filters = { "channels__channelprofilemembership__enabled": True, "channels__user_level": 0, - "channels__channelprofilemembership__channel_profile__in": user.channel_profiles.all() + "channels__channelprofilemembership__channel_profile__in": user.channel_profiles.all(), + **hidden_exclusion, } - channel_groups = ChannelGroup.objects.filter(**filters).distinct().annotate(min_channel_number=Min('channels__channel_number')).order_by('min_channel_number') + channel_groups = ChannelGroup.objects.filter(**filters).distinct().annotate(min_channel_number=effective_min).order_by('min_channel_number') else: channel_groups = ChannelGroup.objects.filter( - channels__isnull=False, channels__user_level__lte=user.user_level - ).distinct().annotate(min_channel_number=Min('channels__channel_number')).order_by('min_channel_number') + channels__isnull=False, + channels__user_level__lte=user.user_level, + **hidden_exclusion, + ).distinct().annotate(min_channel_number=effective_min).order_by('min_channel_number') for group in channel_groups: response.append( @@ -2146,6 +2174,8 @@ def xc_get_live_categories(user): def xc_get_live_streams(request, user, category_id=None): + from apps.channels.managers import with_effective_values + streams = [] if user.user_level < 10: @@ -2160,7 +2190,7 @@ def xc_get_live_streams(request, user, category_id=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('channel_group', 'logo').order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('channel_group', 'logo') else: # User has specific limited profiles assigned filters = { @@ -2173,14 +2203,20 @@ def xc_get_live_streams(request, user, category_id=None): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channels = Channel.objects.filter(**filters).select_related('channel_group', 'logo').distinct().order_by("channel_number") + base_qs = Channel.objects.filter(**filters).select_related('channel_group', 'logo').distinct() else: if not category_id: - channels = Channel.objects.filter(user_level__lte=user.user_level).select_related('channel_group', 'logo').order_by("channel_number") + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('channel_group', 'logo') else: - channels = Channel.objects.filter( + base_qs = Channel.objects.filter( channel_group__id=category_id, user_level__lte=user.user_level - ).select_related('channel_group', 'logo').order_by("channel_number") + ).select_related('channel_group', 'logo') + + channels = ( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) # Resolve the fallback group ID once to avoid a get_or_create query per null-group channel _default_group_id = None @@ -2196,21 +2232,21 @@ def xc_get_live_streams(request, user, category_id=None): channel_num_map = {} # Maps channel.id -> integer channel number for XC used_numbers = set() # Track all assigned integer channel numbers - # First pass: assign integers for channels that already have integer numbers + # First pass: assign integers for channels that already have integer effective numbers for channel in channels: - if channel.channel_number == int(channel.channel_number): - # Already an integer, use it directly - num = int(channel.channel_number) + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) channel_num_map[channel.id] = num used_numbers.add(num) # Second pass: assign integers for channels with float numbers # Find next available number to avoid collisions for channel in channels: - if channel.channel_number != int(channel.channel_number): + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): # Has decimal component, need to find available integer - # Start from truncated value and increment until we find an unused number - candidate = int(channel.channel_number) + candidate = int(effective_num) while candidate in used_numbers: candidate += 1 channel_num_map[channel.id] = candidate @@ -2219,26 +2255,28 @@ def xc_get_live_streams(request, user, category_id=None): # Build the streams list with the collision-free channel numbers for channel in channels: channel_num_int = channel_num_map[channel.id] + effective_logo = channel.effective_logo_obj + effective_group = channel.effective_channel_group_obj streams.append( { "num": channel_num_int, - "name": channel.name, + "name": channel.effective_name, "stream_type": "live", "stream_id": channel.id, "stream_icon": ( None - if not channel.logo + if not effective_logo else build_absolute_uri_with_port( request, - reverse("api:channels:logo-cache", args=[channel.logo.id]) + reverse("api:channels:logo-cache", args=[effective_logo.id]) ) ), "epg_channel_id": str(channel_num_int), "added": str(int(channel.created_at.timestamp())), "is_adult": int(channel.is_adult), - "category_id": str(channel.channel_group.id if channel.channel_group else _get_default_group_id()), - "category_ids": [channel.channel_group.id if channel.channel_group else _get_default_group_id()], + "category_id": str(effective_group.id if effective_group else _get_default_group_id()), + "category_ids": [effective_group.id if effective_group else _get_default_group_id()], "custom_sid": None, "tv_archive": 0, "direct_source": "", @@ -2250,11 +2288,19 @@ def xc_get_live_streams(request, user, category_id=None): def xc_get_epg(request, user, short=False): + from apps.channels.managers import with_effective_values + channel_id = request.GET.get('stream_id') if not channel_id: raise Http404() channel = None + # Apply effective-value annotation + hidden-exclusion at every channel + # resolution path so a single channel lookup honors the same visibility + # rules as xc_get_live_streams. + def _annotate(qs): + return with_effective_values(qs, select_related_fks=True).exclude(hidden_from_output=True) + if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -2268,7 +2314,7 @@ def xc_get_epg(request, user, short=False): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channel = Channel.objects.filter(**filters).select_related('epg_data__epg_source').first() + channel = _annotate(Channel.objects.filter(**filters).select_related('epg_data__epg_source')).first() else: # User has specific limited profiles assigned filters = { @@ -2280,44 +2326,58 @@ def xc_get_epg(request, user, short=False): # Hide adult content if user preference is set if (user.custom_properties or {}).get('hide_adult_content', False): filters["is_adult"] = False - channel = Channel.objects.filter(**filters).select_related('epg_data__epg_source').distinct().first() + channel = _annotate(Channel.objects.filter(**filters).select_related('epg_data__epg_source').distinct()).first() if not channel: raise Http404() else: - channel = get_object_or_404(Channel.objects.select_related('epg_data__epg_source'), id=channel_id) + channel = _annotate(Channel.objects.filter(id=channel_id).select_related('epg_data__epg_source')).first() + if not channel: + raise Http404() if not channel: raise Http404() # Calculate the collision-free integer channel number for this channel - # This must match the logic in xc_get_live_streams to ensure consistency - # Get all channels in the same category for collision detection - category_channels = Channel.objects.filter( - channel_group=channel.channel_group - ).order_by("channel_number") + # This must match the logic in xc_get_live_streams to ensure consistency. + # The category channels must be filtered by the channel's EFFECTIVE group + # (an override can move a channel into a different group), then annotated + # so the comparison runs on effective numbers. + effective_group = channel.effective_channel_group_obj + category_channels = ( + with_effective_values( + Channel.objects.filter(channel_group=effective_group) if effective_group else Channel.objects.none() + ) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + ) channel_num_map = {} used_numbers = set() - # First pass: assign integers for channels that already have integer numbers + # First pass: assign integers for channels that already have integer effective numbers for ch in category_channels: - if ch.channel_number == int(ch.channel_number): - num = int(ch.channel_number) + effective_num = ch.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) channel_num_map[ch.id] = num used_numbers.add(num) - # Second pass: assign integers for channels with float numbers + # Second pass: assign integers for channels with float effective numbers for ch in category_channels: - if ch.channel_number != int(ch.channel_number): - candidate = int(ch.channel_number) + effective_num = ch.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): + candidate = int(effective_num) while candidate in used_numbers: candidate += 1 channel_num_map[ch.id] = candidate used_numbers.add(candidate) # Get the mapped integer for this specific channel - channel_num_int = channel_num_map.get(channel.id, int(channel.channel_number)) + channel_num_int = channel_num_map.get( + channel.id, + int(channel.effective_channel_number) if channel.effective_channel_number is not None else 0, + ) limit = int(request.GET.get('limit', 4)) user_custom = user.custom_properties or {} @@ -2334,25 +2394,28 @@ def xc_get_epg(request, user, short=False): now = django_timezone.now() lookback_cutoff = now - timedelta(days=prev_days) forward_cutoff = now + timedelta(days=num_days) if num_days > 0 else None - if channel.epg_data: + effective_epg_data = channel.effective_epg_data_obj + effective_name = channel.effective_name + + if effective_epg_data: # Check if this is a dummy EPG that generates on-demand - if channel.epg_data.epg_source and channel.epg_data.epg_source.source_type == 'dummy': - if not channel.epg_data.programs.exists(): + if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + if not effective_epg_data.programs.exists(): # Generate on-demand using custom patterns programs = generate_dummy_programs( channel_id=channel_id, - channel_name=channel.name, - epg_source=channel.epg_data.epg_source + channel_name=effective_name, + epg_source=effective_epg_data.epg_source ) else: # Has stored programs, use them if short: # Short EPG: current and upcoming only (never historical), limited count - programs = channel.epg_data.programs.filter( + programs = effective_epg_data.programs.filter( end_time__gt=now ).order_by('start_time')[:limit] else: - qs = channel.epg_data.programs.filter(end_time__gt=lookback_cutoff) + qs = effective_epg_data.programs.filter(end_time__gt=lookback_cutoff) if forward_cutoff: qs = qs.filter(start_time__lt=forward_cutoff) programs = qs.order_by('start_time') @@ -2360,17 +2423,17 @@ def xc_get_epg(request, user, short=False): # Regular EPG with stored programs if short: # Short EPG: current and upcoming only (never historical), limited count - programs = channel.epg_data.programs.filter( + programs = effective_epg_data.programs.filter( end_time__gt=now ).order_by('start_time')[:limit] else: - qs = channel.epg_data.programs.filter(end_time__gt=lookback_cutoff) + qs = effective_epg_data.programs.filter(end_time__gt=lookback_cutoff) if forward_cutoff: qs = qs.filter(start_time__lt=forward_cutoff) programs = qs.order_by('start_time') else: # No EPG data assigned, generate default dummy - programs = generate_dummy_programs(channel_id=channel_id, channel_name=channel.name, epg_source=None) + programs = generate_dummy_programs(channel_id=channel_id, channel_name=effective_name, epg_source=None) output = {"epg_listings": []} @@ -2391,7 +2454,7 @@ def xc_get_epg(request, user, short=False): # epg_id refers to the EPG source/channel mapping in XC panels # Use the actual EPGData ID when available, otherwise fall back to 0 - epg_id = str(channel.epg_data.id) if channel.epg_data else "0" + epg_id = str(effective_epg_data.id) if effective_epg_data else "0" program_output = { "id": program_id, diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index 7170c5dc..35946815 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -564,6 +564,9 @@ def stream_xc(request, username, password, channel_id): extension = pathlib.Path(channel_id).suffix channel_id = pathlib.Path(channel_id).stem + if not network_access_allowed(request, 'STREAMS', user): + return Response({"error": "Forbidden"}, status=403) + custom_properties = user.custom_properties or {} if "xc_password" not in custom_properties: diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 924f6906..f73f48b4 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -1023,6 +1023,9 @@ def stream_xc_movie(request, username, password, stream_id, extension): user = get_object_or_404(User, username=username) + if not network_access_allowed(request, 'STREAMS', user): + return Response({"error": "Forbidden"}, status=403) + custom_properties = user.custom_properties or {} if "xc_password" not in custom_properties: @@ -1057,6 +1060,9 @@ def stream_xc_episode(request, username, password, stream_id, extension): user = get_object_or_404(User, username=username) + if not network_access_allowed(request, 'STREAMS', user): + return Response({"error": "Forbidden"}, status=403) + custom_properties = user.custom_properties or {} if "xc_password" not in custom_properties: diff --git a/core/management/commands/reset_user_network.py b/core/management/commands/reset_user_network.py new file mode 100644 index 00000000..a6a43243 --- /dev/null +++ b/core/management/commands/reset_user_network.py @@ -0,0 +1,22 @@ +from django.core.management.base import BaseCommand +from apps.accounts.models import User + + +class Command(BaseCommand): + help = "Reset per-user network access restrictions" + + def add_arguments(self, parser): + parser.add_argument('--user', type=str, help='Username to reset (omit to reset all users)') + + def handle(self, *args, **options): + username = options.get('user') + qs = User.objects.filter(username=username) if username else User.objects.all() + count = 0 + for user in qs: + props = user.custom_properties or {} + if 'allowed_networks' in props: + del props['allowed_networks'] + user.custom_properties = props + user.save(update_fields=['custom_properties']) + count += 1 + self.stdout.write(f"Reset allowed_networks for {count} user(s).") diff --git a/dispatcharr/utils.py b/dispatcharr/utils.py index c8fe58ad..02fe7853 100644 --- a/dispatcharr/utils.py +++ b/dispatcharr/utils.py @@ -38,7 +38,7 @@ def get_client_ip(request): return ip -def network_access_allowed(request, settings_key): +def network_access_allowed(request, settings_key, user=None): try: network_access = CoreSettings.objects.get(key=NETWORK_ACCESS_KEY).value except CoreSettings.DoesNotExist: @@ -66,4 +66,19 @@ def network_access_allowed(request, settings_key): network_allowed = True break - return network_allowed + if not network_allowed: + return False + + if user is not None: + user_networks = (getattr(user, 'custom_properties', None) or {}).get('allowed_networks', {}) + raw = user_networks.get(settings_key, '') + if raw: + for cidr in (c.strip() for c in raw.split(',') if c.strip()): + try: + if client_ip in ipaddress.ip_network(cidr, strict=False): + return True + except ValueError: + continue + return False + + return True diff --git a/frontend/src/api.js b/frontend/src/api.js index ff080c88..6c003bf3 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -271,6 +271,77 @@ export default class API { } } + // Repack visible auto-created channels into [start, end]; override + // pins are reservations and hidden non-pinned channels release their + // number. + static async repackGroupChannels(accountId, channelGroupId) { + try { + const params = new URLSearchParams({ + channel_group_id: String(channelGroupId), + }); + const url = `${host}/api/m3u/accounts/${accountId}/repack-group/?${params.toString()}`; + return await request(url, { method: 'POST' }); + } catch (e) { + errorNotification('Failed to re-pack group channels', e); + return null; + } + } + + // Returns occupants whose effective channel_number falls in [start, end]. + // Pass `signal` from an AbortController on per-keystroke calls so an + // out-of-order response cannot overwrite newer state. + static async getChannelsInRange(start, end, { signal } = {}) { + try { + const params = new URLSearchParams({ start: String(start) }); + if (end !== undefined && end !== null && end !== '') { + params.set('end', String(end)); + } + const url = `${host}/api/channels/channels/numbers-in-range/?${params.toString()}`; + return await request(url, { signal }); + } catch (e) { + if (e?.name === 'AbortError') { + throw e; + } + // Silent failure is correct here: the warning is purely advisory and + // should not block the user from saving when the server is flaky. + return { occupants: [] }; + } + } + + // Server-side regex preview for a group's streams. Returns find_matches, + // filter_matches, and exclude_matches plus accurate counts across the + // whole group (capped at 5000 scanned streams server-side). Used by the + // auto-sync gear modal so the user sees real matches and totals rather + // than a small client-side sample. The three patterns are independent; + // any combination can be supplied per call. + static async getStreamsRegexPreview( + channelGroupName, + { find, replace, match, exclude, limit = 10, signal, m3uAccountId } = {} + ) { + try { + const params = new URLSearchParams({ + channel_group: channelGroupName, + }); + if (m3uAccountId !== undefined && m3uAccountId !== null) { + params.set('m3u_account_id', String(m3uAccountId)); + } + if (find) params.set('find', find); + if (replace !== undefined && replace !== null) { + params.set('replace', replace); + } + if (match) params.set('match', match); + if (exclude) params.set('exclude', exclude); + params.set('limit', String(limit)); + const url = `${host}/api/channels/streams/regex-preview/?${params.toString()}`; + return await request(url, { signal }); + } catch (e) { + if (e?.name === 'AbortError') { + throw e; + } + return null; + } + } + static async queryChannels(params) { try { API.lastQueryParams = params; @@ -1341,14 +1412,34 @@ export default class API { } static async deletePlaylist(id) { + // Cascade-deletes auto-created channels owned by the account; the + // response includes the deleted count for the confirmation toast. try { - await request(`${host}/api/m3u/accounts/${id}/`, { + const response = await request(`${host}/api/m3u/accounts/${id}/`, { method: 'DELETE', }); - usePlaylistsStore.getState().removePlaylists([id]); + return response || {}; } catch (e) { errorNotification(`Failed to delete playlist ${id}`, e); + throw e; + } + } + + // Used by the Delete Playlist confirmation dialog to render an accurate + // "Also delete N auto-created channels?" option before the user commits. + static async getPlaylistAutoCreatedChannelsCount(id) { + try { + const response = await request( + `${host}/api/m3u/accounts/${id}/auto-created-channels-count/` + ); + return response; + } catch (e) { + console.error( + `Failed to fetch auto-created channel count for playlist ${id}`, + e + ); + return { count: 0, sample_names: [] }; } } @@ -2983,10 +3074,12 @@ export default class API { } } - static async deleteSeriesRule(tvgId) { + static async deleteSeriesRule(tvgId, title) { try { - const encodedTvgId = encodeURIComponent(tvgId); - await request(`${host}/api/channels/series-rules/${encodedTvgId}/`, { + const params = new URLSearchParams(); + if (tvgId) params.set('tvg_id', tvgId); + if (title) params.set('title', title); + await request(`${host}/api/channels/series-rules/?${params}`, { method: 'DELETE', }); notifications.show({ title: 'Series rule removed' }); @@ -3024,6 +3117,15 @@ export default class API { } } + static async previewSeriesRule(values, { signal } = {}) { + // Throws on error so callers can ignore aborted requests vs real failures. + return request(`${host}/api/channels/series-rules/preview/`, { + method: 'POST', + body: values, + signal, + }); + } + static async bulkRemoveSeriesRecordings({ tvg_id, title = null, diff --git a/frontend/src/components/M3URefreshNotification.jsx b/frontend/src/components/M3URefreshNotification.jsx index bd6319bf..8e24bdb3 100644 --- a/frontend/src/components/M3URefreshNotification.jsx +++ b/frontend/src/components/M3URefreshNotification.jsx @@ -5,7 +5,15 @@ import useStreamsStore from '../store/streams'; import useChannelsStore from '../store/channels'; import useEPGsStore from '../store/epgs'; import useVODStore from '../store/useVODStore'; -import { Stack, Button, Group } from '@mantine/core'; +import { + Stack, + Button, + Group, + Modal, + ScrollArea, + Text, + Code, +} from '@mantine/core'; import API from '../api'; import { useNavigate } from 'react-router-dom'; import { CircleCheck } from 'lucide-react'; @@ -43,6 +51,47 @@ const M3uSetupSuccess = ({ data }) => { ); }; +// One-line outcome summary for the notification body. +const buildAutoSyncSummary = (data) => { + const created = data.channels_created || 0; + const updated = data.channels_updated || 0; + const deleted = data.channels_deleted || 0; + const failed = data.channels_failed || 0; + if (!created && !updated && !deleted && !failed) return null; + const parts = []; + if (created) parts.push(`${created} created`); + if (updated) parts.push(`${updated} updated`); + if (deleted) parts.push(`${deleted} deleted`); + if (failed) parts.push(`${failed} failed`); + return `Auto-sync: ${parts.join(', ')}.`; +}; + +// Human labels for the typed failure reasons attached to each +// failed_stream_details entry. Unknown reasons fall back to the raw +// key so a future reason added on the backend still surfaces in the +// modal without a code change here. +const FAILURE_REASON_LABELS = { + RANGE_EXHAUSTED: 'Channel number range exhausted', + INTEGRITY_ERROR: 'Database integrity error', + OTHER: 'Other errors', +}; + +const FAILURES_PER_GROUP_LIMIT = 50; + +// Group entries by their reason key, preserving insertion order so +// section ordering reflects the sync's encounter order. Entries +// without a reason field bucket into OTHER so a backend that does +// not classify failures still renders correctly. +const groupFailuresByReason = (entries) => { + const buckets = new Map(); + for (const entry of entries) { + const key = entry?.reason || 'OTHER'; + if (!buckets.has(key)) buckets.set(key, []); + buckets.get(key).push(entry); + } + return buckets; +}; + export default function M3URefreshNotification() { const playlists = usePlaylistsStore((s) => s.playlists); const refreshProgress = usePlaylistsStore((s) => s.refreshProgress); @@ -54,6 +103,9 @@ export default function M3URefreshNotification() { const fetchCategories = useVODStore((s) => s.fetchCategories); const [notificationStatus, setNotificationStatus] = useState({}); + // Modal payload for the "Click for details" affordance on syncs that + // produced failed_stream_details. Null when the modal is closed. + const [failureModal, setFailureModal] = useState(null); const handleM3UUpdate = (data) => { // Skip if status hasn't changed @@ -148,7 +200,7 @@ export default function M3URefreshNotification() { const handleProgressNotification = (playlist, data) => { const baseMessage = getActionMessage(data.action); - const message = + let message = data.progress == 0 ? `${baseMessage} starting...` : `${baseMessage} complete!`; @@ -157,11 +209,49 @@ export default function M3URefreshNotification() { triggerPostCompletionFetches(data.action); } + let body = message; + let autoClose = 2000; + // Surface auto-sync counts attached to the parsing-complete event + // so the channel-side outcome appears in the notification body. + if (data.progress == 100 && data.action === 'parsing') { + const autoSyncSummary = buildAutoSyncSummary(data); + const failed = data.channels_failed || 0; + const failedDetails = Array.isArray(data.failed_stream_details) + ? data.failed_stream_details + : []; + if (autoSyncSummary) { + body = ( + + {message} + + {autoSyncSummary} + + {failed > 0 && failedDetails.length > 0 && ( + + )} + + ); + autoClose = failed > 0 ? 12000 : 4000; + } + } + showNotification({ title: `M3U Processing: ${playlist.name}`, - message, + message: body, loading: data.progress == 0, - autoClose: 2000, + autoClose, icon: data.progress == 100 ? : null, }); }; @@ -182,5 +272,61 @@ export default function M3URefreshNotification() { Object.values(refreshProgress).map((data) => handleM3UUpdate(data)); }, [playlists, refreshProgress]); - return <>; + return ( + setFailureModal(null)} + title={ + failureModal + ? `Auto-sync failures: ${failureModal.playlistName}` + : 'Auto-sync failures' + } + size="lg" + > + + + The following streams could not be synced. Failures are grouped by + cause so the most common issues surface first. + + + + {Array.from( + groupFailuresByReason(failureModal?.failedDetails || []).entries() + ).map(([reasonKey, entries]) => { + const label = FAILURE_REASON_LABELS[reasonKey] || reasonKey; + const visible = entries.slice(0, FAILURES_PER_GROUP_LIMIT); + const hidden = entries.length - visible.length; + return ( + + + {label} ({entries.length}) + + {visible.map((entry, idx) => ( + + {[ + entry.stream_name && `Stream: ${entry.stream_name}`, + entry.group && `Group: ${entry.group}`, + entry.error && `Error: ${entry.error}`, + ] + .filter(Boolean) + .join('\n')} + + ))} + {hidden > 0 && ( + + Showing first {visible.length} of {entries.length}. + Remaining {hidden} entries are recorded in the server log. + + )} + + ); + })} + + + + + ); } diff --git a/frontend/src/components/__tests__/M3URefreshNotification.test.jsx b/frontend/src/components/__tests__/M3URefreshNotification.test.jsx index 43375fa7..734d83e4 100644 --- a/frontend/src/components/__tests__/M3URefreshNotification.test.jsx +++ b/frontend/src/components/__tests__/M3URefreshNotification.test.jsx @@ -52,6 +52,19 @@ vi.mock('@mantine/core', async () => { Button: ({ children, onClick }) => ( ), + // Stub for the auto-sync failure-details modal. + Modal: ({ children, opened, onClose, title }) => + opened ? ( +
+ + {children} +
+ ) : null, + ScrollArea: ({ children }) =>
{children}
, + Text: ({ children }) => {children}, + Code: ({ children }) => 
{children}
, }; }); @@ -714,4 +727,128 @@ describe('M3URefreshNotification', () => { ); }); }); + + // The parsing-complete event payload carries auto-sync counts + // (channels_created/updated/deleted/failed) and a failed_stream_details + // array. The component surfaces the counts inline in the notification + // body and exposes a "Click for details" affordance when failures exist. + describe('Auto-sync count rendering on parsing complete', () => { + it('inlines auto-sync summary when counts are present', async () => { + mockPlaylistsStore.refreshProgress = { + 1: { + account: 1, + action: 'parsing', + progress: 100, + status: 'success', + channels_created: 12, + channels_updated: 3, + channels_deleted: 1, + channels_failed: 0, + failed_stream_details: [], + }, + }; + + renderWithProviders(); + + await waitFor(() => { + expect(showNotification).toHaveBeenCalled(); + }); + // The message arg is JSX; render it and confirm both the + // base message and the auto-sync summary text appear. + const call = showNotification.mock.calls.find( + (c) => typeof c[0]?.message === 'object' + ); + expect(call).toBeDefined(); + const { container } = render(<>{call[0].message}); + expect(container.textContent).toContain('Stream parsing complete!'); + expect(container.textContent).toContain('12 created'); + expect(container.textContent).toContain('3 updated'); + expect(container.textContent).toContain('1 deleted'); + }); + + it('shows "Click for details" button when failures exist', async () => { + mockPlaylistsStore.refreshProgress = { + 1: { + account: 1, + action: 'parsing', + progress: 100, + status: 'success', + channels_created: 5, + channels_updated: 0, + channels_deleted: 0, + channels_failed: 2, + failed_stream_details: [ + { + stream_name: 'BadStream1', + group: 'Sports', + error: 'Range exhausted', + }, + { + stream_name: 'BadStream2', + group: 'News', + error: 'Channel number conflict', + }, + ], + }, + }; + + renderWithProviders(); + + await waitFor(() => { + expect(showNotification).toHaveBeenCalled(); + }); + const call = showNotification.mock.calls.find( + (c) => typeof c[0]?.message === 'object' + ); + expect(call).toBeDefined(); + const { container } = render(<>{call[0].message}); + expect(container.textContent).toContain('2 failed'); + expect(container.textContent).toContain('Click for details'); + }); + + it('falls back to plain string body when no auto-sync counts arrive', async () => { + // Older payload shape (or non-parsing actions) get the original + // simple "X complete!" string body, no JSX wrapping. + mockPlaylistsStore.refreshProgress = { + 1: { + account: 1, + action: 'downloading', + progress: 100, + status: 'success', + }, + }; + + renderWithProviders(); + + await waitFor(() => { + expect(showNotification).toHaveBeenCalled(); + }); + const call = showNotification.mock.calls[0]; + expect(typeof call[0].message).toBe('string'); + }); + + it('uses extended autoClose when failures are present', async () => { + mockPlaylistsStore.refreshProgress = { + 1: { + account: 1, + action: 'parsing', + progress: 100, + status: 'success', + channels_created: 1, + channels_failed: 1, + failed_stream_details: [{ stream_name: 'X', group: 'Y', error: 'Z' }], + }, + }; + + renderWithProviders(); + + await waitFor(() => { + expect(showNotification).toHaveBeenCalled(); + }); + const call = showNotification.mock.calls[0]; + // 12000ms when failures > 0; 4000ms when summary present but no + // failures; 2000ms when no auto-sync counts. + expect(call[0].autoClose).toBe(12000); + }); + }); }); diff --git a/frontend/src/components/forms/Channel.jsx b/frontend/src/components/forms/Channel.jsx index cc503209..90b95a42 100644 --- a/frontend/src/components/forms/Channel.jsx +++ b/frontend/src/components/forms/Channel.jsx @@ -33,7 +33,7 @@ import { UnstyledButton, useMantineTheme, } from '@mantine/core'; -import { ListOrdered, SquarePlus, X, Zap } from 'lucide-react'; +import { ListOrdered, SquarePlus, Undo2, X, Zap } from 'lucide-react'; import useEPGsStore from '../../store/epgs'; import { FixedSizeList as List } from 'react-window'; import { USER_LEVEL_LABELS, USER_LEVELS } from '../../constants'; @@ -43,11 +43,18 @@ import { } from '../../utils/notificationUtils.js'; import { addChannel, + clearChannelOverrides, createLogo, getChannelFormDefaultValues, + getFkProviderHint, getFormattedValues, + getProviderFormValue, + getProviderHint, handleEpgUpdate, + isFormFieldOverridden, matchChannelEpg, + OVERRIDABLE_FIELDS, + OVERRIDE_FIELD_LABELS, requeryChannels, } from '../../utils/forms/ChannelUtils.js'; @@ -56,13 +63,48 @@ const validationSchema = Yup.object({ channel_group_id: Yup.string().required('Channel group is required'), }); -const ChannelForm = ({ channel = null, isOpen, onClose }) => { +// Provider hint plus a reset-to-provider icon for auto-synced +// channels; rendered as the field's `description` prop. +const ProviderHintRow = ({ channel, field, formValue, hintText, onReset }) => { + if (!hintText) return null; + const overridden = isFormFieldOverridden(channel, field, formValue); + return ( + + + {hintText} + + {overridden && ( + + + + + + )} + + ); +}; + +const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { const theme = useMantineTheme(); const listRef = useRef(null); const logoListRef = useRef(null); const groupListRef = useRef(null); + // Local copy so in-modal mutations (clear overrides, etc.) update the + // form immediately. Reset on each open from `channelProp`; mutated in + // place with API responses. + const [channel, setChannel] = useState(channelProp); + useEffect(() => { + setChannel(channelProp); + }, [channelProp]); + const channelGroups = useChannelsStore((s) => s.channelGroups); const { @@ -309,7 +351,45 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { resolver: yupResolver(validationSchema), }); + const clearOverrides = async () => { + if (!channel) return; + try { + const updated = await clearChannelOverrides(channel.id); + // Update local state first so the form reflects the cleared + // overrides immediately; the table-store refresh is best-effort. + if (updated && typeof updated === 'object') { + setChannel(updated); + } + requeryChannels(); + showNotification({ + title: 'Overrides Cleared', + message: 'Channel values now follow the provider.', + color: 'green', + }); + } catch (error) { + showNotification({ + title: 'Clear Failed', + message: + error?.body?.detail || error?.message || 'Could not clear overrides.', + color: 'red', + }); + } + }; + + // Computed from live form values so per-field resets update the + // Clear-all button immediately, before submit. + const watchedFormValues = watch(); + const overriddenFieldLabels = useMemo(() => { + if (!channel) return []; + return OVERRIDABLE_FIELDS.filter((field) => + isFormFieldOverridden(channel, field, watchedFormValues[field]) + ).map((field) => OVERRIDE_FIELD_LABELS[field]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [channel, JSON.stringify(watchedFormValues)]); + const hasAnyOverride = overriddenFieldLabels.length > 0; + const onSubmit = async (values) => { + let saveFailed = false; try { const formattedValues = getFormattedValues(values); @@ -324,8 +404,29 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { } } catch (error) { console.error('Error saving channel:', error); + saveFailed = true; + showNotification({ + title: 'Save Failed', + message: + error?.body?.detail || error?.message || 'Failed to save channel.', + color: 'red', + }); } + if (saveFailed) { + // Keep the form open with the user's edits intact so they can correct + // a validation error without retyping. + return; + } + + showNotification({ + title: 'Saved', + message: channel + ? `Channel "${values.name}" updated.` + : `Channel "${values.name}" created.`, + color: 'green', + }); + reset(); requeryChannels(); @@ -408,6 +509,17 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { styles={{ content: { '--mantine-color-body': '#27272A' } }} >
+ {channel?.auto_created && channel?.source_stream && ( + + Auto-created from:{' '} + + {channel.source_stream.account_name || 'Unknown provider'} + + {channel.source_stream.name + ? ` / ${channel.source_stream.name}` + : ''} + + )} { )} } + description={ + + setValue('name', getProviderFormValue(channel, 'name'), { + shouldDirty: true, + }) + } + /> + } {...register('name')} error={errors.name?.message} size="xs" @@ -448,6 +573,24 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { id="channel_group_id" name="channel_group_id" label="Channel Group" + description={ + + setValue( + 'channel_group_id', + getProviderFormValue(channel, 'channel_group_id') + ) + } + /> + } readOnly value={ channelGroups[watch('channel_group_id')] @@ -535,6 +678,27 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { id="stream_profile_id" label="Stream Profile" name="stream_profile_id" + description={ + { + acc[p.id] = p; + return acc; + }, {}) + )} + onReset={() => + setValue( + 'stream_profile_id', + getProviderFormValue(channel, 'stream_profile_id') + ) + } + /> + } value={watch('stream_profile_id')} onChange={(value) => { setValue('stream_profile_id', value); @@ -605,6 +769,24 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { )} } + description={ + + setValue( + 'logo_id', + getProviderFormValue(channel, 'logo_id') + ) + } + /> + } readOnly value={channelLogos[watch('logo_id')]?.name || 'Default'} onClick={() => { @@ -738,6 +920,38 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { /> + + + + setValue('hidden_from_output', event.currentTarget.checked) + } + size="md" + /> + + + {channel?.auto_created && hasAnyOverride && ( + + + + )} @@ -747,6 +961,21 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { id="channel_number" name="channel_number" label="Channel # (blank to auto-assign)" + description={ + + setValue( + 'channel_number', + getProviderFormValue(channel, 'channel_number'), + { shouldDirty: true } + ) + } + /> + } value={watch('channel_number')} onChange={(value) => setValue('channel_number', value)} error={errors.channel_number?.message} @@ -775,6 +1004,21 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { )} } + description={ + + setValue( + 'tvg_id', + getProviderFormValue(channel, 'tvg_id'), + { shouldDirty: true } + ) + } + /> + } {...register('tvg_id')} error={errors.tvg_id?.message} size="xs" @@ -784,6 +1028,21 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { id="tvc_guide_stationid" name="tvc_guide_stationid" label="Gracenote StationId" + description={ + + setValue( + 'tvc_guide_stationid', + getProviderFormValue(channel, 'tvc_guide_stationid'), + { shouldDirty: true } + ) + } + /> + } {...register('tvc_guide_stationid')} error={errors.tvc_guide_stationid?.message} size="xs" @@ -829,6 +1088,24 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { } + description={ + + setValue( + 'epg_data_id', + getProviderFormValue(channel, 'epg_data_id') + ) + } + /> + } readOnly value={(() => { const tvg = tvgsById[watch('epg_data_id')]; diff --git a/frontend/src/components/forms/ChannelBatch.jsx b/frontend/src/components/forms/ChannelBatch.jsx index a227b20c..e0355ea0 100644 --- a/frontend/src/components/forms/ChannelBatch.jsx +++ b/frontend/src/components/forms/ChannelBatch.jsx @@ -54,6 +54,7 @@ import { setChannelNamesFromEpg, setChannelTvgIdsFromEpg, updateChannels, + updateChannelsWithOverrideRouting, } from '../../utils/forms/ChannelBatchUtils.js'; const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { @@ -123,14 +124,40 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { stream_profile_id: '-1', user_level: '-1', is_adult: '-1', + hidden_from_output: '-1', + clear_overrides: '-1', }, }); + // Surfaces auto-vs-manual routing in the selection. Falls back to a + // single total when the table store only has a partial view (e.g. + // cross-page selects). Kept separate from getConfirmationMessage so + // the line does not count against the no-changes guard. + const getSelectionSummary = () => { + const channelsById = useChannelsTableStore + .getState() + .channels.reduce((acc, c) => { + acc[c.id] = c; + return acc; + }, {}); + let autoCount = 0; + let manualCount = 0; + for (const id of channelIds) { + const c = channelsById[id]; + if (!c) continue; + if (c.auto_created) autoCount++; + else manualCount++; + } + const resolved = autoCount + manualCount; + return resolved === channelIds.length + ? `Selection: ${autoCount} auto-synced, ${manualCount} manual` + : `Selection: ${channelIds.length} channels`; + }; + // Build confirmation message based on selected changes const getConfirmationMessage = () => { const values = form.getValues(); - - return [ + const lines = [ getRegexNameChange(regexFind, regexReplace), getChannelGroupChange(selectedChannelGroup, channelGroups), getLogoChange(selectedLogoId, channelLogos), @@ -138,7 +165,18 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { getUserLevelChange(values.user_level, USER_LEVEL_LABELS), getMatureContentChange(values.is_adult), getEpgChange(selectedDummyEpgId, epgs), - ].filter(Boolean); + ]; + if (values.hidden_from_output && values.hidden_from_output !== '-1') { + lines.push( + `• Hidden: ${values.hidden_from_output === 'true' ? 'Yes' : 'No'}` + ); + } + if (values.clear_overrides === 'clear') { + lines.push( + '• Clear all overrides on auto-synced channels in selection, then apply the edits above as new overrides' + ); + } + return lines.filter(Boolean); }; const handleSubmit = () => { @@ -167,14 +205,37 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { setIsSubmitting(true); try { + const formValues = form.getValues(); + const shouldClearOverrides = formValues.clear_overrides === 'clear'; + const values = buildSubmitValues( - form.getValues(), + formValues, selectedChannelGroup, selectedLogoId ); + // Clear runs before the routing PATCH (not in parallel) so a + // late-landing clear cannot wipe the freshly-written override + // fields. + if (shouldClearOverrides && channelIds.length > 0) { + await updateChannels(channelIds, { override: null }); + } + if (Object.keys(values).length > 0) { - await updateChannels(channelIds, values); + // Route auto-created channels to override.X (survives sync) + // and manual channels to direct Channel.X writes; auto_created + // is read from the table store. + const channelsById = useChannelsTableStore + .getState() + .channels.reduce((acc, c) => { + acc[c.id] = c; + return acc; + }, {}); + await updateChannelsWithOverrideRouting( + channelIds, + values, + channelsById + ); } if (regexFind.trim().length > 0) { @@ -197,7 +258,16 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { ]); onClose(); } catch (error) { - console.error('Failed to update channels:', error); + // Keep the form open with the user's edits intact so they can correct + // a validation error without retyping the bulk selection. + showNotification({ + title: 'Bulk Update Failed', + message: + error?.body?.detail || + error?.message || + 'Failed to apply changes to the selected channels.', + color: 'red', + }); } finally { setIsSubmitting(false); } @@ -457,6 +527,9 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { styles={{ hannontent: { '--mantine-color-body': '#27272A' } }} > + + {getSelectionSummary()} + @@ -799,6 +872,31 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { { value: 'false', label: 'No' }, ]} /> + + @@ -905,6 +1003,9 @@ This action cannot be undone.`} style={{ backgroundColor: 'rgba(0, 0, 0, 0.2)' }} > + + {getSelectionSummary()} + {getConfirmationMessage().map((change, index) => ( { + if (!group) return null; + const streamCount = + typeof group.stream_count === 'number' ? group.stream_count : null; + + return ( + + Configure: {group.name} + {streamCount !== null && ( + + ({streamCount} stream{streamCount === 1 ? '' : 's'} available) + + )} + + } + styles={{ content: { '--mantine-color-body': '#27272A' } }} + > + {children} + {/* Done keeps in-memory edits routed into the parent's groupStates + (parent's Save and Refresh persists them). Cancel reverts to + the open-time snapshot. */} + + + + + + ); +}; + +export default GroupConfigureModal; diff --git a/frontend/src/components/forms/ProgramRecordingModal.jsx b/frontend/src/components/forms/ProgramRecordingModal.jsx index 82da2674..1ce58fb9 100644 --- a/frontend/src/components/forms/ProgramRecordingModal.jsx +++ b/frontend/src/components/forms/ProgramRecordingModal.jsx @@ -1,8 +1,9 @@ -import React from 'react'; -import { Modal, Flex, Button } from '@mantine/core'; +import React, { useState } from 'react'; +import { Modal, Flex, Button, Anchor } from '@mantine/core'; import { deleteRecordingById } from '../../utils/cards/RecordingCardUtils.js'; import { deleteSeriesAndRule } from '../../utils/cards/RecordingCardUtils.js'; import { deleteSeriesRuleByTvgId } from '../../utils/guideUtils.js'; +import SeriesRuleEditorModal from './SeriesRuleEditorModal.jsx'; export default function ProgramRecordingModal({ opened, @@ -10,11 +11,14 @@ export default function ProgramRecordingModal({ program, recording, existingRuleMode, + existingRule, onRecordOne, onRecordSeriesAll, onRecordSeriesNew, onExistingRuleModeChange, }) { + const [editorOpen, setEditorOpen] = useState(false); + const handleRemoveRecording = async () => { try { await deleteRecordingById(recording.id); @@ -35,7 +39,7 @@ export default function ProgramRecordingModal({ }; const handleRemoveSeriesRule = async () => { - await deleteSeriesRuleByTvgId(program.tvg_id); + await deleteSeriesRuleByTvgId(program.tvg_id, program.title); onExistingRuleModeChange(null); onClose(); }; @@ -85,6 +89,16 @@ export default function ProgramRecordingModal({ New episodes only + setEditorOpen(true)} + > + Customize rule... + + {recording && ( <> - - - - ))} - - + <> + + + + + + + {(!rules || rules.length === 0) && ( + + No series rules configured + + )} + {rules && + rules.map((r) => ( + + + + + {r.title || r.tvg_id} + + {r.channel_id && ( + + Pinned channel + + )} + + + {renderRuleSummary(r)} + + + + + + + + + ))} + + + + setEditorOpen(false)} + initialRule={editorRule} + onSaved={handleEditorSaved} + /> + ); } diff --git a/frontend/src/components/forms/SeriesRuleEditorModal.jsx b/frontend/src/components/forms/SeriesRuleEditorModal.jsx new file mode 100644 index 00000000..201f285f --- /dev/null +++ b/frontend/src/components/forms/SeriesRuleEditorModal.jsx @@ -0,0 +1,423 @@ +import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { + Modal, + Stack, + Text, + TextInput, + Textarea, + SegmentedControl, + Group, + Button, + Select, + Badge, + Divider, + ScrollArea, + Alert, +} from '@mantine/core'; +import API from '../../api.js'; +import useChannelsStore from '../../store/channels.jsx'; +import useEPGsStore from '../../store/epgs.jsx'; +import { useDebounce } from '../../utils.js'; +import { showNotification } from '../../utils/notificationUtils.js'; + +const TITLE_MODES = [ + { label: 'Exact', value: 'exact' }, + { label: 'Contains', value: 'contains' }, + { label: 'Whole word', value: 'search' }, + { label: 'Regex', value: 'regex' }, +]; + +const DESCRIPTION_MODES = [ + { label: 'Contains', value: 'contains' }, + { label: 'Whole word', value: 'search' }, + { label: 'Regex', value: 'regex' }, +]; + +const EPISODE_MODES = [ + { label: 'All episodes', value: 'all' }, + { label: 'New only', value: 'new' }, +]; + +function formatRange(start, end) { + try { + const s = new Date(start); + const e = new Date(end); + const sameDay = s.toDateString() === e.toDateString(); + const dateStr = s.toLocaleDateString(); + const startStr = s.toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + }); + const endStr = e.toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit', + }); + return sameDay + ? `${dateStr} ${startStr} - ${endStr}` + : `${dateStr} ${startStr} -> ${e.toLocaleString()}`; + } catch { + return `${start} - ${end}`; + } +} + +export default function SeriesRuleEditorModal({ + opened, + onClose, + initialRule, + onSaved, +}) { + const tvgs = useEPGsStore((s) => s.tvgs); + const tvgsById = useEPGsStore((s) => s.tvgsById); + + const [tvgId, setTvgId] = useState(''); + const [mode, setMode] = useState('all'); + const [title, setTitle] = useState(''); + const [titleMode, setTitleMode] = useState('exact'); + const [description, setDescription] = useState(''); + const [descriptionMode, setDescriptionMode] = useState('contains'); + const [channelId, setChannelId] = useState(''); + const [allChannels, setAllChannels] = useState([]); + + const [preview, setPreview] = useState({ + matches: [], + total: 0, + epg_found: true, + }); + const [previewLoading, setPreviewLoading] = useState(false); + const [previewError, setPreviewError] = useState(null); + const [saving, setSaving] = useState(false); + const abortRef = useRef(null); + + // Hydrate state when opened or initial rule changes + useEffect(() => { + if (!opened) return; + setTvgId(String(initialRule?.tvg_id || '')); + setMode(initialRule?.mode || 'all'); + setTitle(initialRule?.title || ''); + setTitleMode(initialRule?.title_mode || 'exact'); + setDescription(initialRule?.description || ''); + setDescriptionMode(initialRule?.description_mode || 'contains'); + setChannelId(initialRule?.channel_id ? String(initialRule.channel_id) : ''); + setPreview({ matches: [], total: 0, epg_found: true }); + setPreviewError(null); + }, [opened, initialRule]); + + // Load channels from the summary API on modal open (same as Recording.jsx). + useEffect(() => { + if (!opened) return; + let cancelled = false; + API.getChannelsSummary() + .then((chans) => { + if (!cancelled) setAllChannels(Array.isArray(chans) ? chans : []); + }) + .catch(() => { + if (!cancelled) setAllChannels([]); + }); + return () => { + cancelled = true; + }; + }, [opened]); + + // Build the payload for both preview and save + const payload = useMemo( + () => ({ + tvg_id: tvgId.trim(), + mode, + title: title.trim(), + title_mode: titleMode, + description: description.trim(), + description_mode: descriptionMode, + ...(channelId ? { channel_id: Number(channelId) } : {}), + }), + [tvgId, mode, title, titleMode, description, descriptionMode, channelId] + ); + + // Debounce the part of the payload that affects preview + const debouncedPreviewKey = useDebounce(payload, 500); + + useEffect(() => { + if (!opened) return; + if ( + !debouncedPreviewKey.tvg_id && + !debouncedPreviewKey.title && + !debouncedPreviewKey.description + ) { + setPreview({ matches: [], total: 0, epg_found: true }); + return; + } + // Abort any in-flight request + if (abortRef.current) abortRef.current.abort(); + const controller = new AbortController(); + abortRef.current = controller; + + setPreviewLoading(true); + setPreviewError(null); + API.previewSeriesRule(debouncedPreviewKey, { signal: controller.signal }) + .then((resp) => { + if (controller.signal.aborted) return; + setPreview(resp || { matches: [], total: 0 }); + }) + .catch((err) => { + if (err?.name === 'AbortError' || controller.signal.aborted) return; + const msg = err?.body?.error || err?.message || 'Preview failed'; + setPreviewError(msg); + setPreview({ matches: [], total: 0, epg_found: true }); + }) + .finally(() => { + if (!controller.signal.aborted) setPreviewLoading(false); + }); + + return () => controller.abort(); + }, [opened, debouncedPreviewKey]); + + // EPG channel options for the tvg_id selector. Deduplicate by tvg_id value + // since the same channel can appear across multiple EPG sources. + const tvgOptions = useMemo(() => { + const seen = new Set(); + const options = []; + for (const t of tvgs || []) { + if (!t.tvg_id || seen.has(t.tvg_id)) continue; + seen.add(t.tvg_id); + options.push({ + value: t.tvg_id, + label: t.name ? `${t.name} (${t.tvg_id})` : t.tvg_id, + }); + } + return options.sort((a, b) => a.label.localeCompare(b.label)); + }, [tvgs]); + + // Channel select options: prefer channels matching the selected tvg_id. + const channelOptions = useMemo(() => { + const sorted = [...allChannels].sort((a, b) => { + const aNum = Number(a.channel_number) || 0; + const bNum = Number(b.channel_number) || 0; + if (aNum !== bNum) return aNum - bNum; + return (a.name || '').localeCompare(b.name || ''); + }); + const matching = []; + const others = []; + for (const c of sorted) { + const item = { + value: String(c.id), + label: c.channel_number + ? `${c.channel_number} - ${c.name || `Channel ${c.id}`}` + : c.name || `Channel ${c.id}`, + }; + const cTvg = c.epg_data_id ? tvgsById?.[c.epg_data_id]?.tvg_id : null; + if (tvgId && cTvg && String(cTvg) === String(tvgId)) { + matching.push(item); + } else { + others.push(item); + } + } + return [...matching, ...others]; + }, [allChannels, tvgsById, tvgId]); + + const canSave = !!(payload.title || payload.description); + + const handleSave = async () => { + setSaving(true); + try { + await API.createSeriesRule(payload); + // Trigger evaluation so matching upcoming programs get scheduled. + try { + await API.evaluateSeriesRules(payload.tvg_id); + await useChannelsStore.getState().fetchRecordings(); + } catch (e) { + console.warn('Failed to evaluate after save', e); + } + showNotification({ title: 'Series rule saved' }); + if (onSaved) await onSaved(); + onClose(); + } finally { + setSaving(false); + } + }; + + return ( + + +