From 2781b21a8902ab0e9d12e15e1ec189ac0e0dab23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Tue, 19 May 2026 22:28:35 +0200 Subject: [PATCH 01/64] feat: built-in XC catch-up (timeshift) support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds native catch-up/timeshift replay for Xtream Codes providers through the same HTTPStreamReader transport pipeline as live TV. Timeshift proxy (apps/timeshift/): - URL cascade: 3 candidate timestamp formats per provider, per-account format cache for fast-forward seek performance - MPEG-TS preamble stripping (shared with HTTPStreamReader) - Stats integration: timeshift viewers appear on /stats with TIMESHIFT badge - Auth via hmac.compare_digest on XC password Catchup detection — denormalized for zero-cost output queries: - Stream.is_catchup + Stream.catchup_days populated at XC import time - Channel.has_catchup + Channel.catchup_days + Channel.catchup_provider_stream_id rolled up via ChannelStream post_save signal (UI path) and explicit SQL after bulk_create (import path) - _xc_channel_entry() reads denormalized fields instead of per-channel custom_properties JSON introspection (eliminates N+1 queries) - Migration 0038 backfills existing data via raw SQL XC API enhancements: - server_info.timezone + start/end + time_now use configured timezone (triple consistency rule — fixes wrong-programme-plays bug) - Dynamic has_archive flag + auto prev_days for catch-up channels - XMLTV timestamps rewritten to local timezone for catch-up clients HTTPStreamReader extended (apps/proxy/live_proxy/input/http_streamer.py): - 1 MB pipe buffer via fcntl F_SETPIPE_SZ (eliminates producer/consumer ping-pong that halved throughput) - Pre-opened response= for URL cascade workflows - strip_ts_preamble= for XC servers emitting PHP warnings before TS - find_ts_sync() as shared utility - Builds on upstream O_NONBLOCK + select() write loop Provider stream_id lookup order: - stream_xc() and xc_get_epg() try internal Channel.id first, fall back to provider stream_id only when needed (avoids unconditional query on every request) Also includes: - VOD provider cascade in stream_vod() — iterates all M3U relations by priority when first provider is at capacity - Defensive null-safety: custom_sid: None → "" in get_live_streams, get_vod_streams, get_vod_info, get_series_info (fixes iPlayTV crash on JSON null for string fields) - Timeshift settings UI (timezone selector, debug toggle) - StreamConnectionCard violet TIMESHIFT badge - Orphan cleanup skips timeshift_* virtual channels --- CHANGELOG.md | 5 + .../migrations/0038_add_catchup_fields.py | 120 ++++ apps/channels/models.py | 32 ++ apps/channels/serializers.py | 1 + apps/channels/signals.py | 26 +- apps/channels/utils.py | 54 ++ apps/m3u/tasks.py | 59 +- apps/output/views.py | 237 +++++++- apps/proxy/live_proxy/channel_status.py | 14 + apps/proxy/live_proxy/constants.py | 2 + apps/proxy/live_proxy/input/http_streamer.py | 289 +++++++--- apps/proxy/live_proxy/server.py | 9 + apps/proxy/live_proxy/views.py | 25 +- apps/timeshift/__init__.py | 0 apps/timeshift/apps.py | 7 + apps/timeshift/helpers.py | 116 ++++ apps/timeshift/tests/__init__.py | 0 apps/timeshift/tests/test_helpers.py | 56 ++ apps/timeshift/tests/test_views.py | 177 ++++++ apps/timeshift/views.py | 512 ++++++++++++++++++ core/models.py | 24 + dispatcharr/settings.py | 1 + dispatcharr/urls.py | 8 + .../components/cards/StreamConnectionCard.jsx | 20 +- .../forms/settings/TimeshiftSettingsForm.jsx | 116 ++++ frontend/src/pages/Settings.jsx | 16 + .../settings/TimeshiftSettingsFormUtils.js | 17 + frontend/src/utils/pages/SettingsUtils.js | 43 ++ 28 files changed, 1867 insertions(+), 119 deletions(-) create mode 100644 apps/channels/migrations/0038_add_catchup_fields.py create mode 100644 apps/timeshift/__init__.py create mode 100644 apps/timeshift/apps.py create mode 100644 apps/timeshift/helpers.py create mode 100644 apps/timeshift/tests/__init__.py create mode 100644 apps/timeshift/tests/test_helpers.py create mode 100644 apps/timeshift/tests/test_views.py create mode 100644 apps/timeshift/views.py create mode 100644 frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx create mode 100644 frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cef64db..8d2b046c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **None** (default): software decode. - **NVIDIA NVDEC (`--cuvid`)**: requires the NVIDIA container toolkit and a supported GPU inside the container. - **Intel Quick Sync (`--qsv`)**: requires an Intel iGPU or ARC GPU with the i915 driver exposed to the container. +- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. + - **`Settings → Timeshift` panel** with four knobs (defaults: `UTC`, `en`, `0`, off): `default_timezone`, `default_language`, `xmltv_prev_days_override`, `debug_logging`. + - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. + - **Byte path uses a dedicated producer thread + `os.pipe`** matching the pattern in `apps/proxy/live_proxy/input/http_streamer.py:HTTPStreamReader`. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. - **HDHR output profile URL support.** HDHomeRun lineup URLs now support an `output_profile` path segment so HDHR clients (Plex, Channels DVR, Emby, etc.) can request a specific transcode profile without any query-parameter support. URL formats accepted: - `/hdhr/output_profile//lineup.json` - output profile only - `/hdhr//output_profile//lineup.json` - channel profile + output profile diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py new file mode 100644 index 00000000..97a0b925 --- /dev/null +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -0,0 +1,120 @@ +"""Add denormalized catch-up fields to Stream and Channel. + +Populated at M3U/XC import time so _xc_channel_entry() can read them as +zero-cost column reads instead of introspecting custom_properties JSON +per channel on every xc_get_live_streams call. +""" + +from django.db import migrations, models + + +def backfill_stream_catchup(apps, schema_editor): + """Derive is_catchup/catchup_days from Stream.custom_properties JSON.""" + from django.db import connection + + with connection.cursor() as cursor: + cursor.execute(""" + UPDATE dispatcharr_channels_stream + SET is_catchup = TRUE, + catchup_days = COALESCE( + (custom_properties->>'tv_archive_duration')::int, 7 + ) + WHERE custom_properties IS NOT NULL + AND custom_properties != 'null'::jsonb + AND ( + custom_properties->>'tv_archive' = '1' + OR custom_properties->>'tv_archive' = 'True' + ) + """) + + +def backfill_channel_catchup(apps, schema_editor): + """Roll up catch-up fields from streams to channels.""" + from django.db import connection + + with connection.cursor() as cursor: + cursor.execute(""" + UPDATE dispatcharr_channels_channel c SET + is_catchup = EXISTS ( + SELECT 1 FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), + catchup_days = COALESCE(( + SELECT MAX(s.catchup_days) FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), 0), + catchup_provider_stream_id = COALESCE(( + SELECT s.custom_properties->>'stream_id' + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ORDER BY cs."order" LIMIT 1 + ), '') + """) + + +class Migration(migrations.Migration): + + dependencies = [ + ("dispatcharr_channels", "0037_auto_sync_overhaul"), + ] + + operations = [ + # Stream fields + migrations.AddField( + model_name="stream", + name="is_catchup", + field=models.BooleanField( + default=False, + db_index=True, + help_text="Whether this stream supports catch-up/timeshift (tv_archive=1)", + ), + ), + migrations.AddField( + model_name="stream", + name="catchup_days", + field=models.PositiveIntegerField( + default=0, + help_text="Number of days of catch-up archive available (tv_archive_duration)", + ), + ), + # Channel fields + migrations.AddField( + model_name="channel", + name="is_catchup", + field=models.BooleanField( + default=False, + db_index=True, + help_text="Whether any stream on this channel supports catch-up (tv_archive=1)", + ), + ), + migrations.AddField( + model_name="channel", + name="catchup_days", + field=models.PositiveIntegerField( + default=0, + help_text="Max catch-up archive days across all streams on this channel", + ), + ), + migrations.AddField( + model_name="channel", + name="catchup_provider_stream_id", + field=models.CharField( + max_length=64, + blank=True, + default="", + help_text="Provider stream_id of the highest-priority catch-up stream", + ), + ), + # Backfill existing data + migrations.RunPython( + backfill_stream_catchup, + reverse_code=migrations.RunPython.noop, + ), + migrations.RunPython( + backfill_channel_catchup, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 59af3e00..8964f067 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -134,6 +134,19 @@ class Stream(models.Model): db_index=True ) + # Denormalized catch-up fields — populated at M3U/XC import time from + # custom_properties["tv_archive"] and ["tv_archive_duration"]. Avoids + # per-channel JSON introspection on every xc_get_live_streams call. + is_catchup = models.BooleanField( + default=False, + db_index=True, + help_text="Whether this stream supports catch-up/timeshift (tv_archive=1)", + ) + catchup_days = models.PositiveIntegerField( + default=0, + help_text="Number of days of catch-up archive available (tv_archive_duration)", + ) + class Meta: # If you use m3u_account, you might do unique_together = ('name','url','m3u_account') verbose_name = "Stream" @@ -374,6 +387,25 @@ class Channel(models.Model): help_text="The M3U account that auto-created this channel" ) + # Denormalized catch-up fields — rolled up from the channel's streams at + # import time and via post_save/post_delete signals on ChannelStream. + # Eliminates per-channel DB queries in _xc_channel_entry(). + is_catchup = models.BooleanField( + default=False, + db_index=True, + help_text="Whether any stream on this channel supports catch-up (tv_archive=1)", + ) + catchup_days = models.PositiveIntegerField( + default=0, + help_text="Max catch-up archive days across all streams on this channel", + ) + catchup_provider_stream_id = models.CharField( + max_length=64, + blank=True, + default="", + help_text="Provider stream_id of the highest-priority catch-up stream (for timeshift URL construction)", + ) + # Hidden channels are excluded from HDHR, M3U, EPG, and XC output queries. # Auto-sync still recognizes them so they are not recreated when their # underlying provider stream persists; this is an output-layer concern, not diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index f5c83527..c0bdc525 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -454,6 +454,7 @@ class ChannelSerializer(serializers.ModelSerializer): "logo_id", "user_level", "is_adult", + "is_catchup", "hidden_from_output", "auto_created", "auto_created_by", diff --git a/apps/channels/signals.py b/apps/channels/signals.py index a1917b25..f1615c9a 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -5,7 +5,7 @@ from django.dispatch import receiver from django.utils.timezone import now, is_aware, make_aware from celery.result import AsyncResult from django_celery_beat.models import ClockedSchedule, PeriodicTask -from .models import Channel, Stream, ChannelProfile, ChannelProfileMembership, Recording +from .models import Channel, Stream, ChannelStream, ChannelProfile, ChannelProfileMembership, Recording from apps.m3u.models import M3UAccount from apps.epg.tasks import parse_programs_for_tvg_id import json @@ -365,3 +365,27 @@ def schedule_task_on_save(sender, instance, created, **kwargs): @receiver(post_delete, sender=Recording) def revoke_task_on_delete(sender, instance, **kwargs): revoke_task(instance.task_id) + + +@receiver([post_save, post_delete], sender=ChannelStream) +def update_channel_catchup_fields(sender, instance, **kwargs): + """Roll up denormalized catch-up fields from streams to the channel. + + Covers the UI/API path (admin adds/removes a stream from a channel). + The bulk-import path bypasses signals and uses explicit SQL instead — + see apps/m3u/tasks.py after ChannelStream.objects.bulk_create(). + """ + channel = instance.channel + best = ( + channel.streams + .filter(is_catchup=True) + .order_by("channelstream__order") + .first() + ) + Channel.objects.filter(pk=channel.pk).update( + is_catchup=best is not None, + catchup_days=best.catchup_days if best else 0, + catchup_provider_stream_id=( + (best.custom_properties or {}).get("stream_id", "") if best else "" + ), + ) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index f41c2b72..2ab343bb 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -16,6 +16,60 @@ def format_channel_number(value, empty=""): return int(value) return value + +def resolve_channel_by_provider_stream_id(provider_stream_id): + """Find a Channel + Stream by the XC provider's stream_id. + + XC clients address channels via the provider's `stream_id` (stored in + `Stream.custom_properties["stream_id"]`), not Dispatcharr's internal + `Channel.id`. Returns `(Channel, Stream)` on hit, `(None, None)` on miss. + """ + from apps.channels.models import Stream + + stream = ( + Stream.objects.filter( + custom_properties__stream_id=str(provider_stream_id), + m3u_account__account_type="XC", + ) + .select_related("m3u_account") + .first() + ) + if stream is None: + return None, None + channel = stream.channels.first() + if channel is None: + return None, None + return channel, stream + + +def get_channel_catchup_info(channel): + """Return catch-up info for a Channel, or None if no stream has tv_archive. + + Walks `channel.streams` in `channelstream__order`, returns a dict for the + first archive-enabled stream: `{stream, props, provider_stream_id, + tv_archive_duration}`. `tv_archive` may be stored as 1, "1", True or + "True" depending on the M3U importer that wrote it. + """ + for stream in channel.streams.order_by("channelstream__order"): + props = stream.custom_properties or {} + if str(props.get("tv_archive")) not in ("1", "True"): + continue + provider_stream_id = props.get("stream_id") + if not provider_stream_id: + continue + try: + archive_days = int(props.get("tv_archive_duration", 7) or 7) + except (TypeError, ValueError): + archive_days = 7 + return { + "stream": stream, + "props": props, + "provider_stream_id": str(provider_stream_id), + "tv_archive_duration": archive_days, + } + return None + + def increment_stream_count(account): with lock: current_usage = active_streams_map.get(account.id, 0) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index db6b1b6e..b48d3c48 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -951,6 +951,16 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, account_type='XC', stream_id=provider_stream_id ) + # Derive denormalized catch-up fields from the XC + # API response at import time so output views can + # read them as zero-cost column reads. + _tv_archive = str(stream.get("tv_archive", "0")) + _is_catchup = _tv_archive in ("1", "True") + try: + _catchup_days = int(stream.get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + _catchup_days = 0 + stream_props = { "name": name, "url": url, @@ -964,6 +974,8 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): "is_stale": False, "stream_id": provider_stream_id, "stream_chno": stream_chno, + "is_catchup": _is_catchup, + "catchup_days": _catchup_days, } if stream_hash not in stream_hashes: @@ -1030,7 +1042,7 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): # Simplified bulk update for better performance Stream.objects.bulk_update( streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'is_catchup', 'catchup_days'], batch_size=150 # Smaller batch size for XC processing ) @@ -1176,6 +1188,16 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): account_type=account_type_for_hash, stream_id=provider_stream_id ) + # Derive catch-up fields from stream attributes (M3U files + # may carry tv_archive via custom attributes). + _attrs = stream_info["attributes"] + _tv_archive_m3u = str(_attrs.get("tv_archive", "0")) + _is_catchup_m3u = _tv_archive_m3u in ("1", "True") + try: + _catchup_days_m3u = int(_attrs.get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + _catchup_days_m3u = 0 + stream_props = { "name": name, "url": url, @@ -1184,11 +1206,13 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): "m3u_account": account, "channel_group_id": int(groups.get(group_title)), "stream_hash": stream_hash, - "custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"], - "is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)), + "custom_properties": {**_attrs, "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else _attrs, + "is_adult": parse_is_adult(_attrs.get("is_adult", 0)), "is_stale": False, "stream_id": provider_stream_id, "stream_chno": channel_num, + "is_catchup": _is_catchup_m3u, + "catchup_days": _catchup_days_m3u, } if stream_hash not in stream_hashes: @@ -1254,7 +1278,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): # Update all streams in a single bulk operation Stream.objects.bulk_update( streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'is_catchup', 'catchup_days'], batch_size=200 ) except Exception as e: @@ -2550,6 +2574,33 @@ def sync_auto_channels(account_id, scan_start_time=None): batch_size=500, ) + # Roll up denormalized catch-up fields from streams to + # channels. bulk_create bypasses Django signals, so this + # explicit SQL is the only path that fires during import. + from django.db import connection as _conn + with _conn.cursor() as _cur: + _cur.execute(""" + UPDATE dispatcharr_channels_channel c SET + is_catchup = EXISTS ( + SELECT 1 FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), + catchup_days = COALESCE(( + SELECT MAX(s.catchup_days) + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), 0), + catchup_provider_stream_id = COALESCE(( + SELECT s.custom_properties->>'stream_id' + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ORDER BY cs."order" LIMIT 1 + ), '') + """) + if profiles_to_assign: ChannelProfileMembership.objects.bulk_create( [ diff --git a/apps/output/views.py b/apps/output/views.py index 1a73923d..131d98a4 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -2,7 +2,10 @@ from django.http import HttpResponse, JsonResponse, Http404, HttpResponseForbidd import json from django.urls import reverse from apps.channels.models import Channel, ChannelProfile, ChannelGroup, Stream -from apps.channels.utils import format_channel_number +from apps.channels.utils import ( + format_channel_number, + resolve_channel_by_provider_stream_id, +) from django.db.models import Prefetch from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods @@ -11,11 +14,12 @@ from apps.accounts.models import User from dispatcharr.utils import network_access_allowed from django.utils import timezone as django_timezone from django.shortcuts import get_object_or_404 -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone as dt_timezone +from zoneinfo import ZoneInfo import html import time from tzlocal import get_localzone -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse import base64 import logging from django.db.models.functions import Lower @@ -23,6 +27,7 @@ import os from apps.m3u.utils import calculate_tuner_count from apps.proxy.utils import get_user_active_connections import regex +from core.models import CoreSettings from core.utils import log_system_event import hashlib @@ -1315,11 +1320,31 @@ def generate_epg(request, profile_name=None, user=None): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 + # Timeshift: prev_days resolution order: + # 1. URL ?prev_days= (explicit, even 0 means "no past") + # 2. user.custom_properties.epg_prev_days + # 3. CoreSettings.timeshift_settings.xmltv_prev_days_override (>0) + # 4. Auto-detect: max provider tv_archive_duration capped at 30 + url_prev = request.GET.get('prev_days') + user_prev = user_custom.get('epg_prev_days') if user_custom else None + if url_prev is not None: + try: + prev_days = max(0, min(int(url_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + elif user_prev not in (None, ""): + try: + prev_days = max(0, min(int(user_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + else: + from apps.timeshift.helpers import compute_provider_archive_days_capped + timeshift_settings = CoreSettings.get_timeshift_settings() + try: + override = int(timeshift_settings.get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + override = 0 + prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() cache_params = ( @@ -1975,6 +2000,34 @@ def _xc_allowed_output_formats(user): return ['ts', 'mp4'] +def _build_xc_server_info(request, hostname, port): + """Build the server_info dict for XC API responses. + + The timezone, time_now, and xc_get_epg start/end fields form a "timezone + triple" that MUST all use the same zone — XC clients (iPlayTV, TiviMate) + use server_info.timezone to interpret start/end strings and calculate seek + offsets into catch-up archives. A mismatch makes the wrong programme play. + Learned from plugin v1.1.4 → v1.2.6 (6 iterations of timezone bugs). + """ + tz_name = CoreSettings.get_timeshift_settings().get("default_timezone", "UTC") + try: + tz = ZoneInfo(tz_name) + except (KeyError, Exception): + # Invalid timezone in settings — fall back to UTC so XC clients + # can still connect instead of getting a 500 error. + tz = ZoneInfo("UTC") + tz_name = "UTC" + return { + "url": hostname, + "server_protocol": request.scheme, + "port": port, + "timezone": tz_name, + "timestamp_now": int(time.time()), + "time_now": datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S"), + "process": True, + } + + def xc_get_info(request, full=False): user = xc_get_user(request) @@ -2007,15 +2060,7 @@ def xc_get_info(request, full=False): "max_connections": str(max_connections), "allowed_output_formats": _xc_allowed_output_formats(user), }, - "server_info": { - "url": hostname, - "server_protocol": request.scheme, - "port": port, - "timezone": get_localzone().key, - "timestamp_now": int(time.time()), - "time_now": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "process": True, - }, + "server_info": _build_xc_server_info(request, hostname, port), } if full == True: @@ -2141,7 +2186,70 @@ def xc_xmltv(request): ) return JsonResponse({'error': 'Unauthorized'}, status=401) - return generate_epg(request, None, user) + response = generate_epg(request, None, user) + return _convert_xmltv_to_local_timezone(response) + + +# Pre-compiled pattern for XMLTV timestamp attributes: `20251128143000 +0000`. +_XMLTV_TS_PATTERN = regex.compile(rb'(\d{14}) ([+-]\d{4})') + + +def _convert_xmltv_to_local_timezone(response): + """Rewrite UTC timestamps in an XMLTV streaming response to the configured + catch-up local time zone. + + Catch-up clients such as TiviMate display the wall-clock part of XMLTV + timestamps verbatim and ignore the trailing offset, so a programme + emitted as `start="20260512170000 +0000"` (correct UTC) ends up shown + as 17:00 even when the user is in Brussels and expects 19:00. Rewriting + the same instant as `start="20260512190000 +0200"` keeps the wall-clock + in the EPG aligned with the user's perception while the timestamp still + points at the same moment. + + The conversion runs only on the `/xmltv.php` catch-up endpoint — the + plain `/output/...epg.xml` consumers keep the UTC representation. + """ + timeshift_settings = CoreSettings.get_timeshift_settings() + tz_name = timeshift_settings.get('default_timezone', 'UTC') + try: + local_tz = ZoneInfo(tz_name) + except Exception: + return response + + # Use the builtin UTC, not ZoneInfo('UTC'), because the latter can pick + # up the host's mis-set /etc/timezone in some Docker setups. + # dt_timezone imported at module level + utc = dt_timezone.utc + + def _convert(match): + ts_bytes, _zone = match.group(1), match.group(2) + try: + ts_str = ts_bytes.decode('ascii') + utc_time = datetime.strptime(ts_str, '%Y%m%d%H%M%S').replace(tzinfo=utc) + local_time = utc_time.astimezone(local_tz) + return local_time.strftime('%Y%m%d%H%M%S %z').encode('ascii') + except Exception: + return match.group(0) + + # generate_epg may return either StreamingHttpResponse (large EPGs) or a + # plain HttpResponse (small EPGs or some error paths) — handle both. + if hasattr(response, 'streaming_content'): + source = response.streaming_content + else: + body = response.content + if isinstance(body, str): + body = body.encode('utf-8') + source = iter([body]) + + def _rewrite(): + for chunk in source: + if isinstance(chunk, str): + chunk = chunk.encode('utf-8') + if b'start="' in chunk or b'stop="' in chunk: + chunk = _XMLTV_TS_PATTERN.sub(_convert, chunk) + yield chunk + + return StreamingHttpResponse(_rewrite(), content_type=response.get('Content-Type', 'application/xml')) def xc_get_live_categories(user): @@ -2290,11 +2398,26 @@ def _xc_channel_entry(channel, channel_num_map, _get_default_group_id, _logo_url effective_logo = channel.effective_logo_obj effective_group = channel.effective_channel_group_obj group_id = effective_group.id if effective_group else _get_default_group_id() + + # Denormalized catch-up fields — populated at M3U/XC import time and + # rolled up via ChannelStream signal. Zero DB queries here. + if channel.is_catchup: + tv_archive = 1 + tv_archive_duration = channel.catchup_days + try: + stream_id_value = int(channel.catchup_provider_stream_id) + except (TypeError, ValueError): + stream_id_value = channel.id + else: + tv_archive = 0 + tv_archive_duration = 0 + stream_id_value = channel.id + return { "num": channel_num_int, "name": channel.effective_name, "stream_type": "live", - "stream_id": channel.id, + "stream_id": stream_id_value, "stream_icon": ( f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" if effective_logo else None @@ -2304,10 +2427,10 @@ def _xc_channel_entry(channel, channel_num_map, _get_default_group_id, _logo_url "is_adult": int(channel.is_adult), "category_id": str(group_id), "category_ids": [group_id], - "custom_sid": None, - "tv_archive": 0, + "custom_sid": "", + "tv_archive": tv_archive, "direct_source": "", - "tv_archive_duration": 0, + "tv_archive_duration": tv_archive_duration, } @@ -2340,6 +2463,21 @@ def xc_get_epg(request, user, short=False): if not channel_id: raise Http404() + # Try internal Channel.id first (most requests). Fall back to provider + # stream_id lookup for catch-up channels whose xc_get_live_streams emits + # the provider's stream_id instead of the internal Channel.id. + resolved_channel_id = channel_id + try: + candidate = int(channel_id) + if Channel.objects.filter(id=candidate).exists(): + resolved_channel_id = candidate + else: + raise ValueError + except (TypeError, ValueError): + provider_channel, _ = resolve_channel_by_provider_stream_id(channel_id) + if provider_channel is not None: + resolved_channel_id = provider_channel.id + channel = None # Apply effective-value annotation + hidden-exclusion at every channel # resolution path so a single channel lookup honors the same visibility @@ -2354,7 +2492,7 @@ def xc_get_epg(request, user, short=False): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": channel_id, + "id": resolved_channel_id, "user_level__lte": user.user_level } # Hide adult content if user preference is set @@ -2364,7 +2502,7 @@ def xc_get_epg(request, user, short=False): else: # User has specific limited profiles assigned filters = { - "id": channel_id, + "id": resolved_channel_id, "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -2377,7 +2515,7 @@ def xc_get_epg(request, user, short=False): if not channel: raise Http404() else: - channel = _annotate(Channel.objects.filter(id=channel_id).select_related('epg_data__epg_source')).first() + channel = _annotate(Channel.objects.filter(id=resolved_channel_id).select_related('epg_data__epg_source')).first() if not channel: raise Http404() @@ -2438,6 +2576,19 @@ def xc_get_epg(request, user, short=False): except (ValueError, TypeError): prev_days = 0 now = django_timezone.now() + + # When a channel supports catch-up, automatically include past programmes + # within the archive window even if the client did not request prev_days. + # XC clients (iPlayTV, TiviMate) call get_simple_data_table without + # prev_days and expect the response to already contain archived entries + # marked with has_archive=1 — they use that list to populate the catch-up + # programme menu. + # Use denormalized catch-up fields (zero DB queries). + _channel_is_catchup = getattr(channel, "is_catchup", False) + _channel_catchup_days = getattr(channel, "catchup_days", 0) + if _channel_is_catchup and prev_days == 0: + prev_days = _channel_catchup_days + lookback_cutoff = now - timedelta(days=prev_days) forward_cutoff = now + timedelta(days=num_days) if num_days > 0 else None effective_epg_data = channel.effective_epg_data_obj @@ -2483,6 +2634,26 @@ def xc_get_epg(request, user, short=False): output = {"epg_listings": []} + # Reuse the denormalized catch-up fields for the has_archive flag. + if _channel_is_catchup: + archive_window = timedelta(days=_channel_catchup_days) + else: + archive_window = None + + # start/end must be in the SAME timezone reported by server_info.timezone. + # XC clients display these strings verbatim AND use server_info.timezone + # to calculate seek offsets into catch-up archives. If these don't match, + # the wrong programme plays. Learned from plugin v1.1.4 → v1.2.6 (6 + # iterations of timezone bugs all converged on this rule). + # start_timestamp/stop_timestamp (epoch) stay in UTC — they're inherently + # timezone-agnostic and clients use them for their own conversions. + _ts_settings = CoreSettings.get_timeshift_settings() + _epg_tz_name = _ts_settings.get("default_timezone", "UTC") + try: + _epg_tz = ZoneInfo(_epg_tz_name) + except Exception: + _epg_tz = None + for program in programs: title = program['title'] if isinstance(program, dict) else program.title description = program['description'] if isinstance(program, dict) else program.description @@ -2507,8 +2678,8 @@ def xc_get_epg(request, user, short=False): "epg_id": epg_id, "title": base64.b64encode((title or "").encode()).decode(), "lang": "", - "start": start.strftime("%Y-%m-%d %H:%M:%S"), - "end": end.strftime("%Y-%m-%d %H:%M:%S"), + "start": start.astimezone(_epg_tz).strftime("%Y-%m-%d %H:%M:%S") if _epg_tz else start.strftime("%Y-%m-%d %H:%M:%S"), + "end": end.astimezone(_epg_tz).strftime("%Y-%m-%d %H:%M:%S") if _epg_tz else end.strftime("%Y-%m-%d %H:%M:%S"), "description": base64.b64encode((description or "").encode()).decode(), "channel_id": str(channel_num_int), "start_timestamp": str(int(start.timestamp())), @@ -2516,10 +2687,18 @@ def xc_get_epg(request, user, short=False): "stream_id": f"{channel_id}", } - if short == False: - program_output["now_playing"] = 1 if start <= django_timezone.now() <= end else 0 + # has_archive tells XC clients which past programmes are available + # for catch-up playback. Always emitted (not gated by short) because + # both get_simple_data_table and get_short_epg callers need it to + # populate the catch-up programme menu. + if archive_window is not None and end < now and end > now - archive_window: + program_output["has_archive"] = 1 + else: program_output["has_archive"] = 0 + if short == False: + program_output["now_playing"] = 1 if start <= now <= end else 0 + output['epg_listings'].append(program_output) return output diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index 9bd1d284..300f3b0d 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -407,6 +407,20 @@ class ChannelStatus: if channel_name: info['channel_name'] = channel_name + info['is_timeshift'] = bool(metadata.get(ChannelMetadataField.IS_TIMESHIFT)) + + for key, field in ( + ('logo_id', ChannelMetadataField.LOGO_ID), + ('m3u_profile_id', ChannelMetadataField.M3U_PROFILE), + ): + raw = metadata.get(field) + if not raw: + continue + try: + info[key] = int(raw) + except (TypeError, ValueError): + pass + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: diff --git a/apps/proxy/live_proxy/constants.py b/apps/proxy/live_proxy/constants.py index b3839508..414e1a2f 100644 --- a/apps/proxy/live_proxy/constants.py +++ b/apps/proxy/live_proxy/constants.py @@ -54,6 +54,8 @@ class ChannelMetadataField: STREAM_ID = "stream_id" CHANNEL_NAME = "channel_name" STREAM_NAME = "stream_name" + IS_TIMESHIFT = "is_timeshift" + LOGO_ID = "logo_id" # Profile fields STREAM_PROFILE = "stream_profile" diff --git a/apps/proxy/live_proxy/input/http_streamer.py b/apps/proxy/live_proxy/input/http_streamer.py index 7efb5d33..da7f9894 100644 --- a/apps/proxy/live_proxy/input/http_streamer.py +++ b/apps/proxy/live_proxy/input/http_streamer.py @@ -1,161 +1,290 @@ """ HTTP Stream Reader - Thread-based HTTP stream reader that writes to a pipe. -This allows us to use the same fetch_chunk() path for both transcode and HTTP streams. +This allows us to use the same fetch_chunk() path for both transcode and HTTP +streams (live TV) and for timeshift/catch-up archives. + +When *response* is supplied the reader skips connection setup and streams the +already-open ``requests.Response`` directly. This is used by the timeshift +view which needs to cascade through multiple candidate URLs before handing +the winning response to the reader. + +When *strip_ts_preamble* is True the first bytes written to the pipe are +aligned to the first MPEG-TS sync chain (0x47 at 188-byte intervals). Some +XC servers emit PHP warnings or HTML error text before the binary stream; +strict demuxers (ExoPlayer / TiviMate) raise ``UnrecognizedInputFormat`` on +those bytes. The live path typically receives clean TS so the flag defaults +to False, but the timeshift path enables it. """ -import threading +import fcntl import os +import select as _select +import threading + import requests from requests.adapters import HTTPAdapter + from ..utils import get_logger logger = get_logger() +# MPEG-TS sync constants (shared with timeshift views) +_TS_PACKET_SIZE = 188 +_TS_SYNC_BYTE = 0x47 +_TS_SYNC_SEARCH_LIMIT = 65536 + +# Linux fcntl constant — increase pipe buffer beyond the 64 KB default. +_F_SETPIPE_SZ = 1031 +_PIPE_TARGET_SIZE = 1 << 20 # 1 MB + + +def find_ts_sync(buf): + """Offset of the first MPEG-TS sync chain in *buf*, or -1. + + A valid chain needs 0x47 at offsets i, i+188 and i+376 — three sync + bytes one packet apart (the standard demuxer probe). + """ + end = len(buf) - 2 * _TS_PACKET_SIZE + for i in range(0, end): + if ( + buf[i] == _TS_SYNC_BYTE + and buf[i + _TS_PACKET_SIZE] == _TS_SYNC_BYTE + and buf[i + 2 * _TS_PACKET_SIZE] == _TS_SYNC_BYTE + ): + return i + return -1 + class HTTPStreamReader: - """Thread-based HTTP stream reader that writes to a pipe""" + """Thread-based HTTP stream reader that writes to an OS pipe. - def __init__(self, url, user_agent=None, chunk_size=8192): + Parameters + ---------- + url : str + Provider URL (used when *response* is None to open a new connection). + user_agent : str, optional + User-Agent header for the upstream request. + chunk_size : int + Chunk size for ``iter_content`` and ``pipe_reader.read``. + response : requests.Response, optional + An already-open streaming response. When provided the reader skips + connection setup and streams this response directly. The caller is + responsible for having opened it with ``stream=True``. + extra_headers : dict, optional + Additional HTTP headers (e.g. Range) forwarded to the provider when + *response* is None. + strip_ts_preamble : bool + Strip non-MPEG-TS bytes before the first 0x47 sync chain. + """ + + def __init__( + self, + url, + user_agent=None, + chunk_size=8192, + *, + response=None, + extra_headers=None, + strip_ts_preamble=False, + ): self.url = url self.user_agent = user_agent self.chunk_size = chunk_size + self._provided_response = response + self._extra_headers = extra_headers or {} + self._strip_ts_preamble = strip_ts_preamble self.session = None - self.response = None + self.response = response # may be pre-set self.thread = None self.pipe_read = None self.pipe_write = None self.running = False def start(self): - """Start the HTTP stream reader thread""" + """Start the HTTP stream reader thread.""" self.pipe_read, self.pipe_write = os.pipe() - # Make the write end non-blocking so that os.write() raises BlockingIOError - # instead of stalling the OS thread when the pipe buffer is full. Without - # this, a full pipe blocks the entire gevent worker (all greenlets freeze) - # because gevent does not patch os.write() on pipes. - import fcntl - flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) - fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) + # Grow the kernel pipe buffer from 64 KB to 1 MB so the producer + # thread is not blocked while the consumer yields to uWSGI. This + # is the single biggest throughput improvement for the timeshift + # path: it turns the producer/consumer from ping-pong alternation + # into genuinely parallel operation. + try: + fcntl.fcntl(self.pipe_write, _F_SETPIPE_SZ, _PIPE_TARGET_SIZE) + except (OSError, ValueError): + pass # non-Linux or unprivileged — keep 64 KB default + + # Make the write end non-blocking so that os.write() raises + # BlockingIOError instead of stalling the OS thread when the pipe + # buffer is full. Without this, a full pipe blocks the entire gevent + # worker (all greenlets freeze) because gevent does not patch + # os.write() on pipes. + # Skip for pre-opened responses (timeshift path) — those run in a + # real OS thread where blocking writes are fine and the 1 MB pipe + # buffer provides sufficient decoupling. + if self._provided_response is None: + flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) + fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) self.running = True self.thread = threading.Thread(target=self._read_stream, daemon=True) self.thread.start() - logger.info(f"Started HTTP stream reader thread for {self.url}") + logger.info("Started HTTP stream reader thread for %s", self.url) return self.pipe_read + # ------------------------------------------------------------------ + # Non-blocking pipe write helper + # ------------------------------------------------------------------ + + def _pipe_write_all(self, data): + """Write *data* to the pipe, handling O_NONBLOCK + partial writes. + + Uses select() on the write fd (gevent-patched — yields to hub) to + wait for space when the pipe is full, instead of blocking the OS + thread. + """ + offset = 0 + while offset < len(data) and self.running: + try: + n = os.write(self.pipe_write, data[offset:]) + offset += n + except BlockingIOError: + _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) + if not writable and not self.running: + return False + except (BrokenPipeError, OSError): + return False + return True + + # ------------------------------------------------------------------ + # Producer thread + # ------------------------------------------------------------------ + def _read_stream(self): - """Thread worker that reads HTTP stream and writes to pipe""" + """Thread worker: read HTTP stream → write to pipe.""" try: - # Build headers - headers = {} - if self.user_agent: - headers['User-Agent'] = self.user_agent + if self._provided_response is not None: + # Re-use the already-open response (timeshift cascade path). + self.response = self._provided_response + else: + # Open a fresh connection (live TV path). + headers = dict(self._extra_headers) + if self.user_agent: + headers["User-Agent"] = self.user_agent - logger.info(f"HTTP reader connecting to {self.url}") + logger.info("HTTP reader connecting to %s", self.url) - # Create session - self.session = requests.Session() + self.session = requests.Session() + adapter = HTTPAdapter( + max_retries=0, pool_connections=1, pool_maxsize=1, + ) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) - # Disable retries for faster failure detection - adapter = HTTPAdapter(max_retries=0, pool_connections=1, pool_maxsize=1) - self.session.mount('http://', adapter) - self.session.mount('https://', adapter) + self.response = self.session.get( + self.url, + headers=headers, + stream=True, + timeout=(5, 30), + ) - # Stream the URL - self.response = self.session.get( - self.url, - headers=headers, - stream=True, - timeout=(5, 30) # 5s connect, 30s read - ) + if self.response.status_code not in (200, 206): + logger.error( + "HTTP %d from %s", self.response.status_code, self.url, + ) + return - if self.response.status_code != 200: - logger.error(f"HTTP {self.response.status_code} from {self.url}") - return + logger.info("HTTP reader connected, streaming data…") - logger.info(f"HTTP reader connected successfully, streaming data...") + # Optional TS preamble stripping — find the first MPEG-TS sync + # chain and discard everything before it. + synced = not self._strip_ts_preamble + sync_buf = bytearray() if not synced else None - import select as _select - - # Stream chunks to pipe chunk_count = 0 for chunk in self.response.iter_content(chunk_size=self.chunk_size): if not self.running: break + if not chunk: + continue - if chunk: - # Write the chunk in a non-blocking loop. The pipe write end is - # set O_NONBLOCK in start(), so os.write() raises BlockingIOError - # instead of stalling the OS thread. We use select.select on the - # write fd (gevent-patched - yields to hub) to wait for space, - # then retry. Partial writes are handled by advancing the offset. - offset = 0 - write_error = False - while offset < len(chunk) and self.running: - try: - n = os.write(self.pipe_write, chunk[offset:]) - offset += n - except BlockingIOError: - _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) - if not writable and not self.running: - write_error = True - break - except OSError as e: - logger.error(f"Pipe write error: {e}") - write_error = True - break - if write_error: - break + # ---- TS preamble strip logic ---- + if not synced: + sync_buf.extend(chunk) + if len(sync_buf) < 2 * _TS_PACKET_SIZE + 1: + continue + offset = find_ts_sync(sync_buf) + if offset >= 0: + chunk = bytes(sync_buf[offset:]) + synced = True + sync_buf = None + elif len(sync_buf) >= _TS_SYNC_SEARCH_LIMIT: + logger.warning( + "Upstream produced %d bytes without TS sync " + "chain; passing through unmodified", + len(sync_buf), + ) + chunk = bytes(sync_buf) + synced = True + sync_buf = None + else: + continue - chunk_count += 1 - if chunk_count % 1000 == 0: - logger.debug(f"HTTP reader streamed {chunk_count} chunks") + # ---- Non-blocking write to pipe ---- + if not self._pipe_write_all(chunk): + break + + chunk_count += 1 + if chunk_count % 1000 == 0: + logger.debug("HTTP reader streamed %d chunks", chunk_count) + + # Flush any remaining sync_buf that never found a chain. + if not synced and sync_buf: + self._pipe_write_all(bytes(sync_buf)) logger.info("HTTP stream ended") - except requests.exceptions.RequestException as e: - logger.error(f"HTTP reader request error: {e}") - except Exception as e: - logger.error(f"HTTP reader unexpected error: {e}", exc_info=True) + except requests.exceptions.RequestException as exc: + logger.error("HTTP reader request error: %s", exc) + except Exception as exc: + logger.error("HTTP reader unexpected error: %s", exc, exc_info=True) finally: self.running = False - # Close write end of pipe to signal EOF try: if self.pipe_write is not None: os.close(self.pipe_write) self.pipe_write = None - except: + except OSError: pass + # ------------------------------------------------------------------ + # Shutdown + # ------------------------------------------------------------------ + def stop(self): - """Stop the HTTP stream reader""" + """Stop the HTTP stream reader.""" logger.info("Stopping HTTP stream reader") self.running = False - # Close response if self.response: try: self.response.close() - except: + except Exception: pass - # Close session if self.session: try: self.session.close() - except: + except Exception: pass - # Close write end of pipe if self.pipe_write is not None: try: os.close(self.pipe_write) self.pipe_write = None - except: + except OSError: pass - # Wait for thread if self.thread and self.thread.is_alive(): self.thread.join(timeout=2.0) diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 879de2ec..14157a9c 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -1807,6 +1807,15 @@ class ProxyServer: try: channel_id = key.split(':')[2] + # Skip channel ids that don't follow Dispatcharr's UUID + # convention. Those are managed by features outside of + # ProxyServer (e.g. timeshift catch-up sessions use a + # `timeshift___` virtual id and are + # owned by the WSGI request that opened them — they + # expire naturally via the clients-set TTL). + if channel_id.startswith("timeshift_"): + continue + # Get metadata first metadata = self.redis_client.hgetall(key) if not metadata: diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index d882bfca..dc42845c 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -601,6 +601,25 @@ def stream_xc(request, username, password, channel_id): if custom_properties["xc_password"] != password: return Response({"error": "Invalid credentials"}, status=401) + # Resolve the channel id. Try the internal Channel.id first (most + # requests). Fall back to provider stream_id lookup for catch-up + # channels whose xc_get_live_streams emits the provider's stream_id. + from apps.channels.utils import resolve_channel_by_provider_stream_id + resolved_internal_id = None + try: + candidate = int(channel_id) + if Channel.objects.filter(id=candidate).exists(): + resolved_internal_id = candidate + except (TypeError, ValueError): + pass + + if resolved_internal_id is None: + provider_channel, _ = resolve_channel_by_provider_stream_id(str(channel_id)) + if provider_channel is not None: + resolved_internal_id = provider_channel.id + else: + return JsonResponse({"error": "Not found"}, status=404) + if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -608,14 +627,14 @@ def stream_xc(request, username, password, channel_id): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": int(channel_id), + "id": resolved_internal_id, "user_level__lte": user.user_level } channel = Channel.objects.filter(**filters).first() else: # User has specific limited profiles assigned filters = { - "id": int(channel_id), + "id": resolved_internal_id, "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -625,7 +644,7 @@ def stream_xc(request, username, password, channel_id): if not channel: return JsonResponse({"error": "Not found"}, status=404) else: - channel = get_object_or_404(Channel, id=channel_id) + channel = get_object_or_404(Channel, id=resolved_internal_id) if extension.lower() == '.mp4': force_format = 'fmp4' diff --git a/apps/timeshift/__init__.py b/apps/timeshift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/timeshift/apps.py b/apps/timeshift/apps.py new file mode 100644 index 00000000..e4ecbfba --- /dev/null +++ b/apps/timeshift/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class TimeshiftConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.timeshift" + verbose_name = "Timeshift" diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py new file mode 100644 index 00000000..97d32b95 --- /dev/null +++ b/apps/timeshift/helpers.py @@ -0,0 +1,116 @@ +"""URL builders + timestamp conversion + archive-days probe for XC catch-up.""" + +import logging +from datetime import datetime + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + +DEFAULT_DURATION_MINUTES = 120 +DURATION_BUFFER_MINUTES = 5 +MAX_DURATION_MINUTES = 480 + +PROVIDER_ARCHIVE_CACHE_TTL_SECONDS = 300 +MAX_AUTO_PREV_DAYS = 30 + + +def compute_provider_archive_days_capped(): + """Largest `tv_archive_duration` advertised by any XC stream (capped, cached). + + Returns 0 when no XC stream advertises catch-up. + """ + def _scan(): + from apps.channels.models import Stream + + max_days = 0 + for props in ( + Stream.objects.filter(m3u_account__account_type="XC") + .exclude(custom_properties__isnull=True) + .values_list("custom_properties", flat=True) + ): + try: + if not int((props or {}).get("tv_archive", 0) or 0): + continue + value = int((props or {}).get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + continue + if value > max_days: + max_days = value + return min(max_days, MAX_AUTO_PREV_DAYS) + + return cache.get_or_set( + "timeshift:provider_archive_days_capped", + _scan, + PROVIDER_ARCHIVE_CACHE_TTL_SECONDS, + ) + + +def get_programme_duration(channel, timestamp_str): + """Duration in minutes of the EPG programme starting at `timestamp_str`. + + `timestamp_str` is `YYYY-MM-DD:HH-MM` in UTC (passed through from the + client URL unchanged — no timezone conversion). Falls back to a + 120-minute default if EPG lookup fails. + """ + try: + dt = datetime.strptime(timestamp_str, "%Y-%m-%d:%H-%M") + if not channel.epg_data: + return DEFAULT_DURATION_MINUTES + + programme = channel.epg_data.programs.filter( + start_time__lte=dt, end_time__gt=dt + ).first() + if not programme: + return DEFAULT_DURATION_MINUTES + + duration_seconds = (programme.end_time - programme.start_time).total_seconds() + duration_minutes = int(duration_seconds / 60) + DURATION_BUFFER_MINUTES + return min(duration_minutes, MAX_DURATION_MINUTES) + except Exception: + return DEFAULT_DURATION_MINUTES + + +def build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes): + """Format A: `/streaming/timeshift.php?username=&password=&stream=&start=&duration=`.""" + return ( + f"{m3u_account.server_url.rstrip('/')}/streaming/timeshift.php" + f"?username={m3u_account.username}" + f"&password={m3u_account.password}" + f"&stream={stream_id}" + f"&start={timestamp}" + f"&duration={duration_minutes}" + ) + + +def build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes): + """Format B: `/timeshift/{user}/{pass}/{duration}/{timestamp}/{stream_id}.ts`.""" + return ( + f"{m3u_account.server_url.rstrip('/')}/timeshift" + f"/{m3u_account.username}" + f"/{m3u_account.password}" + f"/{duration_minutes}" + f"/{timestamp}" + f"/{stream_id}.ts" + ) + + +def format_timestamp_as_sql_datetime(timestamp): + """Reshape `YYYY-MM-DD:HH-MM` to `YYYY-MM-DD HH:MM:SS` without any + timezone conversion. + + Some XC servers refuse the dash-only shape for archives whose recording + is still being finalised and only resolve the SQL-datetime shape. This + function changes only the format — the timestamp value stays in whatever + zone the caller supplied (typically UTC, since clients derive it from the + UTC epoch in the EPG data). + + Do NOT add timezone conversion here — that was the root cause of the + "wrong programme plays" bug (plugin v1.1.4 → v1.2.6 history). + """ + try: + dt = datetime.strptime(timestamp, "%Y-%m-%d:%H-%M") + return dt.strftime("%Y-%m-%d %H:%M:%S") + except Exception as e: + logger.error("Timeshift SQL timestamp reshape failed for %r: %s", timestamp, e) + return timestamp diff --git a/apps/timeshift/tests/__init__.py b/apps/timeshift/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py new file mode 100644 index 00000000..13d3e0a9 --- /dev/null +++ b/apps/timeshift/tests/test_helpers.py @@ -0,0 +1,56 @@ +"""Tests for `apps.timeshift.helpers` — timestamp shape conversion and URL build.""" + +from django.test import TestCase + +from apps.timeshift.helpers import ( + build_timeshift_url_format_a, + build_timeshift_url_format_b, + format_timestamp_as_sql_datetime, +) + + +class _FakeAccount: + def __init__(self): + self.server_url = "http://example.test" + self.username = "user" + self.password = "pass" + + +class TimestampFormatTests(TestCase): + """The SQL-datetime shape is needed for some XC server parsers. The + reshape function changes format only — no timezone conversion.""" + + def test_format_sql_reshapes_without_tz_conversion(self): + # Must reshape only — no timezone shift. The value stays as-is. + self.assertEqual( + format_timestamp_as_sql_datetime("2026-05-12:17-00"), + "2026-05-12 17:00:00", + ) + + def test_format_sql_invalid_falls_back(self): + self.assertEqual(format_timestamp_as_sql_datetime("garbage"), "garbage") + + +class BuildTimeshiftUrlTests(TestCase): + def setUp(self): + self.account = _FakeAccount() + + def test_format_a_passes_dash_shape_unchanged(self): + url = build_timeshift_url_format_a( + self.account, "22372", "2026-05-12:19-00", 40 + ) + self.assertIn("start=2026-05-12:19-00", url) + self.assertIn("stream=22372", url) + self.assertIn("duration=40", url) + + def test_format_a_passes_sql_shape_unchanged(self): + url = build_timeshift_url_format_a( + self.account, "22372", "2026-05-12 19:00:00", 40 + ) + self.assertIn("start=2026-05-12 19:00:00", url) + + def test_format_b_path_with_dash_shape(self): + url = build_timeshift_url_format_b( + self.account, "22372", "2026-05-12:19-00", 40 + ) + self.assertIn("/40/2026-05-12:19-00/22372.ts", url) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py new file mode 100644 index 00000000..b8daf30a --- /dev/null +++ b/apps/timeshift/tests/test_views.py @@ -0,0 +1,177 @@ +"""Tests for the timeshift proxy view, focused on upstream status mapping.""" + +from unittest.mock import MagicMock, patch + +from django.test import RequestFactory, TestCase + +from apps.timeshift import views +from apps.proxy.live_proxy.input.http_streamer import find_ts_sync as _find_ts_sync, _TS_PACKET_SIZE + + +class FindTsSyncTests(TestCase): + """Locate the first MPEG-TS sync chain so a leading HTML/PHP preamble + can be skipped before the bytes reach a strict demuxer (ExoPlayer).""" + + def test_returns_zero_when_buffer_already_aligned(self): + buf = b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + self.assertEqual(_find_ts_sync(buf), 0) + + def test_returns_offset_of_first_chain_after_preamble(self): + preamble = b"
\nWarning" + aligned = b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + self.assertEqual(_find_ts_sync(preamble + aligned), len(preamble)) + + def test_returns_minus_one_when_no_chain_exists(self): + # Three lone 0x47 bytes that are NOT spaced at 188 — must not be + # mistaken for a sync chain. + self.assertEqual(_find_ts_sync(b"\x47\x00\x00\x47\x00\x00\x47" * 50), -1) + + def test_returns_minus_one_for_short_buffer(self): + self.assertEqual(_find_ts_sync(b"\x47" * 10), -1) + + +class _FakeReader: + """Stand-in for HTTPStreamReader that yields EOF immediately via a pipe.""" + + def __init__(self, *args, **kwargs): + self.pipe_read = None + self.pipe_write = None + + def start(self): + import os + self.pipe_read, self.pipe_write = os.pipe() + os.close(self.pipe_write) # EOF immediately + self.pipe_write = None + return self.pipe_read + + def stop(self): + pass + + +def _fake_upstream(status_code, *, content_type="video/mp2t", body=b""): + resp = MagicMock() + resp.status_code = status_code + resp.headers = {"Content-Type": content_type} + resp.iter_content = MagicMock(return_value=iter([body] if body else [])) + resp.close = MagicMock() + return resp + + +class StreamFromProviderStatusMappingTests(TestCase): + """`_stream_from_provider` must translate upstream HTTP status codes into + semantically correct Django responses so downstream IPTV clients react + the right way (notably: stop retrying on 404).""" + + def setUp(self): + self.factory = RequestFactory() + self.kwargs = dict( + candidate_urls=[ + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12:17-00", + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12 17:00:00", + "http://example.test/timeshift/u/p/60/2026-05-12:17-00/1.ts", + ], + user_agent="test-agent", + range_header=None, + virtual_channel_id="timeshift_1_2026-05-12-17-00_1", + client_id="timeshift_test123", + client_ip="127.0.0.1", + user=None, + channel_display_name="Test", + timestamp_utc="2026-05-12:17-00", + channel_logo_id=None, + m3u_profile_id=None, + debug=False, + ) + + @patch.object(views, "_open_upstream") + def test_all_candidates_404_returns_404(self, mocked_open): + mocked_open.return_value = _fake_upstream(404) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 404) + # Every candidate is attempted before giving up. + self.assertEqual(mocked_open.call_count, 3) + + @patch.object(views, "_open_upstream") + def test_upstream_403_short_circuits_loop(self, mocked_open): + # 403 is decisive (auth) — no retry of further candidates. + mocked_open.return_value = _fake_upstream(403) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 403) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_upstream_500_short_circuits_loop(self, mocked_open): + mocked_open.return_value = _fake_upstream(500) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_first_candidate_succeeds(self, mocked_open): + mocked_open.side_effect = [_fake_upstream(200, body=b"\x47" * 188)] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_second_candidate_succeeds_after_404(self, mocked_open): + # Primary 404 → second candidate 200 → streams successfully. + mocked_open.side_effect = [ + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 2) + + @patch.object(views, "_open_upstream") + def test_third_candidate_succeeds_after_400_then_404(self, mocked_open): + mocked_open.side_effect = [ + _fake_upstream(400), + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 3) + + @patch.object(views, "_open_upstream") + def test_cache_promotes_winning_index_to_first(self, mocked_open): + """Once a candidate succeeds for an account, the next request reorders + the list so the cached winner is tried first — saving cascade + overhead on fast-forward.""" + # First request: candidate index 1 wins after index 0 returns 404. + mocked_open.side_effect = [ + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + kwargs = dict(self.kwargs, account_id=999) + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + r1 = views._stream_from_provider(**kwargs) + self.assertEqual(r1.status_code, 200) + self.assertEqual(mocked_open.call_count, 2) + + # Second request: cached winner (index 1) is tried first, succeeds + # immediately — no cascade. + mocked_open.reset_mock() + mocked_open.side_effect = [_fake_upstream(200, body=b"\x47" * 188)] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + r2 = views._stream_from_provider(**kwargs) + self.assertEqual(r2.status_code, 200) + self.assertEqual(mocked_open.call_count, 1) + # Confirm the URL used is the SQL-datetime candidate (index 1 in the + # original list set up in setUp), not the dash-only one (index 0). + self.assertIn("17:00:00", mocked_open.call_args_list[0][0][0]) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py new file mode 100644 index 00000000..87345164 --- /dev/null +++ b/apps/timeshift/views.py @@ -0,0 +1,512 @@ +"""XC catch-up (timeshift) HTTP view — proxies the provider's +`/streaming/timeshift.php` endpoint to clients like iPlayTV and TiviMate. + +URL parameter quirk matching what iPlayTV / TiviMate emit: + Position "stream_id" -> EPG channel number, IGNORED here. + Position "duration" -> actually the provider's stream_id. +""" + +import hmac +import logging +import os +import threading +import time +import uuid + +import requests +from django.http import ( + Http404, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + StreamingHttpResponse, +) + +from apps.accounts.models import User +from apps.channels.utils import ( + get_channel_catchup_info, + resolve_channel_by_provider_stream_id, +) +from apps.proxy.live_proxy.config_helper import ConfigHelper +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState +from apps.proxy.live_proxy.redis_keys import RedisKeys +from apps.proxy.live_proxy.utils import get_client_ip +from apps.proxy.live_proxy.input.http_streamer import HTTPStreamReader +from core.models import CoreSettings +from core.utils import RedisClient + +from .helpers import ( + build_timeshift_url_format_a, + build_timeshift_url_format_b, + format_timestamp_as_sql_datetime, + get_programme_duration, +) + +logger = logging.getLogger(__name__) + +CLIENT_TTL_SECONDS = 60 + + +def timeshift_proxy(request, username, password, stream_id, timestamp, duration): # noqa: ARG001 stream_id + # The "duration" URL slot is the provider's stream_id — see module docstring. + provider_stream_id = duration[:-3] if duration.endswith(".ts") else duration + + user = _authenticate_user(username, password) + if user is None: + return HttpResponseForbidden("Invalid credentials") + + channel, _ = resolve_channel_by_provider_stream_id(provider_stream_id) + if channel is None: + raise Http404("Channel not found") + + if not _user_can_access_channel(user, channel): + return HttpResponseForbidden("Access denied") + + catchup = get_channel_catchup_info(channel) + if catchup is None: + return HttpResponseBadRequest("Timeshift not supported for this channel") + + catchup_stream = catchup["stream"] + props = catchup["props"] + m3u_account = catchup_stream.m3u_account + if m3u_account is None or m3u_account.account_type != "XC": + return HttpResponseBadRequest("Channel not from Xtream Codes provider") + + timeshift_settings = CoreSettings.get_timeshift_settings() + debug = bool(timeshift_settings.get("debug_logging", False)) + + # The client-supplied timestamp is already in UTC (derived from the + # start_timestamp epoch in the EPG data). Pass it through to the + # provider as-is — no timezone conversion. Converting here was the + # root cause of the "wrong programme plays" bug: the provider indexes + # archives in UTC, so shifting to Brussels meant requesting a programme + # 1-2 hours later than intended. + # Learned from plugin history v1.1.4 → v1.2.6: 6 iterations of timezone + # bugs all converged on "never convert the provider URL timestamp". + local_timestamp = timestamp + sql_timestamp = format_timestamp_as_sql_datetime(timestamp) + duration_minutes = get_programme_duration(channel, local_timestamp) + stream_id_value = (props or {}).get("stream_id") + if stream_id_value is None: + return HttpResponseBadRequest("Cannot build timeshift URL") + + # Build a small set of upstream candidates. Different XC servers (or even + # different code paths inside the same server) accept different timestamp + # shapes and different URL layouts: the dash-only shape covers most + # finalised archives, the SQL-datetime shape unlocks a separate path that + # some servers use for archives still being indexed, and the path layout + # (Format B) is the only one some Stalker-style portals expose. We try + # them in order until one accepts the request — see `_stream_from_provider` + # for the iteration logic. + candidate_urls = [ + build_timeshift_url_format_a(m3u_account, stream_id_value, local_timestamp, duration_minutes), + build_timeshift_url_format_a(m3u_account, stream_id_value, sql_timestamp, duration_minutes), + build_timeshift_url_format_b(m3u_account, stream_id_value, local_timestamp, duration_minutes), + ] + + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + safe_ts = timestamp.replace(":", "-").replace("/", "-") + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{provider_stream_id}" + client_id = f"timeshift_{uuid.uuid4().hex[:16]}" + client_ip = get_client_ip(request) + range_header = request.META.get("HTTP_RANGE") + + # Resolve channel logo + M3U default profile so the Stats card can render + # the same logo + M3U profile name as for live streams. + channel_logo_id = getattr(channel, "logo_id", None) + default_profile = m3u_account.profiles.filter(is_default=True).first() + m3u_profile_id = default_profile.id if default_profile is not None else None + + # If the user is switching from live TV to timeshift on the same channel, + # the live stream still holds the provider connection (~1s teardown). + # Force-stop it synchronously before opening the timeshift upstream so + # the provider slot is free. No-op if no live stream is active. + from apps.proxy.live_proxy.services.channel_service import ChannelService + ChannelService.stop_channel(str(channel.uuid)) + + if debug: + logger.info( + "Timeshift request: channel=%s ts=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, + local_timestamp, + provider_stream_id, + virtual_channel_id, + client_id, + range_header or "(none)", + ) + + return _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + debug=debug, + account_id=m3u_account.id, + ) + + +# --------------------------------------------------------------------------- +# Authentication, lookup, access control +# --------------------------------------------------------------------------- + + +def _authenticate_user(username, password): + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + return None + expected = (user.custom_properties or {}).get("xc_password") + if not expected: + return None + if not hmac.compare_digest(str(expected), str(password)): + return None + return user + + +def _user_can_access_channel(user, channel): + if user.user_level < channel.user_level: + return False + if user.user_level >= User.UserLevel.ADMIN: + return True + profile_count = user.channel_profiles.count() + if profile_count == 0: + return True + return ( + type(channel).objects.filter( + id=channel.id, + channelprofilemembership__enabled=True, + channelprofilemembership__channel_profile__in=user.channel_profiles.all(), + ) + .exists() + ) + + +# --------------------------------------------------------------------------- +# Stats integration (direct Redis writes, no ClientManager instance) +# --------------------------------------------------------------------------- + + +def _register_stats_client( + redis_client, + virtual_channel_id, + client_id, + client_ip, + user_agent, + user, + *, + channel_display_name, + timestamp_utc, + primary_url, + channel_logo_id=None, + m3u_profile_id=None, +): + """Write the same Redis keys the live proxy's ClientManager writes so the + catch-up viewer appears on `/stats`. `is_timeshift=1` toggles the badge.""" + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + now = str(time.time()) + client_payload = { + "user_agent": user_agent or "unknown", + "ip_address": client_ip, + "connected_at": now, + "last_active": now, + "user_id": str(user.id) if user is not None else "0", + "username": user.username if user is not None else "unknown", + } + metadata_payload = { + ChannelMetadataField.STATE: ChannelState.ACTIVE, + ChannelMetadataField.INIT_TIME: now, + ChannelMetadataField.OWNER: "timeshift", + ChannelMetadataField.CHANNEL_NAME: channel_display_name or "Timeshift", + ChannelMetadataField.STREAM_NAME: f"Catch-up @ {timestamp_utc} UTC" if timestamp_utc else "Catch-up", + ChannelMetadataField.URL: _redact_url(primary_url) if primary_url else "", + ChannelMetadataField.IS_TIMESHIFT: "1", + } + if channel_logo_id is not None: + metadata_payload[ChannelMetadataField.LOGO_ID] = str(channel_logo_id) + if m3u_profile_id is not None: + metadata_payload[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) + try: + pipe = redis_client.pipeline(transaction=False) + pipe.hset(client_key, mapping=client_payload) + pipe.expire(client_key, CLIENT_TTL_SECONDS) + pipe.sadd(client_set_key, client_id) + pipe.expire(client_set_key, CLIENT_TTL_SECONDS) + pipe.hset(metadata_key, mapping=metadata_payload) + pipe.expire(metadata_key, CLIENT_TTL_SECONDS) + pipe.execute() + except Exception as exc: + logger.warning("Timeshift stats register failed: %s", exc) + + +def _heartbeat_stats_client(redis_client, virtual_channel_id, client_id, bytes_delta=0): + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + try: + pipe = redis_client.pipeline(transaction=False) + pipe.hset(client_key, "last_active", str(time.time())) + pipe.expire(client_key, CLIENT_TTL_SECONDS) + pipe.expire(client_set_key, CLIENT_TTL_SECONDS) + if bytes_delta > 0: + pipe.hincrby(metadata_key, ChannelMetadataField.TOTAL_BYTES, bytes_delta) + pipe.expire(metadata_key, CLIENT_TTL_SECONDS) + pipe.execute() + except Exception: + pass + + +def _unregister_stats_client(redis_client, virtual_channel_id, client_id): + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + try: + redis_client.srem(client_set_key, client_id) + redis_client.delete(client_key) + if (redis_client.scard(client_set_key) or 0) == 0: + redis_client.delete(client_set_key) + redis_client.delete(metadata_key) + except Exception as exc: + logger.warning("Timeshift stats unregister failed: %s", exc) + + +# --------------------------------------------------------------------------- +# Provider streaming +# --------------------------------------------------------------------------- + + + +def _open_upstream(url, user_agent, range_header): + """Open the upstream HTTP request (status + headers known synchronously).""" + headers = {} + if user_agent: + headers["User-Agent"] = user_agent + if range_header: + headers["Range"] = range_header + return requests.get( + url, + headers=headers, + stream=True, + timeout=ConfigHelper.connection_timeout(), + ) + + +_format_cache_lock = threading.Lock() +_format_cache = {} + + +def _get_cached_format_index(account_id): + """Return the index of the URL shape that last worked for this account, + or None if we haven't seen one succeed yet. + + Caching the winning shape lets a fast-forward session — where the player + reissues GETs at one-second intervals as the user scrubs — skip the + cascade after the first successful candidate and go straight to the + working URL on every subsequent request. + """ + if account_id is None: + return None + with _format_cache_lock: + return _format_cache.get(account_id) + + +def _set_cached_format_index(account_id, index): + if account_id is None: + return + with _format_cache_lock: + _format_cache[account_id] = index + + +def _stream_from_provider( + *, + candidate_urls, + user_agent, + range_header, + virtual_channel_id, + client_id, + client_ip, + user, + channel_display_name, + timestamp_utc, + channel_logo_id, + m3u_profile_id, + debug, + account_id=None, +): + # Use 256 KB chunks: amortises per-yield uWSGI/gevent overhead. + chunk_size = max(ConfigHelper.chunk_size(), 262144) + + # Reorder the candidates so the cached winning shape is tried first. + # The cache is per-account; within a single fast-forward session every + # seek lands in the same archive entry and therefore the same parser + # path, so the first attempt almost always succeeds. + cached_index = _get_cached_format_index(account_id) + if cached_index is not None and 0 <= cached_index < len(candidate_urls): + ordered_urls = [candidate_urls[cached_index]] + [ + u for i, u in enumerate(candidate_urls) if i != cached_index + ] + original_indices = [cached_index] + [ + i for i in range(len(candidate_urls)) if i != cached_index + ] + else: + ordered_urls = list(candidate_urls) + original_indices = list(range(len(candidate_urls))) + + # Try each candidate URL until one returns a streamable status. We keep + # going on 400/404 (the two refusal modes XC servers use when the URL + # shape doesn't match their parser); a 403, 5xx or connection error is + # decisive and short-circuits the loop. + upstream = None + last_status = None + last_url = ordered_urls[0] + winning_index = None + for url, orig_idx in zip(ordered_urls, original_indices): + try: + response = _open_upstream(url, user_agent, range_header) + except requests.exceptions.RequestException as exc: + logger.error("Timeshift provider unreachable: %s", exc) + return HttpResponseBadRequest("Provider connection error") + last_status = response.status_code + last_url = url + if response.status_code in (200, 206): + upstream = response + winning_index = orig_idx + break + response.close() + if response.status_code not in (400, 404): + # Decisive failure (403, 5xx, …) — no point trying alternative URLs. + break + + if winning_index is not None: + _set_cached_format_index(account_id, winning_index) + + if upstream is None: + logger.error("Timeshift upstream rejected: status=%s url=%s", + last_status, _redact_url(last_url)) + # Mirror semantically meaningful upstream statuses so clients react + # correctly: 404 stops retry loops (catch-up not yet indexed), 403 + # surfaces auth problems. Everything else stays a generic 400. + if last_status == 404: + return HttpResponseNotFound("Catch-up not available yet") + if last_status == 403: + return HttpResponseForbidden("Provider denied access") + return HttpResponseBadRequest(f"Provider error: {last_status}") + + content_type = upstream.headers.get("Content-Type", "video/mp2t") + content_range = upstream.headers.get("Content-Range", "") + status = upstream.status_code + + redis_client = RedisClient.get_client() + _register_stats_client( + redis_client, + virtual_channel_id, + client_id, + client_ip, + user_agent, + user, + channel_display_name=channel_display_name, + timestamp_utc=timestamp_utc, + primary_url=last_url, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + ) + + # Use HTTPStreamReader (same component as live TV) with a 1 MB pipe + # buffer and TS preamble stripping. The producer thread reads from the + # provider and writes to the pipe; the WSGI greenlet reads from the + # pipe and yields to the client. + reader = HTTPStreamReader( + url=last_url, + chunk_size=chunk_size, + response=upstream, + strip_ts_preamble=True, + ) + pipe_read_fd = reader.start() + pipe_reader = os.fdopen(pipe_read_fd, "rb", buffering=0) + + def stream_generator(): + last_heartbeat = time.time() + bytes_since_heartbeat = 0 + total_yielded = 0 + loop_start = time.time() + try: + while True: + data = pipe_reader.read(chunk_size) + if not data: + break + yield data + bytes_since_heartbeat += len(data) + total_yielded += len(data) + + now = time.time() + if now - last_heartbeat >= 5: + _heartbeat_stats_client( + redis_client, virtual_channel_id, client_id, + bytes_delta=bytes_since_heartbeat, + ) + last_heartbeat = now + bytes_since_heartbeat = 0 + except GeneratorExit: + pass + except Exception: + logger.exception("Timeshift stream loop error") + finally: + elapsed = time.time() - loop_start + if debug and total_yielded > 0: + mbps = (total_yielded * 8) / elapsed / 1_000_000 if elapsed > 0 else 0 + logger.info( + "Timeshift disconnect: vid=%s client=%s yielded=%d bytes in %.1fs (%.2f Mbps avg)", + virtual_channel_id, client_id, total_yielded, elapsed, mbps, + ) + reader.stop() + try: + pipe_reader.close() + except Exception: + pass + _unregister_stats_client(redis_client, virtual_channel_id, client_id) + + response = StreamingHttpResponse( + stream_generator(), + content_type=content_type, + status=status, + ) + # Tell nginx not to buffer this streaming response. Without this, the + # default uwsgi_buffering=on under `location /` throttles the response + # roughly to half the upstream rate (back-pressure from buffer flushes + # propagates back into the generator). + response["X-Accel-Buffering"] = "no" + # Forward Content-Range so iPlayTV / TiviMate seek cursor stays correct. + # Content-Length is intentionally NOT forwarded: chunked transfer is the + # safer interchange format for long-lived streaming. + if content_range: + response["Content-Range"] = content_range + response["Accept-Ranges"] = "bytes" + return response + + +def _redact_url(url): + """Strip credentials from a URL for safe logging.""" + if not url or "://" not in url: + return url + scheme, rest = url.split("://", 1) + if "@" in rest: + rest = rest.split("@", 1)[1] + return f"{scheme}://{rest.split('?')[0]}/..." diff --git a/core/models.py b/core/models.py index 0ec4efda..40c17a7f 100644 --- a/core/models.py +++ b/core/models.py @@ -195,6 +195,14 @@ NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" USER_LIMITS_SETTINGS_KEY = "user_limit_settings" +TIMESHIFT_SETTINGS_KEY = "timeshift_settings" + +TIMESHIFT_DEFAULTS = { + "default_timezone": "UTC", + "default_language": "en", + "xmltv_prev_days_override": 0, + "debug_logging": False, +} class CoreSettings(models.Model): @@ -436,6 +444,22 @@ class CoreSettings(models.Model): "terminate_oldest": True, }) + # Timeshift Settings (XC catch-up) + @classmethod + def get_timeshift_settings(cls): + """Return all timeshift-related settings, falling back to neutral defaults.""" + stored = cls._get_group(TIMESHIFT_SETTINGS_KEY, {}) or {} + merged = dict(TIMESHIFT_DEFAULTS) + for key, value in stored.items(): + if key in merged and value is not None: + merged[key] = value + return merged + + @classmethod + def update_timeshift_settings(cls, updates): + clean = {k: v for k, v in (updates or {}).items() if k in TIMESHIFT_DEFAULTS} + return cls._update_group(TIMESHIFT_SETTINGS_KEY, "Timeshift Settings", clean) + class SystemEvent(models.Model): """ diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index e8818108..e95f91cd 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -97,6 +97,7 @@ INSTALLED_APPS = [ "django_filters", "django_celery_beat", "apps.plugins", + "apps.timeshift.apps.TimeshiftConfig", ] # EPG Processing optimization settings diff --git a/dispatcharr/urls.py b/dispatcharr/urls.py index 9123f3da..d8e462a5 100644 --- a/dispatcharr/urls.py +++ b/dispatcharr/urls.py @@ -7,6 +7,7 @@ from .routing import websocket_urlpatterns from apps.output.views import xc_player_api, xc_panel_api, xc_get, xc_xmltv from apps.proxy.live_proxy.views import stream_xc from apps.proxy.vod_proxy.views import stream_xc_movie, stream_xc_episode +from apps.timeshift.views import timeshift_proxy urlpatterns = [ # API Routes @@ -41,6 +42,13 @@ urlpatterns = [ stream_xc, name="xc_stream_endpoint", ), + # XC catch-up (timeshift). The "duration" slot is actually the provider's + # stream_id — matches what iPlayTV / TiviMate emit (see apps/timeshift/views). + path( + "timeshift/////", + timeshift_proxy, + name="timeshift_proxy", + ), # XC VOD endpoints path( "movie///.", diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 30509a86..794045f4 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -26,6 +26,7 @@ import { HardDriveDownload, HardDriveUpload, Radio, + Rewind, SquareX, Timer, Users, @@ -496,7 +497,7 @@ const StreamConnectionCard = ({ ); }, - mantineExpandButtonProps: ({ row, table }) => ({ + mantineExpandButtonProps: ({ row }) => ({ size: 'xs', style: { transform: row.getIsExpanded() ? 'rotate(180deg)' : 'rotate(-90deg)', @@ -533,7 +534,10 @@ const StreamConnectionCard = ({ }, [channel.name, channel.stream_id]); const channelName = - channel.name || previewedStream?.name || 'Unnamed Channel'; + channel.name || + channel.channel_name || + previewedStream?.name || + 'Unnamed Channel'; const uptime = channel.uptime || 0; const bitrates = channel.bitrates || []; const totalBytes = channel.total_bytes || 0; @@ -745,6 +749,18 @@ const StreamConnectionCard = ({ {/* Add stream information badges */} + {channel.is_timeshift && ( + + } + > + TIMESHIFT + + + )} {channel.resolution && ( diff --git a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx new file mode 100644 index 00000000..59db1e62 --- /dev/null +++ b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx @@ -0,0 +1,116 @@ +import useSettingsStore from '../../../store/settings.jsx'; +import React, { useEffect, useState, useMemo } from 'react'; +import { + getChangedSettings, + parseSettings, + saveChangedSettings, +} from '../../../utils/pages/SettingsUtils.js'; +import { + Button, + NumberInput, + Select, + Stack, + Switch, + Text, + TextInput, +} from '@mantine/core'; +import { useForm } from '@mantine/form'; +import { buildTimeZoneOptions } from '../../../utils/dateTimeUtils.js'; +import { + getTimeshiftSettingsFormInitialValues, + getTimeshiftSettingsFormValidation, +} from '../../../utils/forms/settings/TimeshiftSettingsFormUtils.js'; + +const TimeshiftSettingsForm = React.memo(({ active }) => { + const settings = useSettingsStore((s) => s.settings); + const [saved, setSaved] = useState(false); + + const form = useForm({ + mode: 'controlled', + initialValues: getTimeshiftSettingsFormInitialValues(), + validate: getTimeshiftSettingsFormValidation(), + }); + + const tzOptions = useMemo( + () => + buildTimeZoneOptions( + form.getValues().timeshift_default_timezone || 'UTC' + ), + [] + ); + + useEffect(() => { + if (!active) setSaved(false); + }, [active]); + + useEffect(() => { + if (settings) { + const parsed = parseSettings(settings); + // Fall back explicitly to the neutral defaults so the Select shows + // "UTC (now UTC+00:00)" when no setting has been saved yet. + form.setValues({ + timeshift_default_timezone: parsed.timeshift_default_timezone || 'UTC', + timeshift_default_language: parsed.timeshift_default_language || 'en', + xmltv_prev_days_override: parsed.xmltv_prev_days_override ?? 0, + timeshift_debug_logging: !!parsed.timeshift_debug_logging, + }); + } + }, [settings]); + + const onSubmit = async () => { + setSaved(false); + const changed = getChangedSettings(form.getValues(), settings); + try { + await saveChangedSettings(settings, changed); + setSaved(true); + } catch (error) { + console.error('Error saving timeshift settings:', error); + } + }; + + return ( +
+ + + XC catch-up uses these settings to convert UTC EPG timestamps into the + provider's local time and to set the XMLTV lookback window. The + defaults are neutral (UTC, en, off) — adjust to match your provider. + + - { step={1} {...form.getInputProps('xmltv_prev_days_override')} /> - diff --git a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js index 3b5813f5..880c32b0 100644 --- a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js @@ -1,17 +1,9 @@ export const getTimeshiftSettingsFormInitialValues = () => { return { - timeshift_default_timezone: 'UTC', - timeshift_default_language: 'en', xmltv_prev_days_override: 0, - timeshift_debug_logging: false, }; }; export const getTimeshiftSettingsFormValidation = () => { - return { - timeshift_default_language: (value) => - value && !/^[a-z]{2}$/i.test(value) - ? 'Must be a 2-letter ISO 639-1 code' - : null, - }; + return {}; }; diff --git a/frontend/src/utils/pages/SettingsUtils.js b/frontend/src/utils/pages/SettingsUtils.js index 28a5764c..e796af96 100644 --- a/frontend/src/utils/pages/SettingsUtils.js +++ b/frontend/src/utils/pages/SettingsUtils.js @@ -70,12 +70,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'auto_import_mapped_files', 'enable_ip_lookup', ]; - const timeshiftFields = [ - 'timeshift_default_timezone', - 'timeshift_default_language', - 'xmltv_prev_days_override', - 'timeshift_debug_logging', - ]; + const timeshiftFields = ['xmltv_prev_days_override']; for (const formKey in changedSettings) { let value = changedSettings[formKey]; @@ -142,23 +137,12 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'comskip_enabled', 'schedule_enabled', 'auto_import_mapped_files', - 'timeshift_debug_logging', 'enable_ip_lookup', ]; if (booleanFields.includes(formKey) && value != null) { value = typeof value === 'boolean' ? value : Boolean(value); } - // Map UI form keys (with timeshift_ prefix) to the storage subkeys used in - // CoreSettings.value. The prefix lets us share form keys across groups - // without ambiguity but the JSON storage uses short keys. - const timeshiftFormKeyToStorage = { - timeshift_default_timezone: 'default_timezone', - timeshift_default_language: 'default_language', - xmltv_prev_days_override: 'xmltv_prev_days_override', - timeshift_debug_logging: 'debug_logging', - }; - // Route to appropriate group if (streamFields.includes(formKey)) { groupedChanges.stream_settings[formKey] = value; @@ -171,8 +155,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { } else if (systemFields.includes(formKey)) { groupedChanges.system_settings[formKey] = value; } else if (timeshiftFields.includes(formKey)) { - const storageKey = timeshiftFormKeyToStorage[formKey]; - groupedChanges.timeshift_settings[storageKey] = value; + groupedChanges.timeshift_settings[formKey] = value; } } @@ -388,26 +371,16 @@ export const parseSettings = (settings) => { : true; } - // Timeshift settings - direct mapping with timeshift_ prefixed form keys + // Timeshift settings - direct key mapping. + // The only key is the XMLTV lookback override: the XC API surface is strictly + // UTC (no timezone setting) and verbose logging follows the standard logger level. const timeshiftSettings = settings['timeshift_settings']?.value; - parsed.timeshift_default_timezone = - timeshiftSettings && timeshiftSettings.default_timezone - ? String(timeshiftSettings.default_timezone) - : 'UTC'; - parsed.timeshift_default_language = - timeshiftSettings && timeshiftSettings.default_language - ? String(timeshiftSettings.default_language) - : 'en'; parsed.xmltv_prev_days_override = timeshiftSettings && timeshiftSettings.xmltv_prev_days_override != null ? typeof timeshiftSettings.xmltv_prev_days_override === 'number' ? timeshiftSettings.xmltv_prev_days_override : parseInt(timeshiftSettings.xmltv_prev_days_override, 10) || 0 : 0; - parsed.timeshift_debug_logging = - timeshiftSettings && timeshiftSettings.debug_logging != null - ? Boolean(timeshiftSettings.debug_logging) - : false; // Proxy and network access are already grouped objects if (settings['proxy_settings']?.value) { From 2909c9b339b6b41dfec1b296055b1dff161fb42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Wed, 10 Jun 2026 22:05:01 +0200 Subject: [PATCH 10/64] chore(timeshift): pre-review hardening and dead-code cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - http_streamer.py: restore HTTPStreamReader to its upstream form and keep only the find_ts_sync() addition. The response=/extra_headers=/ strip_ts_preamble= extensions had no remaining callers since the timeshift view moved to direct iter_content streaming (delta shrinks from +235/-83 lines to +28). - Unify the catchup_days semantics everywhere: a channel's archive depth is MAX(catchup_days) over its CATCH-UP streams only. The SQL rollup now uses a FILTER (WHERE s.is_catchup) aggregate and the ChannelStream signal uses the same MAX aggregation (it previously took the first stream by order, and the rollup aggregated over all streams including non-catchup ones). - Migration backfill: also accept the lowercase 'true' that ->> extraction yields for JSON booleans. - get_channel_catchup_info(): drop the tv_archive_duration key — its only caller never used it. - Document why timeshift termination fails closed when Redis is unavailable (denying the new stream is what protects the provider connection limit), and update the stale stop-key comment (5 s cadence, not 100 chunks). --- .../migrations/0038_add_catchup_fields.py | 4 + apps/channels/signals.py | 15 +- apps/channels/utils.py | 1 - apps/m3u/tasks.py | 4 +- apps/proxy/live_proxy/input/http_streamer.py | 431 +++++++----------- apps/proxy/utils.py | 8 +- 6 files changed, 183 insertions(+), 280 deletions(-) diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py index 50e1ad01..d6cdc335 100644 --- a/apps/channels/migrations/0038_add_catchup_fields.py +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -26,6 +26,10 @@ def backfill_stream_catchup(apps, schema_editor): AND custom_properties != 'null'::jsonb AND ( custom_properties->>'tv_archive' = '1' + -- JSON booleans extract as lowercase 'true' via ->>; the + -- 'True' spelling covers Python-str values stored by + -- older import code. + OR custom_properties->>'tv_archive' = 'true' OR custom_properties->>'tv_archive' = 'True' ) """) diff --git a/apps/channels/signals.py b/apps/channels/signals.py index f1f3f0ca..c896bd13 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -382,16 +382,19 @@ def update_channel_catchup_fields(sender, instance, **kwargs): Covers the UI/API path (admin adds/removes a stream from a channel). The bulk-import path bypasses signals and uses explicit SQL instead — - see apps/m3u/tasks.py after ChannelStream.objects.bulk_create(). + see rollup_channel_catchup_fields() in apps/m3u/tasks.py. Same semantics + as that rollup: is_catchup = any catch-up stream, catchup_days = MAX of + the catch-up streams' archive depth. """ + from django.db.models import Max + channel = instance.channel - best = ( + max_days = ( channel.streams .filter(is_catchup=True) - .order_by("channelstream__order") - .first() + .aggregate(max_days=Max("catchup_days"))["max_days"] ) Channel.objects.filter(pk=channel.pk).update( - is_catchup=best is not None, - catchup_days=best.catchup_days if best else 0, + is_catchup=max_days is not None, + catchup_days=max_days or 0, ) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index d5dea651..043a8a36 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -41,7 +41,6 @@ def get_channel_catchup_info(channel): return { "stream": stream, "props": stream.custom_properties or {}, - "tv_archive_duration": channel.catchup_days or 7, } diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 2e3cde6d..7ebe2ad0 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1845,7 +1845,9 @@ def rollup_channel_catchup_fields(account_id): SELECT cs.channel_id, bool_or(s.is_catchup) AS any_catchup, - MAX(s.catchup_days) AS max_days + -- Only catch-up streams contribute their archive depth: + -- a non-catchup stream may carry a stale catchup_days value. + MAX(s.catchup_days) FILTER (WHERE s.is_catchup) AS max_days FROM dispatcharr_channels_channelstream cs JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id WHERE cs.channel_id IN ( diff --git a/apps/proxy/live_proxy/input/http_streamer.py b/apps/proxy/live_proxy/input/http_streamer.py index cedb09d9..d90297c5 100644 --- a/apps/proxy/live_proxy/input/http_streamer.py +++ b/apps/proxy/live_proxy/input/http_streamer.py @@ -1,48 +1,182 @@ """ HTTP Stream Reader - Thread-based HTTP stream reader that writes to a pipe. -This allows us to use the same fetch_chunk() path for both transcode and HTTP -streams (live TV) and for timeshift/catch-up archives. - -When *response* is supplied the reader skips connection setup and streams the -already-open ``requests.Response`` directly. This is used by the timeshift -view which needs to cascade through multiple candidate URLs before handing -the winning response to the reader. - -When *strip_ts_preamble* is True the first bytes written to the pipe are -aligned to the first MPEG-TS sync chain (0x47 at 188-byte intervals). Some -XC servers emit PHP warnings or HTML error text before the binary stream; -strict demuxers (ExoPlayer / TiviMate) raise ``UnrecognizedInputFormat`` on -those bytes. The live path typically receives clean TS so the flag defaults -to False, but the timeshift path enables it. +This allows us to use the same fetch_chunk() path for both transcode and HTTP streams. """ -import fcntl -import os -import select as _select import threading - +import os import requests from requests.adapters import HTTPAdapter - from ..utils import get_logger logger = get_logger() -# MPEG-TS sync constants (shared with timeshift views) + +class HTTPStreamReader: + """Thread-based HTTP stream reader that writes to a pipe""" + + def __init__(self, url, user_agent=None, chunk_size=8192): + self.url = url + self.user_agent = user_agent + self.chunk_size = chunk_size + self.session = None + self.response = None + self.thread = None + self.pipe_read = None + self.pipe_write = None + self.running = False + + def start(self): + """Start the HTTP stream reader thread""" + self.pipe_read, self.pipe_write = os.pipe() + + # Make the write end non-blocking so that os.write() raises BlockingIOError + # instead of stalling the OS thread when the pipe buffer is full. Without + # this, a full pipe blocks the entire gevent worker (all greenlets freeze) + # because gevent does not patch os.write() on pipes. + import fcntl + flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) + fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + self.running = True + self.thread = threading.Thread(target=self._read_stream, daemon=True) + self.thread.start() + + logger.info(f"Started HTTP stream reader thread for {self.url}") + return self.pipe_read + + def _read_stream(self): + """Thread worker that reads HTTP stream and writes to pipe""" + try: + # Build headers + headers = {} + if self.user_agent: + headers['User-Agent'] = self.user_agent + + logger.info(f"HTTP reader connecting to {self.url}") + + # Create session + self.session = requests.Session() + + # Disable retries for faster failure detection + adapter = HTTPAdapter(max_retries=0, pool_connections=1, pool_maxsize=1) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) + + # Stream the URL + self.response = self.session.get( + self.url, + headers=headers, + stream=True, + timeout=(5, 30) # 5s connect, 30s read + ) + + if self.response.status_code != 200: + logger.error(f"HTTP {self.response.status_code} from {self.url}") + return + + logger.info(f"HTTP reader connected successfully, streaming data...") + + import select as _select + + # Stream chunks to pipe + chunk_count = 0 + for chunk in self.response.iter_content(chunk_size=self.chunk_size): + if not self.running: + break + + if chunk: + # Write the chunk in a non-blocking loop. The pipe write end is + # set O_NONBLOCK in start(), so os.write() raises BlockingIOError + # instead of stalling the OS thread. We use select.select on the + # write fd (gevent-patched - yields to hub) to wait for space, + # then retry. Partial writes are handled by advancing the offset. + offset = 0 + write_error = False + while offset < len(chunk) and self.running: + try: + n = os.write(self.pipe_write, chunk[offset:]) + offset += n + except BlockingIOError: + _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) + if not writable and not self.running: + write_error = True + break + except OSError as e: + logger.error(f"Pipe write error: {e}") + write_error = True + break + if write_error: + break + + chunk_count += 1 + if chunk_count % 1000 == 0: + logger.debug(f"HTTP reader streamed {chunk_count} chunks") + + logger.info("HTTP stream ended") + + except requests.exceptions.RequestException as e: + logger.error(f"HTTP reader request error: {e}") + except Exception as e: + logger.error(f"HTTP reader unexpected error: {e}", exc_info=True) + finally: + self.running = False + # Close write end of pipe to signal EOF + try: + if self.pipe_write is not None: + os.close(self.pipe_write) + self.pipe_write = None + except: + pass + + def stop(self): + """Stop the HTTP stream reader""" + logger.info("Stopping HTTP stream reader") + self.running = False + + # Close response + if self.response: + try: + self.response.close() + except: + pass + + # Close session + if self.session: + try: + self.session.close() + except: + pass + + # Close write end of pipe + if self.pipe_write is not None: + try: + os.close(self.pipe_write) + self.pipe_write = None + except: + pass + + # Wait for thread + if self.thread and self.thread.is_alive(): + self.thread.join(timeout=2.0) + + +# --------------------------------------------------------------------------- +# MPEG-TS sync detection (used by the timeshift catch-up proxy) +# --------------------------------------------------------------------------- + _TS_PACKET_SIZE = 188 _TS_SYNC_BYTE = 0x47 -_TS_SYNC_SEARCH_LIMIT = 65536 - -# Linux fcntl constant — increase pipe buffer beyond the 64 KB default. -_F_SETPIPE_SZ = 1031 -_PIPE_TARGET_SIZE = 1 << 20 # 1 MB def find_ts_sync(buf): """Offset of the first MPEG-TS sync chain in *buf*, or -1. A valid chain needs 0x47 at offsets i, i+188 and i+376 — three sync - bytes one packet apart (the standard demuxer probe). + bytes one packet apart (the standard demuxer probe). The timeshift + proxy peeks at the first upstream bytes with this to reject HTTP-200 + PHP error pages and to strip any pre-sync preamble before the bytes + reach a strict demuxer (ExoPlayer). """ end = len(buf) - 2 * _TS_PACKET_SIZE for i in range(0, end): @@ -53,248 +187,3 @@ def find_ts_sync(buf): ): return i return -1 - - -class HTTPStreamReader: - """Thread-based HTTP stream reader that writes to an OS pipe. - - Parameters - ---------- - url : str - Provider URL (used when *response* is None to open a new connection). - user_agent : str, optional - User-Agent header for the upstream request. - chunk_size : int - Chunk size for ``iter_content`` and ``pipe_reader.read``. - response : requests.Response, optional - An already-open streaming response. When provided the reader skips - connection setup and streams this response directly. The caller is - responsible for having opened it with ``stream=True``. - extra_headers : dict, optional - Additional HTTP headers (e.g. Range) forwarded to the provider when - *response* is None. - strip_ts_preamble : bool - Strip non-MPEG-TS bytes before the first 0x47 sync chain. - """ - - def __init__( - self, - url, - user_agent=None, - chunk_size=8192, - *, - response=None, - extra_headers=None, - strip_ts_preamble=False, - ): - self.url = url - self.user_agent = user_agent - self.chunk_size = chunk_size - self._provided_response = response - self._extra_headers = extra_headers or {} - self._strip_ts_preamble = strip_ts_preamble - self.session = None - self.response = response # may be pre-set - self.thread = None - self.pipe_read = None - self.pipe_write = None - self.running = False - - def start(self): - """Start the HTTP stream reader thread.""" - self.pipe_read, self.pipe_write = os.pipe() - - # Grow the kernel pipe buffer from 64 KB to 1 MB so the producer - # thread is not blocked while the consumer yields to uWSGI. This - # is the single biggest throughput improvement for the timeshift - # path: it turns the producer/consumer from ping-pong alternation - # into genuinely parallel operation. - try: - fcntl.fcntl(self.pipe_write, _F_SETPIPE_SZ, _PIPE_TARGET_SIZE) - except (OSError, ValueError): - pass # non-Linux or unprivileged — keep 64 KB default - - # Make the write end non-blocking so that os.write() raises - # BlockingIOError instead of stalling the OS thread when the pipe - # buffer is full. Without this, a full pipe blocks the entire gevent - # worker (all greenlets freeze) because gevent does not patch - # os.write() on pipes. - # Skip for pre-opened responses (timeshift path) — those run in a - # real OS thread where blocking writes are fine and the 1 MB pipe - # buffer provides sufficient decoupling. - if self._provided_response is None: - flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) - fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - self.running = True - self.thread = threading.Thread(target=self._read_stream, daemon=True) - self.thread.start() - - logger.info("Started HTTP stream reader thread for %s", self.url) - return self.pipe_read - - # ------------------------------------------------------------------ - # Non-blocking pipe write helper - # ------------------------------------------------------------------ - - def _pipe_write_all(self, data): - """Write *data* to the pipe, handling O_NONBLOCK + partial writes. - - Uses select() on the write fd (gevent-patched — yields to hub) to - wait for space when the pipe is full, instead of blocking the OS - thread. - """ - offset = 0 - while offset < len(data) and self.running: - try: - n = os.write(self.pipe_write, data[offset:]) - offset += n - except BlockingIOError: - _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) - if not writable and not self.running: - return False - except (BrokenPipeError, OSError): - return False - return True - - # ------------------------------------------------------------------ - # Producer thread - # ------------------------------------------------------------------ - - def _read_stream(self): - """Thread worker: read HTTP stream → write to pipe.""" - try: - if self._provided_response is not None: - # Re-use the already-open response (timeshift cascade path). - self.response = self._provided_response - else: - # Open a fresh connection (live TV path). - headers = dict(self._extra_headers) - if self.user_agent: - headers["User-Agent"] = self.user_agent - - logger.info("HTTP reader connecting to %s", self.url) - - self.session = requests.Session() - adapter = HTTPAdapter( - max_retries=0, pool_connections=1, pool_maxsize=1, - ) - self.session.mount("http://", adapter) - self.session.mount("https://", adapter) - - self.response = self.session.get( - self.url, - headers=headers, - stream=True, - timeout=(5, 30), - ) - - if self.response.status_code not in (200, 206): - logger.error( - "HTTP %d from %s", self.response.status_code, self.url, - ) - return - - logger.info("HTTP reader connected, streaming data…") - - # Optional TS preamble stripping — find the first MPEG-TS sync - # chain and discard everything before it. - synced = not self._strip_ts_preamble - sync_buf = bytearray() if not synced else None - - # If the caller peeked at the response body (e.g. to validate TS - # sync before accepting the response), prepend those bytes so they - # aren't lost. They've already been consumed from the raw socket. - peek_data = getattr(self.response, "_peek_data", None) - - chunk_count = 0 - chunks_iter = self.response.iter_content(chunk_size=self.chunk_size) - if peek_data: - import itertools - chunks_iter = itertools.chain([peek_data], chunks_iter) - - for chunk in chunks_iter: - if not self.running: - break - if not chunk: - continue - - # ---- TS preamble strip logic ---- - if not synced: - sync_buf.extend(chunk) - if len(sync_buf) < 2 * _TS_PACKET_SIZE + 1: - continue - offset = find_ts_sync(sync_buf) - if offset >= 0: - chunk = bytes(sync_buf[offset:]) - synced = True - sync_buf = None - elif len(sync_buf) >= _TS_SYNC_SEARCH_LIMIT: - logger.warning( - "Upstream produced %d bytes without TS sync " - "chain; passing through unmodified", - len(sync_buf), - ) - chunk = bytes(sync_buf) - synced = True - sync_buf = None - else: - continue - - # ---- Non-blocking write to pipe ---- - if not self._pipe_write_all(chunk): - break - - chunk_count += 1 - if chunk_count % 1000 == 0: - logger.debug("HTTP reader streamed %d chunks", chunk_count) - - # Flush any remaining sync_buf that never found a chain. - if not synced and sync_buf: - self._pipe_write_all(bytes(sync_buf)) - - logger.info("HTTP stream ended") - - except requests.exceptions.RequestException as exc: - logger.error("HTTP reader request error: %s", exc) - except Exception as exc: - logger.error("HTTP reader unexpected error: %s", exc, exc_info=True) - finally: - self.running = False - try: - if self.pipe_write is not None: - os.close(self.pipe_write) - self.pipe_write = None - except OSError: - pass - - # ------------------------------------------------------------------ - # Shutdown - # ------------------------------------------------------------------ - - def stop(self): - """Stop the HTTP stream reader.""" - logger.info("Stopping HTTP stream reader") - self.running = False - - if self.response: - try: - self.response.close() - except Exception: - pass - - if self.session: - try: - self.session.close() - except Exception: - pass - - if self.pipe_write is not None: - try: - os.close(self.pipe_write) - self.pipe_write = None - except OSError: - pass - - if self.thread and self.thread.is_alive(): - self.thread.join(timeout=2.0) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 5c54c7b1..32e2b674 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -65,9 +65,15 @@ def attempt_stream_termination(user_id, requesting_client_id, active_connections elif t['type'] == 'timeshift': # Timeshift uses the same Redis key pattern as live # (RedisKeys.client_stop). The stream_generator in - # apps/timeshift/views.py checks this key every 100 chunks. + # apps/timeshift/views.py polls this key on its 5-second + # heartbeat cadence. redis_client = RedisClient.get_client() if not redis_client: + # Without Redis the stop key cannot be set, so the old + # stream cannot be terminated. Failing the whole attempt + # is the safe outcome: the caller then denies the NEW + # stream instead of letting the user exceed the provider's + # connection limit (which escalates to an IP ban). return False stop_key = RedisKeys.client_stop(t['media_id'], t['client_id']) redis_client.setex(stop_key, 60, "true") From 1703d6a70371f1cab202276766c4ff068ac3f3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Thu, 11 Jun 2026 20:08:16 +0200 Subject: [PATCH 11/64] =?UTF-8?q?feat(timeshift):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20network=20gate,=20catch-up=20failover,=20live=5Fpro?= =?UTF-8?q?xy=20revert,=20setting=20moved=20to=20proxy=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements all four points from the latest review, plus hardening from a pre-submission audit pass. 1. Access control: timeshift_proxy now enforces network_access_allowed(request, "STREAMS", user) — same key and placement as the live XC stream endpoint. 2. Catch-up failover: the proxy walks the channel's catch-up streams in channelstream order (get_channel_catchup_streams), mirroring live playback. Each attempt carries its own provider context: account credentials, provider stream id, reported server_info timezone (the UTC->provider conversion is recomputed per attempt), user-agent, and the per-account URL-format cache. The first streamable response wins; if all providers fail the last failure is returned. Ban-safety is per account: a decisive auth/ban-class failure (401/403/406) marks the account and skips its remaining streams (e.g. FHD/HD variants of the same channel) instead of hammering a banning provider, while other accounts — different hosts — are still tried. Streams from disabled M3U accounts are excluded, same as live dispatch. Redirects stay enabled on purpose (XC providers legitimately 302 to load-balanced streaming nodes); the 3xx decisive branch is kept as defense-in-depth and documented as such. 3. apps/proxy/live_proxy/views.py restored byte-identical to upstream — the leftover channel-id wrapper from the removed provider-stream-id fallback is gone (zero-line diff). 4. Single remaining setting relocated: xmltv_prev_days_override now lives in proxy_settings (backend default in get_proxy_settings, consumed by the XMLTV prev_days resolution). The timeshift_settings group, TIMESHIFT_DEFAULTS, get_timeshift_settings, the Settings → Timeshift form and tab are all removed; the field appears under Settings → Proxy Settings (0 = auto-detect, capped at 30). Audit hardening in the same pass: - Updated the proxy-settings defaults unit test for the new key (would have failed CI otherwise). - Migration backfills use schema_editor.connection instead of the global connection (multi-database correctness). - CHANGELOG and module docstring brought in line with the final architecture (PATH-first cascade, failover, setting under Proxy Settings). - Tests grown to 69 backend tests: failover success/exhaustion/skip semantics, decisive-account skip vs soft-failure retry, per-stream timezone conversion (different zones per provider), 406/connection-error cascade paths, stream-limit and no-eligible-stream outcomes, network-gate 403, server_info strict-UTC guarantee, EPG duration window resolution, and DB-backed coverage of xc_password auth, user_level access and the failover stream ordering (catch-up-only, active accounts, channelstream order). The format-cache test now runs on an isolated locmem cache. --- CHANGELOG.md | 3 +- .../migrations/0038_add_catchup_fields.py | 8 +- apps/channels/utils.py | 26 +- apps/output/views.py | 6 +- apps/proxy/live_proxy/views.py | 13 +- apps/timeshift/tests/test_helpers.py | 50 +++ apps/timeshift/tests/test_views.py | 413 +++++++++++++++++- apps/timeshift/views.py | 240 +++++----- core/models.py | 29 +- .../components/cards/StreamConnectionCard.jsx | 2 +- .../forms/settings/ProxySettingsForm.jsx | 5 +- .../forms/settings/TimeshiftSettingsForm.jsx | 74 ---- frontend/src/constants.js | 5 + frontend/src/pages/Settings.jsx | 16 - .../forms/settings/ProxySettingsFormUtils.js | 1 + .../settings/TimeshiftSettingsFormUtils.js | 9 - .../__tests__/ProxySettingsFormUtils.test.js | 2 + frontend/src/utils/pages/SettingsUtils.js | 17 +- 18 files changed, 624 insertions(+), 295 deletions(-) delete mode 100644 frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx delete mode 100644 frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc264f7..1b26cd77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,7 +156,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - - **`Settings → Timeshift` panel** with a single knob: `xmltv_prev_days_override` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. + - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. + - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. - **HDHR output profile URL support.** HDHomeRun lineup URLs now support an `output_profile` path segment so HDHR clients (Plex, Channels DVR, Emby, etc.) can request a specific transcode profile without any query-parameter support. URL formats accepted: diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py index d6cdc335..2ec76333 100644 --- a/apps/channels/migrations/0038_add_catchup_fields.py +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -10,9 +10,7 @@ from django.db import migrations, models def backfill_stream_catchup(apps, schema_editor): """Derive is_catchup/catchup_days from Stream.custom_properties JSON.""" - from django.db import connection - - with connection.cursor() as cursor: + with schema_editor.connection.cursor() as cursor: cursor.execute(""" UPDATE dispatcharr_channels_stream SET is_catchup = TRUE, @@ -37,9 +35,7 @@ def backfill_stream_catchup(apps, schema_editor): def backfill_channel_catchup(apps, schema_editor): """Roll up catch-up fields from streams to channels.""" - from django.db import connection - - with connection.cursor() as cursor: + with schema_editor.connection.cursor() as cursor: cursor.execute(""" UPDATE dispatcharr_channels_channel c SET is_catchup = EXISTS ( diff --git a/apps/channels/utils.py b/apps/channels/utils.py index 043a8a36..2efcef62 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -24,24 +24,22 @@ def format_channel_number(value, empty=""): return value -def get_channel_catchup_info(channel): - """Return catch-up info for a Channel, or None if catch-up is unavailable.""" - if not getattr(channel, "is_catchup", False): - return None +def get_channel_catchup_streams(channel): + """Ordered catch-up streams for a Channel (empty list if unavailable). - stream = ( - channel.streams.filter(is_catchup=True) + Returned in ``channelstream__order`` so the timeshift proxy can fail over: + it tries each stream's provider in turn until one serves the archive — + mirroring how live playback walks a channel's stream list. Streams from + disabled M3U accounts are excluded, same as live dispatch. + """ + if not getattr(channel, "is_catchup", False): + return [] + + return list( + channel.streams.filter(is_catchup=True, m3u_account__is_active=True) .order_by("channelstream__order") .select_related("m3u_account") - .first() ) - if stream is None: - return None - - return { - "stream": stream, - "props": stream.custom_properties or {}, - } def increment_stream_count(account): diff --git a/apps/output/views.py b/apps/output/views.py index 0cc20e3f..6ba06e9f 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -1323,7 +1323,7 @@ def generate_epg(request, profile_name=None, user=None): # Timeshift: prev_days resolution order: # 1. URL ?prev_days= (explicit, even 0 means "no past") # 2. user.custom_properties.epg_prev_days - # 3. CoreSettings.timeshift_settings.xmltv_prev_days_override (>0) + # 3. CoreSettings.proxy_settings.xmltv_prev_days_override (>0) # 4. Auto-detect: max provider tv_archive_duration capped at 30 url_prev = request.GET.get('prev_days') user_prev = user_custom.get('epg_prev_days') if user_custom else None @@ -1339,9 +1339,9 @@ def generate_epg(request, profile_name=None, user=None): prev_days = 0 else: from apps.timeshift.helpers import compute_provider_archive_days_capped - timeshift_settings = CoreSettings.get_timeshift_settings() + proxy_settings = CoreSettings.get_proxy_settings() try: - override = int(timeshift_settings.get("xmltv_prev_days_override", 0) or 0) + override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) except (TypeError, ValueError): override = 0 prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 15a4a8ab..d60d9703 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -609,13 +609,6 @@ def stream_xc(request, username, password, channel_id): if custom_properties["xc_password"] != password: return Response({"error": "Invalid credentials"}, status=401) - # Clients always receive channel.id from get_live_streams — straightforward - # int lookup, no provider stream_id fallback needed. - try: - resolved_internal_id = int(channel_id) - except (TypeError, ValueError): - return JsonResponse({"error": "Not found"}, status=404) - if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -623,14 +616,14 @@ def stream_xc(request, username, password, channel_id): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": resolved_internal_id, + "id": int(channel_id), "user_level__lte": user.user_level } channel = Channel.objects.filter(**filters).first() else: # User has specific limited profiles assigned filters = { - "id": resolved_internal_id, + "id": int(channel_id), "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -640,7 +633,7 @@ def stream_xc(request, username, password, channel_id): if not channel: return JsonResponse({"error": "Not found"}, status=404) else: - channel = get_object_or_404(Channel, id=resolved_internal_id) + channel = get_object_or_404(Channel, id=channel_id) if extension.lower() == '.mp4': force_format = 'fmp4' diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index 3f4b10ea..abd3089f 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -171,3 +171,53 @@ class ConvertTimestampToProviderTzTests(TestCase): convert_timestamp_to_provider_tz("garbage", "Europe/Brussels"), "garbage", ) + + +class GetProgrammeDurationTests(TestCase): + """Duration window resolution: programme length + buffer, capped, with a + safe default whenever the EPG lookup cannot resolve.""" + + def _channel_with_programme(self, minutes): + from datetime import datetime, timedelta, timezone as dt_timezone + from unittest.mock import MagicMock + + start = datetime(2026, 6, 8, 17, 0, tzinfo=dt_timezone.utc) + programme = MagicMock( + start_time=start, end_time=start + timedelta(minutes=minutes) + ) + channel = MagicMock() + channel.epg_data.programs.filter.return_value.first.return_value = programme + return channel + + def test_duration_is_programme_length_plus_buffer(self): + from apps.timeshift.helpers import get_programme_duration + # 40-minute programme + 5-minute buffer. + self.assertEqual( + get_programme_duration(self._channel_with_programme(40), "2026-06-08:17-00"), + 45, + ) + + def test_duration_capped_at_max(self): + from apps.timeshift.helpers import get_programme_duration + self.assertEqual( + get_programme_duration(self._channel_with_programme(1000), "2026-06-08:17-00"), + 480, + ) + + def test_no_epg_data_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + channel = MagicMock(epg_data=None) + self.assertEqual(get_programme_duration(channel, "2026-06-08:17-00"), 120) + + def test_no_matching_programme_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + channel = MagicMock() + channel.epg_data.programs.filter.return_value.first.return_value = None + self.assertEqual(get_programme_duration(channel, "2026-06-08:17-00"), 120) + + def test_garbage_timestamp_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + self.assertEqual(get_programme_duration(MagicMock(), "garbage"), 120) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 4db30a66..2f2b6fbb 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch -from django.test import RequestFactory, TestCase +from django.test import RequestFactory, TestCase, override_settings from apps.timeshift import views from apps.proxy.live_proxy.input.http_streamer import find_ts_sync as _find_ts_sync @@ -171,14 +171,16 @@ class StreamFromProviderStatusMappingTests(TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(mocked_open.call_count, 3) + @override_settings(CACHES={ + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"} + }) @patch.object(views, "_open_upstream") def test_cache_promotes_winning_index_to_first(self, mocked_open): """Once a candidate succeeds for an account, the next request reorders the list so the cached winner is tried first — saving cascade overhead on fast-forward.""" - # The format cache lives in django.core.cache (Redis-backed), which - # persists across test runs — clear this account's key so the first - # request starts from the unordered candidate list. + # locmem cache: isolates this test from the shared Redis-backed django + # cache (which persists across runs and parallel test sessions). from django.core.cache import cache as django_cache django_cache.delete(views._FORMAT_CACHE_KEY.format(999)) @@ -252,6 +254,22 @@ class RedactUrlTests(TestCase): self.assertIsNone(views._redact_url(None)) +def _make_catchup_stream(provider_tz="Europe/Brussels", *, account_id=9, + stream_id="22372", account_type="XC", profile_id=31): + """Build a mocked catch-up Stream with its own provider context.""" + profile = MagicMock() + profile.id = profile_id + profile.custom_properties = {"server_info": {"timezone": provider_tz}} + m3u_account = MagicMock() + m3u_account.account_type = account_type + m3u_account.id = account_id + m3u_account.profiles.filter.return_value.first.return_value = profile + stream = MagicMock() + stream.m3u_account = m3u_account + stream.custom_properties = {"stream_id": stream_id} if stream_id else {} + return stream + + class TimeshiftProxyTimestampWiringTests(TestCase): """`timeshift_proxy` must convert the client's UTC timestamp to the serving provider's zone for the upstream URL, while keeping the ORIGINAL @@ -261,26 +279,15 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def setUp(self): self.factory = RequestFactory() - def _make_catchup(self, provider_tz): - profile = MagicMock() - profile.id = 31 - profile.custom_properties = {"server_info": {"timezone": provider_tz}} - m3u_account = MagicMock() - m3u_account.account_type = "XC" - m3u_account.id = 9 - m3u_account.profiles.filter.return_value.first.return_value = profile - stream = MagicMock() - stream.m3u_account = m3u_account - return {"stream": stream, "props": {"stream_id": "22372"}} - def _call(self, timestamp, provider_tz="Europe/Brussels"): request = self.factory.get(f"/timeshift/u/p/8/{timestamp}/8.ts") - sentinel = MagicMock() + sentinel = MagicMock(status_code=200) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ - patch.object(views, "get_channel_catchup_info", - return_value=self._make_catchup(provider_tz)), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream(provider_tz)]), \ patch.object(views, "get_programme_duration", return_value=40) as duration_mock, \ patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ @@ -309,12 +316,378 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def test_invalid_timestamp_rejected_before_upstream(self): request = self.factory.get("/timeshift/u/p/8/garbage/8.ts") with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ - patch.object(views, "get_channel_catchup_info") as catchup_mock, \ + patch.object(views, "get_channel_catchup_streams") as catchup_mock, \ patch.object(views, "_stream_from_provider") as stream_mock: channel_cls.objects.get.return_value = MagicMock(id=8) response = views.timeshift_proxy(request, "u", "p", "8", "garbage", "8.ts") self.assertEqual(response.status_code, 400) catchup_mock.assert_not_called() stream_mock.assert_not_called() + + def test_network_access_denied_returns_403(self): + # Same network-level gate as the live XC endpoint: when the request's + # network is not allowed for STREAMS, nothing else runs. + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=False) as gate, \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_stream_from_provider") as stream_mock: + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(gate.call_args[0][1], "STREAMS") + channel_cls.objects.get.assert_not_called() + stream_mock.assert_not_called() + + +class TimeshiftProxyFailoverTests(TestCase): + """When the first catch-up stream's provider cannot serve the archive, + the proxy must fail over to the channel's next catch-up stream — each + attempt with its own provider context.""" + + def setUp(self): + self.factory = RequestFactory() + + def _call(self, streams, provider_responses): + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "build_timeshift_candidate_urls", + return_value=["http://example.test/x.ts"]) as build_mock, \ + patch.object(views, "check_user_stream_limits", return_value=True) as limits_mock, \ + patch.object(views, "_stream_from_provider", + side_effect=provider_responses) as stream_mock: + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + return response, stream_mock, build_mock, limits_mock + + def test_second_stream_serves_after_first_fails(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + ok = MagicMock(status_code=200) + response, stream_mock, build_mock, _ = self._call( + streams, [MagicMock(status_code=404), ok] + ) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + # Each attempt used its own provider context. + self.assertEqual( + [c.args[0] for c in build_mock.call_args_list], + [streams[0].m3u_account, streams[1].m3u_account], + ) + self.assertEqual( + [c.args[1] for c in build_mock.call_args_list], ["111", "222"] + ) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 2] + ) + + def test_all_streams_fail_returns_last_failure(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + last = MagicMock(status_code=404) + response, stream_mock, _, _ = self._call( + streams, [MagicMock(status_code=400), last] + ) + self.assertIs(response, last) + self.assertEqual(stream_mock.call_count, 2) + + def test_non_xc_and_missing_stream_id_are_skipped(self): + streams = [ + _make_catchup_stream(account_id=1, account_type="M3U"), + _make_catchup_stream(account_id=2, stream_id=None), + _make_catchup_stream(account_id=3, stream_id="333"), + ] + ok = MagicMock(status_code=200) + response, stream_mock, _, _ = self._call(streams, [ok]) + self.assertIs(response, ok) + # Only the eligible third stream produced an upstream attempt. + self.assertEqual(stream_mock.call_count, 1) + self.assertEqual(stream_mock.call_args.kwargs["account_id"], 3) + + def test_stream_limits_checked_once_for_the_request(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + _, _, _, limits_mock = self._call( + streams, [MagicMock(status_code=404), MagicMock(status_code=200)] + ) + self.assertEqual(limits_mock.call_count, 1) + + +class TimeshiftProxyFailoverHardeningTests(TestCase): + """Ban-safety and per-provider context guarantees of the failover loop.""" + + def setUp(self): + self.factory = RequestFactory() + + def _call(self, streams, provider_responses, limits=True): + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "build_timeshift_candidate_urls", + return_value=["http://example.test/x.ts"]) as build_mock, \ + patch.object(views, "check_user_stream_limits", return_value=limits), \ + patch.object(views, "_stream_from_provider", + side_effect=provider_responses) as stream_mock: + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + return response, stream_mock, build_mock + + def test_decisive_failure_skips_same_accounts_other_streams(self): + # Account 1 carries two variants (e.g. FHD + HD). A decisive + # (auth/ban-class) failure on the first must NOT retry account 1's + # second stream — that would hammer a banning provider — but a + # DIFFERENT account stays fair game. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=1, stream_id="112"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + decisive = MagicMock(status_code=403, timeshift_decisive=True) + ok = MagicMock(status_code=200) + response, stream_mock, _ = self._call(streams, [decisive, ok]) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 2] + ) + + def test_soft_failure_still_tries_same_accounts_other_streams(self): + # A soft failure (404: this stream's archive missing) is stream- + # specific — the same account's other variant may still have it. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=1, stream_id="112"), + ] + soft = MagicMock(status_code=404, timeshift_decisive=False) + ok = MagicMock(status_code=200) + response, stream_mock, _ = self._call(streams, [soft, ok]) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 1] + ) + + def test_each_stream_uses_its_own_provider_timezone(self): + # June: 17:00 UTC = 19:00 Brussels (CEST) but 13:00 New York (EDT). + # The converted timestamp must be recomputed per attempt. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", + provider_tz="Europe/Brussels"), + _make_catchup_stream(account_id=2, stream_id="222", + provider_tz="America/New_York"), + ] + response, _, build_mock = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False), + MagicMock(status_code=200)], + ) + self.assertEqual(response.status_code, 200) + self.assertEqual( + [c.args[2] for c in build_mock.call_args_list], + ["2026-06-08:19-00", "2026-06-08:13-00"], + ) + + def test_stream_limit_exceeded_returns_403_before_upstream(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111")] + response, stream_mock, _ = self._call(streams, [], limits=False) + self.assertEqual(response.status_code, 403) + stream_mock.assert_not_called() + + def test_no_catchup_streams_returns_400(self): + response, stream_mock, _ = self._call([], []) + self.assertEqual(response.status_code, 400) + stream_mock.assert_not_called() + + def test_all_streams_ineligible_returns_400(self): + streams = [ + _make_catchup_stream(account_id=1, account_type="M3U"), + _make_catchup_stream(account_id=2, stream_id=None), + ] + response, stream_mock, _ = self._call(streams, []) + self.assertEqual(response.status_code, 400) + stream_mock.assert_not_called() + + +class XcServerInfoUtcTests(TestCase): + """The XC server_info 'timezone triple' guarantee the timeshift chain + relies on: server_info.timezone is always UTC and time_now is UTC + wall-clock. (Tested here because catch-up seek correctness depends on + it: clients build the timeshift URL from this declared zone.)""" + + def test_server_info_is_strictly_utc(self): + from datetime import datetime, timezone as dt_timezone + from apps.output.views import _build_xc_server_info + + request = MagicMock(scheme="http") + info = _build_xc_server_info(request, "example.test", "9191") + self.assertEqual(info["timezone"], "UTC") + reported = datetime.strptime(info["time_now"], "%Y-%m-%d %H:%M:%S") + now_utc = datetime.now(dt_timezone.utc).replace(tzinfo=None) + self.assertLess(abs((now_utc - reported).total_seconds()), 60) + self.assertIsInstance(info["timestamp_now"], int) + + +class StreamFromProviderDecisiveEdgeTests(TestCase): + """Remaining decisive-status and transport-error paths of the cascade.""" + + def setUp(self): + self.kwargs = dict( + candidate_urls=[ + "http://example.test/timeshift/u/p/60/2026-05-12:17-00/1.ts", + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12_17-00", + ], + user_agent="test-agent", + range_header=None, + virtual_channel_id="timeshift_1_2026-05-12-17-00_1", + client_id="timeshift_test456", + client_ip="127.0.0.1", + user=None, + channel_display_name="Test", + timestamp_utc="2026-05-12:17-00", + channel_logo_id=None, + m3u_profile_id=None, + debug=False, + ) + + @patch.object(views, "_open_upstream") + def test_406_is_decisive_and_marks_response(self, mocked_open): + # 406 = IP-wide block in the XC ban escalation — single attempt, + # generic 400 to the client, and the failover loop must see the + # decisive marker so it skips this account's other streams. + mocked_open.return_value = _fake_upstream(406) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + self.assertTrue(response.timeshift_decisive) + + @patch.object(views, "_open_upstream") + def test_404_failure_is_not_decisive(self, mocked_open): + mocked_open.return_value = _fake_upstream(404) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 404) + self.assertFalse(response.timeshift_decisive) + + @patch.object(views, "_open_upstream") + def test_connection_error_returns_400_after_single_attempt(self, mocked_open): + import requests as _requests + mocked_open.side_effect = _requests.exceptions.ConnectionError("boom") + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + # Transport errors are host-level, not auth/ban-class: the failover + # loop may still try a different account. + self.assertFalse(getattr(response, "timeshift_decisive", False)) + + +class CatchupStreamsDbTests(TestCase): + """get_channel_catchup_streams: the function that defines the failover + order — channelstream order, catch-up streams only, active accounts only.""" + + @classmethod + def setUpTestData(cls): + from apps.channels.models import Channel, ChannelStream, Stream + from apps.m3u.models import M3UAccount + + cls.active = M3UAccount.objects.create( + name="ts-test-active", server_url="http://example.test", + account_type="XC", is_active=True, + ) + cls.inactive = M3UAccount.objects.create( + name="ts-test-inactive", server_url="http://example.test", + account_type="XC", is_active=False, + ) + cls.channel = Channel.objects.create(name="ts-test-channel", is_catchup=True) + + def add(name, account, *, catchup, order): + s = Stream.objects.create( + name=name, url=f"http://example.test/{name}", + m3u_account=account, is_catchup=catchup, + ) + ChannelStream.objects.create(channel=cls.channel, stream=s, order=order) + return s + + cls.s_inactive = add("s-inactive", cls.inactive, catchup=True, order=0) + cls.s_second = add("s-second", cls.active, catchup=True, order=2) + cls.s_first = add("s-first", cls.active, catchup=True, order=1) + cls.s_live_only = add("s-live-only", cls.active, catchup=False, order=3) + + def test_ordered_active_catchup_streams_only(self): + from apps.channels.utils import get_channel_catchup_streams + + result = get_channel_catchup_streams(self.channel) + # Inactive-account and non-catchup streams excluded; channelstream order. + self.assertEqual([s.id for s in result], [self.s_first.id, self.s_second.id]) + + def test_channel_without_catchup_flag_returns_empty(self): + from apps.channels.models import Channel + from apps.channels.utils import get_channel_catchup_streams + + ch = Channel.objects.create(name="ts-test-nocatchup", is_catchup=False) + self.assertEqual(get_channel_catchup_streams(ch), []) + + +class AuthHelpersDbTests(TestCase): + """_authenticate_user (xc_password custom property) and + _user_can_access_channel (user_level gate) — exercised against real models + instead of being mocked away.""" + + @classmethod + def setUpTestData(cls): + from apps.accounts.models import User + from apps.channels.models import Channel + + cls.viewer = User.objects.create( + username="ts-test-viewer", user_level=0, + custom_properties={"xc_password": "right-pass"}, + ) + cls.no_xc = User.objects.create( + username="ts-test-noxc", user_level=10, + custom_properties={}, + ) + cls.basic_channel = Channel.objects.create(name="ts-test-basic", user_level=0) + cls.admin_channel = Channel.objects.create(name="ts-test-adult", user_level=10) + + def test_valid_xc_password_authenticates(self): + user = views._authenticate_user("ts-test-viewer", "right-pass") + self.assertIsNotNone(user) + self.assertEqual(user.id, self.viewer.id) + + def test_wrong_xc_password_rejected(self): + self.assertIsNone(views._authenticate_user("ts-test-viewer", "wrong")) + + def test_user_without_xc_password_rejected(self): + # Accounts with no xc_password set (e.g. admins) must be denied even + # if the caller guesses any string — there is nothing to compare to. + self.assertIsNone(views._authenticate_user("ts-test-noxc", "")) + self.assertIsNone(views._authenticate_user("ts-test-noxc", "anything")) + + def test_unknown_username_rejected(self): + self.assertIsNone(views._authenticate_user("ts-test-ghost", "x")) + + def test_user_level_gate(self): + # Level-0 viewer with no profiles: allowed on level-0, denied on level-10. + self.assertTrue(views._user_can_access_channel(self.viewer, self.basic_channel)) + self.assertFalse(views._user_can_access_channel(self.viewer, self.admin_channel)) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 43e0e80a..651ede15 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -1,5 +1,7 @@ -"""XC catch-up (timeshift) HTTP view — proxies the provider's -`/streaming/timeshift.php` endpoint to clients like iPlayTV and TiviMate. +"""XC catch-up (timeshift) HTTP view — proxies the provider's catch-up +archive (PATH `/timeshift/.../{id}.ts` form first, `/streaming/timeshift.php` +query form as fallback) to clients like iPlayTV and TiviMate, with +multi-provider failover across the channel's catch-up streams. URL parameter quirk matching what iPlayTV / TiviMate emit: Position "stream_id" -> EPG channel number, IGNORED here. @@ -25,7 +27,7 @@ from django.http import ( from apps.accounts.models import User from apps.channels.models import Channel -from apps.channels.utils import get_channel_catchup_info +from apps.channels.utils import get_channel_catchup_streams from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState from apps.proxy.live_proxy.redis_keys import RedisKeys @@ -33,6 +35,7 @@ from apps.proxy.live_proxy.utils import get_client_ip from apps.proxy.live_proxy.input.http_streamer import find_ts_sync from apps.proxy.utils import check_user_stream_limits from core.utils import RedisClient +from dispatcharr.utils import network_access_allowed from .helpers import ( build_timeshift_candidate_urls, @@ -56,6 +59,10 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if user is None: return HttpResponseForbidden("Invalid credentials") + # Same network-level gate as the live XC stream endpoint (stream_xc). + if not network_access_allowed(request, "STREAMS", user): + return HttpResponseForbidden("Access denied") + try: channel = Channel.objects.get(id=int(raw_id)) except (Channel.DoesNotExist, ValueError, TypeError): @@ -71,115 +78,132 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if parse_catchup_timestamp(timestamp) is None: return HttpResponseBadRequest("Invalid timestamp") - catchup = get_channel_catchup_info(channel) - if catchup is None: + catchup_streams = get_channel_catchup_streams(channel) + if not catchup_streams: return HttpResponseBadRequest("Timeshift not supported for this channel") - catchup_stream = catchup["stream"] - props = catchup["props"] - m3u_account = catchup_stream.m3u_account - if m3u_account is None or m3u_account.account_type != "XC": - return HttpResponseBadRequest("Channel not from Xtream Codes provider") - # Verbose timeshift logging follows the standard logger level (set via # DISPATCHARR_LOG_LEVEL / the logging config), not a per-feature toggle. debug = logger.isEnabledFor(logging.DEBUG) - stream_id_value = (props or {}).get("stream_id") - if stream_id_value is None: - return HttpResponseBadRequest("Cannot build timeshift URL") - # The client supplies the catch-up timestamp in UTC — Dispatcharr's XC API # surface is strictly UTC (server_info.timezone="UTC", EPG start/end in UTC), - # and the client builds the URL from those UTC strings. Real XC providers, - # however, index their archive in their OWN local zone (the - # server_info.timezone they report on auth). So convert the UTC instant to - # the serving provider's zone before building the upstream URL — empirically - # a provider reads `17-00` as 17:00 LOCAL, not UTC, so skipping this would - # seek ~1-2h off. The EPG duration lookup stays on the original UTC timestamp - # (the EPG is UTC too). This is the ONLY timezone conversion in the chain. + # and the client builds the URL from those UTC strings. The EPG duration + # lookup therefore uses the original UTC value (programmes are stored in + # UTC); the per-provider conversion happens inside the failover loop below. duration_minutes = get_programme_duration(channel, timestamp) - # Default profile carries the provider's server_info (its reported timezone) - # and is reused below for the Stats card's M3U profile name. - default_profile = m3u_account.profiles.filter(is_default=True).first() - provider_tz_name = None - if default_profile is not None: - _server_info = (default_profile.custom_properties or {}).get("server_info") or {} - if isinstance(_server_info, dict): - provider_tz_name = _server_info.get("timezone") - provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) - if debug and provider_timestamp != timestamp: - logger.debug( - "Timeshift tz convert: %s UTC -> %s (provider tz=%s)", - timestamp, provider_timestamp, provider_tz_name, - ) - - # Ordered upstream candidates, PATH form first — see - # build_timeshift_candidate_urls() for the full rationale (the QUERY form is - # a last-resort fallback because some providers return LIVE on it, ignoring - # the requested timestamp). We try them in order until one returns a - # streamable MPEG-TS response — see `_stream_from_provider`. - candidate_urls = build_timeshift_candidate_urls( - m3u_account, stream_id_value, provider_timestamp, duration_minutes - ) - - try: - user_agent = m3u_account.get_user_agent().user_agent - except AttributeError: - user_agent = "" - safe_ts = timestamp.replace(":", "-").replace("/", "-") - virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" client_id = f"timeshift_{uuid.uuid4().hex[:16]}" client_ip = get_client_ip(request) range_header = request.META.get("HTTP_RANGE") - - # Channel logo + M3U default-profile id for the Stats card (default_profile - # was already resolved above for the provider timezone). channel_logo_id = getattr(channel, "logo_id", None) - m3u_profile_id = default_profile.id if default_profile is not None else None - # Free the provider slot before connecting upstream. - # 1. Enforce user stream limits — this terminates the oldest active stream - # (live, timeshift, or VOD) via the Redis stop-key mechanism, same as - # the live and VOD proxy entry points. - if not check_user_stream_limits(user, client_id, media_id=virtual_channel_id): + # Enforce user stream limits once, before connecting upstream — this + # terminates the oldest active stream (live, timeshift, or VOD) via the + # Redis stop-key mechanism, same as the live and VOD proxy entry points. + # (No explicit ChannelService.stop_channel(): that would kill other users' + # live streams on the same channel. The media_id only matters for the + # live same-channel exemption, so a channel-scoped id is enough.) + if not check_user_stream_limits( + user, client_id, media_id=f"timeshift_{channel.id}_{safe_ts}" + ): return HttpResponseForbidden("Stream limit exceeded") - # Note: no explicit ChannelService.stop_channel() here. The - # check_user_stream_limits() call above already terminates the oldest - # active stream (live, timeshift, or VOD) via the Redis stop-key - # mechanism — same pattern as the live and VOD proxy entry points. - # Calling stop_channel() unconditionally would kill other users' live - # streams on the same channel. + # Failover: try each catch-up stream in channelstream order — mirroring how + # live playback walks a channel's stream list. Every attempt carries ITS + # OWN provider context (account, provider stream id, reported timezone, + # user-agent, per-account format cache). Ban safety is per ACCOUNT: when an + # account fails decisively (401/403/406 — auth or ban-class status), its + # other catch-up streams (e.g. FHD/HD variants of the same channel) are + # skipped too, so the cascade never hammers a banning provider; a DIFFERENT + # account is a different host and remains safe to try. + last_response = None + decisive_accounts = set() + for catchup_stream in catchup_streams: + m3u_account = catchup_stream.m3u_account + if m3u_account is None or m3u_account.account_type != "XC": + continue + if m3u_account.id in decisive_accounts: + continue - if debug: - logger.debug( - "Timeshift request: channel=%s ts=%s provider_sid=%s vid=%s client=%s range=%s", - channel.name, - timestamp, - stream_id_value, - virtual_channel_id, - client_id, - range_header or "(none)", + stream_id_value = (catchup_stream.custom_properties or {}).get("stream_id") + if stream_id_value is None: + continue + + # Real XC providers index their archive in their OWN local zone (the + # server_info.timezone they report on auth — stored on the default + # profile). Convert the UTC instant to that zone for the upstream URL — + # empirically a provider reads `17-00` as 17:00 LOCAL, not UTC, so + # skipping this would seek 1-2h off. This is the ONLY timezone + # conversion in the chain, and it is per-provider. + default_profile = m3u_account.profiles.filter(is_default=True).first() + provider_tz_name = None + if default_profile is not None: + _server_info = (default_profile.custom_properties or {}).get("server_info") or {} + if isinstance(_server_info, dict): + provider_tz_name = _server_info.get("timezone") + provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) + + # Ordered upstream candidates, PATH form first — see + # build_timeshift_candidate_urls() for the full rationale (the QUERY + # form is a last-resort fallback because some providers return LIVE on + # it, ignoring the requested timestamp). + candidate_urls = build_timeshift_candidate_urls( + m3u_account, stream_id_value, provider_timestamp, duration_minutes ) - return _stream_from_provider( - candidate_urls=candidate_urls, - user_agent=user_agent, - range_header=range_header, - virtual_channel_id=virtual_channel_id, - client_id=client_id, - client_ip=client_ip, - user=user, - channel_display_name=channel.name, - timestamp_utc=timestamp, - channel_logo_id=channel_logo_id, - m3u_profile_id=m3u_profile_id, - debug=debug, - account_id=m3u_account.id, - ) + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" + m3u_profile_id = default_profile.id if default_profile is not None else None + + if debug: + logger.debug( + "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " + "account=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, timestamp, provider_tz_name, provider_timestamp, + m3u_account.id, stream_id_value, virtual_channel_id, client_id, + range_header or "(none)", + ) + + response = _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + debug=debug, + account_id=m3u_account.id, + ) + if response.status_code < 400: + return response + + last_response = response + if getattr(response, "timeshift_decisive", False): + decisive_accounts.add(m3u_account.id) + logger.warning( + "Timeshift attempt failed (HTTP %d%s) on account %s for channel %s — " + "trying next catch-up stream", + response.status_code, + ", decisive: skipping this account's other streams" + if m3u_account.id in decisive_accounts else "", + m3u_account.id, channel.name, + ) + + if last_response is not None: + return last_response + # Streams existed but none was usable (non-XC accounts / missing stream_id). + return HttpResponseBadRequest("Cannot build timeshift URL") # --------------------------------------------------------------------------- @@ -320,7 +344,15 @@ def _unregister_stats_client(redis_client, virtual_channel_id, client_id): def _open_upstream(url, user_agent, range_header): - """Open the upstream HTTP request (status + headers known synchronously).""" + """Open the upstream HTTP request (status + headers known synchronously). + + Redirects are followed on purpose: XC providers routinely 302 from the + main host to a load-balanced streaming node carrying a session token, so + disabling redirects would break normal catch-up. The cascade therefore + observes the FINAL status; a ban-onset 302 that redirects somewhere + non-streamable surfaces as a failed TS-sync peek (soft reject) or an + error status. + """ # identity: the TS-sync peek reads raw bytes (response.raw), which are NOT # transparently decompressed — a gzip-encoded body would fail the sync check. headers = {"Accept-Encoding": "identity"} @@ -402,6 +434,7 @@ def _stream_from_provider( last_status = None last_url = ordered_urls[0] winning_index = None + decisive_failure = False for url, orig_idx in zip(ordered_urls, original_indices): try: response = _open_upstream(url, user_agent, range_header) @@ -451,8 +484,12 @@ def _stream_from_provider( continue response.close() # Decisive statuses where trying other URL shapes can't help and may - # escalate an IP ban: auth failures (401/403), IP-level block (406), and - # any redirect (3xx) — for XC providers a 302 is the first sign of a ban. + # escalate an IP ban: auth failures (401/403) and IP-level block (406). + # The 3xx case is defense-in-depth only: requests follows redirects + # transparently (XC providers legitimately 302 to load-balanced + # streaming nodes, so redirects MUST be followed), meaning a 3xx can + # only surface here if that ever changes — but if one does, it is the + # documented first sign of an IP ban and must stop the cascade. # A 5xx is different: it is usually format-specific — some XC servers run # PHP with display_errors off, so the "Undefined array key" warning that # another server emits as 200+text becomes a hard 500. The very next @@ -460,6 +497,7 @@ def _stream_from_provider( # giving up (this is why catch-up appeared broken on providers that 500). code = response.status_code if code in (401, 403, 406) or 300 <= code < 400: + decisive_failure = True break if winning_index is not None: @@ -472,10 +510,16 @@ def _stream_from_provider( # correctly: 404 stops retry loops (catch-up not yet indexed), 403 # surfaces auth problems. Everything else stays a generic 400. if last_status == 404: - return HttpResponseNotFound("Catch-up not available yet") - if last_status == 403: - return HttpResponseForbidden("Provider denied access") - return HttpResponseBadRequest("Provider error") + failure = HttpResponseNotFound("Catch-up not available yet") + elif last_status == 403: + failure = HttpResponseForbidden("Provider denied access") + else: + failure = HttpResponseBadRequest("Provider error") + # Tell the failover loop whether this account failed DECISIVELY + # (auth/ban-class status): its other catch-up streams must be skipped, + # while a different provider remains safe to try. + failure.timeshift_decisive = decisive_failure + return failure content_type = upstream.headers.get("Content-Type", "video/mp2t") content_range = upstream.headers.get("Content-Range", "") diff --git a/core/models.py b/core/models.py index 922a1e3e..ffb5d2c7 100644 --- a/core/models.py +++ b/core/models.py @@ -195,16 +195,6 @@ NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" USER_LIMITS_SETTINGS_KEY = "user_limit_settings" -TIMESHIFT_SETTINGS_KEY = "timeshift_settings" - -# The XC API surface is strictly UTC, so there is no timezone setting here: -# server_info.timezone, the EPG start/end strings and time_now are all UTC, and -# any provider-local catch-up conversion happens at proxy time against the -# serving provider's own server_info.timezone (apps/timeshift/views). -# Verbose timeshift logging follows the standard logger DEBUG level, not a toggle. -TIMESHIFT_DEFAULTS = { - "xmltv_prev_days_override": 0, -} class CoreSettings(models.Model): @@ -406,6 +396,9 @@ class CoreSettings(models.Model): "channel_shutdown_delay": 0, "channel_init_grace_period": 5, "new_client_behind_seconds": 5, + # XC catch-up: forced XMLTV lookback window in days (0 = auto-detect + # from the providers' largest tv_archive_duration, capped at 30). + "xmltv_prev_days_override": 0, }) # System Settings @@ -447,22 +440,6 @@ class CoreSettings(models.Model): "terminate_oldest": True, }) - # Timeshift Settings (XC catch-up) - @classmethod - def get_timeshift_settings(cls): - """Return all timeshift-related settings, falling back to neutral defaults. - - Keys not in TIMESHIFT_DEFAULTS are ignored, so stale stored values from - removed settings are filtered out automatically. Writes go through the - generic settings API (same path as every other settings group). - """ - stored = cls._get_group(TIMESHIFT_SETTINGS_KEY, {}) or {} - merged = dict(TIMESHIFT_DEFAULTS) - for key, value in stored.items(): - if key in merged and value is not None: - merged[key] = value - return merged - class SystemEvent(models.Model): """ diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index f2ae0b8f..ae85b410 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -450,7 +450,7 @@ const StreamConnectionCard = ({ ); }, - mantineExpandButtonProps: ({ row }) => ({ + mantineExpandButtonProps: ({ row, table }) => ({ size: 'xs', style: { transform: row.getIsExpanded() ? 'rotate(180deg)' : 'rotate(-90deg)', diff --git a/frontend/src/components/forms/settings/ProxySettingsForm.jsx b/frontend/src/components/forms/settings/ProxySettingsForm.jsx index 52440769..9b2c2c62 100644 --- a/frontend/src/components/forms/settings/ProxySettingsForm.jsx +++ b/frontend/src/components/forms/settings/ProxySettingsForm.jsx @@ -25,6 +25,7 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { 'channel_shutdown_delay', 'channel_init_grace_period', 'new_client_behind_seconds', + 'xmltv_prev_days_override', ].includes(key); }; const isFloatField = (key) => { @@ -39,7 +40,9 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { ? 300 : key === 'new_client_behind_seconds' ? 120 - : 60; + : key === 'xmltv_prev_days_override' + ? 30 + : 60; }; return ( <> diff --git a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx deleted file mode 100644 index 93d0652c..00000000 --- a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx +++ /dev/null @@ -1,74 +0,0 @@ -import useSettingsStore from '../../../store/settings.jsx'; -import React, { useEffect, useState } from 'react'; -import { - getChangedSettings, - parseSettings, - saveChangedSettings, -} from '../../../utils/pages/SettingsUtils.js'; -import { Button, NumberInput, Stack, Text } from '@mantine/core'; -import { useForm } from '@mantine/form'; -import { - getTimeshiftSettingsFormInitialValues, - getTimeshiftSettingsFormValidation, -} from '../../../utils/forms/settings/TimeshiftSettingsFormUtils.js'; - -const TimeshiftSettingsForm = React.memo(({ active }) => { - const settings = useSettingsStore((s) => s.settings); - const [saved, setSaved] = useState(false); - - const form = useForm({ - mode: 'controlled', - initialValues: getTimeshiftSettingsFormInitialValues(), - validate: getTimeshiftSettingsFormValidation(), - }); - - useEffect(() => { - if (!active) setSaved(false); - }, [active]); - - useEffect(() => { - if (settings) { - const parsed = parseSettings(settings); - form.setValues({ - xmltv_prev_days_override: parsed.xmltv_prev_days_override ?? 0, - }); - } - }, [settings]); - - const onSubmit = async () => { - setSaved(false); - const changed = getChangedSettings(form.getValues(), settings); - try { - await saveChangedSettings(settings, changed); - setSaved(true); - } catch (error) { - console.error('Error saving timeshift settings:', error); - } - }; - - return ( - - - - XC catch-up serves EPG and archive timestamps in UTC. Each provider's - own timezone is applied automatically at request time, so there is no - timezone to configure here. The only setting is the XMLTV lookback - window. - - - - - - ); -}); - -export default TimeshiftSettingsForm; diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 92aba531..32aa4bed 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -60,6 +60,11 @@ export const PROXY_SETTINGS_OPTIONS = { description: 'Seconds of received buffer to start behind live when a new client connects (0 = start at live). Note: this is chunk receive time, not video duration.', }, + xmltv_prev_days_override: { + label: 'XMLTV prev_days Override (catch-up)', + description: + 'Days of past programmes in the XC EPG output. 0 = auto-detect from the providers’ tv_archive_duration (capped at 30).', + }, }; export const USER_LIMITS_OPTIONS = { diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index ab4792e4..29d46dd8 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -48,9 +48,6 @@ const SystemSettingsForm = React.lazy( const NavOrderForm = React.lazy( () => import('../components/forms/settings/NavOrderForm.jsx') ); -const TimeshiftSettingsForm = React.lazy( - () => import('../components/forms/settings/TimeshiftSettingsForm.jsx') -); const SettingsPage = () => { const authUser = useAuthStore((s) => s.user); @@ -235,19 +232,6 @@ const SettingsPage = () => { - - - Timeshift - - - }> - - - - - )} diff --git a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js index eddbba91..93cccc9b 100644 --- a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js @@ -15,5 +15,6 @@ export const getProxySettingDefaults = () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, + xmltv_prev_days_override: 0, }; }; diff --git a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js deleted file mode 100644 index 880c32b0..00000000 --- a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js +++ /dev/null @@ -1,9 +0,0 @@ -export const getTimeshiftSettingsFormInitialValues = () => { - return { - xmltv_prev_days_override: 0, - }; -}; - -export const getTimeshiftSettingsFormValidation = () => { - return {}; -}; diff --git a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js index 21c48269..c1fd0908 100644 --- a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js +++ b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js @@ -62,6 +62,7 @@ describe('ProxySettingsFormUtils', () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, + xmltv_prev_days_override: 0, }); }); @@ -82,6 +83,7 @@ describe('ProxySettingsFormUtils', () => { expect(typeof result.channel_shutdown_delay).toBe('number'); expect(typeof result.channel_init_grace_period).toBe('number'); expect(typeof result.new_client_behind_seconds).toBe('number'); + expect(typeof result.xmltv_prev_days_override).toBe('number'); }); }); }); diff --git a/frontend/src/utils/pages/SettingsUtils.js b/frontend/src/utils/pages/SettingsUtils.js index e796af96..acdc6cae 100644 --- a/frontend/src/utils/pages/SettingsUtils.js +++ b/frontend/src/utils/pages/SettingsUtils.js @@ -24,7 +24,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { dvr_settings: {}, backup_settings: {}, system_settings: {}, - timeshift_settings: {}, }; // Map of field prefixes to their groups @@ -70,7 +69,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'auto_import_mapped_files', 'enable_ip_lookup', ]; - const timeshiftFields = ['xmltv_prev_days_override']; for (const formKey in changedSettings) { let value = changedSettings[formKey]; @@ -127,7 +125,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'retention_count', 'schedule_day_of_week', 'max_system_events', - 'xmltv_prev_days_override', ]; if (numericFields.includes(formKey) && value != null) { value = typeof value === 'number' ? value : parseInt(value, 10); @@ -154,8 +151,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { groupedChanges.backup_settings[formKey] = value; } else if (systemFields.includes(formKey)) { groupedChanges.system_settings[formKey] = value; - } else if (timeshiftFields.includes(formKey)) { - groupedChanges.timeshift_settings[formKey] = value; } } @@ -371,18 +366,8 @@ export const parseSettings = (settings) => { : true; } - // Timeshift settings - direct key mapping. - // The only key is the XMLTV lookback override: the XC API surface is strictly - // UTC (no timezone setting) and verbose logging follows the standard logger level. - const timeshiftSettings = settings['timeshift_settings']?.value; - parsed.xmltv_prev_days_override = - timeshiftSettings && timeshiftSettings.xmltv_prev_days_override != null - ? typeof timeshiftSettings.xmltv_prev_days_override === 'number' - ? timeshiftSettings.xmltv_prev_days_override - : parseInt(timeshiftSettings.xmltv_prev_days_override, 10) || 0 - : 0; - // Proxy and network access are already grouped objects + // (xmltv_prev_days_override lives inside proxy_settings) if (settings['proxy_settings']?.value) { parsed.proxy_settings = settings['proxy_settings'].value; } From c91f4f9a6e606e9676728d257f20d0181ac3df94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Fri, 12 Jun 2026 16:36:02 +0200 Subject: [PATCH 12/64] feat(timeshift): provider pool accounting, per-channel session takeover, rollup self-heal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns catch-up with how live and VOD manage provider capacity: - Reserve a provider profile slot (connection_pool) before every upstream connect, walking the account's active profiles default-first when the default is at capacity (profile_full/credential_full are transient and never mark the account decisive). Credentials for the reserved profile are resolved via get_transformed_credentials — the same credential extraction live playback uses — so pool accounting and real upstream usage always agree. All eligible streams blocked on capacity alone returns 503 (the VOD pool-exhausted precedent). - Release exactly once via a one-shot Redis ownership token consumed with a transactional GET+DEL: the generator finally, the response-close wrapper, failed failover attempts and session takeover all share it, so no path can double-decrement and a client disconnecting before the first chunk still releases (Django registers the iterator's close() as a resource closer). Failures between reservation and the streaming response owning the slot release before propagating; an ownership-token write failure releases directly and reports transient unavailability. - One catch-up session per user and channel: a new request (programme jump or seek) displaces the user's previous session on that channel — its slot is released synchronously, its stats are unregistered, and its generator is stopped through the standard stop-key mechanism — so rapid seeking cannot stack upstream provider connections. - rollup_channel_catchup_fields self-heals channels left flagged is_catchup with no remaining catch-up stream (outside the account-scoped CTE), covering bulk removals on manual/multi-provider channels regardless of how the link rows disappeared. A regression test locks the ChannelStream post_delete signal firing on queryset bulk deletes. Backend test suite grows to 94 (slot reservation/release on every failover outcome, profile walk, mixed capacity-vs-upstream precedence, exception-path release, token exactly-once semantics, takeover scoping and ordering, never-started-generator release, rollup self-heal). --- CHANGELOG.md | 4 +- apps/m3u/tasks.py | 25 +- apps/timeshift/helpers.py | 39 +- apps/timeshift/tests/test_helpers.py | 26 +- apps/timeshift/tests/test_views.py | 533 ++++++++++++++++++++++++++- apps/timeshift/views.py | 347 ++++++++++++++--- 6 files changed, 893 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c070643b..4e66fcf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,9 +159,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **NVIDIA NVDEC (`--cuvid`)**: requires the NVIDIA container toolkit and a supported GPU inside the container. - **Intel Quick Sync (`--qsv`)**: requires an Intel iGPU or ARC GPU with the i915 driver exposed to the container. - **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. - - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. + - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. + - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 7eb6443b..8dcc8b68 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1818,7 +1818,8 @@ def rollup_channel_catchup_fields(account_id): Updates ``is_catchup`` and ``catchup_days`` on every Channel that has at least one Stream belonging to *account_id* (auto-created or manually - assigned). + assigned), then self-heals any channel left flagged with no catch-up + stream at all. Called from the account-refresh pipeline after streams have been written so that both new and existing channels reflect the current @@ -1859,6 +1860,28 @@ def rollup_channel_catchup_fields(account_id): WHERE c.id = agg.channel_id """, [account_id]) + # Self-heal: reset channels still flagged as catch-up that no longer + # have ANY catch-up stream. The CTE above only reaches channels that + # still hold at least one stream from this account, so a channel whose + # last stream was just removed (stale cleanup, manual unassignment) + # falls outside its scope. The ChannelStream post_delete signal + # normally resets those rows already — this pass guarantees the + # invariant regardless of how the link rows disappeared (raw SQL, + # signal-less bulk operations, historic staleness). Idempotent and + # cheap: ``is_catchup`` is indexed and TRUE on a small minority of + # channels. + cur.execute(""" + UPDATE dispatcharr_channels_channel c + SET is_catchup = FALSE, catchup_days = 0 + WHERE c.is_catchup = TRUE + AND NOT EXISTS ( + SELECT 1 + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ) + """) + @shared_task def sync_auto_channels(account_id, scan_start_time=None): diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py index 041b6ce5..1075946a 100644 --- a/apps/timeshift/helpers.py +++ b/apps/timeshift/helpers.py @@ -1,6 +1,7 @@ """URL builders + timestamp conversion + archive-days probe for XC catch-up.""" import logging +from collections import namedtuple from datetime import datetime, timezone from urllib.parse import quote from zoneinfo import ZoneInfo @@ -9,6 +10,16 @@ from django.core.cache import cache logger = logging.getLogger(__name__) +#: Resolved upstream credentials for one catch-up attempt. Produced from +#: ``get_transformed_credentials(account, reserved_profile)`` so the URL is +#: built with the credentials of the profile whose pool slot was actually +#: reserved — alternate profiles express different logins as URL regex +#: transforms, and using the raw account fields for them would desynchronize +#: pool accounting from real upstream usage. +TimeshiftCredentials = namedtuple( + "TimeshiftCredentials", ("server_url", "username", "password") +) + DEFAULT_DURATION_MINUTES = 120 DURATION_BUFFER_MINUTES = 5 MAX_DURATION_MINUTES = 480 @@ -129,33 +140,33 @@ def get_programme_duration(channel, timestamp_str): return DEFAULT_DURATION_MINUTES -def build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes): """Format A: `/streaming/timeshift.php?username=&password=&stream=&start=&duration=`.""" # Credentials are URL-encoded: a `&`, `/` or `#` in the password would # otherwise corrupt the URL structure. return ( - f"{m3u_account.server_url.rstrip('/')}/streaming/timeshift.php" - f"?username={quote(str(m3u_account.username), safe='')}" - f"&password={quote(str(m3u_account.password), safe='')}" + f"{creds.server_url.rstrip('/')}/streaming/timeshift.php" + f"?username={quote(str(creds.username), safe='')}" + f"&password={quote(str(creds.password), safe='')}" f"&stream={stream_id}" f"&start={timestamp}" f"&duration={duration_minutes}" ) -def build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes): """Format B: `/timeshift/{user}/{pass}/{duration}/{timestamp}/{stream_id}.ts`.""" return ( - f"{m3u_account.server_url.rstrip('/')}/timeshift" - f"/{quote(str(m3u_account.username), safe='')}" - f"/{quote(str(m3u_account.password), safe='')}" + f"{creds.server_url.rstrip('/')}/timeshift" + f"/{quote(str(creds.username), safe='')}" + f"/{quote(str(creds.password), safe='')}" f"/{duration_minutes}" f"/{timestamp}" f"/{stream_id}.ts" ) -def build_timeshift_candidate_urls(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_candidate_urls(creds, stream_id, timestamp, duration_minutes): """Ordered upstream catch-up candidates — PATH form first. Two URL layouts exist on XC servers and they do NOT behave the same: @@ -180,12 +191,12 @@ def build_timeshift_candidate_urls(m3u_account, stream_id, timestamp, duration_m sql_ts = format_timestamp_as_sql_datetime(timestamp) return [ # PATH form first — it seeks the archive correctly. - build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes), - build_timeshift_url_format_b(m3u_account, stream_id, underscore_ts, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, underscore_ts, duration_minutes), # QUERY form fallback — may return LIVE on some providers (see above). - build_timeshift_url_format_a(m3u_account, stream_id, underscore_ts, duration_minutes), - build_timeshift_url_format_a(m3u_account, stream_id, sql_ts, duration_minutes), - build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, underscore_ts, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, sql_ts, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes), ] diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index abd3089f..2c29e337 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -3,6 +3,7 @@ from django.test import TestCase from apps.timeshift.helpers import ( + TimeshiftCredentials, build_timeshift_candidate_urls, build_timeshift_url_format_a, build_timeshift_url_format_b, @@ -12,11 +13,10 @@ from apps.timeshift.helpers import ( ) -class _FakeAccount: - def __init__(self): - self.server_url = "http://example.test" - self.username = "user" - self.password = "pass" +def _make_creds(): + # The builders consume resolved per-profile credentials, never an account + # object — get_transformed_credentials() produces these in the view. + return TimeshiftCredentials("http://example.test", "user", "pass") class TimestampFormatTests(TestCase): @@ -56,11 +56,11 @@ class TimestampFormatTests(TestCase): class BuildTimeshiftUrlTests(TestCase): def setUp(self): - self.account = _FakeAccount() + self.creds = _make_creds() def test_format_a_passes_dash_shape_unchanged(self): url = build_timeshift_url_format_a( - self.account, "22372", "2026-05-12:19-00", 40 + self.creds, "22372", "2026-05-12:19-00", 40 ) self.assertIn("start=2026-05-12:19-00", url) self.assertIn("stream=22372", url) @@ -68,13 +68,13 @@ class BuildTimeshiftUrlTests(TestCase): def test_format_a_passes_sql_shape_unchanged(self): url = build_timeshift_url_format_a( - self.account, "22372", "2026-05-12 19:00:00", 40 + self.creds, "22372", "2026-05-12 19:00:00", 40 ) self.assertIn("start=2026-05-12 19:00:00", url) def test_format_b_path_with_dash_shape(self): url = build_timeshift_url_format_b( - self.account, "22372", "2026-05-12:19-00", 40 + self.creds, "22372", "2026-05-12:19-00", 40 ) self.assertIn("/40/2026-05-12:19-00/22372.ts", url) @@ -86,7 +86,7 @@ class CandidateOrderingTests(TestCase): "catch-up plays the live stream instead of the requested programme" bug.""" def setUp(self): - self.account = _FakeAccount() + self.creds = _make_creds() def _is_path_form(self, url): return "/timeshift/" in url and url.endswith(".ts") and "timeshift.php" not in url @@ -95,7 +95,7 @@ class CandidateOrderingTests(TestCase): return "timeshift.php?" in url def test_every_path_candidate_precedes_every_query_candidate(self): - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12:19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12:19-00", 40) path_indices = [i for i, u in enumerate(urls) if self._is_path_form(u)] query_indices = [i for i, u in enumerate(urls) if self._is_query_form(u)] # Each URL is classified as exactly one form. @@ -105,14 +105,14 @@ class CandidateOrderingTests(TestCase): self.assertLess(max(path_indices), min(query_indices)) def test_first_candidate_is_path_form_with_canonical_dash_timestamp(self): - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12:19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12:19-00", 40) self.assertTrue(self._is_path_form(urls[0])) # Canonical colon-dash timestamp, passed through unchanged. self.assertIn("/40/2026-05-12:19-00/22372.ts", urls[0]) def test_accepts_underscore_input_timestamp(self): # Client may send the underscore shape; PATH form still leads. - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12_19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12_19-00", 40) self.assertTrue(self._is_path_form(urls[0])) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 2f2b6fbb..59b8fd0b 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -255,21 +255,91 @@ class RedactUrlTests(TestCase): def _make_catchup_stream(provider_tz="Europe/Brussels", *, account_id=9, - stream_id="22372", account_type="XC", profile_id=31): - """Build a mocked catch-up Stream with its own provider context.""" + stream_id="22372", account_type="XC", profile_id=31, + extra_profiles=()): + """Build a mocked catch-up Stream with its own provider context. + + The default (tz-bearing) profile leads the active-profile list the view + walks; ``extra_profiles`` appends alternate (non-default) profiles for + capacity-walk tests. + """ profile = MagicMock() profile.id = profile_id + profile.is_default = True profile.custom_properties = {"server_info": {"timezone": provider_tz}} m3u_account = MagicMock() m3u_account.account_type = account_type m3u_account.id = account_id - m3u_account.profiles.filter.return_value.first.return_value = profile + m3u_account.profiles.filter.return_value = [profile, *extra_profiles] stream = MagicMock() stream.m3u_account = m3u_account stream.custom_properties = {"stream_id": stream_id} if stream_id else {} return stream +def _make_alt_profile(profile_id): + """A non-default active profile for the capacity walk.""" + profile = MagicMock() + profile.id = profile_id + profile.is_default = False + profile.custom_properties = {} + return profile + + +class _FakeRedis: + """Just enough of the redis-py surface for the slot-token protocol: + setex/get/delete plus a transactional pipeline doing GET+DEL.""" + + def __init__(self): + self.store = {} + + def setex(self, key, ttl, value): + self.store[key] = str(value) + + def set(self, key, value): + self.store[key] = str(value) + + def get(self, key): + return self.store.get(key) + + def delete(self, *keys): + return sum(1 for k in keys if self.store.pop(k, None) is not None) + + def exists(self, key): + return 1 if key in self.store else 0 + + def pipeline(self, transaction=False): + return _FakeRedisPipeline(self) + + +class _FakeRedisPipeline: + def __init__(self, redis): + self._redis = redis + self._ops = [] + + def get(self, key): + self._ops.append(("get", key)) + + def delete(self, key): + self._ops.append(("delete", key)) + + def execute(self): + results = [] + for op, key in self._ops: + if op == "get": + results.append(self._redis.get(key)) + else: + results.append(self._redis.delete(key)) + self._ops = [] + return results + + +def _fake_creds(acc, prof): + """Distinguishable per-account credentials, mirroring what + get_transformed_credentials returns for the reserved profile.""" + return (f"http://a{acc.id}.test", f"u{acc.id}", "p") + + class TimeshiftProxyTimestampWiringTests(TestCase): """`timeshift_proxy` must convert the client's UTC timestamp to the serving provider's zone for the upstream URL, while keeping the ORIGINAL @@ -292,7 +362,13 @@ class TimeshiftProxyTimestampWiringTests(TestCase): patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", return_value=sentinel) as stream_mock: + redis_cls.get_client.return_value = _FakeRedis() channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) response = views.timeshift_proxy(request, "u", "p", "8", timestamp, "8.ts") return response, sentinel, build_mock, duration_mock, stream_mock @@ -363,12 +439,20 @@ class TimeshiftProxyFailoverTests(TestCase): patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=True) as limits_mock, \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", + side_effect=_fake_creds) as creds_mock, \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", side_effect=provider_responses) as stream_mock: + redis_cls.get_client.return_value = _FakeRedis() channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) response = views.timeshift_proxy( request, "u", "p", "8", "2026-06-08:17-00", "8.ts" ) + self.creds_mock = creds_mock return response, stream_mock, build_mock, limits_mock def test_second_stream_serves_after_first_fails(self): @@ -382,9 +466,14 @@ class TimeshiftProxyFailoverTests(TestCase): ) self.assertIs(response, ok) self.assertEqual(stream_mock.call_count, 2) - # Each attempt used its own provider context. + # Each attempt used its own provider context: credentials resolved per + # account/profile (via get_transformed_credentials) and its stream id. self.assertEqual( [c.args[0] for c in build_mock.call_args_list], + [("http://a1.test", "u1", "p"), ("http://a2.test", "u2", "p")], + ) + self.assertEqual( + [c.args[0] for c in self.creds_mock.call_args_list], [streams[0].m3u_account, streams[1].m3u_account], ) self.assertEqual( @@ -430,14 +519,27 @@ class TimeshiftProxyFailoverTests(TestCase): self.assertEqual(limits_mock.call_count, 1) -class TimeshiftProxyFailoverHardeningTests(TestCase): - """Ban-safety and per-provider context guarantees of the failover loop.""" +class _ProxyLoopTestMixin: + """Shared driver for tests exercising the failover loop end to end — + pool reservation, credential resolution and Redis are all controlled.""" def setUp(self): self.factory = RequestFactory() - def _call(self, streams, provider_responses, limits=True): + def _call(self, streams, provider_responses, limits=True, reserve_results=None, + build_side_effect=None): request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + self.fake_redis = _FakeRedis() + reserve_kwargs = ( + {"side_effect": reserve_results} + if reserve_results is not None + else {"return_value": (True, 1, None)} + ) + build_kwargs = ( + {"side_effect": build_side_effect} + if build_side_effect is not None + else {"return_value": ["http://example.test/x.ts"]} + ) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ @@ -445,16 +547,32 @@ class TimeshiftProxyFailoverHardeningTests(TestCase): patch.object(views, "get_channel_catchup_streams", return_value=streams), \ patch.object(views, "get_programme_duration", return_value=40), \ patch.object(views, "build_timeshift_candidate_urls", - return_value=["http://example.test/x.ts"]) as build_mock, \ + **build_kwargs) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=limits), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", **reserve_kwargs) as reserve_mock, \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "get_transformed_credentials", + side_effect=_fake_creds) as creds_mock, \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", side_effect=provider_responses) as stream_mock: + redis_cls.get_client.return_value = self.fake_redis channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + # Exposed before the call so raising tests can still assert on them. + self.reserve_mock = reserve_mock + self.release_mock = release_mock + self.creds_mock = creds_mock + self.stream_mock = stream_mock response = views.timeshift_proxy( request, "u", "p", "8", "2026-06-08:17-00", "8.ts" ) return response, stream_mock, build_mock + +class TimeshiftProxyFailoverHardeningTests(_ProxyLoopTestMixin, TestCase): + """Ban-safety and per-provider context guarantees of the failover loop.""" + def test_decisive_failure_skips_same_accounts_other_streams(self): # Account 1 carries two variants (e.g. FHD + HD). A decisive # (auth/ban-class) failure on the first must NOT retry account 1's @@ -691,3 +809,402 @@ class AuthHelpersDbTests(TestCase): # Level-0 viewer with no profiles: allowed on level-0, denied on level-10. self.assertTrue(views._user_can_access_channel(self.viewer, self.basic_channel)) self.assertFalse(views._user_can_access_channel(self.viewer, self.admin_channel)) + + +class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): + """Provider pool participation: a profile slot is reserved before any + upstream attempt and released exactly once afterwards — the same + accounting contract live (Channel.get_stream) and VOD follow.""" + + def _slot_token_keys(self): + return [k for k in self.fake_redis.store if k.startswith("timeshift_slot:")] + + def test_reserve_called_with_default_profile_before_upstream(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, stream_mock, _ = self._call(streams, [MagicMock(status_code=200)]) + self.assertEqual(response.status_code, 200) + self.reserve_mock.assert_called_once() + reserved_profile = self.reserve_mock.call_args.args[0] + self.assertEqual(reserved_profile.id, 31) + # The reserved profile's id is what reaches the stats metadata. + self.assertEqual(stream_mock.call_args.kwargs["m3u_profile_id"], 31) + + def test_slot_released_after_failed_attempt(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, _, _ = self._call( + streams, [MagicMock(status_code=404, timeshift_decisive=False)] + ) + self.assertEqual(response.status_code, 404) + # The one-shot token was consumed and the pool counter decremented. + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_slot_kept_on_success_for_the_streaming_session(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, _, _ = self._call(streams, [MagicMock(status_code=200)]) + self.assertEqual(response.status_code, 200) + # Slot still owned by the (mocked) streaming session: token present, + # nothing released yet. + self.release_mock.assert_not_called() + self.assertEqual(len(self._slot_token_keys()), 1) + + def test_decisive_failure_releases_slot_and_skips_account(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=1, stream_id="112", profile_id=31), + ] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=403, timeshift_decisive=True)] + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(stream_mock.call_count, 1) + self.release_mock.assert_called_once_with(31, self.fake_redis) + # Decisive skip means the second stream never reserved a slot. + self.assertEqual(self.reserve_mock.call_count, 1) + + def test_profile_full_walks_to_next_profile_same_account(self): + alt = _make_alt_profile(32) + streams = [_make_catchup_stream( + account_id=1, stream_id="111", profile_id=31, extra_profiles=(alt,) + )] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=200)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 200) + # Default profile full -> alternate profile reserved and used. + self.assertEqual( + [c.args[0].id for c in self.reserve_mock.call_args_list], [31, 32] + ) + self.assertEqual(stream_mock.call_args.kwargs["m3u_profile_id"], 32) + # Credentials were resolved for the RESERVED (alternate) profile. + self.assertIs(self.creds_mock.call_args.args[1], alt) + + def test_all_profiles_full_returns_503_without_upstream_attempt(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, stream_mock, _ = self._call( + streams, [], + reserve_results=[ + (False, 1, "profile_full"), + (False, 1, "credential_full"), + ], + ) + # Pool capacity exhausted everywhere: 503 (VOD's pool-exhausted + # status), and crucially the provider was never contacted. + self.assertEqual(response.status_code, 503) + stream_mock.assert_not_called() + self.release_mock.assert_not_called() + + def test_capacity_failure_is_not_decisive_for_the_account(self): + # profile_full on account 1's first stream must NOT mark account 1 + # decisive — capacity is transient, unlike a ban-class status. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=1, stream_id="112", profile_id=31), + ] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=200)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 200) + # Second stream of the SAME account still got its reservation attempt. + self.assertEqual(self.reserve_mock.call_count, 2) + self.assertEqual(stream_mock.call_count, 1) + + def test_account_without_active_default_profile_is_skipped(self): + # Mirrors live dispatch: no active default profile -> skip the account + # without reserving anything. + stream = _make_catchup_stream(account_id=1, stream_id="111") + stream.m3u_account.profiles.filter.return_value = [_make_alt_profile(32)] + response, stream_mock, _ = self._call([stream], []) + self.assertEqual(response.status_code, 400) + self.reserve_mock.assert_not_called() + stream_mock.assert_not_called() + + def test_exception_from_provider_releases_slot(self): + # An unexpected exception between reserve and response construction + # must release the slot before propagating — otherwise the counter + # (no TTL) leaks until the next Redis flush. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with self.assertRaises(RuntimeError): + self._call(streams, RuntimeError("boom")) + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_exception_before_upstream_releases_slot(self): + # Same guarantee for failures BEFORE the upstream call (URL building, + # credential resolution, user-agent lookup) — the guarded window + # starts right after the reservation. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with self.assertRaises(RuntimeError): + self._call(streams, [], build_side_effect=RuntimeError("boom")) + self.stream_mock.assert_not_called() + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_token_store_failure_releases_directly(self): + # If the ownership token cannot be written, no release path could + # ever free the slot — the view must release it directly and report + # transient unavailability instead of streaming unaccounted. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with patch.object(views, "_store_slot_token", return_value=False): + response, stream_mock, _ = self._call(streams, []) + self.assertEqual(response.status_code, 503) + stream_mock.assert_not_called() + self.release_mock.assert_called_once_with(31, self.fake_redis) + + def test_mixed_capacity_then_upstream_failure_returns_failure(self): + # Mixed outcome: one stream capacity-blocked, another actually tried + # upstream and failed -> the REAL upstream failure wins over 503 + # (capacity was not the sole blocker). + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, _, _ = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 404) + + def test_mixed_upstream_failure_then_capacity_returns_failure(self): + # Same in the opposite order. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, _, _ = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False)], + reserve_results=[(True, 1, None), (False, 1, "profile_full")], + ) + self.assertEqual(response.status_code, 404) + + +class TimeshiftSlotTokenTests(TestCase): + """The one-shot release token: exactly-once semantics across all release + paths (generator finally, response close, takeover, failed attempt).""" + + def setUp(self): + self.redis = _FakeRedis() + + def test_release_happens_exactly_once(self): + views._store_slot_token(self.redis, "timeshift_abc", 31) + with patch.object(views, "release_profile_slot") as release_mock: + self.assertTrue(views._release_slot_token(self.redis, "timeshift_abc")) + self.assertFalse(views._release_slot_token(self.redis, "timeshift_abc")) + release_mock.assert_called_once_with(31, self.redis) + + def test_release_without_token_is_noop(self): + with patch.object(views, "release_profile_slot") as release_mock: + self.assertFalse(views._release_slot_token(self.redis, "timeshift_ghost")) + release_mock.assert_not_called() + + def test_release_without_redis_is_noop(self): + with patch.object(views, "release_profile_slot") as release_mock: + self.assertFalse(views._release_slot_token(None, "timeshift_abc")) + release_mock.assert_not_called() + + def test_wrapper_close_releases_even_when_generator_never_started(self): + # The WSGI layer can close the response before the first chunk is + # pulled; closing a never-started generator runs NO body code, so the + # generator's own finally cannot be the only release point. + finally_ran = [] + + def gen(): + try: + yield b"x" + finally: + finally_ran.append(True) + + on_close = MagicMock() + wrapper = views._SlotReleasingStream(gen(), on_close) + wrapper.close() + on_close.assert_called_once() + self.assertEqual(finally_ran, []) # proves the leak this wrapper fixes + + def test_streaming_response_close_invokes_wrapper_close(self): + # Locks the Django contract the wrapper relies on: an iterator with a + # close() method is registered as a resource closer of the response. + from django.http import StreamingHttpResponse + + on_close = MagicMock() + wrapper = views._SlotReleasingStream(iter([b"x"]), on_close) + response = StreamingHttpResponse(wrapper, content_type="video/mp2t") + response.close() + on_close.assert_called_once() + + +class TimeshiftTakeoverTests(TestCase): + """One catch-up session per user+channel: a new request displaces the + user's previous session on the same channel — synchronous slot release + + stats unregister + stop key — and never touches other users, other + channels, or live sessions.""" + + def setUp(self): + self.redis = _FakeRedis() + self.user = MagicMock(id=5) + + def _conn(self, media_id, client_id, conn_type="timeshift"): + return { + "media_id": media_id, + "client_id": client_id, + "connected_at": 0.0, + "type": conn_type, + } + + def test_displaces_only_same_channel_timeshift_sessions(self): + views._store_slot_token(self.redis, "timeshift_old1", 31) + views._store_slot_token(self.redis, "timeshift_other", 41) + connections = [ + self._conn("timeshift_8_2026-06-08-17-00_111", "timeshift_old1"), + self._conn("timeshift_9_2026-06-08-17-00_222", "timeshift_other"), + self._conn("42", "live_client", conn_type="live"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections) as conns_mock, \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + conns_mock.assert_called_once_with(5) + # Channel 8's old session: slot released, stats dropped, stop key set. + release_mock.assert_called_once_with(31, self.redis) + unregister_mock.assert_called_once_with( + self.redis, "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" + ) + from apps.proxy.live_proxy.redis_keys import RedisKeys + stop_key = RedisKeys.client_stop( + "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" + ) + self.assertIn(stop_key, self.redis.store) + # Channel 9's session untouched: token still present, no stop key. + self.assertIn("timeshift_slot:timeshift_other", self.redis.store) + + def test_channel_id_prefix_cannot_match_other_channels(self): + # Channel 8 must not displace channel 80/81 sessions (prefix ends + # with an underscore). + connections = [ + self._conn("timeshift_80_2026-06-08-17-00_111", "timeshift_c80"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections), \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + release_mock.assert_not_called() + unregister_mock.assert_not_called() + + def test_noop_without_redis_or_user(self): + with patch.object(views, "get_user_active_connections") as conns_mock: + views._terminate_previous_timeshift_sessions(None, self.user, 8) + views._terminate_previous_timeshift_sessions(self.redis, None, 8) + conns_mock.assert_not_called() + + def test_proxy_runs_takeover_before_stream_limit_check(self): + # Order matters: with terminate_on_limit_exceeded=False a seek must + # displace its own predecessor BEFORE the limit check counts it, or + # the user's own seek gets denied. + call_order = [] + request = RequestFactory().get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "_terminate_previous_timeshift_sessions", + side_effect=lambda *a: call_order.append("takeover")) as takeover_mock, \ + patch.object(views, "check_user_stream_limits", + side_effect=lambda *a, **k: call_order.append("limits") or False): + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(call_order, ["takeover", "limits"]) + self.assertEqual(takeover_mock.call_args.args[2], 8) + + +class RollupSelfHealDbTests(TestCase): + """Catch-up flag consistency after stream removal — the review-#4 point 1 + guarantees: the ChannelStream signal handles bulk deletes (locked by a + regression test) and the rollup self-heals any channel left flagged with + no catch-up stream, regardless of how the link rows disappeared.""" + + @classmethod + def setUpTestData(cls): + from apps.m3u.models import M3UAccount + + cls.account = M3UAccount.objects.create( + name="ts-rollup-account", server_url="http://example.test", + account_type="XC", is_active=True, + ) + + def _make_channel_with_catchup_stream(self, name, days=5): + from apps.channels.models import Channel, ChannelStream, Stream + + channel = Channel.objects.create(name=name) + stream = Stream.objects.create( + name=f"{name}-stream", url=f"http://example.test/{name}", + m3u_account=self.account, is_catchup=True, catchup_days=days, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + return channel, stream + + def test_bulk_stream_delete_resets_channel_flags_via_signal(self): + # cleanup_streams() removes stale streams with a queryset bulk delete; + # the cascaded ChannelStream rows still fire post_delete (signal + # listeners disable Django's fast-delete path), which must reset the + # channel's denormalized catch-up fields. + from apps.channels.models import Stream + + channel, stream = self._make_channel_with_catchup_stream("ts-rollup-bulk") + channel.refresh_from_db() + self.assertTrue(channel.is_catchup) + self.assertEqual(channel.catchup_days, 5) + + Stream.objects.filter(id=stream.id).delete() + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) + + def test_rollup_self_heals_stale_channel_without_streams(self): + # Simulate staleness no signal caught (raw SQL delete, historic data): + # a flagged channel with zero catch-up streams must be reset by the + # rollup even though no remaining stream ties it to the account. + from apps.channels.models import Channel + from apps.m3u.tasks import rollup_channel_catchup_fields + + channel = Channel.objects.create(name="ts-rollup-stale") + Channel.objects.filter(pk=channel.pk).update(is_catchup=True, catchup_days=9) + + rollup_channel_catchup_fields(self.account.id) + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) + + def test_rollup_keeps_and_corrects_channels_with_catchup_streams(self): + # The self-heal pass must not touch channels that legitimately have + # catch-up streams — and the account-scoped pass still corrects their + # values. + from apps.channels.models import Channel + from apps.m3u.tasks import rollup_channel_catchup_fields + + channel, _ = self._make_channel_with_catchup_stream("ts-rollup-valid", days=7) + # Knock the denormalized values out of sync (bypasses signals). + Channel.objects.filter(pk=channel.pk).update(is_catchup=False, catchup_days=0) + + rollup_channel_catchup_fields(self.account.id) + + channel.refresh_from_db() + self.assertTrue(channel.is_catchup) + self.assertEqual(channel.catchup_days, 7) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 651ede15..f158df4b 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -3,6 +3,13 @@ archive (PATH `/timeshift/.../{id}.ts` form first, `/streaming/timeshift.php` query form as fallback) to clients like iPlayTV and TiviMate, with multi-provider failover across the channel's catch-up streams. +Provider capacity accounting follows the same contract as live and VOD: a +profile slot is reserved through ``apps.m3u.connection_pool`` before any +upstream connect and released exactly once when the session ends (one-shot +Redis token), so Dispatcharr's pool counters always match real upstream usage. +A user gets ONE active catch-up session per channel — a seek displaces the +previous one. + URL parameter quirk matching what iPlayTV / TiviMate emit: Position "stream_id" -> EPG channel number, IGNORED here. Position "duration" -> Dispatcharr's Channel.id (the XC API emits @@ -19,6 +26,7 @@ import requests from django.core.cache import cache from django.http import ( Http404, + HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseNotFound, @@ -28,16 +36,19 @@ from django.http import ( from apps.accounts.models import User from apps.channels.models import Channel from apps.channels.utils import get_channel_catchup_streams +from apps.m3u.connection_pool import release_profile_slot, reserve_profile_slot +from apps.m3u.tasks import get_transformed_credentials from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.utils import get_client_ip from apps.proxy.live_proxy.input.http_streamer import find_ts_sync -from apps.proxy.utils import check_user_stream_limits +from apps.proxy.utils import check_user_stream_limits, get_user_active_connections from core.utils import RedisClient from dispatcharr.utils import network_access_allowed from .helpers import ( + TimeshiftCredentials, build_timeshift_candidate_urls, convert_timestamp_to_provider_tz, get_programme_duration, @@ -99,6 +110,18 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) range_header = request.META.get("HTTP_RANGE") channel_logo_id = getattr(channel, "logo_id", None) + redis_client = RedisClient.get_client() + + # One active catch-up session per user+channel: a new request (programme + # jump, or a seek re-request on the same programme) DISPLACES the user's + # previous session on this channel. Its provider slot is released + # synchronously — so the reservation below cannot collide with the dying + # session on max_connections=1 providers — and its generator is stopped + # via the same Redis stop-key the stream-limit path uses. Runs BEFORE the + # stream-limit check so a seek displaces its own predecessor instead of + # being denied when terminate_on_limit_exceeded is off. + _terminate_previous_timeshift_sessions(redis_client, user, channel.id) + # Enforce user stream limits once, before connecting upstream — this # terminates the oldest active stream (live, timeshift, or VOD) via the # Redis stop-key mechanism, same as the live and VOD proxy entry points. @@ -120,6 +143,7 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) # account is a different host and remains safe to try. last_response = None decisive_accounts = set() + capacity_blocked = False for catchup_stream in catchup_streams: m3u_account = catchup_stream.m3u_account if m3u_account is None or m3u_account.account_type != "XC": @@ -131,63 +155,158 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if stream_id_value is None: continue + # Provider profile walk — same ordering as live's Channel.get_stream(): + # active profiles only, default first; an account without an active + # default profile is skipped entirely, mirroring live dispatch. + m3u_profiles = list(m3u_account.profiles.filter(is_active=True)) + default_profile = next((p for p in m3u_profiles if p.is_default), None) + if default_profile is None: + logger.debug( + "Timeshift: account %s has no active default profile, skipping", + m3u_account.id, + ) + continue + profile_walk = [default_profile] + [ + p for p in m3u_profiles if not p.is_default + ] + # Real XC providers index their archive in their OWN local zone (the # server_info.timezone they report on auth — stored on the default # profile). Convert the UTC instant to that zone for the upstream URL — # empirically a provider reads `17-00` as 17:00 LOCAL, not UTC, so # skipping this would seek 1-2h off. This is the ONLY timezone - # conversion in the chain, and it is per-provider. - default_profile = m3u_account.profiles.filter(is_default=True).first() + # conversion in the chain, and it is per-provider (account-level: all + # profiles log into the same server, so the default profile's zone + # applies to every profile of the walk). provider_tz_name = None - if default_profile is not None: - _server_info = (default_profile.custom_properties or {}).get("server_info") or {} - if isinstance(_server_info, dict): - provider_tz_name = _server_info.get("timezone") + _server_info = (default_profile.custom_properties or {}).get("server_info") or {} + if isinstance(_server_info, dict): + provider_tz_name = _server_info.get("timezone") provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) - # Ordered upstream candidates, PATH form first — see - # build_timeshift_candidate_urls() for the full rationale (the QUERY - # form is a last-resort fallback because some providers return LIVE on - # it, ignoring the requested timestamp). - candidate_urls = build_timeshift_candidate_urls( - m3u_account, stream_id_value, provider_timestamp, duration_minutes - ) + # Reserve a provider profile slot BEFORE connecting upstream — the + # same accounting contract live (Channel.get_stream) and VOD follow, + # so the Redis pool counters always match real upstream usage. Walk + # to the next profile only on RESERVATION failure (profile_full / + # credential_full — transient capacity, never ban-class); an upstream + # failure instead moves to the next catch-up STREAM, because probing + # the same server again through an alternate profile is wasteful and + # ban-adjacent. + reserved_profile = None + for profile in profile_walk: + if redis_client is None: + # Without Redis no accounting is possible — proceed unpooled + # rather than blocking playback (stats writes are equally + # fail-open). Deployments always run Redis; this is a dev-only + # path. + reserved_profile = profile + break + reserved, _count, reason = reserve_profile_slot(profile, redis_client) + if reserved: + reserved_profile = profile + break + logger.info( + "Timeshift: profile %s %s on account %s — trying next profile", + profile.id, reason or "unavailable", m3u_account.id, + ) + if reserved_profile is None: + capacity_blocked = True + logger.warning( + "Timeshift: all profiles at capacity on account %s for channel %s", + m3u_account.id, channel.name, + ) + continue + token_stored = False + if redis_client is not None: + token_stored = _store_slot_token( + redis_client, client_id, reserved_profile.id + ) + if not token_stored: + # Redis hiccup right after the INCR: without a token no + # release path could ever free this slot, so release it + # directly NOW (race-free: the session is not yet visible to + # the takeover scan) and report transient unavailability + # instead of streaming unaccounted. + try: + release_profile_slot(reserved_profile.id, redis_client) + except Exception as exc: + logger.error( + "Timeshift: could not release slot for profile %s " + "after token-store failure: %s", reserved_profile.id, exc, + ) + capacity_blocked = True + continue + + # From here until the streaming response owns the slot, ANY failure + # must release the reservation before propagating — otherwise the + # pool counter (which has no TTL) leaks until the next Redis flush. try: - user_agent = m3u_account.get_user_agent().user_agent - except AttributeError: - user_agent = "" + # Build the upstream URLs with the RESERVED profile's credentials, + # resolved the same way live playback does (credential extraction + # via the profile's URL transform). Using the raw account fields + # for a non-default profile would consume that profile's slot + # while authenticating with the default login — double-occupying + # the provider connection the pool thinks is free. + server_url, xc_username, xc_password = get_transformed_credentials( + m3u_account, reserved_profile + ) + creds = TimeshiftCredentials(server_url, xc_username, xc_password) - virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" - m3u_profile_id = default_profile.id if default_profile is not None else None - - if debug: - logger.debug( - "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " - "account=%s provider_sid=%s vid=%s client=%s range=%s", - channel.name, timestamp, provider_tz_name, provider_timestamp, - m3u_account.id, stream_id_value, virtual_channel_id, client_id, - range_header or "(none)", + # Ordered upstream candidates, PATH form first — see + # build_timeshift_candidate_urls() for the full rationale (the + # QUERY form is a last-resort fallback because some providers + # return LIVE on it, ignoring the requested timestamp). + candidate_urls = build_timeshift_candidate_urls( + creds, stream_id_value, provider_timestamp, duration_minutes ) - response = _stream_from_provider( - candidate_urls=candidate_urls, - user_agent=user_agent, - range_header=range_header, - virtual_channel_id=virtual_channel_id, - client_id=client_id, - client_ip=client_ip, - user=user, - channel_display_name=channel.name, - timestamp_utc=timestamp, - channel_logo_id=channel_logo_id, - m3u_profile_id=m3u_profile_id, - debug=debug, - account_id=m3u_account.id, - ) + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" + + if debug: + logger.debug( + "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " + "account=%s profile=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, timestamp, provider_tz_name, provider_timestamp, + m3u_account.id, reserved_profile.id, stream_id_value, + virtual_channel_id, client_id, range_header or "(none)", + ) + + response = _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=reserved_profile.id, + debug=debug, + account_id=m3u_account.id, + redis_client=redis_client, + ) + except Exception: + if token_stored: + # A False return here means someone else (a takeover) already + # consumed the token and released the slot — nothing to do. + _release_slot_token(redis_client, client_id) + raise if response.status_code < 400: + # The reserved slot is now owned by the streaming response: it is + # released (exactly once, via the one-shot token) when the + # generator finishes or the WSGI layer closes the response. return response + # Failed attempt: free the slot before trying the next stream. + _release_slot_token(redis_client, client_id) last_response = response if getattr(response, "timeshift_decisive", False): decisive_accounts.add(m3u_account.id) @@ -202,6 +321,11 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if last_response is not None: return last_response + if capacity_blocked: + # Every eligible stream failed on pool capacity alone (no upstream + # attempt was made) — mirror the VOD proxy's pool-exhausted status so + # clients back off instead of treating it as a permanent failure. + return HttpResponse("No available stream slot", status=503) # Streams existed but none was usable (non-XC accounts / missing stream_id). return HttpResponseBadRequest("Cannot build timeshift URL") @@ -242,6 +366,127 @@ def _user_can_access_channel(user, channel): ) +# --------------------------------------------------------------------------- +# Provider pool slot ownership (one-shot release token) + session takeover +# --------------------------------------------------------------------------- + +# Maps a timeshift client to the profile slot it reserved. Consumed (GET+DEL, +# atomically) by whichever release path runs first — the generator's finally, +# the response-close wrapper, a failed failover attempt, or a takeover by the +# user's next request on the same channel — so the slot is released exactly +# once even across uWSGI workers. Note the TTL does NOT recover the slot: if a +# worker dies mid-session the token eventually expires while the counter (no +# TTL) stays consumed — like the rest of the pool, the recovery for +# crash-leaked counters is the Redis flush at container start +# (scripts/wait_for_redis.py). +_SLOT_TOKEN_KEY = "timeshift_slot:{client_id}" +_SLOT_TOKEN_TTL = 24 * 3600 + + +def _store_slot_token(redis_client, client_id, profile_id): + """Record slot ownership; returns False when the token could not be + written (the caller must then release the reservation itself — without a + token no other path ever could).""" + if redis_client is None: + return False + try: + redis_client.setex( + _SLOT_TOKEN_KEY.format(client_id=client_id), + _SLOT_TOKEN_TTL, + str(profile_id), + ) + return True + except Exception as exc: + logger.warning("Timeshift slot token store failed for %s: %s", client_id, exc) + return False + + +def _release_slot_token(redis_client, client_id): + """Release the profile slot owned by *client_id*, exactly once. + + GET+DEL inside a MULTI/EXEC transaction: concurrent release attempts are + serialized by Redis, so only the one that actually deletes a live token + decrements the pool counters. Returns True when this call performed the + release. + """ + if redis_client is None: + return False + key = _SLOT_TOKEN_KEY.format(client_id=client_id) + try: + pipe = redis_client.pipeline(transaction=True) + pipe.get(key) + pipe.delete(key) + token_value, deleted = pipe.execute() + if not token_value or not deleted: + return False + release_profile_slot(int(token_value), redis_client) + return True + except Exception as exc: + logger.warning("Timeshift slot release failed for %s: %s", client_id, exc) + return False + + +def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): + """Displace the user's previous catch-up session(s) on this channel. + + For each of the user's active timeshift sessions whose virtual channel id + belongs to *channel_id*: release its provider slot NOW (synchronously — + the dying generator's own release becomes a no-op thanks to the one-shot + token), drop its stats keys so the stream-limit count no longer includes + it, and set the stop key its generator polls on the 5-second heartbeat. + The accounting is therefore correct immediately; only the old TCP socket + closes lazily (≤ ~5 s). + """ + if redis_client is None or user is None: + return + prefix = f"timeshift_{channel_id}_" + try: + for conn in get_user_active_connections(user.id): + if conn.get("type") != "timeshift": + continue + media_id = str(conn.get("media_id") or "") + if not media_id.startswith(prefix): + continue + old_client_id = conn.get("client_id") + logger.info( + "Timeshift takeover: displacing session %s on %s", + old_client_id, media_id, + ) + _release_slot_token(redis_client, old_client_id) + _unregister_stats_client(redis_client, media_id, old_client_id) + stop_key = RedisKeys.client_stop(media_id, old_client_id) + redis_client.setex(stop_key, 60, "true") + except Exception as exc: + logger.warning("Timeshift takeover check failed: %s", exc) + + +class _SlotReleasingStream: + """Iterator wrapper whose close() always releases the reserved slot. + + Django registers ``streaming_content.close`` in the response's resource + closers, and the WSGI layer guarantees close() runs — including when the + client disconnects before the first chunk, in which case the generator + never starts and its ``finally`` would never execute. The one-shot token + makes the duplicate call from an already-finished generator a no-op. + """ + + def __init__(self, generator, on_close): + self._generator = generator + self._on_close = on_close + + def __iter__(self): + return self + + def __next__(self): + return next(self._generator) + + def close(self): + try: + self._generator.close() + finally: + self._on_close() + + # --------------------------------------------------------------------------- # Stats integration (direct Redis writes, no ClientManager instance) # --------------------------------------------------------------------------- @@ -406,6 +651,7 @@ def _stream_from_provider( m3u_profile_id, debug, account_id=None, + redis_client=None, ): # Use 256 KB chunks: amortises per-yield uWSGI/gevent overhead. chunk_size = max(ConfigHelper.chunk_size(), 262144) @@ -525,7 +771,6 @@ def _stream_from_provider( content_range = upstream.headers.get("Content-Range", "") status = upstream.status_code - redis_client = RedisClient.get_client() _register_stats_client( redis_client, virtual_channel_id, @@ -604,9 +849,23 @@ def _stream_from_provider( except Exception: pass _unregister_stats_client(redis_client, virtual_channel_id, client_id) + # Free the provider pool slot this session reserved. One-shot + # token: a no-op if a takeover (the user's next request on this + # channel) already released it. + _release_slot_token(redis_client, client_id) + # The wrapper guarantees session teardown even when the generator never + # starts (client gone before the first chunk — its finally would then + # never run): Django registers the iterator's close() as a resource + # closer, which WSGI always invokes. Both calls are idempotent, so the + # duplicate from a normally-finished generator is harmless. + def _close_session(): + _unregister_stats_client(redis_client, virtual_channel_id, client_id) + _release_slot_token(redis_client, client_id) + + stream_iter = _SlotReleasingStream(stream_generator(), _close_session) response = StreamingHttpResponse( - stream_generator(), + stream_iter, content_type=content_type, status=status, ) From a29704af55015597e4f5215b300a4b584d42f892 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:40:10 -0500 Subject: [PATCH 13/64] fix(channels): account for group_override in the auto-sync range-conflict check The range-conflict warning classified each channel in the configured range as either this config's own auto-sync output or a real conflict, comparing each occupant against the source group. When a group sets a group_override, auto-sync creates its channels in the override target group, so the config's own channels failed that comparison and were misclassified as a conflict. - Add effectiveSyncGroupId to resolve the group the sync's channels actually land in (the group_override target when set, otherwise the source group). - Compare occupants against that effective target, so the config's own output is recognized while genuine conflicts (manual channels, channels from another account, channels in a different group, user-pinned numbers) still warn. - Add Vitest coverage for the helper, the override case, and an over-suppression guard, plus a backend test asserting the numbers-in-range endpoint reports the override target group. --- apps/m3u/tests/test_sync_correctness.py | 28 +++++ .../src/components/forms/LiveGroupFilter.jsx | 13 +- .../src/utils/forms/LiveGroupFilterUtils.js | 12 ++ .../__tests__/LiveGroupFilterUtils.test.js | 111 ++++++++++++++++++ 4 files changed, 161 insertions(+), 3 deletions(-) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 3cc24942..8ae09fa8 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -644,6 +644,34 @@ class NumbersInRangeLookupTests(TestCase): ) self.assertFalse(occupant["has_channel_number_override"]) + def test_group_override_channel_reports_target_group(self): + # When auto-sync routes channels into a different group via + # group_override, the occupant's channel_group_id is the override + # target, not the source group being configured. The frontend relies + # on this to recognize override-routed channels as the config's own + # output (effectiveSyncGroupId), so the warning does not flag them. + account = _make_account() + source = _make_group(name="SourceGrp") + target = _make_group(name="TargetGrp") + Channel.objects.create( + name="Routed", + channel_number=3210, + channel_group=target, + auto_created=True, + auto_created_by=account, + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=3210&end=3210" + ) + + occupant = response.data["occupants"][0] + self.assertEqual(occupant["channel_group_id"], target.id) + self.assertNotEqual(occupant["channel_group_id"], source.id) + self.assertTrue(occupant["auto_created"]) + self.assertEqual(occupant["auto_created_by_account_id"], account.id) + def test_manual_channel_exposed_with_auto_created_false(self): # Manual channels are always a real collision worth surfacing. # The response must flag them with auto_created=False and a null diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index 626ec55e..5ed2be94 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -34,6 +34,7 @@ import { getRegexOptions, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, isGroupVisible, rangeFor, } from '../../utils/forms/LiveGroupFilterUtils.js'; @@ -131,7 +132,12 @@ const LiveGroupFilter = ({ // (in-memory range overlap with sibling groups) sources so the sweep // can refresh form-overlap synchronously without firing HTTP for // groups that did not change. - const scheduleConflictScan = (groupId, rawStart, rawEnd) => { + const scheduleConflictScan = ( + groupId, + rawStart, + rawEnd, + expectedGroupId = groupId + ) => { if (conflictTimersRef.current[groupId]) { clearTimeout(conflictTimersRef.current[groupId]); } @@ -156,7 +162,7 @@ const LiveGroupFilter = ({ ? result.occupants : []; const unexpected = occupants.filter( - (o) => !isExpectedOccupantForGroup(o, groupId, playlist) + (o) => !isExpectedOccupantForGroup(o, expectedGroupId, playlist) ); setConflictSource(groupId, 'occupant', unexpected.length > 0); } catch (e) { @@ -221,7 +227,8 @@ const LiveGroupFilter = ({ scheduleConflictScan( g.channel_group, range.startRaw, - g.auto_sync_channel_end + g.auto_sync_channel_end, + effectiveSyncGroupId(g) ); } } diff --git a/frontend/src/utils/forms/LiveGroupFilterUtils.js b/frontend/src/utils/forms/LiveGroupFilterUtils.js index a1372887..3cb62bf5 100644 --- a/frontend/src/utils/forms/LiveGroupFilterUtils.js +++ b/frontend/src/utils/forms/LiveGroupFilterUtils.js @@ -57,6 +57,18 @@ export const isExpectedOccupantForGroup = ( ); }; +// The group the sync's own channels actually land in. A group_override +// routes auto-created channels into a different ChannelGroup, so the +// conflict check must recognize occupants of that target group as this +// config's own output rather than flagging them against the source group. +export const effectiveSyncGroupId = (group) => { + const override = group?.custom_properties?.group_override; + if (override !== undefined && override !== null && override !== '') { + return Number(override); + } + return group?.channel_group; +}; + export const rangeFor = (g) => { if (!g.enabled || !g.auto_channel_sync) return null; const mode = g.custom_properties?.channel_numbering_mode || 'fixed'; diff --git a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js index feb878e1..86874f23 100644 --- a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js +++ b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js @@ -4,6 +4,7 @@ import { getChannelsInRange, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, rangeFor, abortTimers, getRegexOptions, @@ -227,6 +228,116 @@ describe('LiveGroupFilterUtils', () => { }); }); + // ── effectiveSyncGroupId ─────────────────────────────────────────────────── + describe('effectiveSyncGroupId', () => { + it('returns the source channel_group when there is no override', () => { + expect(effectiveSyncGroupId(makeGroup({ channel_group: 7 }))).toBe(7); + }); + + it('returns the group_override target when set', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('coerces a string-stored group_override to a number', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '9' }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('falls back to the source group when group_override is blank', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '' }, + }); + expect(effectiveSyncGroupId(group)).toBe(7); + }); + + // Regression guard for the group-override range-conflict false positive: + // the auto-sync's own channels land in the override target group, so + // comparing against the source group (pre-fix) flags them as a conflict, + // while comparing against the effective target recognizes them as this + // config's own output. + it("makes group-override occupants count as this group's own", () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const occupant = makeOccupant({ channel_group_id: 9 }); + // Pre-fix comparison (source group) treats own channels as a conflict. + expect( + isExpectedOccupantForGroup( + occupant, + group.channel_group, + makePlaylist() + ) + ).toBe(false); + // Comparing against the effective target recognizes them as expected. + expect( + isExpectedOccupantForGroup( + occupant, + effectiveSyncGroupId(group), + makePlaylist() + ) + ).toBe(true); + }); + + // Guards against over-suppression: resolving the effective target group + // must still surface genuine collisions in an override config's range. + // Only the config's own output (auto-created, this account, in the + // target group, unpinned) is excluded. + it('still flags genuine collisions in a group-override config', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const target = effectiveSyncGroupId(group); + // Manual channel sitting in the range. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 9, auto_created: false }), + target, + makePlaylist() + ) + ).toBe(false); + // Auto-created by a different account. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + auto_created_by_account_id: 999, + }), + target, + makePlaylist() + ) + ).toBe(false); + // A channel in a different group than the override target. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 123 }), + target, + makePlaylist() + ) + ).toBe(false); + // A user-pinned channel number. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + has_channel_number_override: true, + }), + target, + makePlaylist() + ) + ).toBe(false); + }); + }); + // ── rangeFor ────────────────────────────────────────────────────────────── describe('rangeFor', () => { it('returns null when group is disabled', () => { From 3868f02c45e118acc9965745115bf4289eea2b24 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 10:57:06 -0500 Subject: [PATCH 14/64] enhancement(epg): optimize EPG export performance and database interactions - Streamlined `generate_epg()` to incrementally stream EPG data without loading the entire guide, improving memory efficiency. - Reduced database load by deferring the fetching of large `programme_index` blobs, enhancing response times for EPG generation. - Introduced a composite index on `(epg_id, id)` in `ProgramData` to optimize query performance during EPG exports. - Updated tests to ensure proper functionality and performance of new features. --- CHANGELOG.md | 6 + .../0025_programdata_epg_id_index.py | 40 ++ apps/epg/models.py | 5 + apps/output/epg_chunk_cache.py | 230 +++++++ apps/output/test_epg_chunk_cache.py | 187 ++++++ apps/output/tests.py | 292 +++++++-- apps/output/views.py | 573 ++++++++++-------- core/tests.py | 13 +- core/utils.py | 19 + 9 files changed, 1065 insertions(+), 300 deletions(-) create mode 100644 apps/epg/migrations/0025_programdata_epg_id_index.py create mode 100644 apps/output/epg_chunk_cache.py create mode 100644 apps/output/test_epg_chunk_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0517493d..3315456b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. + ### Changed - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. diff --git a/apps/epg/migrations/0025_programdata_epg_id_index.py b/apps/epg/migrations/0025_programdata_epg_id_index.py new file mode 100644 index 00000000..1a350b20 --- /dev/null +++ b/apps/epg/migrations/0025_programdata_epg_id_index.py @@ -0,0 +1,40 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class AddIndexConcurrentlyIfPostgres(AddIndexConcurrently): + """Create the index CONCURRENTLY on PostgreSQL (no table lock on large + tables), falling back to a normal blocking AddIndex on other backends + such as the sqlite dev/test fallback.""" + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_forwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_forwards( + self, app_label, schema_editor, from_state, to_state + ) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_backwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_backwards( + self, app_label, schema_editor, from_state, to_state + ) + + +class Migration(migrations.Migration): + # CREATE INDEX CONCURRENTLY cannot run inside a transaction. + atomic = False + + dependencies = [ + ('epg', '0024_remove_epgsource_api_key_epgsource_password_and_more'), + ] + + operations = [ + AddIndexConcurrentlyIfPostgres( + model_name='programdata', + index=models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 025945db..04c6072c 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -153,6 +153,11 @@ class ProgramData(models.Model): program_id = models.CharField(max_length=64, null=True, blank=True, help_text='Schedules Direct programID (e.g. EP123456789). Null for XMLTV sources.') custom_properties = models.JSONField(default=dict, blank=True, null=True) + class Meta: + indexes = [ + models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ] + def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" diff --git a/apps/output/epg_chunk_cache.py b/apps/output/epg_chunk_cache.py new file mode 100644 index 00000000..92ea6eeb --- /dev/null +++ b/apps/output/epg_chunk_cache.py @@ -0,0 +1,230 @@ +"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" + +import logging +import time + +from django.http import StreamingHttpResponse + +logger = logging.getLogger(__name__) + +STATUS_BUILDING = "building" +STATUS_READY = "ready" +STATUS_ERROR = "error" + +DEFAULT_CACHE_TTL = 300 +DEFAULT_LOCK_TTL = 120 +DEFAULT_POLL_INTERVAL = 0.05 +DEFAULT_MAX_FOLLOWER_WAIT = 600 + + +def _chunks_key(base_key): + return f"{base_key}:chunks" + + +def _ready_key(base_key): + return f"{base_key}:ready" + + +def _status_key(base_key): + return f"{base_key}:status" + + +def _lock_key(base_key): + return f"{base_key}:lock" + + +def _decode_chunk(chunk): + if chunk is None: + return None + if isinstance(chunk, bytes): + return chunk.decode("utf-8") + return chunk + + +def _encode_chunk(chunk): + if isinstance(chunk, bytes): + return chunk + return chunk.encode("utf-8") + + +def _poll_wait(interval): + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.sleep(interval) + return + except ImportError: + pass + time.sleep(interval) + + +def _get_redis(): + from django_redis import get_redis_connection + + return get_redis_connection("default") + + +def _get_status(redis, base_key): + raw = redis.get(_status_key(base_key)) + if raw is None: + return None + return _decode_chunk(raw) + + +def _clear_build_keys(redis, base_key): + redis.delete( + _chunks_key(base_key), + _status_key(base_key), + _ready_key(base_key), + _lock_key(base_key), + ) + + +def _try_acquire_lock(redis, base_key, lock_ttl): + return bool(redis.set(_lock_key(base_key), "1", nx=True, ex=lock_ttl)) + + +def _refresh_build_ttl(redis, base_key, lock_ttl): + redis.expire(_lock_key(base_key), lock_ttl) + redis.expire(_status_key(base_key), lock_ttl) + redis.expire(_chunks_key(base_key), lock_ttl) + + +def _stream_ready(redis, base_key): + offset = 0 + chunks_key = _chunks_key(base_key) + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is None: + break + yield _decode_chunk(chunk) + offset += 1 + + +def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): + """Leader: stream to client and append each chunk to Redis.""" + chunks_key = _chunks_key(base_key) + status_key = _status_key(base_key) + try: + from django.core.cache import cache as django_cache + + django_cache.delete(base_key) # legacy monolithic entry + redis.delete(chunks_key, _ready_key(base_key)) + redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + for chunk in source(): + redis.rpush(chunks_key, _encode_chunk(chunk)) + _refresh_build_ttl(redis, base_key, lock_ttl) + yield chunk + redis.set(status_key, STATUS_READY) + redis.set(_ready_key(base_key), "1") + redis.expire(chunks_key, cache_ttl) + redis.expire(status_key, cache_ttl) + redis.expire(_ready_key(base_key), cache_ttl) + logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + except Exception: + logger.exception("EPG cache build failed for %s", base_key) + redis.delete(chunks_key) + redis.set(status_key, STATUS_ERROR, ex=60) + raise + finally: + redis.delete(_lock_key(base_key)) + + +def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, max_follower_wait): + """Follower: read chunks as the leader writes them.""" + offset = 0 + deadline = time.monotonic() + max_follower_wait + idle_polls = 0 + chunks_key = _chunks_key(base_key) + lock_key = _lock_key(base_key) + + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is not None: + idle_polls = 0 + yield _decode_chunk(chunk) + offset += 1 + continue + + status = _get_status(redis, base_key) + if status == STATUS_READY: + break + + if status == STATUS_ERROR: + _clear_build_keys(redis, base_key) + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + raise RuntimeError("EPG cache build failed") + + if time.monotonic() >= deadline: + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + logger.warning("EPG cache follower timed out after partial read for %s", base_key) + break + + lock_active = bool(redis.exists(lock_key)) + if status != STATUS_BUILDING and not lock_active: + idle_polls += 1 + if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): + if _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache leader lost; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + else: + idle_polls = 0 + + _poll_wait(poll_interval) + + +def stream_epg_response( + cache_key, + source, + *, + cache_ttl=DEFAULT_CACHE_TTL, + lock_ttl=DEFAULT_LOCK_TTL, + poll_interval=DEFAULT_POLL_INTERVAL, + max_follower_wait=DEFAULT_MAX_FOLLOWER_WAIT, + redis=None, +): + """ + Stream XMLTV EPG output with single-flight Redis chunk caching. + + ``source`` must be a callable returning a chunk iterator. Only the leader + invokes it; followers read chunks already written to Redis. + """ + if redis is None: + redis = _get_redis() + + if redis.get(_ready_key(cache_key)): + logger.debug("Serving EPG from chunk cache") + stream = _stream_ready(redis, cache_key) + else: + status = _get_status(redis, cache_key) + if status == STATUS_ERROR: + _clear_build_keys(redis, cache_key) + + if _try_acquire_lock(redis, cache_key, lock_ttl): + logger.debug("Building EPG (cache leader)") + stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) + else: + logger.debug("Following in-flight EPG build") + stream = _stream_follow( + redis, + cache_key, + source, + cache_ttl, + lock_ttl, + poll_interval, + max_follower_wait, + ) + + response = StreamingHttpResponse(stream, content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_epg_chunk_cache.py new file mode 100644 index 00000000..6ca217af --- /dev/null +++ b/apps/output/test_epg_chunk_cache.py @@ -0,0 +1,187 @@ +import threading +import time +from unittest import TestCase + +from apps.output.epg_chunk_cache import ( + STATUS_BUILDING, + STATUS_READY, + _chunks_key, + _lock_key, + _ready_key, + _status_key, + stream_epg_response, +) + + +class FakeRedis: + """Minimal Redis stand-in for chunk-cache unit tests.""" + + def __init__(self): + self._strings = {} + self._lists = {} + self._expires_at = {} + + def _purge_expired(self): + now = time.monotonic() + expired = [key for key, deadline in self._expires_at.items() if deadline <= now] + for key in expired: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def get(self, key): + self._purge_expired() + return self._strings.get(key) + + def set(self, key, value, nx=False, ex=None): + self._purge_expired() + if nx and key in self._strings: + return None + self._strings[key] = value + if ex is not None: + self._expires_at[key] = time.monotonic() + ex + return True + + def delete(self, *keys): + for key in keys: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def exists(self, key): + self._purge_expired() + return key in self._strings or key in self._lists + + def expire(self, key, ttl): + if key in self._strings or key in self._lists: + self._expires_at[key] = time.monotonic() + ttl + return True + + def rpush(self, key, value): + self._lists.setdefault(key, []).append(value) + + def lindex(self, key, offset): + items = self._lists.get(key, []) + if offset < len(items): + return items[offset] + return None + + def llen(self, key): + return len(self._lists.get(key, [])) + + +def _consume(response): + return b"".join(response.streaming_content).decode("utf-8") + + +class EPGChunkCacheTests(TestCase): + def test_leader_caches_chunks_and_sets_ready(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, [1]) + self.assertEqual(redis.get(_ready_key("epg:test")), "1") + self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) + self.assertFalse(redis.exists(_lock_key("epg:test"))) + + def test_cache_hit_skips_source(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + _consume(stream_epg_response("epg:test", source, redis=redis)) + calls.clear() + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, []) + + def test_follower_reads_leader_chunks_without_rebuilding(self): + redis = FakeRedis() + base = "epg:follow" + leader_started = threading.Event() + rebuild_calls = [] + + def slow_source(): + rebuild_calls.append(1) + leader_started.set() + yield "a" + time.sleep(0.05) + yield "b" + + def forbidden_source(): + rebuild_calls.append(2) + yield "SHOULD_NOT_RUN" + + def leader(): + _consume( + stream_epg_response( + base, + slow_source, + redis=redis, + poll_interval=0.01, + ) + ) + + leader_thread = threading.Thread(target=leader) + leader_thread.start() + leader_started.wait(timeout=5) + follower_body = _consume( + stream_epg_response( + base, + forbidden_source, + redis=redis, + poll_interval=0.01, + ) + ) + leader_thread.join(timeout=5) + + self.assertEqual(follower_body, "ab") + self.assertEqual(rebuild_calls, [1]) + + def test_only_one_leader_when_two_clients_start_together(self): + redis = FakeRedis() + build_calls = [] + barrier = threading.Barrier(2) + results = {} + + def source(): + build_calls.append(threading.current_thread().name) + yield "x" + + def worker(): + barrier.wait() + results[threading.current_thread().name] = _consume( + stream_epg_response( + "epg:race", + source, + redis=redis, + poll_interval=0.01, + ) + ) + + threads = [ + threading.Thread(target=worker, name="t1"), + threading.Thread(target=worker, name="t2"), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + self.assertEqual(results["t1"], "x") + self.assertEqual(results["t2"], "x") + self.assertEqual(len(build_calls), 1) diff --git a/apps/output/tests.py b/apps/output/tests.py index b1c7d589..6e2ab395 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,31 +1,114 @@ -from django.test import TestCase, Client +from django.test import TestCase, Client, SimpleTestCase from django.urls import reverse -from apps.channels.models import Channel, ChannelGroup +from unittest.mock import patch +from uuid import uuid4 +from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource import xml.etree.ElementTree as ET +from datetime import timedelta + + +def _response_text(response): + """Read body from HttpResponse or StreamingHttpResponse.""" + if getattr(response, "streaming", False): + return b"".join(response.streaming_content).decode() + return response.content.decode() + + +def _epg_response_without_redis(cache_key, source, **kwargs): + """Test helper: stream EPG directly without Redis chunk caching.""" + from django.http import StreamingHttpResponse + + response = StreamingHttpResponse(source(), content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response + + +class OutputEndpointTestMixin: + """Isolate HTTP endpoint tests from network ACL, logging, DB teardown, and Redis.""" -class OutputM3UTest(TestCase): def setUp(self): + super().setUp() + self._network_patch = patch( + "apps.output.views.network_access_allowed", + return_value=True, + ) + self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._log_event_patch = patch("apps.output.views.log_system_event") + self._close_db_patch = patch("django.db.close_old_connections") + self._epg_cache_patch = patch( + "apps.output.views.stream_epg_response", + side_effect=_epg_response_without_redis, + ) + self._network_patch.start() + self._epg_teardown_patch.start() + self._log_event_patch.start() + self._close_db_patch.start() + self._epg_cache_patch.start() + + def tearDown(self): + from django.core.cache import cache + + cache.clear() + self._epg_cache_patch.stop() + self._close_db_patch.stop() + self._log_event_patch.stop() + self._epg_teardown_patch.stop() + self._network_patch.stop() + super().tearDown() + + def _create_isolated_profile(self, prefix): + """New profiles auto-include every channel via signal; clear that for tests.""" + profile = ChannelProfile.objects.create(name=f"{prefix}-{uuid4().hex[:8]}") + ChannelProfileMembership.objects.filter(channel_profile=profile).delete() + return profile + + def _add_channel_to_profile(self, profile, group, **kwargs): + channel = Channel.objects.create(channel_group=group, **kwargs) + ChannelProfileMembership.objects.create( + channel_profile=profile, + channel=channel, + enabled=True, + ) + return channel + + +class OutputM3UTest(OutputEndpointTestMixin, TestCase): + def setUp(self): + super().setUp() self.client = Client() - + self.group = ChannelGroup.objects.create(name=f"M3U Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("m3u") + self._add_channel_to_profile( + self.profile, + self.group, + channel_number=1.0, + name="Test M3U Channel", + ) + + def _m3u_url(self): + return reverse("output:m3u_endpoint", kwargs={"profile_name": self.profile.name}) + def test_generate_m3u_response(self): """ Test that the M3U endpoint returns a valid M3U file. """ - url = reverse('output:generate_m3u') - response = self.client.get(url) + response = self.client.get(self._m3u_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) self.assertIn("#EXTM3U", content) def test_generate_m3u_response_post_empty_body(self): """ Test that a POST request with an empty body returns 200 OK. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data=None, content_type='application/x-www-form-urlencoded') - content = response.content.decode() + response = self.client.post( + self._m3u_url(), + data=None, + content_type="application/x-www-form-urlencoded", + ) + content = _response_text(response) self.assertEqual(response.status_code, 200, "POST with empty body should return 200 OK") self.assertIn("#EXTM3U", content) @@ -34,35 +117,40 @@ class OutputM3UTest(TestCase): """ Test that a POST request with a non-empty body returns 403 Forbidden. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data={'evilstring': 'muhahaha'}) + response = self.client.post(self._m3u_url(), data={"evilstring": "muhahaha"}) self.assertEqual(response.status_code, 403, "POST with body should return 403 Forbidden") - self.assertIn("POST requests with body are not allowed, body is:", response.content.decode()) + self.assertIn("POST requests with body are not allowed", _response_text(response)) -class OutputEPGXMLEscapingTest(TestCase): +class OutputEPGXMLEscapingTest(OutputEndpointTestMixin, TestCase): """Test XML escaping of channel_id attributes in EPG generation""" def setUp(self): + super().setUp() self.client = Client() - self.group = ChannelGroup.objects.create(name="Test Group") + self.group = ChannelGroup.objects.create(name=f"Test Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("epg-xml") + + def _add_channel(self, **kwargs): + return self._add_channel_to_profile(self.profile, self.group, **kwargs) + + def _epg_url(self, query="tvg_id_source=tvg_id&days=0&prev_days=0"): + base = reverse("output:epg_endpoint", kwargs={"profile_name": self.profile.name}) + return f"{base}?{query}" def test_channel_id_with_ampersand(self): """Test channel ID with ampersand is properly escaped""" - channel = Channel.objects.create( + self._add_channel( channel_number=1.0, name="Test Channel", tvg_id="News & Sports", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) # Should contain escaped ampersand self.assertIn('id="News & Sports"', content) @@ -76,17 +164,15 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_angle_brackets(self): """Test channel ID with < and > characters""" - channel = Channel.objects.create( + self._add_channel( channel_number=2.0, name="HD Channel", tvg_id="Channel ", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Channel <HD>"', content) try: @@ -96,23 +182,28 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_all_special_chars(self): """Test channel ID with all XML special characters""" - channel = Channel.objects.create( + expected_id = 'Test & "Special" ' + self._add_channel( channel_number=3.0, name="Complex Channel", - tvg_id='Test & "Special" ', - channel_group=self.group + tvg_id=expected_id, ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Test & "Special" <Chars>"', content) try: tree = ET.fromstring(content) - # Verify we can find the channel with correct ID in parsed tree - channel_elem = tree.find('.//channel[@id="Test & \\"Special\\" "]') + channel_elem = next( + ( + elem + for elem in tree.findall(".//channel") + if elem.get("id") == expected_id + ), + None, + ) self.assertIsNotNone(channel_elem) except ET.ParseError as e: self.fail(f"Generated EPG with all special chars is not valid XML: {e}") @@ -121,25 +212,144 @@ class OutputEPGXMLEscapingTest(TestCase): """Test that programme elements also have escaped channel attributes""" epg_source = EPGSource.objects.create(name="Test EPG", source_type="dummy") epg_data = EPGData.objects.create(name="Test EPG Data", epg_source=epg_source) - channel = Channel.objects.create( + self._add_channel( channel_number=4.0, name="Program Test", tvg_id="News & Sports", epg_data=epg_data, - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) # Check programme elements have escaped channel attributes self.assertIn('channel="News & Sports"', content) try: tree = ET.fromstring(content) - programmes = tree.findall('.//programme[@channel="News & Sports"]') + programmes = [ + programme + for programme in tree.findall(".//programme") + if programme.get("channel") == "News & Sports" + ] self.assertGreater(len(programmes), 0) except ET.ParseError as e: self.fail(f"Generated EPG with programme elements is not valid XML: {e}") + + def test_programmes_emitted_in_start_time_order(self): + """Programmes for a channel are emitted in start_time order, not insert order.""" + from django.utils import timezone + from apps.epg.models import ProgramData + + epg_source = EPGSource.objects.create(name="Real EPG", source_type="xmltv") + epg_data = EPGData.objects.create(name="Station", epg_source=epg_source, tvg_id="station1") + self._add_channel( + channel_number=149.0, + name="Food Network", + tvg_id="station1", + epg_data=epg_data, + ) + now = timezone.now() + # Insert out of chronological order so id order != start_time order. + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=3), + end_time=now + timedelta(days=3, hours=1), + title="Third", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=1), + end_time=now + timedelta(days=1, hours=1), + title="First", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=2), + end_time=now + timedelta(days=2, hours=1), + title="Second", + tvg_id="station1", + ) + + content = _response_text(self.client.get(self._epg_url("tvg_id_source=tvg_id&days=7"))) + + self.assertLess(content.find('First'), content.find('Second')) + self.assertLess(content.find('Second'), content.find('Third')) + + +class OutputEPGCustomDummyTest(TestCase): + """Custom dummy EPG must not fall back to default when pattern matched but event is outside window.""" + + def setUp(self): + self.group = ChannelGroup.objects.create(name="Sports Group") + + def test_custom_dummy_outside_window_fills_with_ended_programmes(self): + from django.utils import timezone + from apps.output.views import generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + channel_name = ( + "NHL 01: Washington Capitals vs Philadelphia Flyers @ April 16 07:30 PM ET" + ) + now = timezone.now() + lookback = now - timedelta(days=7) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=7, + epg_source=epg_source, + export_lookback=lookback, + export_cutoff=now + timedelta(days=7), + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all(p['end_time'] >= lookback for p in programs), + "All programmes should fall inside the export window", + ) + self.assertTrue( + any('Ended' in p['description'] for p in programs), + "Past events outside the window should still show ended filler", + ) + for program in programs: + start = program['start_time'] + self.assertEqual(start.second, 0) + self.assertEqual(start.microsecond, 0) + self.assertIn( + start.minute, (0, 30), + "Filler programmes should start on half-hour boundaries", + ) + self.assertGreaterEqual(programs[0]['start_time'], lookback) + + +class OutputEPGHelperTest(SimpleTestCase): + def test_ceil_to_half_hour_on_boundary(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=30, second=0, microsecond=0) + self.assertEqual(_ceil_to_half_hour(dt), dt) + + def test_ceil_to_half_hour_rounds_up(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=17, second=42, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) diff --git a/apps/output/views.py b/apps/output/views.py index dba35951..a0875c7f 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,9 +25,55 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib +from apps.output.epg_chunk_cache import stream_epg_response logger = logging.getLogger(__name__) +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -112,6 +158,10 @@ def generate_m3u(request, profile_name=None, user=None): # Check if this is a POST request and the body is not empty (which we don't want to allow) logger.debug("Generating M3U for profile: %s, user: %s, method: %s", profile_name, user.username if user else "Anonymous", request.method) + if request.method == "POST" and request.body: + if request.body.decode() != '{}': + return HttpResponseForbidden("POST requests with body are not allowed.") + # Check cache for recent identical request (helps with double-GET from browsers) from django.core.cache import cache cache_params = f"{profile_name or 'all'}:{user.username if user else 'anonymous'}:{request.GET.urlencode()}" @@ -123,10 +173,6 @@ def generate_m3u(request, profile_name=None, user=None): response = HttpResponse(cached_content, content_type="audio/x-mpegurl") response["Content-Disposition"] = 'attachment; filename="channels.m3u"' return response - # Check if this is a POST request with data (which we don't want to allow) - if request.method == "POST" and request.body: - if request.body.decode() != '{}': - return HttpResponseForbidden("POST requests with body are not allowed.") if user is not None: if user.user_level < 10: @@ -409,7 +455,15 @@ def generate_fallback_programs(channel_id, channel_name, now, num_days, program_ return programs -def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4, epg_source=None): +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): """ Generate dummy EPG programs for channels. @@ -435,29 +489,26 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: custom_programs = generate_custom_dummy_programs( channel_id, channel_name, now, num_days, - epg_source.custom_properties + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, ) - # If custom generation succeeded, return those programs - # If it returned empty (pattern didn't match), check for custom fallback templates - if custom_programs: + if custom_programs is not None: return custom_programs - else: - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # Check if custom fallback templates are provided - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # If custom fallback templates exist, use them instead of default - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - else: - logger.info(f"No custom fallback templates found, using default dummy EPG") + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") # Default humorous program descriptions based on time of day time_descriptions = { @@ -531,7 +582,15 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length return programs -def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, custom_properties): +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): """ Generate programs using custom dummy EPG regex patterns. @@ -616,7 +675,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust if not title_pattern: logger.warning(f"No title_pattern in custom_properties, falling back to default") - return [] # Return empty, will use default + return None logger.debug(f"Title pattern from DB: {repr(title_pattern)}") @@ -633,7 +692,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust except Exception as e: logger.error(f"Invalid title regex pattern after conversion: {e}") logger.error(f"Pattern was: {repr(title_pattern)}") - return [] + return None time_regex = None if time_pattern: @@ -665,7 +724,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust title_match = title_regex.search(channel_name) if not title_match: logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return [] # Return empty, will use default + return None groups = title_match.groupdict() logger.debug(f"Title pattern matched. Groups: {groups}") @@ -917,57 +976,93 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust iterations = num_days for day in range(iterations): - # Start from current time (like standard dummy) instead of midnight - # This ensures programs appear in the guide's current viewing window - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - - if time_info: - # We have an extracted event time - this is when the MAIN event starts - # The extracted time is in the SOURCE timezone (e.g., 8PM ET) - # We need to convert it to UTC for storage - - # Determine which date to use - if date_info: - # Use the extracted date from the channel title - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'] - ).date() - logger.debug(f"Using extracted date: {current_date}") - else: - # No date extracted, use day offset from current time in SOURCE timezone - # This ensures we calculate "today" in the event's timezone, not UTC - # For example: 8:30 PM Central (1:30 AM UTC next day) for a 10 PM ET event - # should use today's date in ET, not tomorrow's date in UTC - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - # Create a naive datetime (no timezone info) representing the event in source timezone + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() event_start_naive = datetime.combine( current_date, datetime.min.time().replace( hour=time_info['hour'], - minute=time_info['minute'] - ) + minute=time_info['minute'], + ), ) - - # Use pytz to localize the naive datetime to the source timezone - # This automatically handles DST! try: - event_start_local = source_tz.localize(event_start_naive) - # Convert to UTC - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) except Exception as e: logger.error(f"Error localizing time to {source_tz}: {e}") - # Fallback: treat as UTC event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + # Pre-generate the main event title and description for reuse if title_template: main_event_title = format_template(title_template, all_groups) @@ -994,15 +1089,13 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) - is_event_day = (day == 0) + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window if is_event_day and not event_happened: - # This is THE day the event happens - # Fill programs BEFORE the event current_time = day_start - while current_time < event_start_utc: + while current_time < event_start_utc and current_time < day_end: program_start_utc = current_time program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) @@ -1084,8 +1177,8 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust event_happened = True - # Fill programs AFTER the event until end of day - current_time = event_end_utc + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) while current_time < day_end: program_start_utc = current_time @@ -1325,18 +1418,10 @@ def generate_dummy_epg( def generate_epg(request, profile_name=None, user=None): """ - Dynamically generate an XMLTV (EPG) file using streaming response to handle keep-alives. + Dynamically generate an XMLTV (EPG) file using a streaming response. Since the EPG data is stored independently of Channels, we group programmes by their associated EPGData record. - This version filters data based on the 'days' parameter and sends keep-alives during processing. """ - # Check cache for recent identical request (helps with double-GET from browsers) - from django.core.cache import cache - # Resolve all effective parameter values once here so they are reused for both - # the cache key and inside epg_generator() via closure. - # The cache key is built from resolved values only — not from the raw query string — - # so equivalent requests (e.g. days=7 via URL param vs. user default of 7) share - # the same cache entry regardless of how the value was supplied. user_custom = (user.custom_properties or {}) if user else {} try: num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) @@ -1356,21 +1441,13 @@ def generate_epg(request, profile_name=None, user=None): ) content_cache_key = f"epg_content:{cache_params}" - cached_content = cache.get(content_cache_key) - if cached_content: - logger.debug("Serving EPG from cache") - response = HttpResponse(cached_content, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response - def epg_generator(): """Generator function that yields EPG data with keep-alives during processing.""" - xml_lines = [] - xml_lines.append('') - xml_lines.append( - '' + yield '\n' + yield ( + '\n' ) # Get channels based on user/profile @@ -1416,14 +1493,19 @@ def generate_epg(request, profile_name=None, user=None): # Resolve effective values at SQL level and exclude hidden channels # so output ordering/display honors user overrides. from apps.channels.managers import with_effective_values - channels = ( + channels = list( with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) ) ) + channel_count = len(channels) # For dummy EPG, use either the specified value or default to 3 days dummy_days = num_days if num_days > 0 else 3 @@ -1465,12 +1547,14 @@ def generate_epg(request, profile_name=None, user=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - dummy_epg_ids_for_program_check = set() + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] - # Process channels for the section for channel in channels: effective_name = channel.effective_name effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id effective_logo = channel.effective_logo_obj effective_number = channel.effective_channel_number @@ -1492,8 +1576,6 @@ def generate_epg(request, profile_name=None, user=None): # Check if this is a custom dummy EPG with channel logo URL template if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if channel.effective_epg_data_id: - dummy_epg_ids_for_program_check.add(channel.effective_epg_data_id) epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties @@ -1548,51 +1630,16 @@ def generate_epg(request, profile_name=None, user=None): tvg_logo = direct_logo else: tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - display_name = effective_name - xml_lines.append(f' ') - xml_lines.append(f' {html.escape(display_name)}') - xml_lines.append(f' ') - xml_lines.append(" ") + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") - # Send all channel definitions - channel_xml = '\n'.join(xml_lines) + '\n' - yield channel_xml - xml_lines = [] # Clear to save memory + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] - dummy_epg_with_programs = set() - if dummy_epg_ids_for_program_check: - dummy_epg_with_programs = set( - ProgramData.objects.filter(epg_id__in=dummy_epg_ids_for_program_check) - .values_list('epg_id', flat=True) - .distinct() - ) - - # Pre-pass: categorize channels into dummy and real EPG groups - dummy_program_list = [] # (channel_id, pattern_match_name, epg_source_or_None) - real_epg_map = {} # epg_data_id -> [channel_id, ...] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_number = channel.effective_channel_number - - # Determine channel_id (same logic as channel section) - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - display_name = effective_epg_data.name if effective_epg_data else effective_name pattern_match_name = effective_name - - # Check if we should use stream name instead of channel name if effective_epg_data and effective_epg_data.epg_source: epg_source = effective_epg_data.epg_source if epg_source.custom_properties: @@ -1619,48 +1666,19 @@ def generate_epg(request, profile_name=None, user=None): if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) else: - if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if effective_epg_data_id in dummy_epg_with_programs: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - else: - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - continue - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - # Emit dummy programmes - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source - ) - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - yield f' \n' - yield f" {html.escape(program['title'])}\n" - if program.get('sub_title'): - yield f" {html.escape(program['sub_title'])}\n" - yield f" {html.escape(program['description'])}\n" - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - yield f" {html.escape(cat)}\n" - if 'date' in custom_data: - yield f" {html.escape(custom_data['date'])}\n" - if custom_data.get('live', False): - yield f" \n" - if custom_data.get('new', False): - yield f" \n" - if 'icon' in custom_data: - yield f' \n' - yield f" \n" + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - # Emit real programmes: single bulk query, chunked to avoid server-side cursor issues. all_epg_ids = list(real_epg_map.keys()) if all_epg_ids: if num_days > 0: @@ -1682,27 +1700,44 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None - is_multi = False - multi_buffer = [] + pending = [] program_batch = [] - batch_size = 1000 - chunk_size = 5000 - # Keyset pagination: track last (epg_id, id) instead of OFFSET - # to avoid skipping/duplicating rows if the table changes mid-stream. + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE last_epg_id = 0 last_id = 0 _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + while True: program_chunk = list( programs_base_qs.filter(epg_id__gte=last_epg_id) .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] ) - if not program_chunk: break - # Advance keyset cursor to last row in this chunk last_row = program_chunk[-1] last_epg_id = last_row['epg_id'] last_id = last_row['id'] @@ -1710,31 +1745,19 @@ def generate_epg(request, profile_name=None, user=None): for prog in program_chunk: epg_id = prog['epg_id'] - # When epg_id changes, flush multi-channel buffer for previous group if epg_id != current_epg_id: - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - multi_buffer = [] - + yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] - is_multi = len(channel_ids_for_epg) > 1 - # Build programme XML for primary channel_id primary_cid = channel_ids_for_epg[0] - start_str = prog['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = prog['end_time'].strftime("%Y%m%d%H%M%S %z") + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') @@ -1932,67 +1955,101 @@ def generate_epg(request, profile_name=None, user=None): program_xml.append(" ") xml_text = '\n'.join(program_xml) - program_batch.append(xml_text) + pending.append((prog['start_time'], prog['id'], xml_text)) - if is_multi: - multi_buffer.append(xml_text) + del program_chunk - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - - # Final flush of multi-channel buffer - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) + yield from flush_pending() if program_batch: yield '\n'.join(program_batch) + '\n' - # Send final closing tag and completion message + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + yield "\n" - # Log system event for EPG download after streaming completes (with deduplication based on client) client_id, client_ip, user_agent = get_client_identifier(request) event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - if not cache.get(event_cache_key): - # `len()` reuses the queryset's iteration cache populated above; - # `count()` would issue a separate SELECT COUNT(*). - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=len(channels), - client_ip=client_ip, - user_agent=user_agent, - ) - cache.set(event_cache_key, True, 2) # Prevent duplicate events for 2 seconds - # Wrapper generator that collects content for caching - def caching_generator(): - collected_content = [] - for chunk in epg_generator(): - collected_content.append(chunk) - yield chunk - # After streaming completes, cache the full content - full_content = ''.join(collected_content) - cache.set(content_cache_key, full_content, 300) - logger.debug("Cached EPG content (%d bytes)", len(full_content)) + def _log_epg_download(): + from django.core.cache import cache as event_cache - response = StreamingHttpResponse( - streaming_content=caching_generator(), - content_type="application/xml" - ) - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_epg_response(content_cache_key, build_epg_stream) def xc_get_user(request): diff --git a/core/tests.py b/core/tests.py index 7b2f1986..33475ae7 100644 --- a/core/tests.py +++ b/core/tests.py @@ -1,6 +1,6 @@ from unittest.mock import patch, MagicMock -from django.test import TestCase +from django.test import TestCase, SimpleTestCase from apps.epg.models import EPGSource from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -301,3 +301,14 @@ class DropDBCommandTlsTest(TestCase): host='localhost', port=5432, autocommit=True, ) + + +class MallocTrimTests(SimpleTestCase): + def test_trim_is_noop_when_libc_has_no_malloc_trim(self): + from core.utils import trim_c_allocator_heap + + fake_libc = MagicMock(spec=[]) + with patch('ctypes.util.find_library', return_value='libc.so.6'), patch( + 'ctypes.CDLL', return_value=fake_libc + ): + self.assertFalse(trim_c_allocator_heap()) diff --git a/core/utils.py b/core/utils.py index 29c613b3..89228088 100644 --- a/core/utils.py +++ b/core/utils.py @@ -546,6 +546,25 @@ def monitor_memory_usage(func): return result return wrapper +def trim_c_allocator_heap(): + """Return unused C heap pages to the OS where supported (glibc malloc_trim).""" + try: + import ctypes + import ctypes.util + + libc_name = ctypes.util.find_library("c") + if not libc_name: + return False + libc = ctypes.CDLL(libc_name) + if not hasattr(libc, "malloc_trim"): + return False + libc.malloc_trim(0) + return True + except Exception: + logger.debug("malloc_trim unavailable or failed", exc_info=True) + return False + + def cleanup_memory(log_usage=False, force_collection=True): """ Comprehensive memory cleanup function to reduce memory footprint From bccee9ebc1b36a140103bf3b715e91b8cb1b7def Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 11:50:52 -0500 Subject: [PATCH 15/64] refactor(epg): extract EPG generation logic into dedicated module - Moved all XMLTV output logic from `apps/output/views.py` to `apps/output/epg.py`, streamlining the codebase and maintaining the existing HTTP endpoint functionality. - Updated related tests and references to ensure proper integration with the new module structure. --- CHANGELOG.md | 2 + apps/output/epg.py | 1745 +++++++++++++++++ ...hunk_cache.py => streaming_chunk_cache.py} | 36 +- ...cache.py => test_streaming_chunk_cache.py} | 30 +- apps/output/tests.py | 11 +- apps/output/views.py | 1714 +--------------- 6 files changed, 1790 insertions(+), 1748 deletions(-) create mode 100644 apps/output/epg.py rename apps/output/{epg_chunk_cache.py => streaming_chunk_cache.py} (81%) rename apps/output/{test_epg_chunk_cache.py => test_streaming_chunk_cache.py} (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3315456b..840cb349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. + - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. ### Performance diff --git a/apps/output/epg.py b/apps/output/epg.py new file mode 100644 index 00000000..354d6e5b --- /dev/null +++ b/apps/output/epg.py @@ -0,0 +1,1745 @@ +"""XMLTV (EPG) output generation. + +Consolidates the EPG export logic that backs the `/epg` endpoint and the XC +XMLTV endpoint: real programme streaming, dummy/custom dummy program +generation, and the streaming XMLTV builder. HTTP endpoints live in views.py +and call into this module; Redis chunk caching lives in streaming_chunk_cache.py. +""" + +import html +import logging +from datetime import datetime, timedelta + +import regex + +from django.db.models import Prefetch +from django.http import Http404 +from django.urls import reverse +from django.utils import timezone as django_timezone + +from apps.channels.models import Channel, ChannelProfile, Stream +from apps.channels.utils import format_channel_number +from apps.epg.models import ProgramData +from apps.output.streaming_chunk_cache import stream_cached_response +from core.utils import build_absolute_uri_with_port, log_system_event + +logger = logging.getLogger(__name__) + +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + +def _ordered_channel_streams(channel): + """Return a channel's streams ordered by channelstream join order.""" + prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') + if prefetched is not None: + return list(prefetched) + return list(channel.streams.all().order_by('channelstream__order')) + + +def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): + """Name used for custom dummy EPG regex matching (channel or stream title). + + Returns (name, stream_lookup_failed). stream_lookup_failed is True only when + name_source is 'stream' but the configured index is missing or out of range. + """ + if custom_props.get('name_source') != 'stream': + return effective_name, False + stream_index = custom_props.get('stream_index', 1) - 1 + streams = _ordered_channel_streams(channel) + if 0 <= stream_index < len(streams): + return streams[stream_index].name, False + return effective_name, True + + +def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): + """ + Generate dummy programs using custom fallback templates when patterns don't match. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel name to use as fallback in templates + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + fallback_title: Custom fallback title template (empty string if not provided) + fallback_description: Custom fallback description template (empty string if not provided) + + Returns: + List of program dictionaries + """ + programs = [] + + # Use custom fallback title or channel name as default + title = fallback_title if fallback_title else channel_name + + # Use custom fallback description or a simple default message + if fallback_description: + description = fallback_description + else: + description = f"EPG information is currently unavailable for {channel_name}" + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": title, + "description": description, + }) + + return programs + + +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): + """ + Generate dummy EPG programs for channels. + + If epg_source is provided and it's a custom dummy EPG with patterns, + use those patterns to generate programs from the channel title. + Otherwise, generate default dummy programs. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title/name + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + epg_source: Optional EPGSource for custom dummy EPG with patterns + + Returns: + List of program dictionaries + """ + # Get current time rounded to hour + now = django_timezone.now() + now = now.replace(minute=0, second=0, microsecond=0) + + # Check if this is a custom dummy EPG with regex patterns + if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: + custom_programs = generate_custom_dummy_programs( + channel_id, channel_name, now, num_days, + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, + ) + if custom_programs is not None: + return custom_programs + + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") + + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") + + # Default humorous program descriptions based on time of day + time_descriptions = { + (0, 4): [ + f"Late Night with {channel_name} - Where insomniacs unite!", + f"The 'Why Am I Still Awake?' Show on {channel_name}", + f"Counting Sheep - A {channel_name} production for the sleepless", + ], + (4, 8): [ + f"Dawn Patrol - Rise and shine with {channel_name}!", + f"Early Bird Special - Coffee not included", + f"Morning Zombies - Before coffee viewing on {channel_name}", + ], + (8, 12): [ + f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", + f"The 'I Should Be Working' Hour on {channel_name}", + f"Productivity Killer - {channel_name}'s daytime programming", + ], + (12, 16): [ + f"Lunchtime Laziness with {channel_name}", + f"The Afternoon Slump - Brought to you by {channel_name}", + f"Post-Lunch Food Coma Theater on {channel_name}", + ], + (16, 20): [ + f"Rush Hour - {channel_name}'s alternative to traffic", + f"The 'What's For Dinner?' Debate on {channel_name}", + f"Evening Escapism - {channel_name}'s remedy for reality", + ], + (20, 24): [ + f"Prime Time Placeholder - {channel_name}'s finest not-programming", + f"The 'Netflix Was Too Complicated' Show on {channel_name}", + f"Family Argument Avoider - Courtesy of {channel_name}", + ], + } + + programs = [] + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + # Get the hour for selecting a description + hour = start_time.hour + + # Find the appropriate time slot for description + for time_range, descriptions in time_descriptions.items(): + start_range, end_range = time_range + if start_range <= hour < end_range: + # Pick a description using the sum of the hour and day as seed + # This makes it somewhat random but consistent for the same timeslot + description = descriptions[(hour + day) % len(descriptions)] + break + else: + # Fallback description if somehow no range matches + description = f"Placeholder program for {channel_name} - EPG data went on vacation" + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": channel_name, + "description": description, + }) + + return programs + + +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): + """ + Generate programs using custom dummy EPG regex patterns. + + Extracts information from channel title using regex patterns and generates + programs based on the extracted data. + + TIMEZONE HANDLING: + ------------------ + The timezone parameter specifies the timezone of the event times in your channel + titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). + DST (Daylight Saving Time) is handled automatically by pytz. + + Examples: + - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" + - Set timezone = "US/Eastern" + - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) + - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title to parse + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + custom_properties: Dict with title_pattern, time_pattern, templates, etc. + - timezone: Timezone name (e.g., 'US/Eastern') + + Returns: + List of program dictionaries with start_time/end_time in UTC + """ + import pytz + + logger.info(f"Generating custom dummy programs for channel: {channel_name}") + + # Extract patterns from custom properties + title_pattern = custom_properties.get('title_pattern', '') + time_pattern = custom_properties.get('time_pattern', '') + date_pattern = custom_properties.get('date_pattern', '') + + # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') + timezone_value = custom_properties.get('timezone', 'UTC') + output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone + program_duration = custom_properties.get('program_duration', 180) # Minutes + title_template = custom_properties.get('title_template', '') + subtitle_template = custom_properties.get('subtitle_template', '') + description_template = custom_properties.get('description_template', '') + + # Templates for upcoming/ended programs + upcoming_title_template = custom_properties.get('upcoming_title_template', '') + upcoming_description_template = custom_properties.get('upcoming_description_template', '') + ended_title_template = custom_properties.get('ended_title_template', '') + ended_description_template = custom_properties.get('ended_description_template', '') + + # Image URL templates + channel_logo_url_template = custom_properties.get('channel_logo_url', '') + program_poster_url_template = custom_properties.get('program_poster_url', '') + + # EPG metadata options + category_string = custom_properties.get('category', '') + # Split comma-separated categories and strip whitespace, filter out empty strings + categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] + include_date = custom_properties.get('include_date', True) + include_live = custom_properties.get('include_live', False) + include_new = custom_properties.get('include_new', False) + + # Parse timezone name + try: + source_tz = pytz.timezone(timezone_value) + logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") + source_tz = pytz.utc + + # Parse output timezone if provided (for display purposes) + output_tz = None + if output_timezone_value: + try: + output_tz = pytz.timezone(output_timezone_value) + logger.debug(f"Using output timezone for display: {output_timezone_value}") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") + output_tz = None + + if not title_pattern: + logger.warning(f"No title_pattern in custom_properties, falling back to default") + return None + + logger.debug(f"Title pattern from DB: {repr(title_pattern)}") + + # Convert PCRE/JavaScript named groups (?) to Python format (?P) + # This handles patterns created with JavaScript regex syntax + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) + logger.debug(f"Converted title pattern: {repr(title_pattern)}") + + # Compile regex patterns using the enhanced regex module + # (supports variable-width lookbehinds like JavaScript) + try: + title_regex = regex.compile(title_pattern) + except Exception as e: + logger.error(f"Invalid title regex pattern after conversion: {e}") + logger.error(f"Pattern was: {repr(title_pattern)}") + return None + + time_regex = None + if time_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) + logger.debug(f"Converted time pattern: {repr(time_pattern)}") + try: + time_regex = regex.compile(time_pattern) + except Exception as e: + logger.warning(f"Invalid time regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(time_pattern)}") + + # Compile date regex if provided + date_regex = None + if date_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) + logger.debug(f"Converted date pattern: {repr(date_pattern)}") + try: + date_regex = regex.compile(date_pattern) + except Exception as e: + logger.warning(f"Invalid date regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(date_pattern)}") + + # Try to match the channel name with the title pattern + # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string + title_match = title_regex.search(channel_name) + if not title_match: + logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") + return None + + groups = title_match.groupdict() + logger.debug(f"Title pattern matched. Groups: {groups}") + + # Helper function to format template with matched groups + def format_template(template, groups, url_encode=False): + """Replace {groupname} placeholders with matched group values + + Args: + template: Template string with {groupname} placeholders + groups: Dict of group names to values + url_encode: If True, URL encode the group values for safe use in URLs + """ + if not template: + return '' + result = template + for key, value in groups.items(): + if url_encode and value: + # URL encode the value to handle spaces and special characters + from urllib.parse import quote + encoded_value = quote(str(value), safe='') + result = result.replace(f'{{{key}}}', encoded_value) + else: + result = result.replace(f'{{{key}}}', str(value) if value else '') + return result + + # Extract time from title if time pattern exists + time_info = None + time_groups = {} + if time_regex: + time_match = time_regex.search(channel_name) + if time_match: + time_groups = time_match.groupdict() + try: + hour = int(time_groups.get('hour')) + # Handle optional minute group - could be None if not captured + minute_value = time_groups.get('minute') + minute = int(minute_value) if minute_value is not None else 0 + ampm = time_groups.get('ampm') + ampm = ampm.lower() if ampm else None + + # Determine if this is 12-hour or 24-hour format + if ampm in ('am', 'pm'): + # 12-hour format: convert to 24-hour + if ampm == 'pm' and hour != 12: + hour += 12 + elif ampm == 'am' and hour == 12: + hour = 0 + logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") + else: + # 24-hour format: hour is already in 24-hour format + # Validate that it's actually a 24-hour time (0-23) + if hour > 23: + logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") + hour = hour % 24 # Wrap around just in case + logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") + + time_info = {'hour': hour, 'minute': minute} + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing time: {e}") + + # Extract date from title if date pattern exists + date_info = None + date_groups = {} + if date_regex: + date_match = date_regex.search(channel_name) + if date_match: + date_groups = date_match.groupdict() + try: + # Support various date group names: month, day, year + month_str = date_groups.get('month', '') + day_str = date_groups.get('day', '') + year_str = date_groups.get('year', '') + + # Parse day - default to current day if empty or invalid + day = int(day_str) if day_str else now.day + + # Parse year - default to current year if empty or invalid (matches frontend behavior) + year = int(year_str) if year_str else now.year + + # Parse month - can be numeric (1-12) or text (Jan, January, etc.) + month = None + if month_str: + if month_str.isdigit(): + month = int(month_str) + else: + # Try to parse text month names + import calendar + month_str_lower = month_str.lower() + # Check full month names + for i, month_name in enumerate(calendar.month_name): + if month_name.lower() == month_str_lower: + month = i + break + # Check abbreviated month names if not found + if month is None: + for i, month_abbr in enumerate(calendar.month_abbr): + if month_abbr.lower() == month_str_lower: + month = i + break + + # Default to current month if not extracted or invalid + if month is None: + month = now.month + + if month and 1 <= month <= 12 and 1 <= day <= 31: + date_info = {'year': year, 'month': month, 'day': day} + logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") + else: + logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing date: {e}") + + # Merge title groups, time groups, and date groups for template formatting + all_groups = {**groups, **time_groups, **date_groups} + + # Add normalized versions of all groups for cleaner URLs + # These remove all non-alphanumeric characters and convert to lowercase + for key, value in list(all_groups.items()): + if value: + # Remove all non-alphanumeric characters (except spaces temporarily) + # then replace spaces with nothing, and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + all_groups[f'{key}_normalize'] = normalized + + # Format channel logo URL if template provided (with URL encoding) + channel_logo_url = None + if channel_logo_url_template: + channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted channel logo URL: {channel_logo_url}") + + # Format program poster URL if template provided (with URL encoding) + program_poster_url = None + if program_poster_url_template: + program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted program poster URL: {program_poster_url}") + + # Add formatted time strings for better display (handles minutes intelligently) + if time_info: + hour_24 = time_info['hour'] + minute = time_info['minute'] + + # Determine the base date to use for placeholders + # If date was extracted, use it; otherwise use current date + if date_info: + base_date = datetime(date_info['year'], date_info['month'], date_info['day']) + else: + base_date = datetime.now() + + # If output_timezone is specified, convert the display time to that timezone + if output_tz: + # Create a datetime in the source timezone using the base date + temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + # Convert to output timezone + temp_date_output = temp_date.astimezone(output_tz) + # Extract converted hour and minute for display + hour_24 = temp_date_output.hour + minute = temp_date_output.minute + logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") + + # Add date placeholders based on the OUTPUT timezone + # This ensures {date}, {month}, {day}, {year} reflect the converted timezone + all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_output.month) + all_groups['day'] = str(temp_date_output.day) + all_groups['year'] = str(temp_date_output.year) + logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") + else: + # No output timezone conversion - use source timezone for date + # Create temp date to get proper date in source timezone using the base date + temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_source.month) + all_groups['day'] = str(temp_date_source.day) + all_groups['year'] = str(temp_date_source.year) + + # Format 24-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime24'] = f"{hour_24}:{minute:02d}" + else: + all_groups['starttime24'] = f"{hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {starttime} placeholder + # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) + ampm = 'AM' if hour_24 < 12 else 'PM' + hour_12 = hour_24 + if hour_24 == 0: + hour_12 = 12 + elif hour_24 > 12: + hour_12 = hour_24 - 12 + + # Format 12-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" + else: + all_groups['starttime'] = f"{hour_12} {ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" + + # Calculate end time based on program duration + # Create a datetime for calculations + temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) + temp_end = temp_start + timedelta(minutes=program_duration) + + # Extract end time components (already in correct timezone if output_tz was applied above) + end_hour_24 = temp_end.hour + end_minute = temp_end.minute + + # Format 24-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" + else: + all_groups['endtime24'] = f"{end_hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {endtime} placeholder + end_ampm = 'AM' if end_hour_24 < 12 else 'PM' + end_hour_12 = end_hour_24 + if end_hour_24 == 0: + end_hour_12 = 12 + elif end_hour_24 > 12: + end_hour_12 = end_hour_24 - 12 + + # Format 12-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + else: + all_groups['endtime'] = f"{end_hour_12} {end_ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + + # Generate programs + programs = [] + + # If we have extracted time AND date, the event happens on a SPECIFIC date + # If we have time but NO date, generate for multiple days (existing behavior) + # All other days and times show "Upcoming" before or "Ended" after + event_happened = False + + # Determine how many iterations we need + if date_info and time_info: + # Specific date extracted - only generate for that one date + iterations = 1 + logger.debug(f"Date extracted, generating single event for specific date") + else: + # No specific date - use num_days (existing behavior) + iterations = num_days + + for day in range(iterations): + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + + # Pre-generate the main event title and description for reuse + if title_template: + main_event_title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + main_event_title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + main_event_subtitle = format_template(subtitle_template, all_groups) + else: + main_event_subtitle = None + + if description_template: + main_event_description = format_template(description_template, all_groups) + else: + main_event_description = main_event_title + + + + # Determine if this day is before, during, or after the event + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window + + if is_event_day and not event_happened: + current_time = day_start + + while current_time < event_start_utc and current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) + + # Use custom upcoming templates if provided, otherwise use defaults + if upcoming_title_template: + upcoming_title = format_template(upcoming_title_template, all_groups) + else: + upcoming_title = main_event_title + + if upcoming_description_template: + upcoming_description = format_template(upcoming_description_template, all_groups) + else: + upcoming_description = f"Upcoming: {main_event_description}" + + # Build custom_properties for upcoming programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": upcoming_title, + "sub_title": None, # No subtitle for filler programs + "description": upcoming_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + + # Add the MAIN EVENT at the extracted time + # Build custom_properties for main event (includes category and live) + main_event_custom_properties = {} + + # Add categories if provided + if categories: + main_event_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = event_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + main_event_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + main_event_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + main_event_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + main_event_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": event_start_utc, + "end_time": event_end_utc, + "title": main_event_title, + "sub_title": main_event_subtitle, + "description": main_event_description, + "custom_properties": main_event_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + event_happened = True + + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom ended templates if provided, otherwise use defaults + if ended_title_template: + ended_title = format_template(ended_title_template, all_groups) + else: + ended_title = main_event_title + + if ended_description_template: + ended_description = format_template(ended_description_template, all_groups) + else: + ended_description = f"Ended: {main_event_description}" + + # Build custom_properties for ended programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": ended_title, + "sub_title": None, # No subtitle for filler programs + "description": ended_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + else: + # This day is either before the event (future days) or after the event happened + # Fill entire day with appropriate message + current_time = day_start + + # If event already happened, all programs show "Ended" + # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" + is_ended = event_happened + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom templates based on whether event has ended or is upcoming + if is_ended: + if ended_title_template: + program_title = format_template(ended_title_template, all_groups) + else: + program_title = main_event_title + + if ended_description_template: + program_description = format_template(ended_description_template, all_groups) + else: + program_description = f"Ended: {main_event_description}" + else: + if upcoming_title_template: + program_title = format_template(upcoming_title_template, all_groups) + else: + program_title = main_event_title + + if upcoming_description_template: + program_description = format_template(upcoming_description_template, all_groups) + else: + program_description = f"Upcoming: {main_event_description}" + + # Build custom_properties (only date for upcoming/ended filler programs) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": program_title, + "sub_title": None, # No subtitle for filler programs + "description": program_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, + }) + + current_time += timedelta(minutes=program_duration) + else: + # No extracted time - fill entire day with regular intervals + # day_start and day_end are already in UTC, so no conversion needed + programs_per_day = max(1, int(24 / (program_duration / 60))) + + for program_num in range(programs_per_day): + program_start_utc = day_start + timedelta(minutes=program_num * program_duration) + program_end_utc = program_start_utc + timedelta(minutes=program_duration) + + if title_template: + title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + subtitle = format_template(subtitle_template, all_groups) + else: + subtitle = None + + if description_template: + description = format_template(description_template, all_groups) + else: + description = title + + # Build custom_properties for this program + program_custom_properties = {} + + # Add categories if provided + if categories: + program_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + program_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + program_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": title, + "sub_title": subtitle, + "description": description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") + return programs + + +def generate_dummy_epg( + channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 +): + """ + Generate dummy EPG programs for channels without EPG data. + Creates program blocks for a specified number of days. + + Args: + channel_id: The channel ID to use in the program entries + channel_name: The name of the channel to use in program titles + xml_lines: Optional list to append lines to, otherwise returns new list + num_days: Number of days to generate EPG data for (default: 1) + program_length_hours: Length of each program block in hours (default: 4) + + Returns: + List of XML lines for the dummy EPG entries + """ + if xml_lines is None: + xml_lines = [] + + for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): + # Format times in XMLTV format + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + + # Create program entry with escaped channel name + xml_lines.append( + f' ' + ) + xml_lines.append(f" {html.escape(program['title'])}") + + # Add subtitle if available + if program.get('sub_title'): + xml_lines.append(f" {html.escape(program['sub_title'])}") + + xml_lines.append(f" {html.escape(program['description'])}") + + # Add custom_properties if present + custom_data = program.get('custom_properties', {}) + + # Categories + if 'categories' in custom_data: + for cat in custom_data['categories']: + xml_lines.append(f" {html.escape(cat)}") + + # Date tag + if 'date' in custom_data: + xml_lines.append(f" {html.escape(custom_data['date'])}") + + # Live tag + if custom_data.get('live', False): + xml_lines.append(f" ") + + # New tag + if custom_data.get('new', False): + xml_lines.append(f" ") + + xml_lines.append(f" ") + + return xml_lines + + +def generate_epg(request, profile_name=None, user=None): + """ + Dynamically generate an XMLTV (EPG) file using a streaming response. + Since the EPG data is stored independently of Channels, we group programmes + by their associated EPGData record. + """ + user_custom = (user.custom_properties or {}) if user else {} + try: + num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) + num_days = max(0, min(num_days, 365)) + except (ValueError, TypeError): + num_days = 0 + try: + prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) + prev_days = max(0, min(prev_days, 30)) + except (ValueError, TypeError): + prev_days = 0 + use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' + tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() + cache_params = ( + f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" + f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" + ) + content_cache_key = f"epg_content:{cache_params}" + + def epg_generator(): + """Generator function that yields EPG data with keep-alives during processing.""" + + yield '\n' + yield ( + '\n' + ) + + # Get channels based on user/profile + if user is not None: + if user.user_level < 10: + user_profile_count = user.channel_profiles.count() + + # If user has ALL profiles or NO profiles, give unrestricted access + if user_profile_count == 0: + # No profile filtering - user sees all channels based on user_level + filters = {"user_level__lte": user.user_level} + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') + else: + # User has specific limited profiles assigned + filters = { + "channelprofilemembership__enabled": True, + "user_level__lte": user.user_level, + "channelprofilemembership__channel_profile__in": user.channel_profiles.all() + } + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() + else: + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') + else: + if profile_name is not None: + try: + channel_profile = ChannelProfile.objects.get(name=profile_name) + except ChannelProfile.DoesNotExist: + logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) + raise Http404(f"Channel profile '{profile_name}' not found") + base_qs = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True, + ).select_related('logo', 'epg_data__epg_source') + else: + base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') + + # Resolve effective values at SQL level and exclude hidden channels + # so output ordering/display honors user overrides. + from apps.channels.managers import with_effective_values + channels = list( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") + .prefetch_related( + Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + ) + ) + channel_count = len(channels) + + # For dummy EPG, use either the specified value or default to 3 days + dummy_days = num_days if num_days > 0 else 3 + + # Calculate cutoff dates for EPG data filtering + now = django_timezone.now() + cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None + lookback_cutoff = now - timedelta(days=prev_days) + + # Build collision-free channel number mapping for XC clients (if user is authenticated) + # XC clients require integer channel numbers, so we need to ensure no conflicts + channel_num_map = {} + if user is not None: + # This is an XC client - build collision-free mapping + used_numbers = set() + + # First pass: assign integers for channels that already have integer numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) + channel_num_map[channel.id] = num + used_numbers.add(num) + + # Second pass: assign integers for channels with float numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): + candidate = int(effective_num) + while candidate in used_numbers: + candidate += 1 + channel_num_map[channel.id] = candidate + used_numbers.add(candidate) + + # Host/port/scheme are constant per request; precompute logo URL prefix once. + _base_url = build_absolute_uri_with_port(request, "") + _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) + _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") + _logo_url_prefix = _base_url + _logo_prefix_raw + "/" + _logo_url_suffix = "/" + _logo_suffix_raw + + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] + + for channel in channels: + effective_name = channel.effective_name + effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id + effective_logo = channel.effective_logo_obj + effective_number = channel.effective_channel_number + + # user is set only for XC clients, which require integer channel numbers + if user is not None: + formatted_channel_number = channel_num_map[channel.id] + else: + formatted_channel_number = format_channel_number(effective_number) + + # Determine the channel ID based on the selected source + if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: + channel_id = channel.effective_tvg_id + elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: + channel_id = channel.effective_tvc_guide_stationid + else: + channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) + + tvg_logo = "" + + # Check if this is a custom dummy EPG with channel logo URL template + if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + channel_logo_url_template = custom_props.get('channel_logo_url', '') + + if channel_logo_url_template: + pattern_match_name, _ = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + + # Try to extract groups from the channel/stream name and build the logo URL + title_pattern = custom_props.get('title_pattern', '') + if title_pattern: + try: + # Convert PCRE/JavaScript named groups to Python format + title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) + title_regex = regex.compile(title_pattern) + title_match = title_regex.search(pattern_match_name) + + if title_match: + groups = title_match.groupdict() + + # Add normalized versions of all groups for cleaner URLs + for key, value in list(groups.items()): + if value: + # Remove all non-alphanumeric characters and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + groups[f'{key}_normalize'] = normalized + + # Format the logo URL template with the matched groups (with URL encoding) + from urllib.parse import quote + for key, value in groups.items(): + if value: + encoded_value = quote(str(value), safe='') + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) + else: + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') + tvg_logo = channel_logo_url_template + logger.debug(f"Built channel logo URL from template: {tvg_logo}") + except Exception as e: + logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") + + # If no custom dummy logo, use regular logo logic + if not tvg_logo and effective_logo: + if use_cached_logos: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + else: + # Use direct URL if available, otherwise fall back to cached version + direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None + if direct_logo: + tvg_logo = direct_logo + else: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") + + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] + + pattern_match_name = effective_name + if effective_epg_data and effective_epg_data.epg_source: + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + if ( + custom_props.get('name_source') == 'stream' + and not stream_lookup_failed + and pattern_match_name != effective_name + ): + stream_index = custom_props.get('stream_index', 1) - 1 + logger.debug( + f"Using stream name for parsing: {pattern_match_name} " + f"(stream index: {stream_index})" + ) + elif stream_lookup_failed: + stream_index = custom_props.get('stream_index', 1) - 1 + logger.warning( + f"Stream index {stream_index} not found for channel " + f"{effective_name}, falling back to channel name" + ) + + if not effective_epg_data: + dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) + else: + real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) + + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE + + all_epg_ids = list(real_epg_map.keys()) + if all_epg_ids: + if num_days > 0: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + start_time__lt=cutoff_date, + ) + else: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + ) + + programs_base_qs = programs_qs.order_by('epg_id', 'id').values( + 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', + 'description', 'custom_properties', + ) + + current_epg_id = None + channel_ids_for_epg = None + pending = [] + program_batch = [] + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE + last_epg_id = 0 + last_id = 0 + _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + + while True: + program_chunk = list( + programs_base_qs.filter(epg_id__gte=last_epg_id) + .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] + ) + if not program_chunk: + break + + last_row = program_chunk[-1] + last_epg_id = last_row['epg_id'] + last_id = last_row['id'] + + for prog in program_chunk: + epg_id = prog['epg_id'] + + if epg_id != current_epg_id: + yield from flush_pending() + current_epg_id = epg_id + channel_ids_for_epg = real_epg_map[epg_id] + + primary_cid = channel_ids_for_epg[0] + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" + + program_xml = [f' '] + program_xml.append(f' {html.escape(prog["title"])}') + + if prog['sub_title']: + program_xml.append(f" {html.escape(prog['sub_title'])}") + + if prog['description']: + program_xml.append(f" {html.escape(prog['description'])}") + + custom_data = prog['custom_properties'] or {} + if custom_data: + + if "categories" in custom_data and custom_data["categories"]: + for category in custom_data["categories"]: + program_xml.append(f" {html.escape(category)}") + + if "keywords" in custom_data and custom_data["keywords"]: + for keyword in custom_data["keywords"]: + program_xml.append(f" {html.escape(keyword)}") + + # onscreen_episode takes priority over episode for the onscreen system + if "onscreen_episode" in custom_data: + program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') + elif "episode" in custom_data: + program_xml.append(f' E{custom_data["episode"]}') + + # Handle dd_progid format + if 'dd_progid' in custom_data: + program_xml.append(f' {html.escape(custom_data["dd_progid"])}') + + # Handle external database IDs + for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: + if f'{system}_id' in custom_data: + program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') + + # Add season and episode numbers in xmltv_ns format if available + if "season" in custom_data and "episode" in custom_data: + season = ( + int(custom_data["season"]) - 1 + if str(custom_data["season"]).isdigit() + else 0 + ) + episode = ( + int(custom_data["episode"]) - 1 + if str(custom_data["episode"]).isdigit() + else 0 + ) + program_xml.append(f' {season}.{episode}.') + + if "language" in custom_data: + program_xml.append(f' {html.escape(custom_data["language"])}') + + if "original_language" in custom_data: + program_xml.append(f' {html.escape(custom_data["original_language"])}') + + if "length" in custom_data and isinstance(custom_data["length"], dict): + length_value = custom_data["length"].get("value", "") + length_units = custom_data["length"].get("units", "minutes") + program_xml.append(f' {html.escape(str(length_value))}') + + if "video" in custom_data and isinstance(custom_data["video"], dict): + program_xml.append(" ") + + if "audio" in custom_data and isinstance(custom_data["audio"], dict): + program_xml.append(" ") + + if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): + for subtitle in custom_data["subtitles"]: + if isinstance(subtitle, dict): + subtitle_type = subtitle.get("type", "") + type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" + program_xml.append(f" ") + if "language" in subtitle: + program_xml.append(f" {html.escape(subtitle['language'])}") + program_xml.append(" ") + + if "rating" in custom_data: + rating_system = custom_data.get("rating_system", "TV Parental Guidelines") + program_xml.append(f' ') + program_xml.append(f' {html.escape(custom_data["rating"])}') + program_xml.append(f" ") + + if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): + for star_rating in custom_data["star_ratings"]: + if isinstance(star_rating, dict) and "value" in star_rating: + system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" + program_xml.append(f" ") + program_xml.append(f" {html.escape(star_rating['value'])}") + program_xml.append(" ") + + if "reviews" in custom_data and isinstance(custom_data["reviews"], list): + for review in custom_data["reviews"]: + if isinstance(review, dict) and "content" in review: + review_type = review.get("type", "text") + attrs = [f'type="{html.escape(review_type)}"'] + if "source" in review: + attrs.append(f'source="{html.escape(review["source"])}"') + if "reviewer" in review: + attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') + attr_str = " ".join(attrs) + program_xml.append(f' {html.escape(review["content"])}') + + if "images" in custom_data and isinstance(custom_data["images"], list): + for image in custom_data["images"]: + if isinstance(image, dict) and "url" in image: + attrs = [] + for attr in ['type', 'size', 'orient', 'system']: + if attr in image: + attrs.append(f'{attr}="{html.escape(image[attr])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f' {html.escape(image["url"])}') + + # Add enhanced credits handling + if "credits" in custom_data: + program_xml.append(" ") + credits = custom_data["credits"] + + for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if role in credits: + people = credits[role] + if isinstance(people, list): + for person in people: + program_xml.append(f" <{role}>{html.escape(person)}") + else: + program_xml.append(f" <{role}>{html.escape(people)}") + + # Handle actors separately to include role and guest attributes + if "actor" in credits: + actors = credits["actor"] + if isinstance(actors, list): + for actor in actors: + if isinstance(actor, dict): + name = actor.get("name", "") + role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" + guest_attr = ' guest="yes"' if actor.get("guest") else "" + program_xml.append(f" {html.escape(name)}") + else: + program_xml.append(f" {html.escape(actor)}") + else: + program_xml.append(f" {html.escape(actors)}") + + program_xml.append(" ") + + if "date" in custom_data: + program_xml.append(f' {html.escape(custom_data["date"])}') + + if "country" in custom_data: + program_xml.append(f' {html.escape(custom_data["country"])}') + + if "icon" in custom_data: + program_xml.append(f' ') + elif "sd_icon" in custom_data: + program_xml.append(f' ') + + # Add special flags as proper tags with enhanced handling + if custom_data.get("previously_shown", False): + prev_shown_details = custom_data.get("previously_shown_details", {}) + attrs = [] + if "start" in prev_shown_details: + attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') + if "channel" in prev_shown_details: + attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f" ") + + if custom_data.get("premiere", False): + premiere_text = custom_data.get("premiere_text", "") + if premiere_text: + program_xml.append(f" {html.escape(premiere_text)}") + else: + program_xml.append(" ") + + if custom_data.get("last_chance", False): + last_chance_text = custom_data.get("last_chance_text", "") + if last_chance_text: + program_xml.append(f" {html.escape(last_chance_text)}") + else: + program_xml.append(" ") + + if custom_data.get("new", False): + program_xml.append(" ") + + if custom_data.get('live', False): + program_xml.append(' ') + + program_xml.append(" ") + + xml_text = '\n'.join(program_xml) + pending.append((prog['start_time'], prog['id'], xml_text)) + + del program_chunk + + yield from flush_pending() + + if program_batch: + yield '\n'.join(program_batch) + '\n' + + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + + yield "\n" + + from apps.output.views import get_client_identifier + + client_id, client_ip, user_agent = get_client_identifier(request) + event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" + + def _log_epg_download(): + from django.core.cache import cache as event_cache + + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_cached_response( + content_cache_key, + build_epg_stream, + content_type="application/xml", + filename="Dispatcharr.xml", + ) diff --git a/apps/output/epg_chunk_cache.py b/apps/output/streaming_chunk_cache.py similarity index 81% rename from apps/output/epg_chunk_cache.py rename to apps/output/streaming_chunk_cache.py index 92ea6eeb..f86bb060 100644 --- a/apps/output/epg_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -1,4 +1,4 @@ -"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" +"""Single-flight Redis chunk cache for large streaming HTTP responses.""" import logging import time @@ -111,7 +111,7 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): try: from django.core.cache import cache as django_cache - django_cache.delete(base_key) # legacy monolithic entry + django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) for chunk in source(): @@ -123,9 +123,9 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): redis.expire(chunks_key, cache_ttl) redis.expire(status_key, cache_ttl) redis.expire(_ready_key(base_key), cache_ttl) - logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + logger.debug("Cached response in %s chunks", redis.llen(chunks_key)) except Exception: - logger.exception("EPG cache build failed for %s", base_key) + logger.exception("Chunk cache build failed for %s", base_key) redis.delete(chunks_key) redis.set(status_key, STATUS_ERROR, ex=60) raise @@ -158,14 +158,14 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - raise RuntimeError("EPG cache build failed") + raise RuntimeError("Chunk cache build failed") if time.monotonic() >= deadline: if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + logger.warning("Chunk cache follower timed out; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - logger.warning("EPG cache follower timed out after partial read for %s", base_key) + logger.warning("Chunk cache follower timed out after partial read for %s", base_key) break lock_active = bool(redis.exists(lock_key)) @@ -173,7 +173,7 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, idle_polls += 1 if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): if _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache leader lost; rebuilding %s", base_key) + logger.warning("Chunk cache leader lost; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return else: @@ -182,10 +182,12 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, _poll_wait(poll_interval) -def stream_epg_response( +def stream_cached_response( cache_key, source, *, + content_type="application/xml", + filename=None, cache_ttl=DEFAULT_CACHE_TTL, lock_ttl=DEFAULT_LOCK_TTL, poll_interval=DEFAULT_POLL_INTERVAL, @@ -193,16 +195,17 @@ def stream_epg_response( redis=None, ): """ - Stream XMLTV EPG output with single-flight Redis chunk caching. + Stream a large response with single-flight Redis chunk caching. ``source`` must be a callable returning a chunk iterator. Only the leader - invokes it; followers read chunks already written to Redis. + invokes it; concurrent followers replay chunks already written to Redis, so + the expensive ``source`` runs at most once per ``cache_key``. """ if redis is None: redis = _get_redis() if redis.get(_ready_key(cache_key)): - logger.debug("Serving EPG from chunk cache") + logger.debug("Serving response from chunk cache") stream = _stream_ready(redis, cache_key) else: status = _get_status(redis, cache_key) @@ -210,10 +213,10 @@ def stream_epg_response( _clear_build_keys(redis, cache_key) if _try_acquire_lock(redis, cache_key, lock_ttl): - logger.debug("Building EPG (cache leader)") + logger.debug("Building response (cache leader)") stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) else: - logger.debug("Following in-flight EPG build") + logger.debug("Following in-flight cache build") stream = _stream_follow( redis, cache_key, @@ -224,7 +227,8 @@ def stream_epg_response( max_follower_wait, ) - response = StreamingHttpResponse(stream, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response = StreamingHttpResponse(stream, content_type=content_type) + if filename: + response["Content-Disposition"] = f'attachment; filename="{filename}"' response["Cache-Control"] = "no-cache" return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_streaming_chunk_cache.py similarity index 84% rename from apps/output/test_epg_chunk_cache.py rename to apps/output/test_streaming_chunk_cache.py index 6ca217af..599501b7 100644 --- a/apps/output/test_epg_chunk_cache.py +++ b/apps/output/test_streaming_chunk_cache.py @@ -2,14 +2,14 @@ import threading import time from unittest import TestCase -from apps.output.epg_chunk_cache import ( +from apps.output.streaming_chunk_cache import ( STATUS_BUILDING, STATUS_READY, _chunks_key, _lock_key, _ready_key, _status_key, - stream_epg_response, + stream_cached_response, ) @@ -74,7 +74,7 @@ def _consume(response): return b"".join(response.streaming_content).decode("utf-8") -class EPGChunkCacheTests(TestCase): +class StreamingChunkCacheTests(TestCase): def test_leader_caches_chunks_and_sets_ready(self): redis = FakeRedis() calls = [] @@ -84,14 +84,14 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, [1]) - self.assertEqual(redis.get(_ready_key("epg:test")), "1") - self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) - self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) - self.assertFalse(redis.exists(_lock_key("epg:test"))) + self.assertEqual(redis.get(_ready_key("cache:test")), "1") + self.assertEqual(redis.get(_status_key("cache:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("cache:test")), 2) + self.assertFalse(redis.exists(_lock_key("cache:test"))) def test_cache_hit_skips_source(self): redis = FakeRedis() @@ -102,16 +102,16 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - _consume(stream_epg_response("epg:test", source, redis=redis)) + _consume(stream_cached_response("cache:test", source, redis=redis)) calls.clear() - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, []) def test_follower_reads_leader_chunks_without_rebuilding(self): redis = FakeRedis() - base = "epg:follow" + base = "cache:follow" leader_started = threading.Event() rebuild_calls = [] @@ -128,7 +128,7 @@ class EPGChunkCacheTests(TestCase): def leader(): _consume( - stream_epg_response( + stream_cached_response( base, slow_source, redis=redis, @@ -140,7 +140,7 @@ class EPGChunkCacheTests(TestCase): leader_thread.start() leader_started.wait(timeout=5) follower_body = _consume( - stream_epg_response( + stream_cached_response( base, forbidden_source, redis=redis, @@ -165,8 +165,8 @@ class EPGChunkCacheTests(TestCase): def worker(): barrier.wait() results[threading.current_thread().name] = _consume( - stream_epg_response( - "epg:race", + stream_cached_response( + "cache:race", source, redis=redis, poll_interval=0.01, diff --git a/apps/output/tests.py b/apps/output/tests.py index 6e2ab395..5f4fd5f3 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -34,16 +34,18 @@ class OutputEndpointTestMixin: "apps.output.views.network_access_allowed", return_value=True, ) - self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._epg_teardown_patch = patch("apps.output.epg._epg_export_teardown") self._log_event_patch = patch("apps.output.views.log_system_event") + self._epg_log_event_patch = patch("apps.output.epg.log_system_event") self._close_db_patch = patch("django.db.close_old_connections") self._epg_cache_patch = patch( - "apps.output.views.stream_epg_response", + "apps.output.epg.stream_cached_response", side_effect=_epg_response_without_redis, ) self._network_patch.start() self._epg_teardown_patch.start() self._log_event_patch.start() + self._epg_log_event_patch.start() self._close_db_patch.start() self._epg_cache_patch.start() @@ -53,6 +55,7 @@ class OutputEndpointTestMixin: cache.clear() self._epg_cache_patch.stop() self._close_db_patch.stop() + self._epg_log_event_patch.stop() self._log_event_patch.stop() self._epg_teardown_patch.stop() self._network_patch.stop() @@ -339,14 +342,14 @@ class OutputEPGCustomDummyTest(TestCase): class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=30, second=0, microsecond=0) self.assertEqual(_ceil_to_half_hour(dt), dt) def test_ceil_to_half_hour_rounds_up(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=17, second=42, microsecond=123456) aligned = _ceil_to_half_hour(dt) diff --git a/apps/output/views.py b/apps/output/views.py index a0875c7f..4224fb35 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,54 +25,10 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib -from apps.output.epg_chunk_cache import stream_epg_response +from apps.output.epg import generate_epg, generate_dummy_programs logger = logging.getLogger(__name__) -_EPG_CHANNEL_XML_BATCH_SIZE = 200 -_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 -_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 - - -def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): - if end_time < lookback_cutoff: - return False - if cutoff_date is not None and start_time >= cutoff_date: - return False - return True - - -def _ceil_to_half_hour(dt): - """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) - - -def _epg_export_teardown(): - from django.db import close_old_connections - - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() - def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -384,1674 +340,6 @@ def generate_m3u(request, profile_name=None, user=None): return response -def _ordered_channel_streams(channel): - """Return a channel's streams ordered by channelstream join order.""" - prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') - if prefetched is not None: - return list(prefetched) - return list(channel.streams.all().order_by('channelstream__order')) - - -def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): - """Name used for custom dummy EPG regex matching (channel or stream title). - - Returns (name, stream_lookup_failed). stream_lookup_failed is True only when - name_source is 'stream' but the configured index is missing or out of range. - """ - if custom_props.get('name_source') != 'stream': - return effective_name, False - stream_index = custom_props.get('stream_index', 1) - 1 - streams = _ordered_channel_streams(channel) - if 0 <= stream_index < len(streams): - return streams[stream_index].name, False - return effective_name, True - - -def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): - """ - Generate dummy programs using custom fallback templates when patterns don't match. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel name to use as fallback in templates - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - fallback_title: Custom fallback title template (empty string if not provided) - fallback_description: Custom fallback description template (empty string if not provided) - - Returns: - List of program dictionaries - """ - programs = [] - - # Use custom fallback title or channel name as default - title = fallback_title if fallback_title else channel_name - - # Use custom fallback description or a simple default message - if fallback_description: - description = fallback_description - else: - description = f"EPG information is currently unavailable for {channel_name}" - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": title, - "description": description, - }) - - return programs - - -def generate_dummy_programs( - channel_id, - channel_name, - num_days=1, - program_length_hours=4, - epg_source=None, - export_lookback=None, - export_cutoff=None, -): - """ - Generate dummy EPG programs for channels. - - If epg_source is provided and it's a custom dummy EPG with patterns, - use those patterns to generate programs from the channel title. - Otherwise, generate default dummy programs. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title/name - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - epg_source: Optional EPGSource for custom dummy EPG with patterns - - Returns: - List of program dictionaries - """ - # Get current time rounded to hour - now = django_timezone.now() - now = now.replace(minute=0, second=0, microsecond=0) - - # Check if this is a custom dummy EPG with regex patterns - if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: - custom_programs = generate_custom_dummy_programs( - channel_id, channel_name, now, num_days, - epg_source.custom_properties, - export_lookback=export_lookback, - export_cutoff=export_cutoff, - ) - if custom_programs is not None: - return custom_programs - - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() - - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - logger.info(f"No custom fallback templates found, using default dummy EPG") - - # Default humorous program descriptions based on time of day - time_descriptions = { - (0, 4): [ - f"Late Night with {channel_name} - Where insomniacs unite!", - f"The 'Why Am I Still Awake?' Show on {channel_name}", - f"Counting Sheep - A {channel_name} production for the sleepless", - ], - (4, 8): [ - f"Dawn Patrol - Rise and shine with {channel_name}!", - f"Early Bird Special - Coffee not included", - f"Morning Zombies - Before coffee viewing on {channel_name}", - ], - (8, 12): [ - f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", - f"The 'I Should Be Working' Hour on {channel_name}", - f"Productivity Killer - {channel_name}'s daytime programming", - ], - (12, 16): [ - f"Lunchtime Laziness with {channel_name}", - f"The Afternoon Slump - Brought to you by {channel_name}", - f"Post-Lunch Food Coma Theater on {channel_name}", - ], - (16, 20): [ - f"Rush Hour - {channel_name}'s alternative to traffic", - f"The 'What's For Dinner?' Debate on {channel_name}", - f"Evening Escapism - {channel_name}'s remedy for reality", - ], - (20, 24): [ - f"Prime Time Placeholder - {channel_name}'s finest not-programming", - f"The 'Netflix Was Too Complicated' Show on {channel_name}", - f"Family Argument Avoider - Courtesy of {channel_name}", - ], - } - - programs = [] - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - # Get the hour for selecting a description - hour = start_time.hour - - # Find the appropriate time slot for description - for time_range, descriptions in time_descriptions.items(): - start_range, end_range = time_range - if start_range <= hour < end_range: - # Pick a description using the sum of the hour and day as seed - # This makes it somewhat random but consistent for the same timeslot - description = descriptions[(hour + day) % len(descriptions)] - break - else: - # Fallback description if somehow no range matches - description = f"Placeholder program for {channel_name} - EPG data went on vacation" - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": channel_name, - "description": description, - }) - - return programs - - -def generate_custom_dummy_programs( - channel_id, - channel_name, - now, - num_days, - custom_properties, - export_lookback=None, - export_cutoff=None, -): - """ - Generate programs using custom dummy EPG regex patterns. - - Extracts information from channel title using regex patterns and generates - programs based on the extracted data. - - TIMEZONE HANDLING: - ------------------ - The timezone parameter specifies the timezone of the event times in your channel - titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). - DST (Daylight Saving Time) is handled automatically by pytz. - - Examples: - - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" - - Set timezone = "US/Eastern" - - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) - - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title to parse - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - custom_properties: Dict with title_pattern, time_pattern, templates, etc. - - timezone: Timezone name (e.g., 'US/Eastern') - - Returns: - List of program dictionaries with start_time/end_time in UTC - """ - import pytz - - logger.info(f"Generating custom dummy programs for channel: {channel_name}") - - # Extract patterns from custom properties - title_pattern = custom_properties.get('title_pattern', '') - time_pattern = custom_properties.get('time_pattern', '') - date_pattern = custom_properties.get('date_pattern', '') - - # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') - timezone_value = custom_properties.get('timezone', 'UTC') - output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone - program_duration = custom_properties.get('program_duration', 180) # Minutes - title_template = custom_properties.get('title_template', '') - subtitle_template = custom_properties.get('subtitle_template', '') - description_template = custom_properties.get('description_template', '') - - # Templates for upcoming/ended programs - upcoming_title_template = custom_properties.get('upcoming_title_template', '') - upcoming_description_template = custom_properties.get('upcoming_description_template', '') - ended_title_template = custom_properties.get('ended_title_template', '') - ended_description_template = custom_properties.get('ended_description_template', '') - - # Image URL templates - channel_logo_url_template = custom_properties.get('channel_logo_url', '') - program_poster_url_template = custom_properties.get('program_poster_url', '') - - # EPG metadata options - category_string = custom_properties.get('category', '') - # Split comma-separated categories and strip whitespace, filter out empty strings - categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] - include_date = custom_properties.get('include_date', True) - include_live = custom_properties.get('include_live', False) - include_new = custom_properties.get('include_new', False) - - # Parse timezone name - try: - source_tz = pytz.timezone(timezone_value) - logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") - source_tz = pytz.utc - - # Parse output timezone if provided (for display purposes) - output_tz = None - if output_timezone_value: - try: - output_tz = pytz.timezone(output_timezone_value) - logger.debug(f"Using output timezone for display: {output_timezone_value}") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") - output_tz = None - - if not title_pattern: - logger.warning(f"No title_pattern in custom_properties, falling back to default") - return None - - logger.debug(f"Title pattern from DB: {repr(title_pattern)}") - - # Convert PCRE/JavaScript named groups (?) to Python format (?P) - # This handles patterns created with JavaScript regex syntax - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) - logger.debug(f"Converted title pattern: {repr(title_pattern)}") - - # Compile regex patterns using the enhanced regex module - # (supports variable-width lookbehinds like JavaScript) - try: - title_regex = regex.compile(title_pattern) - except Exception as e: - logger.error(f"Invalid title regex pattern after conversion: {e}") - logger.error(f"Pattern was: {repr(title_pattern)}") - return None - - time_regex = None - if time_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) - logger.debug(f"Converted time pattern: {repr(time_pattern)}") - try: - time_regex = regex.compile(time_pattern) - except Exception as e: - logger.warning(f"Invalid time regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(time_pattern)}") - - # Compile date regex if provided - date_regex = None - if date_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) - logger.debug(f"Converted date pattern: {repr(date_pattern)}") - try: - date_regex = regex.compile(date_pattern) - except Exception as e: - logger.warning(f"Invalid date regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(date_pattern)}") - - # Try to match the channel name with the title pattern - # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string - title_match = title_regex.search(channel_name) - if not title_match: - logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return None - - groups = title_match.groupdict() - logger.debug(f"Title pattern matched. Groups: {groups}") - - # Helper function to format template with matched groups - def format_template(template, groups, url_encode=False): - """Replace {groupname} placeholders with matched group values - - Args: - template: Template string with {groupname} placeholders - groups: Dict of group names to values - url_encode: If True, URL encode the group values for safe use in URLs - """ - if not template: - return '' - result = template - for key, value in groups.items(): - if url_encode and value: - # URL encode the value to handle spaces and special characters - from urllib.parse import quote - encoded_value = quote(str(value), safe='') - result = result.replace(f'{{{key}}}', encoded_value) - else: - result = result.replace(f'{{{key}}}', str(value) if value else '') - return result - - # Extract time from title if time pattern exists - time_info = None - time_groups = {} - if time_regex: - time_match = time_regex.search(channel_name) - if time_match: - time_groups = time_match.groupdict() - try: - hour = int(time_groups.get('hour')) - # Handle optional minute group - could be None if not captured - minute_value = time_groups.get('minute') - minute = int(minute_value) if minute_value is not None else 0 - ampm = time_groups.get('ampm') - ampm = ampm.lower() if ampm else None - - # Determine if this is 12-hour or 24-hour format - if ampm in ('am', 'pm'): - # 12-hour format: convert to 24-hour - if ampm == 'pm' and hour != 12: - hour += 12 - elif ampm == 'am' and hour == 12: - hour = 0 - logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") - else: - # 24-hour format: hour is already in 24-hour format - # Validate that it's actually a 24-hour time (0-23) - if hour > 23: - logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") - hour = hour % 24 # Wrap around just in case - logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") - - time_info = {'hour': hour, 'minute': minute} - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing time: {e}") - - # Extract date from title if date pattern exists - date_info = None - date_groups = {} - if date_regex: - date_match = date_regex.search(channel_name) - if date_match: - date_groups = date_match.groupdict() - try: - # Support various date group names: month, day, year - month_str = date_groups.get('month', '') - day_str = date_groups.get('day', '') - year_str = date_groups.get('year', '') - - # Parse day - default to current day if empty or invalid - day = int(day_str) if day_str else now.day - - # Parse year - default to current year if empty or invalid (matches frontend behavior) - year = int(year_str) if year_str else now.year - - # Parse month - can be numeric (1-12) or text (Jan, January, etc.) - month = None - if month_str: - if month_str.isdigit(): - month = int(month_str) - else: - # Try to parse text month names - import calendar - month_str_lower = month_str.lower() - # Check full month names - for i, month_name in enumerate(calendar.month_name): - if month_name.lower() == month_str_lower: - month = i - break - # Check abbreviated month names if not found - if month is None: - for i, month_abbr in enumerate(calendar.month_abbr): - if month_abbr.lower() == month_str_lower: - month = i - break - - # Default to current month if not extracted or invalid - if month is None: - month = now.month - - if month and 1 <= month <= 12 and 1 <= day <= 31: - date_info = {'year': year, 'month': month, 'day': day} - logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") - else: - logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing date: {e}") - - # Merge title groups, time groups, and date groups for template formatting - all_groups = {**groups, **time_groups, **date_groups} - - # Add normalized versions of all groups for cleaner URLs - # These remove all non-alphanumeric characters and convert to lowercase - for key, value in list(all_groups.items()): - if value: - # Remove all non-alphanumeric characters (except spaces temporarily) - # then replace spaces with nothing, and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - all_groups[f'{key}_normalize'] = normalized - - # Format channel logo URL if template provided (with URL encoding) - channel_logo_url = None - if channel_logo_url_template: - channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted channel logo URL: {channel_logo_url}") - - # Format program poster URL if template provided (with URL encoding) - program_poster_url = None - if program_poster_url_template: - program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted program poster URL: {program_poster_url}") - - # Add formatted time strings for better display (handles minutes intelligently) - if time_info: - hour_24 = time_info['hour'] - minute = time_info['minute'] - - # Determine the base date to use for placeholders - # If date was extracted, use it; otherwise use current date - if date_info: - base_date = datetime(date_info['year'], date_info['month'], date_info['day']) - else: - base_date = datetime.now() - - # If output_timezone is specified, convert the display time to that timezone - if output_tz: - # Create a datetime in the source timezone using the base date - temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - # Convert to output timezone - temp_date_output = temp_date.astimezone(output_tz) - # Extract converted hour and minute for display - hour_24 = temp_date_output.hour - minute = temp_date_output.minute - logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") - - # Add date placeholders based on the OUTPUT timezone - # This ensures {date}, {month}, {day}, {year} reflect the converted timezone - all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_output.month) - all_groups['day'] = str(temp_date_output.day) - all_groups['year'] = str(temp_date_output.year) - logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") - else: - # No output timezone conversion - use source timezone for date - # Create temp date to get proper date in source timezone using the base date - temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_source.month) - all_groups['day'] = str(temp_date_source.day) - all_groups['year'] = str(temp_date_source.year) - - # Format 24-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime24'] = f"{hour_24}:{minute:02d}" - else: - all_groups['starttime24'] = f"{hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {starttime} placeholder - # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) - ampm = 'AM' if hour_24 < 12 else 'PM' - hour_12 = hour_24 - if hour_24 == 0: - hour_12 = 12 - elif hour_24 > 12: - hour_12 = hour_24 - 12 - - # Format 12-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" - else: - all_groups['starttime'] = f"{hour_12} {ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" - - # Calculate end time based on program duration - # Create a datetime for calculations - temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) - temp_end = temp_start + timedelta(minutes=program_duration) - - # Extract end time components (already in correct timezone if output_tz was applied above) - end_hour_24 = temp_end.hour - end_minute = temp_end.minute - - # Format 24-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" - else: - all_groups['endtime24'] = f"{end_hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {endtime} placeholder - end_ampm = 'AM' if end_hour_24 < 12 else 'PM' - end_hour_12 = end_hour_24 - if end_hour_24 == 0: - end_hour_12 = 12 - elif end_hour_24 > 12: - end_hour_12 = end_hour_24 - 12 - - # Format 12-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - else: - all_groups['endtime'] = f"{end_hour_12} {end_ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - - # Generate programs - programs = [] - - # If we have extracted time AND date, the event happens on a SPECIFIC date - # If we have time but NO date, generate for multiple days (existing behavior) - # All other days and times show "Upcoming" before or "Ended" after - event_happened = False - - # Determine how many iterations we need - if date_info and time_info: - # Specific date extracted - only generate for that one date - iterations = 1 - logger.debug(f"Date extracted, generating single event for specific date") - else: - # No specific date - use num_days (existing behavior) - iterations = num_days - - for day in range(iterations): - event_overlaps_window = True - if date_info and time_info: - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'], - ).date() - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - event_overlaps_window = _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ) - if not event_overlaps_window: - logger.debug( - "Custom dummy event outside export window; filling window only: %s", - channel_name, - ) - event_happened = event_end_utc < lookback - day_start = _ceil_to_half_hour(lookback) - if export_cutoff is not None: - day_end = export_cutoff - else: - day_end = now + timedelta(days=num_days if num_days > 0 else 3) - else: - day_start = source_tz.localize( - datetime.combine(current_date, datetime.min.time()) - ).astimezone(pytz.utc) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - else: - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - - if day_start >= day_end: - continue - - if time_info: - if not date_info: - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_local = source_tz.localize(event_start_naive) - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - if not _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ): - continue - else: - logger.debug(f"Using extracted date: {current_date}") - - # Pre-generate the main event title and description for reuse - if title_template: - main_event_title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - main_event_title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - main_event_subtitle = format_template(subtitle_template, all_groups) - else: - main_event_subtitle = None - - if description_template: - main_event_description = format_template(description_template, all_groups) - else: - main_event_description = main_event_title - - - - # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) when it falls inside the window - is_event_day = (day == 0) and event_overlaps_window - - if is_event_day and not event_happened: - current_time = day_start - - while current_time < event_start_utc and current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) - - # Use custom upcoming templates if provided, otherwise use defaults - if upcoming_title_template: - upcoming_title = format_template(upcoming_title_template, all_groups) - else: - upcoming_title = main_event_title - - if upcoming_description_template: - upcoming_description = format_template(upcoming_description_template, all_groups) - else: - upcoming_description = f"Upcoming: {main_event_description}" - - # Build custom_properties for upcoming programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": upcoming_title, - "sub_title": None, # No subtitle for filler programs - "description": upcoming_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - - # Add the MAIN EVENT at the extracted time - # Build custom_properties for main event (includes category and live) - main_event_custom_properties = {} - - # Add categories if provided - if categories: - main_event_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = event_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - main_event_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - main_event_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - main_event_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - main_event_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": event_start_utc, - "end_time": event_end_utc, - "title": main_event_title, - "sub_title": main_event_subtitle, - "description": main_event_description, - "custom_properties": main_event_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - event_happened = True - - # Fill programs AFTER the event until end of export day window - current_time = max(event_end_utc, day_start) - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom ended templates if provided, otherwise use defaults - if ended_title_template: - ended_title = format_template(ended_title_template, all_groups) - else: - ended_title = main_event_title - - if ended_description_template: - ended_description = format_template(ended_description_template, all_groups) - else: - ended_description = f"Ended: {main_event_description}" - - # Build custom_properties for ended programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": ended_title, - "sub_title": None, # No subtitle for filler programs - "description": ended_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - else: - # This day is either before the event (future days) or after the event happened - # Fill entire day with appropriate message - current_time = day_start - - # If event already happened, all programs show "Ended" - # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" - is_ended = event_happened - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom templates based on whether event has ended or is upcoming - if is_ended: - if ended_title_template: - program_title = format_template(ended_title_template, all_groups) - else: - program_title = main_event_title - - if ended_description_template: - program_description = format_template(ended_description_template, all_groups) - else: - program_description = f"Ended: {main_event_description}" - else: - if upcoming_title_template: - program_title = format_template(upcoming_title_template, all_groups) - else: - program_title = main_event_title - - if upcoming_description_template: - program_description = format_template(upcoming_description_template, all_groups) - else: - program_description = f"Upcoming: {main_event_description}" - - # Build custom_properties (only date for upcoming/ended filler programs) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": program_title, - "sub_title": None, # No subtitle for filler programs - "description": program_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, - }) - - current_time += timedelta(minutes=program_duration) - else: - # No extracted time - fill entire day with regular intervals - # day_start and day_end are already in UTC, so no conversion needed - programs_per_day = max(1, int(24 / (program_duration / 60))) - - for program_num in range(programs_per_day): - program_start_utc = day_start + timedelta(minutes=program_num * program_duration) - program_end_utc = program_start_utc + timedelta(minutes=program_duration) - - if title_template: - title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - subtitle = format_template(subtitle_template, all_groups) - else: - subtitle = None - - if description_template: - description = format_template(description_template, all_groups) - else: - description = title - - # Build custom_properties for this program - program_custom_properties = {} - - # Add categories if provided - if categories: - program_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - program_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - program_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": title, - "sub_title": subtitle, - "description": description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") - return programs - - -def generate_dummy_epg( - channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 -): - """ - Generate dummy EPG programs for channels without EPG data. - Creates program blocks for a specified number of days. - - Args: - channel_id: The channel ID to use in the program entries - channel_name: The name of the channel to use in program titles - xml_lines: Optional list to append lines to, otherwise returns new list - num_days: Number of days to generate EPG data for (default: 1) - program_length_hours: Length of each program block in hours (default: 4) - - Returns: - List of XML lines for the dummy EPG entries - """ - if xml_lines is None: - xml_lines = [] - - for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): - # Format times in XMLTV format - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - - # Create program entry with escaped channel name - xml_lines.append( - f' ' - ) - xml_lines.append(f" {html.escape(program['title'])}") - - # Add subtitle if available - if program.get('sub_title'): - xml_lines.append(f" {html.escape(program['sub_title'])}") - - xml_lines.append(f" {html.escape(program['description'])}") - - # Add custom_properties if present - custom_data = program.get('custom_properties', {}) - - # Categories - if 'categories' in custom_data: - for cat in custom_data['categories']: - xml_lines.append(f" {html.escape(cat)}") - - # Date tag - if 'date' in custom_data: - xml_lines.append(f" {html.escape(custom_data['date'])}") - - # Live tag - if custom_data.get('live', False): - xml_lines.append(f" ") - - # New tag - if custom_data.get('new', False): - xml_lines.append(f" ") - - xml_lines.append(f" ") - - return xml_lines - - -def generate_epg(request, profile_name=None, user=None): - """ - Dynamically generate an XMLTV (EPG) file using a streaming response. - Since the EPG data is stored independently of Channels, we group programmes - by their associated EPGData record. - """ - user_custom = (user.custom_properties or {}) if user else {} - try: - num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) - num_days = max(0, min(num_days, 365)) - except (ValueError, TypeError): - num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 - use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' - tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() - cache_params = ( - f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" - f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" - ) - content_cache_key = f"epg_content:{cache_params}" - - def epg_generator(): - """Generator function that yields EPG data with keep-alives during processing.""" - - yield '\n' - yield ( - '\n' - ) - - # Get channels based on user/profile - if user is not None: - if user.user_level < 10: - user_profile_count = user.channel_profiles.count() - - # If user has ALL profiles or NO profiles, give unrestricted access - if user_profile_count == 0: - # No profile filtering - user sees all channels based on user_level - filters = {"user_level__lte": user.user_level} - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') - else: - # User has specific limited profiles assigned - filters = { - "channelprofilemembership__enabled": True, - "user_level__lte": user.user_level, - "channelprofilemembership__channel_profile__in": user.channel_profiles.all() - } - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() - else: - base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') - else: - if profile_name is not None: - try: - channel_profile = ChannelProfile.objects.get(name=profile_name) - except ChannelProfile.DoesNotExist: - logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) - raise Http404(f"Channel profile '{profile_name}' not found") - base_qs = Channel.objects.filter( - channelprofilemembership__channel_profile=channel_profile, - channelprofilemembership__enabled=True, - ).select_related('logo', 'epg_data__epg_source') - else: - base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') - - # Resolve effective values at SQL level and exclude hidden channels - # so output ordering/display honors user overrides. - from apps.channels.managers import with_effective_values - channels = list( - with_effective_values(base_qs, select_related_fks=True) - .exclude(hidden_from_output=True) - .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") - .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) - ) - ) - channel_count = len(channels) - - # For dummy EPG, use either the specified value or default to 3 days - dummy_days = num_days if num_days > 0 else 3 - - # Calculate cutoff dates for EPG data filtering - now = django_timezone.now() - cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None - lookback_cutoff = now - timedelta(days=prev_days) - - # Build collision-free channel number mapping for XC clients (if user is authenticated) - # XC clients require integer channel numbers, so we need to ensure no conflicts - channel_num_map = {} - if user is not None: - # This is an XC client - build collision-free mapping - used_numbers = set() - - # First pass: assign integers for channels that already have integer numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num == int(effective_num): - num = int(effective_num) - channel_num_map[channel.id] = num - used_numbers.add(num) - - # Second pass: assign integers for channels with float numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num != int(effective_num): - candidate = int(effective_num) - while candidate in used_numbers: - candidate += 1 - channel_num_map[channel.id] = candidate - used_numbers.add(candidate) - - # Host/port/scheme are constant per request; precompute logo URL prefix once. - _base_url = build_absolute_uri_with_port(request, "") - _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) - _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") - _logo_url_prefix = _base_url + _logo_prefix_raw + "/" - _logo_url_suffix = "/" + _logo_suffix_raw - - dummy_program_list = [] - real_epg_map = {} - channel_xml_batch = [] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_logo = channel.effective_logo_obj - effective_number = channel.effective_channel_number - - # user is set only for XC clients, which require integer channel numbers - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - - # Determine the channel ID based on the selected source - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - tvg_logo = "" - - # Check if this is a custom dummy EPG with channel logo URL template - if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - channel_logo_url_template = custom_props.get('channel_logo_url', '') - - if channel_logo_url_template: - pattern_match_name, _ = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - - # Try to extract groups from the channel/stream name and build the logo URL - title_pattern = custom_props.get('title_pattern', '') - if title_pattern: - try: - # Convert PCRE/JavaScript named groups to Python format - title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) - title_regex = regex.compile(title_pattern) - title_match = title_regex.search(pattern_match_name) - - if title_match: - groups = title_match.groupdict() - - # Add normalized versions of all groups for cleaner URLs - for key, value in list(groups.items()): - if value: - # Remove all non-alphanumeric characters and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - groups[f'{key}_normalize'] = normalized - - # Format the logo URL template with the matched groups (with URL encoding) - from urllib.parse import quote - for key, value in groups.items(): - if value: - encoded_value = quote(str(value), safe='') - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) - else: - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') - tvg_logo = channel_logo_url_template - logger.debug(f"Built channel logo URL from template: {tvg_logo}") - except Exception as e: - logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") - - # If no custom dummy logo, use regular logo logic - if not tvg_logo and effective_logo: - if use_cached_logos: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - else: - # Use direct URL if available, otherwise fall back to cached version - direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None - if direct_logo: - tvg_logo = direct_logo - else: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - channel_xml_batch.append(f' ') - channel_xml_batch.append(f' {html.escape(effective_name)}') - channel_xml_batch.append(f' ') - channel_xml_batch.append(" ") - - if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: - yield '\n'.join(channel_xml_batch) + '\n' - channel_xml_batch = [] - - pattern_match_name = effective_name - if effective_epg_data and effective_epg_data.epg_source: - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - if ( - custom_props.get('name_source') == 'stream' - and not stream_lookup_failed - and pattern_match_name != effective_name - ): - stream_index = custom_props.get('stream_index', 1) - 1 - logger.debug( - f"Using stream name for parsing: {pattern_match_name} " - f"(stream index: {stream_index})" - ) - elif stream_lookup_failed: - stream_index = custom_props.get('stream_index', 1) - 1 - logger.warning( - f"Stream index {stream_index} not found for channel " - f"{effective_name}, falling back to channel name" - ) - - if not effective_epg_data: - dummy_program_list.append((channel_id, pattern_match_name, None)) - elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - else: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - - if channel_xml_batch: - yield '\n'.join(channel_xml_batch) + '\n' - - del channels - del channel_num_map - - batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - - all_epg_ids = list(real_epg_map.keys()) - if all_epg_ids: - if num_days > 0: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - start_time__lt=cutoff_date, - ) - else: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - ) - - programs_base_qs = programs_qs.order_by('epg_id', 'id').values( - 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', - 'description', 'custom_properties', - ) - - current_epg_id = None - channel_ids_for_epg = None - pending = [] - program_batch = [] - chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE - last_epg_id = 0 - last_id = 0 - _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") - - def flush_pending(): - nonlocal program_batch, pending - if not pending: - return - pending.sort(key=lambda row: (row[0], row[1])) - escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None - ) - for _, _, xml_text in pending: - program_batch.append(xml_text) - if escaped_primary: - for cid in channel_ids_for_epg[1:]: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{html.escape(cid)}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - pending.clear() - - while True: - program_chunk = list( - programs_base_qs.filter(epg_id__gte=last_epg_id) - .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] - ) - if not program_chunk: - break - - last_row = program_chunk[-1] - last_epg_id = last_row['epg_id'] - last_id = last_row['id'] - - for prog in program_chunk: - epg_id = prog['epg_id'] - - if epg_id != current_epg_id: - yield from flush_pending() - current_epg_id = epg_id - channel_ids_for_epg = real_epg_map[epg_id] - - primary_cid = channel_ids_for_epg[0] - # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format - # directly instead of strftime("%Y%m%d%H%M%S %z"), which is - # ~10x slower and dominates XML build over 750k rows. - st = prog['start_time'] - et = prog['end_time'] - start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" - stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - - program_xml = [f' '] - program_xml.append(f' {html.escape(prog["title"])}') - - if prog['sub_title']: - program_xml.append(f" {html.escape(prog['sub_title'])}") - - if prog['description']: - program_xml.append(f" {html.escape(prog['description'])}") - - custom_data = prog['custom_properties'] or {} - if custom_data: - - if "categories" in custom_data and custom_data["categories"]: - for category in custom_data["categories"]: - program_xml.append(f" {html.escape(category)}") - - if "keywords" in custom_data and custom_data["keywords"]: - for keyword in custom_data["keywords"]: - program_xml.append(f" {html.escape(keyword)}") - - # onscreen_episode takes priority over episode for the onscreen system - if "onscreen_episode" in custom_data: - program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') - elif "episode" in custom_data: - program_xml.append(f' E{custom_data["episode"]}') - - # Handle dd_progid format - if 'dd_progid' in custom_data: - program_xml.append(f' {html.escape(custom_data["dd_progid"])}') - - # Handle external database IDs - for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: - if f'{system}_id' in custom_data: - program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') - - # Add season and episode numbers in xmltv_ns format if available - if "season" in custom_data and "episode" in custom_data: - season = ( - int(custom_data["season"]) - 1 - if str(custom_data["season"]).isdigit() - else 0 - ) - episode = ( - int(custom_data["episode"]) - 1 - if str(custom_data["episode"]).isdigit() - else 0 - ) - program_xml.append(f' {season}.{episode}.') - - if "language" in custom_data: - program_xml.append(f' {html.escape(custom_data["language"])}') - - if "original_language" in custom_data: - program_xml.append(f' {html.escape(custom_data["original_language"])}') - - if "length" in custom_data and isinstance(custom_data["length"], dict): - length_value = custom_data["length"].get("value", "") - length_units = custom_data["length"].get("units", "minutes") - program_xml.append(f' {html.escape(str(length_value))}') - - if "video" in custom_data and isinstance(custom_data["video"], dict): - program_xml.append(" ") - - if "audio" in custom_data and isinstance(custom_data["audio"], dict): - program_xml.append(" ") - - if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): - for subtitle in custom_data["subtitles"]: - if isinstance(subtitle, dict): - subtitle_type = subtitle.get("type", "") - type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" - program_xml.append(f" ") - if "language" in subtitle: - program_xml.append(f" {html.escape(subtitle['language'])}") - program_xml.append(" ") - - if "rating" in custom_data: - rating_system = custom_data.get("rating_system", "TV Parental Guidelines") - program_xml.append(f' ') - program_xml.append(f' {html.escape(custom_data["rating"])}') - program_xml.append(f" ") - - if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): - for star_rating in custom_data["star_ratings"]: - if isinstance(star_rating, dict) and "value" in star_rating: - system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" - program_xml.append(f" ") - program_xml.append(f" {html.escape(star_rating['value'])}") - program_xml.append(" ") - - if "reviews" in custom_data and isinstance(custom_data["reviews"], list): - for review in custom_data["reviews"]: - if isinstance(review, dict) and "content" in review: - review_type = review.get("type", "text") - attrs = [f'type="{html.escape(review_type)}"'] - if "source" in review: - attrs.append(f'source="{html.escape(review["source"])}"') - if "reviewer" in review: - attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') - attr_str = " ".join(attrs) - program_xml.append(f' {html.escape(review["content"])}') - - if "images" in custom_data and isinstance(custom_data["images"], list): - for image in custom_data["images"]: - if isinstance(image, dict) and "url" in image: - attrs = [] - for attr in ['type', 'size', 'orient', 'system']: - if attr in image: - attrs.append(f'{attr}="{html.escape(image[attr])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f' {html.escape(image["url"])}') - - # Add enhanced credits handling - if "credits" in custom_data: - program_xml.append(" ") - credits = custom_data["credits"] - - for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if role in credits: - people = credits[role] - if isinstance(people, list): - for person in people: - program_xml.append(f" <{role}>{html.escape(person)}") - else: - program_xml.append(f" <{role}>{html.escape(people)}") - - # Handle actors separately to include role and guest attributes - if "actor" in credits: - actors = credits["actor"] - if isinstance(actors, list): - for actor in actors: - if isinstance(actor, dict): - name = actor.get("name", "") - role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" - guest_attr = ' guest="yes"' if actor.get("guest") else "" - program_xml.append(f" {html.escape(name)}") - else: - program_xml.append(f" {html.escape(actor)}") - else: - program_xml.append(f" {html.escape(actors)}") - - program_xml.append(" ") - - if "date" in custom_data: - program_xml.append(f' {html.escape(custom_data["date"])}') - - if "country" in custom_data: - program_xml.append(f' {html.escape(custom_data["country"])}') - - if "icon" in custom_data: - program_xml.append(f' ') - elif "sd_icon" in custom_data: - program_xml.append(f' ') - - # Add special flags as proper tags with enhanced handling - if custom_data.get("previously_shown", False): - prev_shown_details = custom_data.get("previously_shown_details", {}) - attrs = [] - if "start" in prev_shown_details: - attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') - if "channel" in prev_shown_details: - attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f" ") - - if custom_data.get("premiere", False): - premiere_text = custom_data.get("premiere_text", "") - if premiere_text: - program_xml.append(f" {html.escape(premiere_text)}") - else: - program_xml.append(" ") - - if custom_data.get("last_chance", False): - last_chance_text = custom_data.get("last_chance_text", "") - if last_chance_text: - program_xml.append(f" {html.escape(last_chance_text)}") - else: - program_xml.append(" ") - - if custom_data.get("new", False): - program_xml.append(" ") - - if custom_data.get('live', False): - program_xml.append(' ') - - program_xml.append(" ") - - xml_text = '\n'.join(program_xml) - pending.append((prog['start_time'], prog['id'], xml_text)) - - del program_chunk - - yield from flush_pending() - - if program_batch: - yield '\n'.join(program_batch) + '\n' - - del real_epg_map - - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source, - export_lookback=lookback_cutoff, - export_cutoff=cutoff_date, - ) - if not dummy_programs: - continue - dummy_batch = [] - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - lines = [ - f' ', - f" {html.escape(program['title'])}", - ] - if program.get('sub_title'): - lines.append(f" {html.escape(program['sub_title'])}") - lines.append(f" {html.escape(program['description'])}") - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - lines.append(f" {html.escape(cat)}") - if 'date' in custom_data: - lines.append(f" {html.escape(custom_data['date'])}") - if custom_data.get('live', False): - lines.append(" ") - if custom_data.get('new', False): - lines.append(" ") - if 'icon' in custom_data: - lines.append(f' ') - lines.append(" ") - dummy_batch.append('\n'.join(lines)) - if len(dummy_batch) >= batch_size: - yield '\n'.join(dummy_batch) + '\n' - dummy_batch = [] - del dummy_programs - if dummy_batch: - yield '\n'.join(dummy_batch) + '\n' - - del dummy_program_list - - yield "\n" - - client_id, client_ip, user_agent = get_client_identifier(request) - event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - - def _log_epg_download(): - from django.core.cache import cache as event_cache - - if not event_cache.get(event_cache_key): - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=channel_count, - client_ip=client_ip, - user_agent=user_agent, - ) - event_cache.set(event_cache_key, True, 2) - - try: - from core.utils import _is_gevent_monkey_patched - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_log_epg_download) - else: - _log_epg_download() - except Exception: - _log_epg_download() - - def build_epg_stream(): - try: - yield from epg_generator() - finally: - _epg_export_teardown() - - return stream_epg_response(content_cache_key, build_epg_stream) - - def xc_get_user(request): username = request.GET.get("username") password = request.GET.get("password") From 107a891359fc0319bae9389e6fa608b0e3d556b4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 13:20:17 -0500 Subject: [PATCH 16/64] enhancement(epg): optimize XMLTV escaping and channel prefetch for improved performance --- CHANGELOG.md | 1 + apps/output/epg.py | 10 +++++----- apps/output/streaming_chunk_cache.py | 7 ++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840cb349..825a06fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. diff --git a/apps/output/epg.py b/apps/output/epg.py index 354d6e5b..4e11fbe8 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1188,7 +1188,7 @@ def generate_epg(request, profile_name=None, user=None): # once per channel (was ~13s of the request on large guides). .defer("epg_data__epg_source__programme_index") .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) ) channel_count = len(channels) @@ -1386,6 +1386,7 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None + escaped_primary_cid = None pending = [] program_batch = [] chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE @@ -1399,8 +1400,7 @@ def generate_epg(request, profile_name=None, user=None): return pending.sort(key=lambda row: (row[0], row[1])) escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None + escaped_primary_cid if len(channel_ids_for_epg) > 1 else None ) for _, _, xml_text in pending: program_batch.append(xml_text) @@ -1435,8 +1435,8 @@ def generate_epg(request, profile_name=None, user=None): yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] + escaped_primary_cid = html.escape(channel_ids_for_epg[0]) - primary_cid = channel_ids_for_epg[0] # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format # directly instead of strftime("%Y%m%d%H%M%S %z"), which is # ~10x slower and dominates XML build over 750k rows. @@ -1445,7 +1445,7 @@ def generate_epg(request, profile_name=None, user=None): start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - program_xml = [f' '] + program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') if prog['sub_title']: diff --git a/apps/output/streaming_chunk_cache.py b/apps/output/streaming_chunk_cache.py index f86bb060..4373f5ce 100644 --- a/apps/output/streaming_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -114,9 +114,14 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + refresh_interval = max(1, lock_ttl // 4) + last_refresh = 0.0 for chunk in source(): redis.rpush(chunks_key, _encode_chunk(chunk)) - _refresh_build_ttl(redis, base_key, lock_ttl) + now = time.monotonic() + if now - last_refresh >= refresh_interval: + _refresh_build_ttl(redis, base_key, lock_ttl) + last_refresh = now yield chunk redis.set(status_key, STATUS_READY) redis.set(_ready_key(base_key), "1") From 0dc8898e8b413764012ef14e80e0a0329c8dd7d1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:10:27 -0500 Subject: [PATCH 17/64] refactor(epg): separate programme index into dedicated table for optimized data handling - Moved the `programme_index` from the `EPGSource` model to a new `EPGSourceIndex` table, ensuring that the large JSON blob is only loaded when explicitly accessed, thus improving query performance and memory efficiency. - Updated related queries and API views to utilize the new structure, including adjustments to EPG generation and import logic to prevent unnecessary data loading. - Enhanced memory management in the EPG grid endpoint to reduce worker RSS during response handling. --- CHANGELOG.md | 7 ++- apps/epg/api_views.py | 25 ++++++----- apps/epg/migrations/0026_epgsourceindex.py | 52 ++++++++++++++++++++++ apps/epg/models.py | 37 ++++++++++++--- apps/epg/tasks.py | 17 ++++--- apps/epg/tests/test_epg_import_api.py | 7 ++- apps/epg/tests/test_programme_index.py | 1 - apps/output/epg.py | 38 +++++----------- apps/output/tests.py | 12 ++++- core/tasks.py | 8 +++- core/tests.py | 6 +-- core/utils.py | 24 ++++++++++ 12 files changed, 169 insertions(+), 65 deletions(-) create mode 100644 apps/epg/migrations/0026_epgsourceindex.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 825a06fc..0ad1ef3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed +- **`programme_index` moved off `EPGSource` into a dedicated `EPGSourceIndex` table.** The multi-MB byte-offset index was repeatedly pulled into web and Celery workers by ordinary `EPGSource` queries and `select_related` JOINs (which ignore manager-level `defer()`). It now lives in a one-to-one `EPGSourceIndex` row, read only when explicitly accessed through the `EPGSource.programme_index` property, so no list, detail, or JOIN query can load it by accident. Migration `0026` copies existing indices across. No API or index-build behavior change. + - **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. @@ -29,7 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. -- **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. +- **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index fbc419c5..1e0bfc41 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -64,8 +64,6 @@ class EPGSourceViewSet(viewsets.ModelViewSet): from apps.channels.models import Channel return EPGSource.objects.select_related( "refresh_task__crontab", "refresh_task__interval" - ).defer( - 'programme_index' ).annotate( has_channels=Exists( Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk')) @@ -1082,13 +1080,19 @@ class EPGGridAPIView(APIView): f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" ) - # Combine regular and dummy programs - all_programs = list(serialized_programs) + dummy_programs + # Combine regular and dummy programs in place to avoid copying the large list + serialized_programs.extend(dummy_programs) logger.debug( - f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)." + f"EPGGridAPIView: Returning {len(serialized_programs)} total programs (including {len(dummy_programs)} dummy programs)." ) - return Response({"data": all_programs}, status=status.HTTP_200_OK) + # The grid materializes tens of thousands of program dicts plus the + # rendered JSON; trim once the response is sent so worker RSS does not + # ratchet up per request. + from core.utils import spawn_memory_trim + response = Response({"data": serialized_programs}, status=status.HTTP_200_OK) + response._resource_closers.append(spawn_memory_trim) + return response # ───────────────────────────── @@ -1119,7 +1123,7 @@ class EPGImportAPIView(APIView): epg_id = request.data.get("id", None) force = bool(request.data.get("force", False)) - # Reject dummy sources without loading programme_index (multi-MB JSON). + # Reject dummy sources with a narrow existence query, no full row load. if epg_id is not None: from .models import EPGSource @@ -1236,10 +1240,9 @@ class CurrentProgramsAPIView(APIView): # Limit to 50 IDs per request epg_data_ids = epg_data_ids[:50] - # Defer the multi-MB programme_index the JOIN would pull once per row. The lookup reads it via a targeted refresh_from_db - epg_data_entries = EPGData.objects.select_related('epg_source').defer( - 'epg_source__programme_index' - ).filter(id__in=epg_data_ids) + epg_data_entries = EPGData.objects.select_related('epg_source').filter( + id__in=epg_data_ids + ) # Batch-fetch current programs for all requested EPG entries in one query db_programs = ProgramData.objects.filter( diff --git a/apps/epg/migrations/0026_epgsourceindex.py b/apps/epg/migrations/0026_epgsourceindex.py new file mode 100644 index 00000000..75d9d4eb --- /dev/null +++ b/apps/epg/migrations/0026_epgsourceindex.py @@ -0,0 +1,52 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def copy_index_forward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + rows = list( + EPGSource.objects.exclude(programme_index__isnull=True).values_list( + 'id', 'programme_index' + ) + ) + for source_id, data in rows: + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': data} + ) + + +def copy_index_backward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + for source_id, data in EPGSourceIndex.objects.values_list('source_id', 'data'): + EPGSource.objects.filter(id=source_id).update(programme_index=data) + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0025_programdata_epg_id_index'), + ] + + operations = [ + migrations.CreateModel( + name='EPGSourceIndex', + fields=[ + ('source', models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name='index_record', + serialize=False, + to='epg.epgsource', + )), + ('data', models.JSONField(blank=True, default=None, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.RunPython(copy_index_forward, copy_index_backward), + migrations.RemoveField( + model_name='epgsource', + name='programme_index', + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 04c6072c..b85453af 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -62,12 +62,6 @@ class EPGSource(models.Model): blank=True, help_text="Last status message, including success results or error information" ) - programme_index = models.JSONField( - null=True, - blank=True, - default=None, - help_text="Byte-offset index mapping tvg_id to file positions, built after each EPG refresh" - ) created_at = models.DateTimeField( auto_now_add=True, help_text="Time when this source was created" @@ -124,6 +118,37 @@ class EPGSource(models.Model): kwargs['update_fields'].remove('updated_at') super().save(*args, **kwargs) + @property + def programme_index(self): + """Byte-offset index for this source, read on demand from the separate + EPGSourceIndex table so the multi-MB blob is never pulled into EPGSource + queries or select_related JOINs. Returns the stored dict or None.""" + return ( + EPGSourceIndex.objects.filter(source_id=self.pk) + .values_list('data', flat=True) + .first() + ) + + +class EPGSourceIndex(models.Model): + """Byte-offset programme index for an EPGSource, stored in its own table. + + Kept out of EPGSource so the multi-MB JSON blob is only loaded when read + explicitly, never when querying or joining EPGSource rows. + """ + source = models.OneToOneField( + EPGSource, + on_delete=models.CASCADE, + related_name='index_record', + primary_key=True, + ) + data = models.JSONField(null=True, blank=True, default=None) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return f"Programme index for source {self.source_id}" + + class EPGData(models.Model): tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=512) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 9661c5b9..83aaa275 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -26,7 +26,7 @@ from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 +from .models import EPGSource, EPGSourceIndex, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 from core.utils import ( acquire_task_lock, is_task_lock_held, @@ -653,7 +653,9 @@ def _refresh_epg_data_impl(source_id, force=False): if source.source_type == 'xmltv': # Invalidate the byte-offset index before downloading the new file # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) + EPGSourceIndex.objects.update_or_create( + source_id=source.id, defaults={'data': None} + ) if not fetch_xmltv(source): logger.error(f"Failed to fetch XMLTV for source {source.name}") return @@ -4484,7 +4486,7 @@ def _programme_to_dict(elem, start_time, end_time): def build_programme_index(source_id): """ Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map. - Persists the result to EPGSource.programme_index. Most XMLTV files group programmes + Persists the result to the EPGSourceIndex table. Most XMLTV files group programmes by channel, but some split a channel across multiple non-contiguous blocks, so we record block starts up to _OFFSET_CAP and mark only channels that exceed the cap as interleaved. @@ -4573,7 +4575,9 @@ def build_programme_index(source_id): 'channels': index, 'interleaved_channels': sorted(interleaved_channels), } - EPGSource.objects.filter(id=source_id).update(programme_index=result) + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': result} + ) @shared_task @@ -4620,9 +4624,8 @@ def find_current_program_for_tvg_id(epg_or_id): return None now = timezone.now() - # Force a fresh read of the DB-backed index to avoid using stale related-object - # state when an EPG refresh invalidates/rebuilds the index concurrently. - source.refresh_from_db(fields=['programme_index']) + # The property reads the EPGSourceIndex table fresh on each access, so a + # concurrent refresh invalidating/rebuilding the index can't serve stale state. index = source.programme_index if index is not None: diff --git a/apps/epg/tests/test_epg_import_api.py b/apps/epg/tests/test_epg_import_api.py index c58f2a45..cc96c214 100644 --- a/apps/epg/tests/test_epg_import_api.py +++ b/apps/epg/tests/test_epg_import_api.py @@ -7,7 +7,7 @@ from django.test import TestCase from rest_framework import status from rest_framework.test import APIClient -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex User = get_user_model() @@ -47,7 +47,10 @@ class EPGImportAPITests(TestCase): name="Large Index XMLTV", source_type="xmltv", url="http://example.com/epg.xml", - programme_index={ + ) + EPGSourceIndex.objects.create( + source=source, + data={ "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)}, "interleaved_channels": [], }, diff --git a/apps/epg/tests/test_programme_index.py b/apps/epg/tests/test_programme_index.py index 5ece0ae1..829afc87 100644 --- a/apps/epg/tests/test_programme_index.py +++ b/apps/epg/tests/test_programme_index.py @@ -519,7 +519,6 @@ class FindCurrentProgramTests(TestCase): name="No Index", source_type="xmltv", file_path=FIXTURE_XML, - programme_index=None, ) epg = EPGData.objects.create( tvg_id="channel.current", diff --git a/apps/output/epg.py b/apps/output/epg.py index 4e11fbe8..626b21aa 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -40,34 +40,20 @@ def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cut def _ceil_to_half_hour(dt): """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) + original = dt.replace(microsecond=0) + aligned = dt.replace(second=0, microsecond=0) + remainder = aligned.minute % 30 + if remainder != 0: + aligned += timedelta(minutes=30 - remainder) + if aligned < original: + aligned += timedelta(minutes=30) + return aligned def _epg_export_teardown(): - from django.db import close_old_connections + from core.utils import spawn_memory_trim - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() + spawn_memory_trim(close_connections=True) def _ordered_channel_streams(channel): @@ -1183,10 +1169,6 @@ def generate_epg(request, profile_name=None, user=None): with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) diff --git a/apps/output/tests.py b/apps/output/tests.py index 5f4fd5f3..870ff765 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -355,4 +355,14 @@ class OutputEPGHelperTest(SimpleTestCase): aligned = _ceil_to_half_hour(dt) self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) - self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + def test_ceil_to_half_hour_past_boundary_second(self): + from django.utils import timezone + from apps.output.epg import _ceil_to_half_hour + + dt = timezone.now().replace(minute=0, second=52, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) diff --git a/core/tasks.py b/core/tasks.py index b562f335..a64469b4 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -398,12 +398,16 @@ def scan_and_process_files(): def _rebuild_programme_indices(): """Queue index builds for active EPG sources that are missing their DB index.""" try: + from django.db.models import Q from apps.epg.tasks import build_programme_index_task sources = EPGSource.objects.filter( is_active=True, - programme_index__isnull=True, - ).exclude(source_type__in=('dummy', 'schedules_direct')) + ).exclude( + source_type__in=('dummy', 'schedules_direct') + ).filter( + Q(index_record__isnull=True) | Q(index_record__data__isnull=True) + ) count = 0 for source in sources: diff --git a/core/tests.py b/core/tests.py index 33475ae7..2af52f07 100644 --- a/core/tests.py +++ b/core/tests.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock from django.test import TestCase, SimpleTestCase -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -37,14 +37,10 @@ class DispatcharrUserAgentTests(TestCase): class ProgrammeIndexRebuildTests(TestCase): def test_startup_rebuild_does_not_lock_out_queued_build_task(self): - EPGSource.objects.update( - programme_index={"channels": {}, "interleaved_channels": []} - ) source = EPGSource.objects.create( name="Missing Index", source_type="xmltv", is_active=True, - programme_index=None, ) class FakeRedis: diff --git a/core/utils.py b/core/utils.py index 89228088..44504b6a 100644 --- a/core/utils.py +++ b/core/utils.py @@ -608,6 +608,30 @@ def cleanup_memory(log_usage=False, force_collection=True): pass logger.trace("Memory cleanup complete for django") + +def spawn_memory_trim(close_connections=False): + """Reclaim a request's heap pages: GC, then return freed C pages to the OS. + + On gevent uWSGI workers the trim runs in a spawned greenlet so it never + blocks the caller; Celery prefork workers (no gevent hub) run it inline. + Set close_connections=True when called from a streaming generator's teardown + so the pooled DB connection is released first. + """ + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if close_connections: + from django.db import close_old_connections + close_old_connections() + + if _is_gevent_monkey_patched(): + import gevent + gevent.spawn(_run) + else: + _run() + + def safe_upload_path(filename: str, base_dir) -> str: """Return a safe absolute path for an uploaded file within base_dir. From e984b234e34fd5ad4d5804c2fef8348c8cff3a65 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:27:15 -0500 Subject: [PATCH 18/64] enhancement(epg): add export lookback and cutoff parameters to EPG generation - Updated the `generate_dummy_programs` function to include `export_lookback` and `export_cutoff` parameters, allowing for more precise control over the EPG data generation window. - Added a new test case to verify that future events are correctly represented in the grid window, ensuring upcoming fillers are displayed as expected. --- apps/epg/api_views.py | 4 +++- apps/output/tests.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 1e0bfc41..cdb39a47 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -984,7 +984,9 @@ class EPGGridAPIView(APIView): channel_name=name_to_parse, num_days=1, program_length_hours=4, - epg_source=epg_source + epg_source=epg_source, + export_lookback=one_hour_ago, + export_cutoff=twenty_four_hours_later, ) # Custom dummy should always return data (either from patterns or fallback) diff --git a/apps/output/tests.py b/apps/output/tests.py index 870ff765..e713df21 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -338,6 +338,55 @@ class OutputEPGCustomDummyTest(TestCase): ) self.assertGreaterEqual(programs[0]['start_time'], lookback) + def test_custom_dummy_future_event_fills_grid_window_with_upcoming(self): + """Grid-style window: future event should show upcoming filler, not empty.""" + from django.utils import timezone + from apps.output.epg import _programme_overlaps_export_window, generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy Future", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + now = timezone.now() + grid_start = now - timedelta(hours=1) + grid_end = now + timedelta(hours=24) + future = now + timedelta(days=3) + channel_name = ( + f"NHL 01: Washington Capitals vs Philadelphia Flyers @ " + f"{future.strftime('%B')} {future.day} 07:30 PM ET" + ) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=1, + epg_source=epg_source, + export_lookback=grid_start, + export_cutoff=grid_end, + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all( + _programme_overlaps_export_window( + p["start_time"], p["end_time"], grid_start, grid_end + ) + for p in programs + ), + "All programmes should overlap the grid query window", + ) + self.assertTrue( + any("Upcoming" in p.get("description", "") for p in programs), + "Future events outside the window should show upcoming filler", + ) + class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): From 7b6adf62d7f84e69f2d121b6d24fa2e17bea499d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:15:29 -0500 Subject: [PATCH 19/64] enhancement(vod): optimize VOD stream retrieval and performance - Improved `xc_get_vod_streams` and `xc_get_series` functions to utilize a new helper method for fetching distinct relations based on account priority, significantly reducing memory usage and response times for large libraries. - Updated the handling of VOD streams to avoid unnecessary ORM instantiation, enhancing performance for endpoints dealing with extensive movie and series data. - Added comprehensive tests to ensure the correctness of the new logic and verify that the highest priority relations are correctly selected. --- CHANGELOG.md | 1 + apps/output/tests.py | 483 ++++++++++++++++++++++++++++++++++++++++++- apps/output/views.py | 189 ++++++++++------- 3 files changed, 595 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad1ef3b..6bc8054f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. diff --git a/apps/output/tests.py b/apps/output/tests.py index e713df21..30b22a18 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,9 +1,23 @@ -from django.test import TestCase, Client, SimpleTestCase +from django.test import TestCase, Client, SimpleTestCase, RequestFactory from django.urls import reverse +from unittest import skipUnless from unittest.mock import patch from uuid import uuid4 +from django.db import connection +from django.test.utils import CaptureQueriesContext from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource +from apps.accounts.models import User +from apps.m3u.models import M3UAccount +from apps.output.views import xc_get_series, xc_get_vod_streams +from apps.vod.models import ( + M3UMovieRelation, + M3USeriesRelation, + Movie, + Series, + VODCategory, + VODLogo, +) import xml.etree.ElementTree as ET from datetime import timedelta @@ -415,3 +429,470 @@ class OutputEPGHelperTest(SimpleTestCase): self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + +class XcVodSeriesDistinctTests(TestCase): + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0, is_active=True): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + is_active=is_active, + ) + + def test_vod_streams_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + movie = Movie.objects.create(name="Shared Movie", year=2020) + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + stream_id="low-stream", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + stream_id="high-stream", + container_extension="mp4", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Shared Movie") + self.assertEqual(streams[0]["container_extension"], "mp4") + + def test_vod_streams_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}", priority=1) + inactive = self._account( + f"inactive-{uuid4().hex[:6]}", priority=99, is_active=False + ) + active_movie = Movie.objects.create(name="Active Movie") + inactive_movie = Movie.objects.create(name="Inactive Only Movie") + M3UMovieRelation.objects.create( + m3u_account=active, + movie=active_movie, + stream_id="active-1", + ) + M3UMovieRelation.objects.create( + m3u_account=inactive, + movie=inactive_movie, + stream_id="inactive-1", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + names = {s["name"] for s in streams} + self.assertEqual(names, {"Active Movie"}) + + def test_vod_streams_category_filter(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + action_movie = Movie.objects.create(name="Action Movie") + comedy_movie = Movie.objects.create(name="Comedy Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=action_movie, + category=action, + stream_id="action-1", + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=comedy_movie, + category=comedy, + stream_id="comedy-1", + ) + + streams = xc_get_vod_streams(self.request, self.user, category_id=action.id) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Action Movie") + self.assertEqual(streams[0]["category_id"], str(action.id)) + + def test_vod_streams_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + zebra = Movie.objects.create(name="Zebra Film") + apple = Movie.objects.create(name="Apple Film") + M3UMovieRelation.objects.create( + m3u_account=account, movie=zebra, stream_id="z-1" + ) + M3UMovieRelation.objects.create( + m3u_account=account, movie=apple, stream_id="a-1" + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual([s["name"] for s in streams], ["Apple Film", "Zebra Film"]) + + def test_vod_streams_includes_metadata_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create( + name="Rich Movie", + description="A plot", + genre="Drama", + year=2021, + rating="8", + custom_properties={ + "director": "Dir", + "actors": "Cast", + "release_date": "2021-01-01", + "youtube_trailer": "yt123", + }, + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="rich-1", + container_extension="avi", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["plot"], "A plot") + self.assertEqual(stream["genre"], "Drama") + self.assertEqual(stream["year"], 2021) + self.assertEqual(stream["director"], "Dir") + self.assertEqual(stream["cast"], "Cast") + self.assertEqual(stream["release_date"], "2021-01-01") + self.assertEqual(stream["trailer"], "yt123") + self.assertEqual(stream["container_extension"], "avi") + + def test_vod_streams_stream_icon_uses_logo_id_without_logo_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Poster", url="http://example.com/poster.png") + movie = Movie.objects.create(name="Logo Movie", logo=logo) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="logo-1", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIn(f"/{logo.id}/", stream["stream_icon"]) + + def test_series_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + series = Series.objects.create(name="Shared Series", year=2019) + M3USeriesRelation.objects.create( + m3u_account=low, + series=series, + external_series_id="low-series", + ) + high_rel = M3USeriesRelation.objects.create( + m3u_account=high, + series=series, + external_series_id="high-series", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["name"], "Shared Series") + self.assertEqual(results[0]["series_id"], high_rel.id) + + def test_series_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}") + inactive = self._account(f"inactive-{uuid4().hex[:6]}", is_active=False) + active_series = Series.objects.create(name="Active Series") + inactive_series = Series.objects.create(name="Inactive Only Series") + M3USeriesRelation.objects.create( + m3u_account=active, + series=active_series, + external_series_id="active-s", + ) + M3USeriesRelation.objects.create( + m3u_account=inactive, + series=inactive_series, + external_series_id="inactive-s", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual({r["name"] for r in results}, {"Active Series"}) + + def test_series_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + z = Series.objects.create(name="Zulu Show") + a = Series.objects.create(name="Alpha Show") + M3USeriesRelation.objects.create( + m3u_account=account, series=z, external_series_id="z" + ) + M3USeriesRelation.objects.create( + m3u_account=account, series=a, external_series_id="a" + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual([r["name"] for r in results], ["Alpha Show", "Zulu Show"]) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_vod_streams_dedupe_query_avoids_movie_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Query Shape Movie") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="qs-1" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_vod_streams(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_movie"', distinct_queries[0]["sql"]) + self.assertNotIn('"vod_vodlogo"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_movie"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_series_dedupe_query_avoids_series_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Query Shape Series") + M3USeriesRelation.objects.create( + m3u_account=account, series=series, external_series_id="qs-s" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_series(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_series"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_series"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + +XC_VOD_STREAM_KEYS = frozenset({ + "num", "name", "stream_type", "stream_id", "stream_icon", "rating", + "rating_5based", "added", "is_adult", "tmdb_id", "imdb_id", "trailer", + "plot", "genre", "year", "director", "cast", "release_date", "category_id", + "category_ids", "container_extension", "custom_sid", "direct_source", +}) + +XC_SERIES_KEYS = frozenset({ + "num", "name", "series_id", "cover", "plot", "cast", "director", "genre", + "release_date", "releaseDate", "last_modified", "rating", "rating_5based", + "backdrop_path", "youtube_trailer", "episode_run_time", "category_id", + "category_ids", "tmdb_id", "imdb_id", +}) + + +class XcVodSeriesRegressionTests(TestCase): + """Full output-shape and edge-case regressions for XC list endpoints.""" + + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-reg-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + ) + + def test_vod_streams_empty_library(self): + self.assertEqual(xc_get_vod_streams(self.request, self.user), []) + + def test_series_empty_library(self): + self.assertEqual(xc_get_series(self.request, self.user), []) + + def test_vod_streams_response_keys(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Schema Movie", rating="10") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="schema-1" + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(set(stream.keys()), XC_VOD_STREAM_KEYS) + self.assertEqual(stream["stream_type"], "movie") + self.assertEqual(stream["stream_id"], movie.id) + self.assertEqual(stream["rating_5based"], 5.0) + self.assertEqual(stream["custom_sid"], None) + self.assertEqual(stream["direct_source"], "") + + def test_vod_streams_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Sparse Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="sparse-1", + container_extension=None, + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIsNone(stream["stream_icon"]) + self.assertEqual(stream["category_id"], "0") + self.assertEqual(stream["category_ids"], []) + self.assertEqual(stream["container_extension"], "mp4") + self.assertEqual(stream["plot"], "") + self.assertEqual(stream["trailer"], "") + self.assertEqual(stream["tmdb_id"], "") + self.assertEqual(stream["imdb_id"], "") + + def test_vod_streams_category_from_winning_relation(self): + """Category must come from the highest-priority relation, not any relation.""" + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + movie = Movie.objects.create(name="Dual Category Movie") + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + category=action, + stream_id="low-cat", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + category=comedy, + stream_id="high-cat", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["category_id"], str(comedy.id)) + self.assertEqual(stream["category_ids"], [comedy.id]) + + def test_series_response_keys_and_metadata(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Cover", url="http://example.com/cover.png") + category = VODCategory.objects.create(name="Drama", category_type="series") + series = Series.objects.create( + name="Schema Series", + description="Series plot", + genre="Sci-Fi", + year=2022, + rating="8", + tmdb_id="tm123", + imdb_id="tt123", + logo=logo, + custom_properties={ + "cast": "Actor A", + "director": "Director B", + "release_date": "2022-06-01", + "backdrop_path": ["/img1.jpg"], + "youtube_trailer": "yt-series", + "episode_run_time": "45", + }, + ) + relation = M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + category=category, + external_series_id="schema-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(set(row.keys()), XC_SERIES_KEYS) + self.assertEqual(row["series_id"], relation.id) + self.assertIn(f"/{logo.id}/", row["cover"]) + self.assertEqual(row["plot"], "Series plot") + self.assertEqual(row["cast"], "Actor A") + self.assertEqual(row["director"], "Director B") + self.assertEqual(row["genre"], "Sci-Fi") + self.assertEqual(row["release_date"], "2022-06-01") + self.assertEqual(row["releaseDate"], "2022-06-01") + self.assertEqual(row["backdrop_path"], ["/img1.jpg"]) + self.assertEqual(row["youtube_trailer"], "yt-series") + self.assertEqual(row["episode_run_time"], "45") + self.assertEqual(row["tmdb_id"], "tm123") + self.assertEqual(row["imdb_id"], "tt123") + self.assertEqual(row["category_id"], str(category.id)) + self.assertEqual(row["category_ids"], [category.id]) + self.assertEqual(row["last_modified"], str(int(relation.updated_at.timestamp()))) + + def test_series_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Sparse Series") + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="sparse-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertIsNone(row["cover"]) + self.assertEqual(row["category_id"], "0") + self.assertEqual(row["category_ids"], []) + self.assertEqual(row["release_date"], "") + self.assertEqual(row["releaseDate"], "") + self.assertEqual(row["backdrop_path"], []) + self.assertEqual(row["youtube_trailer"], "") + self.assertEqual(row["episode_run_time"], "") + + def test_series_release_date_falls_back_to_year(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Year Only", year=2018) + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="year-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(row["release_date"], "2018") + self.assertEqual(row["releaseDate"], "2018") + + def test_priority_tiebreaker_uses_lower_relation_id(self): + """Same priority: DISTINCT ON tie-breaks on relation id ascending.""" + a1 = self._account(f"a1-{uuid4().hex[:6]}", priority=5) + a2 = self._account(f"a2-{uuid4().hex[:6]}", priority=5) + movie = Movie.objects.create(name="Tie Movie") + first = M3UMovieRelation.objects.create( + m3u_account=a1, + movie=movie, + stream_id="first", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=a2, + movie=movie, + stream_id="second", + container_extension="mp4", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["container_extension"], first.container_extension) diff --git a/apps/output/views.py b/apps/output/views.py index 4224fb35..699c9f95 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -918,6 +918,73 @@ def xc_get_epg(request, user, short=False): return output +XC_MOVIE_VALUE_FIELDS = ( + 'id', 'movie_id', 'category_id', 'container_extension', + 'movie__id', 'movie__name', 'movie__rating', 'movie__created_at', + 'movie__tmdb_id', 'movie__imdb_id', 'movie__description', 'movie__genre', + 'movie__year', 'movie__custom_properties', 'movie__logo_id', +) + +XC_SERIES_VALUE_FIELDS = ( + 'id', 'series_id', 'category_id', 'updated_at', + 'series__id', 'series__name', 'series__description', 'series__genre', + 'series__year', 'series__rating', 'series__custom_properties', 'series__logo_id', + 'series__tmdb_id', 'series__imdb_id', +) + + +def _xc_fetch_priority_distinct_relations( + *, + manager, + rel_filters, + distinct_field, + value_fields, + order_by_name_field, +): + """ + Return one row dict per distinct content ID (highest account priority wins). + + On PostgreSQL, dedupe on narrow relation rows first, then fetch display + columns via values() (no ORM model instantiation). That avoids sorting + wide joined rows during DISTINCT ON and reduces parallel worker /dev/shm + pressure in Docker. + """ + from django.db import connection, transaction + + narrow_qs = manager.filter(**rel_filters) + + def _fetch_by_ids(ids): + return list( + manager.filter(pk__in=ids) + .values(*value_fields) + .order_by(Lower(order_by_name_field)) + ) + + if connection.vendor == 'postgresql': + winning_ids_qs = ( + narrow_qs + .order_by(distinct_field, '-m3u_account__priority', 'id') + .distinct(distinct_field) + .values('pk') + ) + with transaction.atomic(): + #with connection.cursor() as cursor: + #cursor.execute("SET LOCAL max_parallel_workers_per_gather = 0") + winning_ids = list(winning_ids_qs.values_list('pk', flat=True)) + if not winning_ids: + return [] + return _fetch_by_ids(winning_ids) + + seen = {} + for row in narrow_qs.values(*value_fields).order_by('-m3u_account__priority', 'id'): + key = row[distinct_field] + if key not in seen: + seen[key] = row + rows = list(seen.values()) + rows.sort(key=lambda r: (r[order_by_name_field] or '').lower()) + return rows + + def xc_get_vod_categories(user): """Get VOD categories for XtreamCodes API""" from apps.vod.models import VODCategory, M3UMovieRelation @@ -943,33 +1010,19 @@ def xc_get_vod_categories(user): def xc_get_vod_streams(request, user, category_id=None): """Get VOD streams (movies) for XtreamCodes API""" from apps.vod.models import M3UMovieRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3UMovieRelation.objects - .filter(**rel_filters) - .select_related('movie', 'movie__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3UMovieRelation.objects, + rel_filters=rel_filters, + distinct_field='movie_id', + value_fields=XC_MOVIE_VALUE_FIELDS, + order_by_name_field='movie__name', ) - if connection.vendor == 'postgresql': - # DISTINCT ON returns one row per movie (highest-priority active relation) - # in a single query. ORDER BY must lead with the DISTINCT field. - relations = list( - base_qs.order_by('movie_id', '-m3u_account__priority', 'id') - .distinct('movie_id') - ) - else: - # SQLite fallback: fetch all matching relations, deduplicate in Python. - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.movie_id not in seen: - seen[rel.movie_id] = rel - relations = list(seen.values()) - # Precompute logo URL prefix/suffix once (mirrors _xc_live_streams_setup) # so each row only needs a string concat instead of reverse() + URI build. _base_url = build_absolute_uri_with_port(request, "") @@ -978,44 +1031,40 @@ def xc_get_vod_streams(request, user, category_id=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - # Sort by name (DISTINCT ON forces ORDER BY movie_id; SQLite path is unsorted). - relations.sort(key=lambda r: (r.movie.name or "").lower()) - streams = [] append = streams.append - for num, relation in enumerate(relations, 1): - movie = relation.movie - custom_props = movie.custom_properties or {} - category = relation.category - category_id_str = str(category.id) if category else "0" - category_id_list = [category.id] if category else [] - rating = movie.rating - logo = movie.logo + for num, row in enumerate(relations, 1): + custom_props = row['movie__custom_properties'] or {} + category_id = row['category_id'] + category_id_str = str(category_id) if category_id else "0" + category_id_list = [category_id] if category_id else [] + rating = row['movie__rating'] + logo_id = row['movie__logo_id'] append({ "num": num, - "name": movie.name, + "name": row['movie__name'], "stream_type": "movie", - "stream_id": movie.id, + "stream_id": row['movie__id'], "stream_icon": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), "rating": rating or "0", "rating_5based": round(float(rating or 0) / 2, 2) if rating else 0, - "added": str(int(movie.created_at.timestamp())), + "added": str(int(row['movie__created_at'].timestamp())), "is_adult": 0, - "tmdb_id": movie.tmdb_id or "", - "imdb_id": movie.imdb_id or "", + "tmdb_id": row['movie__tmdb_id'] or "", + "imdb_id": row['movie__imdb_id'] or "", "trailer": custom_props.get('youtube_trailer') or "", - "plot": movie.description or "", - "genre": movie.genre or "", - "year": movie.year or "", + "plot": row['movie__description'] or "", + "genre": row['movie__genre'] or "", + "year": row['movie__year'] or "", "director": custom_props.get('director', ''), "cast": custom_props.get('actors', ''), "release_date": custom_props.get('release_date', ''), "category_id": category_id_str, "category_ids": category_id_list, - "container_extension": relation.container_extension or "mp4", + "container_extension": row['container_extension'] or "mp4", "custom_sid": None, "direct_source": "", }) @@ -1048,72 +1097,58 @@ def xc_get_series_categories(user): def xc_get_series(request, user, category_id=None): """Get series list for XtreamCodes API""" from apps.vod.models import M3USeriesRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3USeriesRelation.objects - .filter(**rel_filters) - .select_related('series', 'series__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3USeriesRelation.objects, + rel_filters=rel_filters, + distinct_field='series_id', + value_fields=XC_SERIES_VALUE_FIELDS, + order_by_name_field='series__name', ) - if connection.vendor == 'postgresql': - relations = list( - base_qs.order_by('series_id', '-m3u_account__priority', 'id') - .distinct('series_id') - ) - else: - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.series_id not in seen: - seen[rel.series_id] = rel - relations = list(seen.values()) - _base_url = build_absolute_uri_with_port(request, "") _sample_logo_path = reverse("api:vod:vodlogo-cache", args=[0]) _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - relations.sort(key=lambda r: (r.series.name or "").lower()) - series_list = [] append = series_list.append - for num, relation in enumerate(relations, 1): - series = relation.series - custom_props = series.custom_properties or {} - category = relation.category - rating = series.rating - logo = series.logo - year_str = str(series.year) if series.year else "" + for num, row in enumerate(relations, 1): + custom_props = row['series__custom_properties'] or {} + category_id = row['category_id'] + rating = row['series__rating'] + logo_id = row['series__logo_id'] + year_str = str(row['series__year']) if row['series__year'] else "" release_date = custom_props.get('release_date', year_str) append({ "num": num, - "name": series.name, - "series_id": relation.id, + "name": row['series__name'], + "series_id": row['id'], "cover": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), - "plot": series.description or "", + "plot": row['series__description'] or "", "cast": custom_props.get('cast', ''), "director": custom_props.get('director', ''), - "genre": series.genre or "", + "genre": row['series__genre'] or "", "release_date": release_date, "releaseDate": release_date, - "last_modified": str(int(relation.updated_at.timestamp())), + "last_modified": str(int(row['updated_at'].timestamp())), "rating": str(rating or "0"), "rating_5based": str(round(float(rating or 0) / 2, 2)) if rating else "0", "backdrop_path": custom_props.get('backdrop_path', []), "youtube_trailer": custom_props.get('youtube_trailer', ''), "episode_run_time": custom_props.get('episode_run_time', ''), - "category_id": str(category.id) if category else "0", - "category_ids": [category.id] if category else [], - "tmdb_id": series.tmdb_id or "", - "imdb_id": series.imdb_id or "", + "category_id": str(category_id) if category_id else "0", + "category_ids": [category_id] if category_id else [], + "tmdb_id": row['series__tmdb_id'] or "", + "imdb_id": row['series__imdb_id'] or "", }) return series_list From 53fa1e42a0a10ae8fcc078d9bb978e464e0190e2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:54:12 -0500 Subject: [PATCH 20/64] enhancement(tests): introduce isolated backend test settings and improve test documentation - Added `dispatcharr.settings_test` for isolated testing, automatically creating an empty PostgreSQL test database and ensuring transaction isolation during tests. - Updated `manage.py` to switch to the test settings when running tests. - Enhanced the CONTRIBUTING.md file with detailed instructions on running the backend test suite and handling Celery tasks in tests. - Refactored test cases to use static methods for `worker_id` in `test_process_label.py` and adjusted assertions in `test_user_preferences.py` for better clarity and correctness. --- CHANGELOG.md | 4 +++ CONTRIBUTING.md | 14 +++++++- dispatcharr/settings_test.py | 58 ++++++++++++++++++++++++++++++++++ manage.py | 7 +++- tests/test_process_label.py | 4 +-- tests/test_user_preferences.py | 4 +-- 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 dispatcharr/settings_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc8054f..dfdeeda5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. + ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 725e21e8..6da68a61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,7 +144,19 @@ Untested code is significantly less likely to be merged. - Use Django's `TestCase` for unit/integration tests. - Test files live at `apps//tests/`. -- Run the test suite with: `uv run python manage.py test` +- Run the backend test suite with: + + ```bash + python manage.py test + ``` + + `manage.py` automatically uses `dispatcharr.settings_test`, which creates an empty PostgreSQL database `test_` (same engine as production), runs migrations, and rolls back each test in a transaction. Your live VOD/channels data is not used. + + Optional: `TEST_USE_SQLITE=1` for machines without Postgres (some PostgreSQL-only tests skip automatically). + + Tests that exercise Celery task bodies should use `@override_settings(CELERY_TASK_ALWAYS_EAGER=True)` locally. Global eager mode is off because `post_save` signals on M3U/EPG models call `.delay()` and would break `TestCase` transaction isolation. + +- Do **not** override with `--settings=dispatcharr.settings` on a live instance. ### Frontend diff --git a/dispatcharr/settings_test.py b/dispatcharr/settings_test.py new file mode 100644 index 00000000..fe4225fa --- /dev/null +++ b/dispatcharr/settings_test.py @@ -0,0 +1,58 @@ +""" +Django settings for running the backend test suite in isolation. + +Always use this module instead of dispatcharr.settings when running tests: + + python manage.py test + +`manage.py` selects this module automatically for the ``test`` command. + +Django creates a separate empty database (``test_``) and runs +migrations — your live data under /data/db is not used. + +Why not dispatcharr.settings? +- Production/AIO points at the live ``dispatcharr`` database. +- django-db-geventpool breaks TestCase transaction isolation on pooled connections. + +SQLite (``TEST_USE_SQLITE=1``) is an optional fallback for machines without +Postgres; production and CI should use the default Postgres test database. +""" +import os + +from dispatcharr.settings import * # noqa: F401,F403 + +# Fast password hashing for tests. +PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +# Do NOT run Celery tasks inline during tests. post_save signals on M3UAccount and +# EPGSource call .delay(); eager mode runs them inside TestCase transactions and +# closes/poisons the DB connection for subsequent queries in the same test. +CELERY_TASK_ALWAYS_EAGER = False +CELERY_TASK_EAGER_PROPAGATES = False + +_use_sqlite = os.environ.get("TEST_USE_SQLITE", "").lower() in ("1", "true", "yes") + +if _use_sqlite: + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } + } +else: + # Default: PostgreSQL with Django-managed test_dispatcharr (matches production). + # Uses the standard backend (not geventpool) so TestCase transactions isolate. + _pg_name = os.environ.get("POSTGRES_DB", "dispatcharr") + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": _pg_name, + "USER": os.environ.get("POSTGRES_USER", "dispatch"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), + "HOST": os.environ.get("POSTGRES_HOST", "localhost"), + "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), + "TEST": { + "NAME": "test_" + _pg_name, + }, + } + } diff --git a/manage.py b/manage.py index f3c22dfd..f0a85a39 100644 --- a/manage.py +++ b/manage.py @@ -5,7 +5,12 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') + # Use isolated test DB settings for `manage.py test` (empty test_). + # Override with --settings=... on the command line if needed. + if len(sys.argv) > 1 and sys.argv[1] == "test": + os.environ["DJANGO_SETTINGS_MODULE"] = "dispatcharr.settings_test" + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dispatcharr.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/tests/test_process_label.py b/tests/test_process_label.py index bb4c8429..e4525a85 100644 --- a/tests/test_process_label.py +++ b/tests/test_process_label.py @@ -17,13 +17,13 @@ class ProcessLabelTests(SimpleTestCase): self.assertEqual(role, "uwsgi") def test_uwsgi_labeled_when_worker_module_present(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 2})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 2)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "uwsgi") def test_uwsgi_master_not_labeled_as_uwsgi(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 0})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 0)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "django") diff --git a/tests/test_user_preferences.py b/tests/test_user_preferences.py index 5749011a..80dcc992 100644 --- a/tests/test_user_preferences.py +++ b/tests/test_user_preferences.py @@ -128,13 +128,13 @@ class UserPreferencesAPITests(TestCase): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) def test_patch_me_cannot_escalate_privileges(self): - """Test PATCH /me/ rejects attempts to change user_level or is_staff""" + """PATCH /me/ ignores privilege fields; they are stripped before save.""" original_level = self.user.user_level data = {"user_level": 99, "is_staff": True, "is_superuser": True} response = self.client.patch(self.me_url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.status_code, status.HTTP_200_OK) self.user.refresh_from_db() self.assertEqual(self.user.user_level, original_level) From 6b4453e9ca7d439730f20f1dea086a49c4b5e9c3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 21:51:24 -0500 Subject: [PATCH 21/64] fix: Readd prev_days logic after dev merge. --- apps/output/epg.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/apps/output/epg.py b/apps/output/epg.py index 626b21aa..b017b8e5 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1100,11 +1100,32 @@ def generate_epg(request, profile_name=None, user=None): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 + # prev_days resolution order: + # 1. URL ?prev_days= (explicit, even 0 means "no past") + # 2. user.custom_properties.epg_prev_days + # 3. CoreSettings.proxy_settings.xmltv_prev_days_override (>0) + # 4. Auto-detect: max provider tv_archive_duration capped at 30 + url_prev = request.GET.get('prev_days') + user_prev = user_custom.get('epg_prev_days') if user_custom else None + if url_prev is not None: + try: + prev_days = max(0, min(int(url_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + elif user_prev not in (None, ""): + try: + prev_days = max(0, min(int(user_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + else: + from apps.timeshift.helpers import compute_provider_archive_days_capped + from core.models import CoreSettings + proxy_settings = CoreSettings.get_proxy_settings() + try: + override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + override = 0 + prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() cache_params = ( From 1b7840b71541ca95a6fcb533d48fa7f145c9fdd3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 07:52:13 -0500 Subject: [PATCH 22/64] changelog: Update for pr 1331 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdeeda5..0ea59832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. +- **Auto-sync range conflict warning no longer flags a group's own channels when using a channel-group override.** The M3U group settings "Range conflict" check compared occupant channel groups against the source group being configured. Auto-sync with `group_override` creates channels in the override target group, so those channels were misclassified as conflicts. The frontend now resolves the effective target group (override target when set, otherwise the source group) for classification. Genuine conflicts—manual channels, other accounts, different groups, or user-pinned numbers—still surface. (Fixes #1331) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.27.0] - 2026-06-16 From 578c50ea962d12f7d9f956c910accca431fc3e4f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 08:01:13 -0500 Subject: [PATCH 23/64] changelog: corrected --- CHANGELOG.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea59832..ce3d0b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance -- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. @@ -28,9 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. -### Performance - -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. ### Fixed From 971065b8a874f5837c818fe28267b2da554324bd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:14:00 -0500 Subject: [PATCH 24/64] chore(dependencies): update Django, requests, gevent, torch, sentence-transformers, and lxml to latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a003bc5c..8d4fad78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,18 +6,18 @@ license = "AGPL-3.0-only" requires-python = ">=3.13" dynamic = ["version"] dependencies = [ - "Django==6.0.5", + "Django==6.0.6", "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", - "requests==2.33.1", + "requests==2.34.2", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", "streamlink", "python-vlc", "yt-dlp", - "gevent==26.4.0", + "gevent==26.5.0", "django-db-geventpool", "daphne", "uwsgi", @@ -28,14 +28,14 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.11.0+cpu", - "sentence-transformers==5.4.1", + "torch==2.12.1+cpu", + "sentence-transformers==5.6.0", "channels", "channels-redis==4.3.0", "django-filter", "django-redis", "django-celery-beat>=2.9.0", - "lxml==6.1.0", + "lxml==6.1.1", "packaging", ] From 107246ff0b43d676e4cffc0219b4a047ca1c77d4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:17:38 -0500 Subject: [PATCH 25/64] changelog: Update for dependency updates. --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3d0b06..b2645f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- Updated `Django` 6.0.5 → 6.0.6, resolving the following CVEs: + - **CVE-2026-6873**: Signed cookie salt namespace collision in `HttpRequest.get_signed_cookie()`. + - **CVE-2026-7666**: Potential unencrypted email transmission via STARTTLS in the SMTP backend. + - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. + - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. + - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. + ### Added - **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. @@ -29,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. +- Dependency updates: + - `Django` 6.0.5 → 6.0.6 (security patch; see Security section) + - `requests` 2.33.1 → 2.34.2 + - `gevent` 26.4.0 → 26.5.0 + - `torch` 2.11.0+cpu → 2.12.1+cpu + - `sentence-transformers` 5.4.1 → 5.6.0 + - `lxml` 6.1.0 → 6.1.1 + ### Fixed From e483fc203b2645b5c5f9cd7b5530ade64a36f3c6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:14:00 -0500 Subject: [PATCH 26/64] chore(dependencies): update Django, requests, gevent, torch, sentence-transformers, and lxml to latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a003bc5c..8d4fad78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,18 +6,18 @@ license = "AGPL-3.0-only" requires-python = ">=3.13" dynamic = ["version"] dependencies = [ - "Django==6.0.5", + "Django==6.0.6", "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", - "requests==2.33.1", + "requests==2.34.2", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", "streamlink", "python-vlc", "yt-dlp", - "gevent==26.4.0", + "gevent==26.5.0", "django-db-geventpool", "daphne", "uwsgi", @@ -28,14 +28,14 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.11.0+cpu", - "sentence-transformers==5.4.1", + "torch==2.12.1+cpu", + "sentence-transformers==5.6.0", "channels", "channels-redis==4.3.0", "django-filter", "django-redis", "django-celery-beat>=2.9.0", - "lxml==6.1.0", + "lxml==6.1.1", "packaging", ] From 6b6eb11cc03ad1bb65b321c2b27c92b869fbe24b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:31:15 -0500 Subject: [PATCH 27/64] chore(dependencies): update Vite, esbuild and js-yaml to latest versions in package.json --- CHANGELOG.md | 4 + frontend/package-lock.json | 240 +++++++++++++++++++------------------ frontend/package.json | 7 +- 3 files changed, 133 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2645f45..f7bedbf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. +- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (1 low, 2 moderate, 1 high): + - Updated `vite` 7.3.2 → 7.3.5, resolving **moderate** NTLMv2 hash disclosure via UNC path handling on Windows ([GHSA-v6wh-96g9-6wx3](https://github.com/advisories/GHSA-v6wh-96g9-6wx3)) and **high** `server.fs.deny` bypass on Windows alternate paths ([GHSA-fx2h-pf6j-xcff](https://github.com/advisories/GHSA-fx2h-pf6j-xcff)) + - Updated `js-yaml` 4.1.1 → 5.1.0, resolving **moderate** quadratic-complexity DoS in merge key handling via repeated aliases ([GHSA-h67p-54hq-rp68](https://github.com/advisories/GHSA-h67p-54hq-rp68)) + - Updated `esbuild` 0.27.3 → 0.28.1, resolving **low** arbitrary file read when running the development server on Windows ([GHSA-g7r4-m6w7-qqqr](https://github.com/advisories/GHSA-g7r4-m6w7-qqqr)) ### Added diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8ca15bc7..ad8ad268 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -57,7 +57,7 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" } }, @@ -595,9 +595,9 @@ "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", "cpu": [ "ppc64" ], @@ -612,9 +612,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", "cpu": [ "arm" ], @@ -629,9 +629,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", "cpu": [ "arm64" ], @@ -646,9 +646,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", "cpu": [ "x64" ], @@ -663,9 +663,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", "cpu": [ "arm64" ], @@ -680,9 +680,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", "cpu": [ "x64" ], @@ -697,9 +697,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", "cpu": [ "arm64" ], @@ -714,9 +714,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", "cpu": [ "x64" ], @@ -731,9 +731,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", "cpu": [ "arm" ], @@ -748,9 +748,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", "cpu": [ "arm64" ], @@ -765,9 +765,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", "cpu": [ "ia32" ], @@ -782,9 +782,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", "cpu": [ "loong64" ], @@ -799,9 +799,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", "cpu": [ "mips64el" ], @@ -816,9 +816,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", "cpu": [ "ppc64" ], @@ -833,9 +833,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", "cpu": [ "riscv64" ], @@ -850,9 +850,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", "cpu": [ "s390x" ], @@ -867,9 +867,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", "cpu": [ "x64" ], @@ -884,9 +884,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", "cpu": [ "arm64" ], @@ -901,9 +901,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", "cpu": [ "x64" ], @@ -918,9 +918,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", "cpu": [ "arm64" ], @@ -935,9 +935,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", "cpu": [ "x64" ], @@ -952,9 +952,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", "cpu": [ "arm64" ], @@ -969,9 +969,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", "cpu": [ "x64" ], @@ -986,9 +986,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", "cpu": [ "arm64" ], @@ -1003,9 +1003,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", "cpu": [ "ia32" ], @@ -1020,9 +1020,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", "cpu": [ "x64" ], @@ -3163,9 +3163,9 @@ "license": "MIT" }, "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3176,32 +3176,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" } }, "node_modules/escape-string-regexp": { @@ -3814,16 +3814,26 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.1.0.tgz", + "integrity": "sha512-s8VA5jkR8f22S3NAXmhKPFqGUduqZGlsufabVOgN14iTdw/RXcym7bKkbwjxLK9Yw2lEvvmJjFp119+KPeo8Kg==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsdom": { @@ -5425,9 +5435,9 @@ } }, "node_modules/vite": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", - "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.5.tgz", + "integrity": "sha512-KuOaNhcnGFN2zIPGA7wRmzF+lJA1sea7rHq17aiJ++9lzY1WWG6Jpwqwe1KNbRVPIqHmr8GLYx7jbrQcN/7/ww==", "dev": true, "license": "MIT", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 2c1b78f5..8675351d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -61,16 +61,17 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" }, "resolutions": { - "vite": "7.1.7", + "vite": "7.3.5", "react": "19.1.0", "react-dom": "19.1.0" }, "overrides": { - "js-yaml": "^4.1.1", + "esbuild": "^0.28.1", + "js-yaml": "^5.1.0", "minimatch": "^10.2.1" } } From e2ceef521779467f1f4947f492dc1d040d5404cb Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:46:37 -0500 Subject: [PATCH 28/64] changelog: refactor xmltv changes and link issue. --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7bedbf8..059e0d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,11 +27,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. -- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. -- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. -- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **XMLTV EPG export is faster and no longer balloons worker memory.** `generate_epg()` was reworked end-to-end for large guides. (Fixes #1366) + - Streams incrementally: on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker); repeat requests within 300s stream chunks back from Redis. `malloc_trim` runs after cold builds. + - Channel streams are prefetched once (only `id`/`name`) instead of one query per custom-dummy channel; dummy `EPGData` programme existence is bulk-checked in a single query. + - The primary channel id is escaped once per `epg_id` group instead of once per programme (~750k fewer `html.escape` calls on a large guide). + - The channel query no longer JOINs multi-MB `programme_index` blobs per channel (~13s saved on a ~2000-channel guide; indices live in `EPGSourceIndex`). + - Programme export uses `(epg_id, id)` keyset pagination with a per-source `start_time` sort; a matching composite index on `ProgramData` (created `CONCURRENTLY` on PostgreSQL) lets each chunk use an ordered index range scan instead of re-sorting every chunk. - **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed From c5e50167285504e03487447d0bb3b2e0f8d12144 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:02:31 -0500 Subject: [PATCH 29/64] fix(dvr): Fix in-progress DVR playback to prevent jumping to live edge and update FloatingVideo component for improved error handling. Added tests to verify HLS configuration behavior. (Fixes #1329) --- CHANGELOG.md | 1 + frontend/src/components/FloatingVideo.jsx | 115 +++++++++--------- .../__tests__/FloatingVideo.test.jsx | 94 ++++++++++++++ 3 files changed, 154 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 059e0d23..5365e79a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **In-progress DVR playback no longer jumps to the live edge.** The floating video player still opens in-progress recordings at the start of the seekable range, but hls.js was configured with `liveMaxLatencyDurationCount: 10`, which forced the playhead forward to "now" shortly after playback began. Live-edge sync is now disabled for recording HLS so users can watch, pause, and scrub from the beginning while the recording continues. (Fixes #1329) - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 507e0caf..e49c3bb2 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -333,65 +333,68 @@ export default function FloatingVideo() { let hls = null; if (isHls && Hls.isSupported()) { - hls = new Hls({ - // Open at the very beginning of the recording rather than the live - // edge. Without this, an in-progress recording would start at "now" - // and hide everything already recorded. hls.js applies this AFTER - // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is - // also kept as a safety net for the Safari native-HLS path and for - // edge cases where this initial-position logic loses to the user's - // first interaction. - startPosition: 0, - // Allow seeking back to the start of the recording, regardless of - // current playhead position. Recordings can be hours long and the - // user may want to scrub anywhere; we explicitly disable buffer - // eviction by setting a very large back-buffer length. - backBufferLength: 90 * 60, // 90 minutes - maxBufferLength: 60, - maxMaxBufferLength: 600, - // For an in-progress recording, hls.js refreshes the playlist on - // its target-duration cadence; let it follow the live edge but keep - // the full DVR window seekable. - liveSyncDurationCount: 3, - liveMaxLatencyDurationCount: 10, - enableWorker: true, - lowLatencyMode: false, - // Inject the JWT into every playlist + segment XHR. Read the token - // from the auth store at request time rather than capturing the - // closure value at hls.js init, so a refreshed access token mid- - // playback is picked up on the next segment fetch. - xhrSetup: (xhr) => { - const token = useAuthStore.getState().accessToken; - if (token) { - xhr.setRequestHeader('Authorization', `Bearer ${token}`); - } - }, - }); - hls.on(Hls.Events.ERROR, (_evt, data) => { - if (data.fatal) { - // eslint-disable-next-line no-console - console.error('HLS fatal error:', data.type, data.details); - if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { - try { - hls.startLoad(); - } catch { - // ignore + try { + hls = new Hls({ + // Open at the very beginning of the recording rather than the live + // edge. Without this, an in-progress recording would start at "now" + // and hide everything already recorded. hls.js applies this AFTER + // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is + // also kept as a safety net for the Safari native-HLS path and for + // edge cases where this initial-position logic loses to the user's + // first interaction. + startPosition: 0, + // Allow seeking back to the start of the recording, regardless of + // current playhead position. Recordings can be hours long and the + // user may want to scrub anywhere; we explicitly disable buffer + // eviction by setting a very large back-buffer length. + backBufferLength: 90 * 60, // 90 minutes + maxBufferLength: 60, + maxMaxBufferLength: 600, + // Leave liveMaxLatencyDurationCount at the hls.js default (Infinity). + // A finite value forces the playhead to the live edge during playback. + enableWorker: true, + lowLatencyMode: false, + // Inject the JWT into every playlist + segment XHR. Read the token + // from the auth store at request time rather than capturing the + // closure value at hls.js init, so a refreshed access token mid- + // playback is picked up on the next segment fetch. + xhrSetup: (xhr) => { + const token = useAuthStore.getState().accessToken; + if (token) { + xhr.setRequestHeader('Authorization', `Bearer ${token}`); } - } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { - try { - hls.recoverMediaError(); - } catch { - // ignore + }, + }); + hls.on(Hls.Events.ERROR, (_evt, data) => { + if (data.fatal) { + // eslint-disable-next-line no-console + console.error('HLS fatal error:', data.type, data.details); + if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { + try { + hls.startLoad(); + } catch { + // ignore + } + } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { + try { + hls.recoverMediaError(); + } catch { + // ignore + } + } else { + setLoadError(`HLS playback error: ${data.details || data.type}`); } - } else { - setLoadError(`HLS playback error: ${data.details || data.type}`); } - } - }); - hls.attachMedia(video); - hls.on(Hls.Events.MEDIA_ATTACHED, () => { - hls.loadSource(streamUrl); - }); + }); + hls.attachMedia(video); + hls.on(Hls.Events.MEDIA_ATTACHED, () => { + hls.loadSource(streamUrl); + }); + } catch (error) { + setIsLoading(false); + setLoadError(`HLS initialization error: ${error.message}`); + return; + } } else if (isHls && video.canPlayType('application/vnd.apple.mpegurl')) { // Safari path: native HLS support, including seekable DVR windows. video.src = streamUrl; diff --git a/frontend/src/components/__tests__/FloatingVideo.test.jsx b/frontend/src/components/__tests__/FloatingVideo.test.jsx index f99a3dcc..9683fc59 100644 --- a/frontend/src/components/__tests__/FloatingVideo.test.jsx +++ b/frontend/src/components/__tests__/FloatingVideo.test.jsx @@ -20,8 +20,49 @@ vi.mock('mpegts.js', () => ({ }, })); +const mockHlsInstance = { + attachMedia: vi.fn(), + loadSource: vi.fn(), + destroy: vi.fn(), + on: vi.fn(), +}; + +let capturedHlsConfig = null; +let forceHlsInitError = false; + +vi.mock('hls.js', () => ({ + default: class MockHls { + static isSupported = vi.fn(() => true); + + static Events = { + ERROR: 'error', + MEDIA_ATTACHED: 'media_attached', + }; + + static ErrorTypes = { + NETWORK_ERROR: 'networkError', + MEDIA_ERROR: 'mediaError', + }; + + constructor(config) { + if (forceHlsInitError) { + throw new Error('Illegal hls.js config'); + } + capturedHlsConfig = config; + Object.assign(this, mockHlsInstance); + } + }, +})); + +vi.mock('../../store/auth', () => ({ + default: { + getState: vi.fn(() => ({ accessToken: 'test-token' })), + }, +})); + // Import the mocked module after mocking const mpegts = (await import('mpegts.js')).default; +const Hls = (await import('hls.js')).default; // Mock react-draggable vi.mock('react-draggable', () => ({ @@ -53,6 +94,8 @@ describe('FloatingVideo', () => { beforeEach(async () => { vi.clearAllMocks(); + capturedHlsConfig = null; + forceHlsInitError = false; // Mock HTMLVideoElement methods HTMLVideoElement.prototype.load = vi.fn(); @@ -239,6 +282,57 @@ describe('FloatingVideo', () => { expect(video.poster).toBe('http://example.com/poster.jpg'); }); + it('should disable live-edge sync for in-progress recording HLS', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + + render(); + + expect(capturedHlsConfig).toEqual( + expect.objectContaining({ + startPosition: 0, + }) + ); + expect(capturedHlsConfig).not.toHaveProperty( + 'liveMaxLatencyDurationCount' + ); + expect(capturedHlsConfig).not.toHaveProperty('liveSyncDurationCount'); + }); + + it('shows an in-player error when hls.js config is invalid', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + forceHlsInitError = true; + + render(); + + expect( + screen.getByText(/HLS initialization error: Illegal hls.js config/i) + ).toBeInTheDocument(); + }); + it('should show metadata overlay', () => { const { container } = render(); const video = container.querySelector('video'); From 562393b77e50d56ae51a2772b2d148d5a16aeca1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:32:48 -0500 Subject: [PATCH 30/64] fix(recording playback): Enhance completed DVR recording playback by allowing JWT token via query parameter for native