From 2781b21a8902ab0e9d12e15e1ec189ac0e0dab23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Tue, 19 May 2026 22:28:35 +0200 Subject: [PATCH 01/31] feat: built-in XC catch-up (timeshift) support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds native catch-up/timeshift replay for Xtream Codes providers through the same HTTPStreamReader transport pipeline as live TV. Timeshift proxy (apps/timeshift/): - URL cascade: 3 candidate timestamp formats per provider, per-account format cache for fast-forward seek performance - MPEG-TS preamble stripping (shared with HTTPStreamReader) - Stats integration: timeshift viewers appear on /stats with TIMESHIFT badge - Auth via hmac.compare_digest on XC password Catchup detection — denormalized for zero-cost output queries: - Stream.is_catchup + Stream.catchup_days populated at XC import time - Channel.has_catchup + Channel.catchup_days + Channel.catchup_provider_stream_id rolled up via ChannelStream post_save signal (UI path) and explicit SQL after bulk_create (import path) - _xc_channel_entry() reads denormalized fields instead of per-channel custom_properties JSON introspection (eliminates N+1 queries) - Migration 0038 backfills existing data via raw SQL XC API enhancements: - server_info.timezone + start/end + time_now use configured timezone (triple consistency rule — fixes wrong-programme-plays bug) - Dynamic has_archive flag + auto prev_days for catch-up channels - XMLTV timestamps rewritten to local timezone for catch-up clients HTTPStreamReader extended (apps/proxy/live_proxy/input/http_streamer.py): - 1 MB pipe buffer via fcntl F_SETPIPE_SZ (eliminates producer/consumer ping-pong that halved throughput) - Pre-opened response= for URL cascade workflows - strip_ts_preamble= for XC servers emitting PHP warnings before TS - find_ts_sync() as shared utility - Builds on upstream O_NONBLOCK + select() write loop Provider stream_id lookup order: - stream_xc() and xc_get_epg() try internal Channel.id first, fall back to provider stream_id only when needed (avoids unconditional query on every request) Also includes: - VOD provider cascade in stream_vod() — iterates all M3U relations by priority when first provider is at capacity - Defensive null-safety: custom_sid: None → "" in get_live_streams, get_vod_streams, get_vod_info, get_series_info (fixes iPlayTV crash on JSON null for string fields) - Timeshift settings UI (timezone selector, debug toggle) - StreamConnectionCard violet TIMESHIFT badge - Orphan cleanup skips timeshift_* virtual channels --- CHANGELOG.md | 5 + .../migrations/0038_add_catchup_fields.py | 120 ++++ apps/channels/models.py | 32 ++ apps/channels/serializers.py | 1 + apps/channels/signals.py | 26 +- apps/channels/utils.py | 54 ++ apps/m3u/tasks.py | 59 +- apps/output/views.py | 237 +++++++- apps/proxy/live_proxy/channel_status.py | 14 + apps/proxy/live_proxy/constants.py | 2 + apps/proxy/live_proxy/input/http_streamer.py | 289 +++++++--- apps/proxy/live_proxy/server.py | 9 + apps/proxy/live_proxy/views.py | 25 +- apps/timeshift/__init__.py | 0 apps/timeshift/apps.py | 7 + apps/timeshift/helpers.py | 116 ++++ apps/timeshift/tests/__init__.py | 0 apps/timeshift/tests/test_helpers.py | 56 ++ apps/timeshift/tests/test_views.py | 177 ++++++ apps/timeshift/views.py | 512 ++++++++++++++++++ core/models.py | 24 + dispatcharr/settings.py | 1 + dispatcharr/urls.py | 8 + .../components/cards/StreamConnectionCard.jsx | 20 +- .../forms/settings/TimeshiftSettingsForm.jsx | 116 ++++ frontend/src/pages/Settings.jsx | 16 + .../settings/TimeshiftSettingsFormUtils.js | 17 + frontend/src/utils/pages/SettingsUtils.js | 43 ++ 28 files changed, 1867 insertions(+), 119 deletions(-) create mode 100644 apps/channels/migrations/0038_add_catchup_fields.py create mode 100644 apps/timeshift/__init__.py create mode 100644 apps/timeshift/apps.py create mode 100644 apps/timeshift/helpers.py create mode 100644 apps/timeshift/tests/__init__.py create mode 100644 apps/timeshift/tests/test_helpers.py create mode 100644 apps/timeshift/tests/test_views.py create mode 100644 apps/timeshift/views.py create mode 100644 frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx create mode 100644 frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cef64db..8d2b046c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **None** (default): software decode. - **NVIDIA NVDEC (`--cuvid`)**: requires the NVIDIA container toolkit and a supported GPU inside the container. - **Intel Quick Sync (`--qsv`)**: requires an Intel iGPU or ARC GPU with the i915 driver exposed to the container. +- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. + - **`Settings → Timeshift` panel** with four knobs (defaults: `UTC`, `en`, `0`, off): `default_timezone`, `default_language`, `xmltv_prev_days_override`, `debug_logging`. + - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. + - **Byte path uses a dedicated producer thread + `os.pipe`** matching the pattern in `apps/proxy/live_proxy/input/http_streamer.py:HTTPStreamReader`. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. - **HDHR output profile URL support.** HDHomeRun lineup URLs now support an `output_profile` path segment so HDHR clients (Plex, Channels DVR, Emby, etc.) can request a specific transcode profile without any query-parameter support. URL formats accepted: - `/hdhr/output_profile//lineup.json` - output profile only - `/hdhr//output_profile//lineup.json` - channel profile + output profile diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py new file mode 100644 index 00000000..97a0b925 --- /dev/null +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -0,0 +1,120 @@ +"""Add denormalized catch-up fields to Stream and Channel. + +Populated at M3U/XC import time so _xc_channel_entry() can read them as +zero-cost column reads instead of introspecting custom_properties JSON +per channel on every xc_get_live_streams call. +""" + +from django.db import migrations, models + + +def backfill_stream_catchup(apps, schema_editor): + """Derive is_catchup/catchup_days from Stream.custom_properties JSON.""" + from django.db import connection + + with connection.cursor() as cursor: + cursor.execute(""" + UPDATE dispatcharr_channels_stream + SET is_catchup = TRUE, + catchup_days = COALESCE( + (custom_properties->>'tv_archive_duration')::int, 7 + ) + WHERE custom_properties IS NOT NULL + AND custom_properties != 'null'::jsonb + AND ( + custom_properties->>'tv_archive' = '1' + OR custom_properties->>'tv_archive' = 'True' + ) + """) + + +def backfill_channel_catchup(apps, schema_editor): + """Roll up catch-up fields from streams to channels.""" + from django.db import connection + + with connection.cursor() as cursor: + cursor.execute(""" + UPDATE dispatcharr_channels_channel c SET + is_catchup = EXISTS ( + SELECT 1 FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), + catchup_days = COALESCE(( + SELECT MAX(s.catchup_days) FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), 0), + catchup_provider_stream_id = COALESCE(( + SELECT s.custom_properties->>'stream_id' + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ORDER BY cs."order" LIMIT 1 + ), '') + """) + + +class Migration(migrations.Migration): + + dependencies = [ + ("dispatcharr_channels", "0037_auto_sync_overhaul"), + ] + + operations = [ + # Stream fields + migrations.AddField( + model_name="stream", + name="is_catchup", + field=models.BooleanField( + default=False, + db_index=True, + help_text="Whether this stream supports catch-up/timeshift (tv_archive=1)", + ), + ), + migrations.AddField( + model_name="stream", + name="catchup_days", + field=models.PositiveIntegerField( + default=0, + help_text="Number of days of catch-up archive available (tv_archive_duration)", + ), + ), + # Channel fields + migrations.AddField( + model_name="channel", + name="is_catchup", + field=models.BooleanField( + default=False, + db_index=True, + help_text="Whether any stream on this channel supports catch-up (tv_archive=1)", + ), + ), + migrations.AddField( + model_name="channel", + name="catchup_days", + field=models.PositiveIntegerField( + default=0, + help_text="Max catch-up archive days across all streams on this channel", + ), + ), + migrations.AddField( + model_name="channel", + name="catchup_provider_stream_id", + field=models.CharField( + max_length=64, + blank=True, + default="", + help_text="Provider stream_id of the highest-priority catch-up stream", + ), + ), + # Backfill existing data + migrations.RunPython( + backfill_stream_catchup, + reverse_code=migrations.RunPython.noop, + ), + migrations.RunPython( + backfill_channel_catchup, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 59af3e00..8964f067 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -134,6 +134,19 @@ class Stream(models.Model): db_index=True ) + # Denormalized catch-up fields — populated at M3U/XC import time from + # custom_properties["tv_archive"] and ["tv_archive_duration"]. Avoids + # per-channel JSON introspection on every xc_get_live_streams call. + is_catchup = models.BooleanField( + default=False, + db_index=True, + help_text="Whether this stream supports catch-up/timeshift (tv_archive=1)", + ) + catchup_days = models.PositiveIntegerField( + default=0, + help_text="Number of days of catch-up archive available (tv_archive_duration)", + ) + class Meta: # If you use m3u_account, you might do unique_together = ('name','url','m3u_account') verbose_name = "Stream" @@ -374,6 +387,25 @@ class Channel(models.Model): help_text="The M3U account that auto-created this channel" ) + # Denormalized catch-up fields — rolled up from the channel's streams at + # import time and via post_save/post_delete signals on ChannelStream. + # Eliminates per-channel DB queries in _xc_channel_entry(). + is_catchup = models.BooleanField( + default=False, + db_index=True, + help_text="Whether any stream on this channel supports catch-up (tv_archive=1)", + ) + catchup_days = models.PositiveIntegerField( + default=0, + help_text="Max catch-up archive days across all streams on this channel", + ) + catchup_provider_stream_id = models.CharField( + max_length=64, + blank=True, + default="", + help_text="Provider stream_id of the highest-priority catch-up stream (for timeshift URL construction)", + ) + # Hidden channels are excluded from HDHR, M3U, EPG, and XC output queries. # Auto-sync still recognizes them so they are not recreated when their # underlying provider stream persists; this is an output-layer concern, not diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index f5c83527..c0bdc525 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -454,6 +454,7 @@ class ChannelSerializer(serializers.ModelSerializer): "logo_id", "user_level", "is_adult", + "is_catchup", "hidden_from_output", "auto_created", "auto_created_by", diff --git a/apps/channels/signals.py b/apps/channels/signals.py index a1917b25..f1615c9a 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -5,7 +5,7 @@ from django.dispatch import receiver from django.utils.timezone import now, is_aware, make_aware from celery.result import AsyncResult from django_celery_beat.models import ClockedSchedule, PeriodicTask -from .models import Channel, Stream, ChannelProfile, ChannelProfileMembership, Recording +from .models import Channel, Stream, ChannelStream, ChannelProfile, ChannelProfileMembership, Recording from apps.m3u.models import M3UAccount from apps.epg.tasks import parse_programs_for_tvg_id import json @@ -365,3 +365,27 @@ def schedule_task_on_save(sender, instance, created, **kwargs): @receiver(post_delete, sender=Recording) def revoke_task_on_delete(sender, instance, **kwargs): revoke_task(instance.task_id) + + +@receiver([post_save, post_delete], sender=ChannelStream) +def update_channel_catchup_fields(sender, instance, **kwargs): + """Roll up denormalized catch-up fields from streams to the channel. + + Covers the UI/API path (admin adds/removes a stream from a channel). + The bulk-import path bypasses signals and uses explicit SQL instead — + see apps/m3u/tasks.py after ChannelStream.objects.bulk_create(). + """ + channel = instance.channel + best = ( + channel.streams + .filter(is_catchup=True) + .order_by("channelstream__order") + .first() + ) + Channel.objects.filter(pk=channel.pk).update( + is_catchup=best is not None, + catchup_days=best.catchup_days if best else 0, + catchup_provider_stream_id=( + (best.custom_properties or {}).get("stream_id", "") if best else "" + ), + ) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index f41c2b72..2ab343bb 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -16,6 +16,60 @@ def format_channel_number(value, empty=""): return int(value) return value + +def resolve_channel_by_provider_stream_id(provider_stream_id): + """Find a Channel + Stream by the XC provider's stream_id. + + XC clients address channels via the provider's `stream_id` (stored in + `Stream.custom_properties["stream_id"]`), not Dispatcharr's internal + `Channel.id`. Returns `(Channel, Stream)` on hit, `(None, None)` on miss. + """ + from apps.channels.models import Stream + + stream = ( + Stream.objects.filter( + custom_properties__stream_id=str(provider_stream_id), + m3u_account__account_type="XC", + ) + .select_related("m3u_account") + .first() + ) + if stream is None: + return None, None + channel = stream.channels.first() + if channel is None: + return None, None + return channel, stream + + +def get_channel_catchup_info(channel): + """Return catch-up info for a Channel, or None if no stream has tv_archive. + + Walks `channel.streams` in `channelstream__order`, returns a dict for the + first archive-enabled stream: `{stream, props, provider_stream_id, + tv_archive_duration}`. `tv_archive` may be stored as 1, "1", True or + "True" depending on the M3U importer that wrote it. + """ + for stream in channel.streams.order_by("channelstream__order"): + props = stream.custom_properties or {} + if str(props.get("tv_archive")) not in ("1", "True"): + continue + provider_stream_id = props.get("stream_id") + if not provider_stream_id: + continue + try: + archive_days = int(props.get("tv_archive_duration", 7) or 7) + except (TypeError, ValueError): + archive_days = 7 + return { + "stream": stream, + "props": props, + "provider_stream_id": str(provider_stream_id), + "tv_archive_duration": archive_days, + } + return None + + def increment_stream_count(account): with lock: current_usage = active_streams_map.get(account.id, 0) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index db6b1b6e..b48d3c48 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -951,6 +951,16 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, account_type='XC', stream_id=provider_stream_id ) + # Derive denormalized catch-up fields from the XC + # API response at import time so output views can + # read them as zero-cost column reads. + _tv_archive = str(stream.get("tv_archive", "0")) + _is_catchup = _tv_archive in ("1", "True") + try: + _catchup_days = int(stream.get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + _catchup_days = 0 + stream_props = { "name": name, "url": url, @@ -964,6 +974,8 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): "is_stale": False, "stream_id": provider_stream_id, "stream_chno": stream_chno, + "is_catchup": _is_catchup, + "catchup_days": _catchup_days, } if stream_hash not in stream_hashes: @@ -1030,7 +1042,7 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): # Simplified bulk update for better performance Stream.objects.bulk_update( streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'is_catchup', 'catchup_days'], batch_size=150 # Smaller batch size for XC processing ) @@ -1176,6 +1188,16 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): account_type=account_type_for_hash, stream_id=provider_stream_id ) + # Derive catch-up fields from stream attributes (M3U files + # may carry tv_archive via custom attributes). + _attrs = stream_info["attributes"] + _tv_archive_m3u = str(_attrs.get("tv_archive", "0")) + _is_catchup_m3u = _tv_archive_m3u in ("1", "True") + try: + _catchup_days_m3u = int(_attrs.get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + _catchup_days_m3u = 0 + stream_props = { "name": name, "url": url, @@ -1184,11 +1206,13 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): "m3u_account": account, "channel_group_id": int(groups.get(group_title)), "stream_hash": stream_hash, - "custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"], - "is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)), + "custom_properties": {**_attrs, "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else _attrs, + "is_adult": parse_is_adult(_attrs.get("is_adult", 0)), "is_stale": False, "stream_id": provider_stream_id, "stream_chno": channel_num, + "is_catchup": _is_catchup_m3u, + "catchup_days": _catchup_days_m3u, } if stream_hash not in stream_hashes: @@ -1254,7 +1278,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): # Update all streams in a single bulk operation Stream.objects.bulk_update( streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'is_catchup', 'catchup_days'], batch_size=200 ) except Exception as e: @@ -2550,6 +2574,33 @@ def sync_auto_channels(account_id, scan_start_time=None): batch_size=500, ) + # Roll up denormalized catch-up fields from streams to + # channels. bulk_create bypasses Django signals, so this + # explicit SQL is the only path that fires during import. + from django.db import connection as _conn + with _conn.cursor() as _cur: + _cur.execute(""" + UPDATE dispatcharr_channels_channel c SET + is_catchup = EXISTS ( + SELECT 1 FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), + catchup_days = COALESCE(( + SELECT MAX(s.catchup_days) + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ), 0), + catchup_provider_stream_id = COALESCE(( + SELECT s.custom_properties->>'stream_id' + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ORDER BY cs."order" LIMIT 1 + ), '') + """) + if profiles_to_assign: ChannelProfileMembership.objects.bulk_create( [ diff --git a/apps/output/views.py b/apps/output/views.py index 1a73923d..131d98a4 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -2,7 +2,10 @@ from django.http import HttpResponse, JsonResponse, Http404, HttpResponseForbidd import json from django.urls import reverse from apps.channels.models import Channel, ChannelProfile, ChannelGroup, Stream -from apps.channels.utils import format_channel_number +from apps.channels.utils import ( + format_channel_number, + resolve_channel_by_provider_stream_id, +) from django.db.models import Prefetch from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods @@ -11,11 +14,12 @@ from apps.accounts.models import User from dispatcharr.utils import network_access_allowed from django.utils import timezone as django_timezone from django.shortcuts import get_object_or_404 -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone as dt_timezone +from zoneinfo import ZoneInfo import html import time from tzlocal import get_localzone -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse import base64 import logging from django.db.models.functions import Lower @@ -23,6 +27,7 @@ import os from apps.m3u.utils import calculate_tuner_count from apps.proxy.utils import get_user_active_connections import regex +from core.models import CoreSettings from core.utils import log_system_event import hashlib @@ -1315,11 +1320,31 @@ def generate_epg(request, profile_name=None, user=None): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 + # Timeshift: prev_days resolution order: + # 1. URL ?prev_days= (explicit, even 0 means "no past") + # 2. user.custom_properties.epg_prev_days + # 3. CoreSettings.timeshift_settings.xmltv_prev_days_override (>0) + # 4. Auto-detect: max provider tv_archive_duration capped at 30 + url_prev = request.GET.get('prev_days') + user_prev = user_custom.get('epg_prev_days') if user_custom else None + if url_prev is not None: + try: + prev_days = max(0, min(int(url_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + elif user_prev not in (None, ""): + try: + prev_days = max(0, min(int(user_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + else: + from apps.timeshift.helpers import compute_provider_archive_days_capped + timeshift_settings = CoreSettings.get_timeshift_settings() + try: + override = int(timeshift_settings.get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + override = 0 + prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() cache_params = ( @@ -1975,6 +2000,34 @@ def _xc_allowed_output_formats(user): return ['ts', 'mp4'] +def _build_xc_server_info(request, hostname, port): + """Build the server_info dict for XC API responses. + + The timezone, time_now, and xc_get_epg start/end fields form a "timezone + triple" that MUST all use the same zone — XC clients (iPlayTV, TiviMate) + use server_info.timezone to interpret start/end strings and calculate seek + offsets into catch-up archives. A mismatch makes the wrong programme play. + Learned from plugin v1.1.4 → v1.2.6 (6 iterations of timezone bugs). + """ + tz_name = CoreSettings.get_timeshift_settings().get("default_timezone", "UTC") + try: + tz = ZoneInfo(tz_name) + except (KeyError, Exception): + # Invalid timezone in settings — fall back to UTC so XC clients + # can still connect instead of getting a 500 error. + tz = ZoneInfo("UTC") + tz_name = "UTC" + return { + "url": hostname, + "server_protocol": request.scheme, + "port": port, + "timezone": tz_name, + "timestamp_now": int(time.time()), + "time_now": datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S"), + "process": True, + } + + def xc_get_info(request, full=False): user = xc_get_user(request) @@ -2007,15 +2060,7 @@ def xc_get_info(request, full=False): "max_connections": str(max_connections), "allowed_output_formats": _xc_allowed_output_formats(user), }, - "server_info": { - "url": hostname, - "server_protocol": request.scheme, - "port": port, - "timezone": get_localzone().key, - "timestamp_now": int(time.time()), - "time_now": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "process": True, - }, + "server_info": _build_xc_server_info(request, hostname, port), } if full == True: @@ -2141,7 +2186,70 @@ def xc_xmltv(request): ) return JsonResponse({'error': 'Unauthorized'}, status=401) - return generate_epg(request, None, user) + response = generate_epg(request, None, user) + return _convert_xmltv_to_local_timezone(response) + + +# Pre-compiled pattern for XMLTV timestamp attributes: `20251128143000 +0000`. +_XMLTV_TS_PATTERN = regex.compile(rb'(\d{14}) ([+-]\d{4})') + + +def _convert_xmltv_to_local_timezone(response): + """Rewrite UTC timestamps in an XMLTV streaming response to the configured + catch-up local time zone. + + Catch-up clients such as TiviMate display the wall-clock part of XMLTV + timestamps verbatim and ignore the trailing offset, so a programme + emitted as `start="20260512170000 +0000"` (correct UTC) ends up shown + as 17:00 even when the user is in Brussels and expects 19:00. Rewriting + the same instant as `start="20260512190000 +0200"` keeps the wall-clock + in the EPG aligned with the user's perception while the timestamp still + points at the same moment. + + The conversion runs only on the `/xmltv.php` catch-up endpoint — the + plain `/output/...epg.xml` consumers keep the UTC representation. + """ + timeshift_settings = CoreSettings.get_timeshift_settings() + tz_name = timeshift_settings.get('default_timezone', 'UTC') + try: + local_tz = ZoneInfo(tz_name) + except Exception: + return response + + # Use the builtin UTC, not ZoneInfo('UTC'), because the latter can pick + # up the host's mis-set /etc/timezone in some Docker setups. + # dt_timezone imported at module level + utc = dt_timezone.utc + + def _convert(match): + ts_bytes, _zone = match.group(1), match.group(2) + try: + ts_str = ts_bytes.decode('ascii') + utc_time = datetime.strptime(ts_str, '%Y%m%d%H%M%S').replace(tzinfo=utc) + local_time = utc_time.astimezone(local_tz) + return local_time.strftime('%Y%m%d%H%M%S %z').encode('ascii') + except Exception: + return match.group(0) + + # generate_epg may return either StreamingHttpResponse (large EPGs) or a + # plain HttpResponse (small EPGs or some error paths) — handle both. + if hasattr(response, 'streaming_content'): + source = response.streaming_content + else: + body = response.content + if isinstance(body, str): + body = body.encode('utf-8') + source = iter([body]) + + def _rewrite(): + for chunk in source: + if isinstance(chunk, str): + chunk = chunk.encode('utf-8') + if b'start="' in chunk or b'stop="' in chunk: + chunk = _XMLTV_TS_PATTERN.sub(_convert, chunk) + yield chunk + + return StreamingHttpResponse(_rewrite(), content_type=response.get('Content-Type', 'application/xml')) def xc_get_live_categories(user): @@ -2290,11 +2398,26 @@ def _xc_channel_entry(channel, channel_num_map, _get_default_group_id, _logo_url effective_logo = channel.effective_logo_obj effective_group = channel.effective_channel_group_obj group_id = effective_group.id if effective_group else _get_default_group_id() + + # Denormalized catch-up fields — populated at M3U/XC import time and + # rolled up via ChannelStream signal. Zero DB queries here. + if channel.is_catchup: + tv_archive = 1 + tv_archive_duration = channel.catchup_days + try: + stream_id_value = int(channel.catchup_provider_stream_id) + except (TypeError, ValueError): + stream_id_value = channel.id + else: + tv_archive = 0 + tv_archive_duration = 0 + stream_id_value = channel.id + return { "num": channel_num_int, "name": channel.effective_name, "stream_type": "live", - "stream_id": channel.id, + "stream_id": stream_id_value, "stream_icon": ( f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" if effective_logo else None @@ -2304,10 +2427,10 @@ def _xc_channel_entry(channel, channel_num_map, _get_default_group_id, _logo_url "is_adult": int(channel.is_adult), "category_id": str(group_id), "category_ids": [group_id], - "custom_sid": None, - "tv_archive": 0, + "custom_sid": "", + "tv_archive": tv_archive, "direct_source": "", - "tv_archive_duration": 0, + "tv_archive_duration": tv_archive_duration, } @@ -2340,6 +2463,21 @@ def xc_get_epg(request, user, short=False): if not channel_id: raise Http404() + # Try internal Channel.id first (most requests). Fall back to provider + # stream_id lookup for catch-up channels whose xc_get_live_streams emits + # the provider's stream_id instead of the internal Channel.id. + resolved_channel_id = channel_id + try: + candidate = int(channel_id) + if Channel.objects.filter(id=candidate).exists(): + resolved_channel_id = candidate + else: + raise ValueError + except (TypeError, ValueError): + provider_channel, _ = resolve_channel_by_provider_stream_id(channel_id) + if provider_channel is not None: + resolved_channel_id = provider_channel.id + channel = None # Apply effective-value annotation + hidden-exclusion at every channel # resolution path so a single channel lookup honors the same visibility @@ -2354,7 +2492,7 @@ def xc_get_epg(request, user, short=False): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": channel_id, + "id": resolved_channel_id, "user_level__lte": user.user_level } # Hide adult content if user preference is set @@ -2364,7 +2502,7 @@ def xc_get_epg(request, user, short=False): else: # User has specific limited profiles assigned filters = { - "id": channel_id, + "id": resolved_channel_id, "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -2377,7 +2515,7 @@ def xc_get_epg(request, user, short=False): if not channel: raise Http404() else: - channel = _annotate(Channel.objects.filter(id=channel_id).select_related('epg_data__epg_source')).first() + channel = _annotate(Channel.objects.filter(id=resolved_channel_id).select_related('epg_data__epg_source')).first() if not channel: raise Http404() @@ -2438,6 +2576,19 @@ def xc_get_epg(request, user, short=False): except (ValueError, TypeError): prev_days = 0 now = django_timezone.now() + + # When a channel supports catch-up, automatically include past programmes + # within the archive window even if the client did not request prev_days. + # XC clients (iPlayTV, TiviMate) call get_simple_data_table without + # prev_days and expect the response to already contain archived entries + # marked with has_archive=1 — they use that list to populate the catch-up + # programme menu. + # Use denormalized catch-up fields (zero DB queries). + _channel_is_catchup = getattr(channel, "is_catchup", False) + _channel_catchup_days = getattr(channel, "catchup_days", 0) + if _channel_is_catchup and prev_days == 0: + prev_days = _channel_catchup_days + lookback_cutoff = now - timedelta(days=prev_days) forward_cutoff = now + timedelta(days=num_days) if num_days > 0 else None effective_epg_data = channel.effective_epg_data_obj @@ -2483,6 +2634,26 @@ def xc_get_epg(request, user, short=False): output = {"epg_listings": []} + # Reuse the denormalized catch-up fields for the has_archive flag. + if _channel_is_catchup: + archive_window = timedelta(days=_channel_catchup_days) + else: + archive_window = None + + # start/end must be in the SAME timezone reported by server_info.timezone. + # XC clients display these strings verbatim AND use server_info.timezone + # to calculate seek offsets into catch-up archives. If these don't match, + # the wrong programme plays. Learned from plugin v1.1.4 → v1.2.6 (6 + # iterations of timezone bugs all converged on this rule). + # start_timestamp/stop_timestamp (epoch) stay in UTC — they're inherently + # timezone-agnostic and clients use them for their own conversions. + _ts_settings = CoreSettings.get_timeshift_settings() + _epg_tz_name = _ts_settings.get("default_timezone", "UTC") + try: + _epg_tz = ZoneInfo(_epg_tz_name) + except Exception: + _epg_tz = None + for program in programs: title = program['title'] if isinstance(program, dict) else program.title description = program['description'] if isinstance(program, dict) else program.description @@ -2507,8 +2678,8 @@ def xc_get_epg(request, user, short=False): "epg_id": epg_id, "title": base64.b64encode((title or "").encode()).decode(), "lang": "", - "start": start.strftime("%Y-%m-%d %H:%M:%S"), - "end": end.strftime("%Y-%m-%d %H:%M:%S"), + "start": start.astimezone(_epg_tz).strftime("%Y-%m-%d %H:%M:%S") if _epg_tz else start.strftime("%Y-%m-%d %H:%M:%S"), + "end": end.astimezone(_epg_tz).strftime("%Y-%m-%d %H:%M:%S") if _epg_tz else end.strftime("%Y-%m-%d %H:%M:%S"), "description": base64.b64encode((description or "").encode()).decode(), "channel_id": str(channel_num_int), "start_timestamp": str(int(start.timestamp())), @@ -2516,10 +2687,18 @@ def xc_get_epg(request, user, short=False): "stream_id": f"{channel_id}", } - if short == False: - program_output["now_playing"] = 1 if start <= django_timezone.now() <= end else 0 + # has_archive tells XC clients which past programmes are available + # for catch-up playback. Always emitted (not gated by short) because + # both get_simple_data_table and get_short_epg callers need it to + # populate the catch-up programme menu. + if archive_window is not None and end < now and end > now - archive_window: + program_output["has_archive"] = 1 + else: program_output["has_archive"] = 0 + if short == False: + program_output["now_playing"] = 1 if start <= now <= end else 0 + output['epg_listings'].append(program_output) return output diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index 9bd1d284..300f3b0d 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -407,6 +407,20 @@ class ChannelStatus: if channel_name: info['channel_name'] = channel_name + info['is_timeshift'] = bool(metadata.get(ChannelMetadataField.IS_TIMESHIFT)) + + for key, field in ( + ('logo_id', ChannelMetadataField.LOGO_ID), + ('m3u_profile_id', ChannelMetadataField.M3U_PROFILE), + ): + raw = metadata.get(field) + if not raw: + continue + try: + info[key] = int(raw) + except (TypeError, ValueError): + pass + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: diff --git a/apps/proxy/live_proxy/constants.py b/apps/proxy/live_proxy/constants.py index b3839508..414e1a2f 100644 --- a/apps/proxy/live_proxy/constants.py +++ b/apps/proxy/live_proxy/constants.py @@ -54,6 +54,8 @@ class ChannelMetadataField: STREAM_ID = "stream_id" CHANNEL_NAME = "channel_name" STREAM_NAME = "stream_name" + IS_TIMESHIFT = "is_timeshift" + LOGO_ID = "logo_id" # Profile fields STREAM_PROFILE = "stream_profile" diff --git a/apps/proxy/live_proxy/input/http_streamer.py b/apps/proxy/live_proxy/input/http_streamer.py index 7efb5d33..da7f9894 100644 --- a/apps/proxy/live_proxy/input/http_streamer.py +++ b/apps/proxy/live_proxy/input/http_streamer.py @@ -1,161 +1,290 @@ """ HTTP Stream Reader - Thread-based HTTP stream reader that writes to a pipe. -This allows us to use the same fetch_chunk() path for both transcode and HTTP streams. +This allows us to use the same fetch_chunk() path for both transcode and HTTP +streams (live TV) and for timeshift/catch-up archives. + +When *response* is supplied the reader skips connection setup and streams the +already-open ``requests.Response`` directly. This is used by the timeshift +view which needs to cascade through multiple candidate URLs before handing +the winning response to the reader. + +When *strip_ts_preamble* is True the first bytes written to the pipe are +aligned to the first MPEG-TS sync chain (0x47 at 188-byte intervals). Some +XC servers emit PHP warnings or HTML error text before the binary stream; +strict demuxers (ExoPlayer / TiviMate) raise ``UnrecognizedInputFormat`` on +those bytes. The live path typically receives clean TS so the flag defaults +to False, but the timeshift path enables it. """ -import threading +import fcntl import os +import select as _select +import threading + import requests from requests.adapters import HTTPAdapter + from ..utils import get_logger logger = get_logger() +# MPEG-TS sync constants (shared with timeshift views) +_TS_PACKET_SIZE = 188 +_TS_SYNC_BYTE = 0x47 +_TS_SYNC_SEARCH_LIMIT = 65536 + +# Linux fcntl constant — increase pipe buffer beyond the 64 KB default. +_F_SETPIPE_SZ = 1031 +_PIPE_TARGET_SIZE = 1 << 20 # 1 MB + + +def find_ts_sync(buf): + """Offset of the first MPEG-TS sync chain in *buf*, or -1. + + A valid chain needs 0x47 at offsets i, i+188 and i+376 — three sync + bytes one packet apart (the standard demuxer probe). + """ + end = len(buf) - 2 * _TS_PACKET_SIZE + for i in range(0, end): + if ( + buf[i] == _TS_SYNC_BYTE + and buf[i + _TS_PACKET_SIZE] == _TS_SYNC_BYTE + and buf[i + 2 * _TS_PACKET_SIZE] == _TS_SYNC_BYTE + ): + return i + return -1 + class HTTPStreamReader: - """Thread-based HTTP stream reader that writes to a pipe""" + """Thread-based HTTP stream reader that writes to an OS pipe. - def __init__(self, url, user_agent=None, chunk_size=8192): + Parameters + ---------- + url : str + Provider URL (used when *response* is None to open a new connection). + user_agent : str, optional + User-Agent header for the upstream request. + chunk_size : int + Chunk size for ``iter_content`` and ``pipe_reader.read``. + response : requests.Response, optional + An already-open streaming response. When provided the reader skips + connection setup and streams this response directly. The caller is + responsible for having opened it with ``stream=True``. + extra_headers : dict, optional + Additional HTTP headers (e.g. Range) forwarded to the provider when + *response* is None. + strip_ts_preamble : bool + Strip non-MPEG-TS bytes before the first 0x47 sync chain. + """ + + def __init__( + self, + url, + user_agent=None, + chunk_size=8192, + *, + response=None, + extra_headers=None, + strip_ts_preamble=False, + ): self.url = url self.user_agent = user_agent self.chunk_size = chunk_size + self._provided_response = response + self._extra_headers = extra_headers or {} + self._strip_ts_preamble = strip_ts_preamble self.session = None - self.response = None + self.response = response # may be pre-set self.thread = None self.pipe_read = None self.pipe_write = None self.running = False def start(self): - """Start the HTTP stream reader thread""" + """Start the HTTP stream reader thread.""" self.pipe_read, self.pipe_write = os.pipe() - # Make the write end non-blocking so that os.write() raises BlockingIOError - # instead of stalling the OS thread when the pipe buffer is full. Without - # this, a full pipe blocks the entire gevent worker (all greenlets freeze) - # because gevent does not patch os.write() on pipes. - import fcntl - flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) - fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) + # Grow the kernel pipe buffer from 64 KB to 1 MB so the producer + # thread is not blocked while the consumer yields to uWSGI. This + # is the single biggest throughput improvement for the timeshift + # path: it turns the producer/consumer from ping-pong alternation + # into genuinely parallel operation. + try: + fcntl.fcntl(self.pipe_write, _F_SETPIPE_SZ, _PIPE_TARGET_SIZE) + except (OSError, ValueError): + pass # non-Linux or unprivileged — keep 64 KB default + + # Make the write end non-blocking so that os.write() raises + # BlockingIOError instead of stalling the OS thread when the pipe + # buffer is full. Without this, a full pipe blocks the entire gevent + # worker (all greenlets freeze) because gevent does not patch + # os.write() on pipes. + # Skip for pre-opened responses (timeshift path) — those run in a + # real OS thread where blocking writes are fine and the 1 MB pipe + # buffer provides sufficient decoupling. + if self._provided_response is None: + flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) + fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) self.running = True self.thread = threading.Thread(target=self._read_stream, daemon=True) self.thread.start() - logger.info(f"Started HTTP stream reader thread for {self.url}") + logger.info("Started HTTP stream reader thread for %s", self.url) return self.pipe_read + # ------------------------------------------------------------------ + # Non-blocking pipe write helper + # ------------------------------------------------------------------ + + def _pipe_write_all(self, data): + """Write *data* to the pipe, handling O_NONBLOCK + partial writes. + + Uses select() on the write fd (gevent-patched — yields to hub) to + wait for space when the pipe is full, instead of blocking the OS + thread. + """ + offset = 0 + while offset < len(data) and self.running: + try: + n = os.write(self.pipe_write, data[offset:]) + offset += n + except BlockingIOError: + _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) + if not writable and not self.running: + return False + except (BrokenPipeError, OSError): + return False + return True + + # ------------------------------------------------------------------ + # Producer thread + # ------------------------------------------------------------------ + def _read_stream(self): - """Thread worker that reads HTTP stream and writes to pipe""" + """Thread worker: read HTTP stream → write to pipe.""" try: - # Build headers - headers = {} - if self.user_agent: - headers['User-Agent'] = self.user_agent + if self._provided_response is not None: + # Re-use the already-open response (timeshift cascade path). + self.response = self._provided_response + else: + # Open a fresh connection (live TV path). + headers = dict(self._extra_headers) + if self.user_agent: + headers["User-Agent"] = self.user_agent - logger.info(f"HTTP reader connecting to {self.url}") + logger.info("HTTP reader connecting to %s", self.url) - # Create session - self.session = requests.Session() + self.session = requests.Session() + adapter = HTTPAdapter( + max_retries=0, pool_connections=1, pool_maxsize=1, + ) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) - # Disable retries for faster failure detection - adapter = HTTPAdapter(max_retries=0, pool_connections=1, pool_maxsize=1) - self.session.mount('http://', adapter) - self.session.mount('https://', adapter) + self.response = self.session.get( + self.url, + headers=headers, + stream=True, + timeout=(5, 30), + ) - # Stream the URL - self.response = self.session.get( - self.url, - headers=headers, - stream=True, - timeout=(5, 30) # 5s connect, 30s read - ) + if self.response.status_code not in (200, 206): + logger.error( + "HTTP %d from %s", self.response.status_code, self.url, + ) + return - if self.response.status_code != 200: - logger.error(f"HTTP {self.response.status_code} from {self.url}") - return + logger.info("HTTP reader connected, streaming data…") - logger.info(f"HTTP reader connected successfully, streaming data...") + # Optional TS preamble stripping — find the first MPEG-TS sync + # chain and discard everything before it. + synced = not self._strip_ts_preamble + sync_buf = bytearray() if not synced else None - import select as _select - - # Stream chunks to pipe chunk_count = 0 for chunk in self.response.iter_content(chunk_size=self.chunk_size): if not self.running: break + if not chunk: + continue - if chunk: - # Write the chunk in a non-blocking loop. The pipe write end is - # set O_NONBLOCK in start(), so os.write() raises BlockingIOError - # instead of stalling the OS thread. We use select.select on the - # write fd (gevent-patched - yields to hub) to wait for space, - # then retry. Partial writes are handled by advancing the offset. - offset = 0 - write_error = False - while offset < len(chunk) and self.running: - try: - n = os.write(self.pipe_write, chunk[offset:]) - offset += n - except BlockingIOError: - _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) - if not writable and not self.running: - write_error = True - break - except OSError as e: - logger.error(f"Pipe write error: {e}") - write_error = True - break - if write_error: - break + # ---- TS preamble strip logic ---- + if not synced: + sync_buf.extend(chunk) + if len(sync_buf) < 2 * _TS_PACKET_SIZE + 1: + continue + offset = find_ts_sync(sync_buf) + if offset >= 0: + chunk = bytes(sync_buf[offset:]) + synced = True + sync_buf = None + elif len(sync_buf) >= _TS_SYNC_SEARCH_LIMIT: + logger.warning( + "Upstream produced %d bytes without TS sync " + "chain; passing through unmodified", + len(sync_buf), + ) + chunk = bytes(sync_buf) + synced = True + sync_buf = None + else: + continue - chunk_count += 1 - if chunk_count % 1000 == 0: - logger.debug(f"HTTP reader streamed {chunk_count} chunks") + # ---- Non-blocking write to pipe ---- + if not self._pipe_write_all(chunk): + break + + chunk_count += 1 + if chunk_count % 1000 == 0: + logger.debug("HTTP reader streamed %d chunks", chunk_count) + + # Flush any remaining sync_buf that never found a chain. + if not synced and sync_buf: + self._pipe_write_all(bytes(sync_buf)) logger.info("HTTP stream ended") - except requests.exceptions.RequestException as e: - logger.error(f"HTTP reader request error: {e}") - except Exception as e: - logger.error(f"HTTP reader unexpected error: {e}", exc_info=True) + except requests.exceptions.RequestException as exc: + logger.error("HTTP reader request error: %s", exc) + except Exception as exc: + logger.error("HTTP reader unexpected error: %s", exc, exc_info=True) finally: self.running = False - # Close write end of pipe to signal EOF try: if self.pipe_write is not None: os.close(self.pipe_write) self.pipe_write = None - except: + except OSError: pass + # ------------------------------------------------------------------ + # Shutdown + # ------------------------------------------------------------------ + def stop(self): - """Stop the HTTP stream reader""" + """Stop the HTTP stream reader.""" logger.info("Stopping HTTP stream reader") self.running = False - # Close response if self.response: try: self.response.close() - except: + except Exception: pass - # Close session if self.session: try: self.session.close() - except: + except Exception: pass - # Close write end of pipe if self.pipe_write is not None: try: os.close(self.pipe_write) self.pipe_write = None - except: + except OSError: pass - # Wait for thread if self.thread and self.thread.is_alive(): self.thread.join(timeout=2.0) diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 879de2ec..14157a9c 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -1807,6 +1807,15 @@ class ProxyServer: try: channel_id = key.split(':')[2] + # Skip channel ids that don't follow Dispatcharr's UUID + # convention. Those are managed by features outside of + # ProxyServer (e.g. timeshift catch-up sessions use a + # `timeshift___` virtual id and are + # owned by the WSGI request that opened them — they + # expire naturally via the clients-set TTL). + if channel_id.startswith("timeshift_"): + continue + # Get metadata first metadata = self.redis_client.hgetall(key) if not metadata: diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index d882bfca..dc42845c 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -601,6 +601,25 @@ def stream_xc(request, username, password, channel_id): if custom_properties["xc_password"] != password: return Response({"error": "Invalid credentials"}, status=401) + # Resolve the channel id. Try the internal Channel.id first (most + # requests). Fall back to provider stream_id lookup for catch-up + # channels whose xc_get_live_streams emits the provider's stream_id. + from apps.channels.utils import resolve_channel_by_provider_stream_id + resolved_internal_id = None + try: + candidate = int(channel_id) + if Channel.objects.filter(id=candidate).exists(): + resolved_internal_id = candidate + except (TypeError, ValueError): + pass + + if resolved_internal_id is None: + provider_channel, _ = resolve_channel_by_provider_stream_id(str(channel_id)) + if provider_channel is not None: + resolved_internal_id = provider_channel.id + else: + return JsonResponse({"error": "Not found"}, status=404) + if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -608,14 +627,14 @@ def stream_xc(request, username, password, channel_id): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": int(channel_id), + "id": resolved_internal_id, "user_level__lte": user.user_level } channel = Channel.objects.filter(**filters).first() else: # User has specific limited profiles assigned filters = { - "id": int(channel_id), + "id": resolved_internal_id, "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -625,7 +644,7 @@ def stream_xc(request, username, password, channel_id): if not channel: return JsonResponse({"error": "Not found"}, status=404) else: - channel = get_object_or_404(Channel, id=channel_id) + channel = get_object_or_404(Channel, id=resolved_internal_id) if extension.lower() == '.mp4': force_format = 'fmp4' diff --git a/apps/timeshift/__init__.py b/apps/timeshift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/timeshift/apps.py b/apps/timeshift/apps.py new file mode 100644 index 00000000..e4ecbfba --- /dev/null +++ b/apps/timeshift/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class TimeshiftConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "apps.timeshift" + verbose_name = "Timeshift" diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py new file mode 100644 index 00000000..97d32b95 --- /dev/null +++ b/apps/timeshift/helpers.py @@ -0,0 +1,116 @@ +"""URL builders + timestamp conversion + archive-days probe for XC catch-up.""" + +import logging +from datetime import datetime + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + +DEFAULT_DURATION_MINUTES = 120 +DURATION_BUFFER_MINUTES = 5 +MAX_DURATION_MINUTES = 480 + +PROVIDER_ARCHIVE_CACHE_TTL_SECONDS = 300 +MAX_AUTO_PREV_DAYS = 30 + + +def compute_provider_archive_days_capped(): + """Largest `tv_archive_duration` advertised by any XC stream (capped, cached). + + Returns 0 when no XC stream advertises catch-up. + """ + def _scan(): + from apps.channels.models import Stream + + max_days = 0 + for props in ( + Stream.objects.filter(m3u_account__account_type="XC") + .exclude(custom_properties__isnull=True) + .values_list("custom_properties", flat=True) + ): + try: + if not int((props or {}).get("tv_archive", 0) or 0): + continue + value = int((props or {}).get("tv_archive_duration", 0) or 0) + except (TypeError, ValueError): + continue + if value > max_days: + max_days = value + return min(max_days, MAX_AUTO_PREV_DAYS) + + return cache.get_or_set( + "timeshift:provider_archive_days_capped", + _scan, + PROVIDER_ARCHIVE_CACHE_TTL_SECONDS, + ) + + +def get_programme_duration(channel, timestamp_str): + """Duration in minutes of the EPG programme starting at `timestamp_str`. + + `timestamp_str` is `YYYY-MM-DD:HH-MM` in UTC (passed through from the + client URL unchanged — no timezone conversion). Falls back to a + 120-minute default if EPG lookup fails. + """ + try: + dt = datetime.strptime(timestamp_str, "%Y-%m-%d:%H-%M") + if not channel.epg_data: + return DEFAULT_DURATION_MINUTES + + programme = channel.epg_data.programs.filter( + start_time__lte=dt, end_time__gt=dt + ).first() + if not programme: + return DEFAULT_DURATION_MINUTES + + duration_seconds = (programme.end_time - programme.start_time).total_seconds() + duration_minutes = int(duration_seconds / 60) + DURATION_BUFFER_MINUTES + return min(duration_minutes, MAX_DURATION_MINUTES) + except Exception: + return DEFAULT_DURATION_MINUTES + + +def build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes): + """Format A: `/streaming/timeshift.php?username=&password=&stream=&start=&duration=`.""" + return ( + f"{m3u_account.server_url.rstrip('/')}/streaming/timeshift.php" + f"?username={m3u_account.username}" + f"&password={m3u_account.password}" + f"&stream={stream_id}" + f"&start={timestamp}" + f"&duration={duration_minutes}" + ) + + +def build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes): + """Format B: `/timeshift/{user}/{pass}/{duration}/{timestamp}/{stream_id}.ts`.""" + return ( + f"{m3u_account.server_url.rstrip('/')}/timeshift" + f"/{m3u_account.username}" + f"/{m3u_account.password}" + f"/{duration_minutes}" + f"/{timestamp}" + f"/{stream_id}.ts" + ) + + +def format_timestamp_as_sql_datetime(timestamp): + """Reshape `YYYY-MM-DD:HH-MM` to `YYYY-MM-DD HH:MM:SS` without any + timezone conversion. + + Some XC servers refuse the dash-only shape for archives whose recording + is still being finalised and only resolve the SQL-datetime shape. This + function changes only the format — the timestamp value stays in whatever + zone the caller supplied (typically UTC, since clients derive it from the + UTC epoch in the EPG data). + + Do NOT add timezone conversion here — that was the root cause of the + "wrong programme plays" bug (plugin v1.1.4 → v1.2.6 history). + """ + try: + dt = datetime.strptime(timestamp, "%Y-%m-%d:%H-%M") + return dt.strftime("%Y-%m-%d %H:%M:%S") + except Exception as e: + logger.error("Timeshift SQL timestamp reshape failed for %r: %s", timestamp, e) + return timestamp diff --git a/apps/timeshift/tests/__init__.py b/apps/timeshift/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py new file mode 100644 index 00000000..13d3e0a9 --- /dev/null +++ b/apps/timeshift/tests/test_helpers.py @@ -0,0 +1,56 @@ +"""Tests for `apps.timeshift.helpers` — timestamp shape conversion and URL build.""" + +from django.test import TestCase + +from apps.timeshift.helpers import ( + build_timeshift_url_format_a, + build_timeshift_url_format_b, + format_timestamp_as_sql_datetime, +) + + +class _FakeAccount: + def __init__(self): + self.server_url = "http://example.test" + self.username = "user" + self.password = "pass" + + +class TimestampFormatTests(TestCase): + """The SQL-datetime shape is needed for some XC server parsers. The + reshape function changes format only — no timezone conversion.""" + + def test_format_sql_reshapes_without_tz_conversion(self): + # Must reshape only — no timezone shift. The value stays as-is. + self.assertEqual( + format_timestamp_as_sql_datetime("2026-05-12:17-00"), + "2026-05-12 17:00:00", + ) + + def test_format_sql_invalid_falls_back(self): + self.assertEqual(format_timestamp_as_sql_datetime("garbage"), "garbage") + + +class BuildTimeshiftUrlTests(TestCase): + def setUp(self): + self.account = _FakeAccount() + + def test_format_a_passes_dash_shape_unchanged(self): + url = build_timeshift_url_format_a( + self.account, "22372", "2026-05-12:19-00", 40 + ) + self.assertIn("start=2026-05-12:19-00", url) + self.assertIn("stream=22372", url) + self.assertIn("duration=40", url) + + def test_format_a_passes_sql_shape_unchanged(self): + url = build_timeshift_url_format_a( + self.account, "22372", "2026-05-12 19:00:00", 40 + ) + self.assertIn("start=2026-05-12 19:00:00", url) + + def test_format_b_path_with_dash_shape(self): + url = build_timeshift_url_format_b( + self.account, "22372", "2026-05-12:19-00", 40 + ) + self.assertIn("/40/2026-05-12:19-00/22372.ts", url) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py new file mode 100644 index 00000000..b8daf30a --- /dev/null +++ b/apps/timeshift/tests/test_views.py @@ -0,0 +1,177 @@ +"""Tests for the timeshift proxy view, focused on upstream status mapping.""" + +from unittest.mock import MagicMock, patch + +from django.test import RequestFactory, TestCase + +from apps.timeshift import views +from apps.proxy.live_proxy.input.http_streamer import find_ts_sync as _find_ts_sync, _TS_PACKET_SIZE + + +class FindTsSyncTests(TestCase): + """Locate the first MPEG-TS sync chain so a leading HTML/PHP preamble + can be skipped before the bytes reach a strict demuxer (ExoPlayer).""" + + def test_returns_zero_when_buffer_already_aligned(self): + buf = b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + self.assertEqual(_find_ts_sync(buf), 0) + + def test_returns_offset_of_first_chain_after_preamble(self): + preamble = b"
\nWarning" + aligned = b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + self.assertEqual(_find_ts_sync(preamble + aligned), len(preamble)) + + def test_returns_minus_one_when_no_chain_exists(self): + # Three lone 0x47 bytes that are NOT spaced at 188 — must not be + # mistaken for a sync chain. + self.assertEqual(_find_ts_sync(b"\x47\x00\x00\x47\x00\x00\x47" * 50), -1) + + def test_returns_minus_one_for_short_buffer(self): + self.assertEqual(_find_ts_sync(b"\x47" * 10), -1) + + +class _FakeReader: + """Stand-in for HTTPStreamReader that yields EOF immediately via a pipe.""" + + def __init__(self, *args, **kwargs): + self.pipe_read = None + self.pipe_write = None + + def start(self): + import os + self.pipe_read, self.pipe_write = os.pipe() + os.close(self.pipe_write) # EOF immediately + self.pipe_write = None + return self.pipe_read + + def stop(self): + pass + + +def _fake_upstream(status_code, *, content_type="video/mp2t", body=b""): + resp = MagicMock() + resp.status_code = status_code + resp.headers = {"Content-Type": content_type} + resp.iter_content = MagicMock(return_value=iter([body] if body else [])) + resp.close = MagicMock() + return resp + + +class StreamFromProviderStatusMappingTests(TestCase): + """`_stream_from_provider` must translate upstream HTTP status codes into + semantically correct Django responses so downstream IPTV clients react + the right way (notably: stop retrying on 404).""" + + def setUp(self): + self.factory = RequestFactory() + self.kwargs = dict( + candidate_urls=[ + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12:17-00", + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12 17:00:00", + "http://example.test/timeshift/u/p/60/2026-05-12:17-00/1.ts", + ], + user_agent="test-agent", + range_header=None, + virtual_channel_id="timeshift_1_2026-05-12-17-00_1", + client_id="timeshift_test123", + client_ip="127.0.0.1", + user=None, + channel_display_name="Test", + timestamp_utc="2026-05-12:17-00", + channel_logo_id=None, + m3u_profile_id=None, + debug=False, + ) + + @patch.object(views, "_open_upstream") + def test_all_candidates_404_returns_404(self, mocked_open): + mocked_open.return_value = _fake_upstream(404) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 404) + # Every candidate is attempted before giving up. + self.assertEqual(mocked_open.call_count, 3) + + @patch.object(views, "_open_upstream") + def test_upstream_403_short_circuits_loop(self, mocked_open): + # 403 is decisive (auth) — no retry of further candidates. + mocked_open.return_value = _fake_upstream(403) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 403) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_upstream_500_short_circuits_loop(self, mocked_open): + mocked_open.return_value = _fake_upstream(500) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_first_candidate_succeeds(self, mocked_open): + mocked_open.side_effect = [_fake_upstream(200, body=b"\x47" * 188)] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_second_candidate_succeeds_after_404(self, mocked_open): + # Primary 404 → second candidate 200 → streams successfully. + mocked_open.side_effect = [ + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 2) + + @patch.object(views, "_open_upstream") + def test_third_candidate_succeeds_after_400_then_404(self, mocked_open): + mocked_open.side_effect = [ + _fake_upstream(400), + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 200) + self.assertEqual(mocked_open.call_count, 3) + + @patch.object(views, "_open_upstream") + def test_cache_promotes_winning_index_to_first(self, mocked_open): + """Once a candidate succeeds for an account, the next request reorders + the list so the cached winner is tried first — saving cascade + overhead on fast-forward.""" + # First request: candidate index 1 wins after index 0 returns 404. + mocked_open.side_effect = [ + _fake_upstream(404), + _fake_upstream(200, body=b"\x47" * 188), + ] + kwargs = dict(self.kwargs, account_id=999) + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + r1 = views._stream_from_provider(**kwargs) + self.assertEqual(r1.status_code, 200) + self.assertEqual(mocked_open.call_count, 2) + + # Second request: cached winner (index 1) is tried first, succeeds + # immediately — no cascade. + mocked_open.reset_mock() + mocked_open.side_effect = [_fake_upstream(200, body=b"\x47" * 188)] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "HTTPStreamReader", _FakeReader): + r2 = views._stream_from_provider(**kwargs) + self.assertEqual(r2.status_code, 200) + self.assertEqual(mocked_open.call_count, 1) + # Confirm the URL used is the SQL-datetime candidate (index 1 in the + # original list set up in setUp), not the dash-only one (index 0). + self.assertIn("17:00:00", mocked_open.call_args_list[0][0][0]) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py new file mode 100644 index 00000000..87345164 --- /dev/null +++ b/apps/timeshift/views.py @@ -0,0 +1,512 @@ +"""XC catch-up (timeshift) HTTP view — proxies the provider's +`/streaming/timeshift.php` endpoint to clients like iPlayTV and TiviMate. + +URL parameter quirk matching what iPlayTV / TiviMate emit: + Position "stream_id" -> EPG channel number, IGNORED here. + Position "duration" -> actually the provider's stream_id. +""" + +import hmac +import logging +import os +import threading +import time +import uuid + +import requests +from django.http import ( + Http404, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + StreamingHttpResponse, +) + +from apps.accounts.models import User +from apps.channels.utils import ( + get_channel_catchup_info, + resolve_channel_by_provider_stream_id, +) +from apps.proxy.live_proxy.config_helper import ConfigHelper +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState +from apps.proxy.live_proxy.redis_keys import RedisKeys +from apps.proxy.live_proxy.utils import get_client_ip +from apps.proxy.live_proxy.input.http_streamer import HTTPStreamReader +from core.models import CoreSettings +from core.utils import RedisClient + +from .helpers import ( + build_timeshift_url_format_a, + build_timeshift_url_format_b, + format_timestamp_as_sql_datetime, + get_programme_duration, +) + +logger = logging.getLogger(__name__) + +CLIENT_TTL_SECONDS = 60 + + +def timeshift_proxy(request, username, password, stream_id, timestamp, duration): # noqa: ARG001 stream_id + # The "duration" URL slot is the provider's stream_id — see module docstring. + provider_stream_id = duration[:-3] if duration.endswith(".ts") else duration + + user = _authenticate_user(username, password) + if user is None: + return HttpResponseForbidden("Invalid credentials") + + channel, _ = resolve_channel_by_provider_stream_id(provider_stream_id) + if channel is None: + raise Http404("Channel not found") + + if not _user_can_access_channel(user, channel): + return HttpResponseForbidden("Access denied") + + catchup = get_channel_catchup_info(channel) + if catchup is None: + return HttpResponseBadRequest("Timeshift not supported for this channel") + + catchup_stream = catchup["stream"] + props = catchup["props"] + m3u_account = catchup_stream.m3u_account + if m3u_account is None or m3u_account.account_type != "XC": + return HttpResponseBadRequest("Channel not from Xtream Codes provider") + + timeshift_settings = CoreSettings.get_timeshift_settings() + debug = bool(timeshift_settings.get("debug_logging", False)) + + # The client-supplied timestamp is already in UTC (derived from the + # start_timestamp epoch in the EPG data). Pass it through to the + # provider as-is — no timezone conversion. Converting here was the + # root cause of the "wrong programme plays" bug: the provider indexes + # archives in UTC, so shifting to Brussels meant requesting a programme + # 1-2 hours later than intended. + # Learned from plugin history v1.1.4 → v1.2.6: 6 iterations of timezone + # bugs all converged on "never convert the provider URL timestamp". + local_timestamp = timestamp + sql_timestamp = format_timestamp_as_sql_datetime(timestamp) + duration_minutes = get_programme_duration(channel, local_timestamp) + stream_id_value = (props or {}).get("stream_id") + if stream_id_value is None: + return HttpResponseBadRequest("Cannot build timeshift URL") + + # Build a small set of upstream candidates. Different XC servers (or even + # different code paths inside the same server) accept different timestamp + # shapes and different URL layouts: the dash-only shape covers most + # finalised archives, the SQL-datetime shape unlocks a separate path that + # some servers use for archives still being indexed, and the path layout + # (Format B) is the only one some Stalker-style portals expose. We try + # them in order until one accepts the request — see `_stream_from_provider` + # for the iteration logic. + candidate_urls = [ + build_timeshift_url_format_a(m3u_account, stream_id_value, local_timestamp, duration_minutes), + build_timeshift_url_format_a(m3u_account, stream_id_value, sql_timestamp, duration_minutes), + build_timeshift_url_format_b(m3u_account, stream_id_value, local_timestamp, duration_minutes), + ] + + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + safe_ts = timestamp.replace(":", "-").replace("/", "-") + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{provider_stream_id}" + client_id = f"timeshift_{uuid.uuid4().hex[:16]}" + client_ip = get_client_ip(request) + range_header = request.META.get("HTTP_RANGE") + + # Resolve channel logo + M3U default profile so the Stats card can render + # the same logo + M3U profile name as for live streams. + channel_logo_id = getattr(channel, "logo_id", None) + default_profile = m3u_account.profiles.filter(is_default=True).first() + m3u_profile_id = default_profile.id if default_profile is not None else None + + # If the user is switching from live TV to timeshift on the same channel, + # the live stream still holds the provider connection (~1s teardown). + # Force-stop it synchronously before opening the timeshift upstream so + # the provider slot is free. No-op if no live stream is active. + from apps.proxy.live_proxy.services.channel_service import ChannelService + ChannelService.stop_channel(str(channel.uuid)) + + if debug: + logger.info( + "Timeshift request: channel=%s ts=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, + local_timestamp, + provider_stream_id, + virtual_channel_id, + client_id, + range_header or "(none)", + ) + + return _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + debug=debug, + account_id=m3u_account.id, + ) + + +# --------------------------------------------------------------------------- +# Authentication, lookup, access control +# --------------------------------------------------------------------------- + + +def _authenticate_user(username, password): + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + return None + expected = (user.custom_properties or {}).get("xc_password") + if not expected: + return None + if not hmac.compare_digest(str(expected), str(password)): + return None + return user + + +def _user_can_access_channel(user, channel): + if user.user_level < channel.user_level: + return False + if user.user_level >= User.UserLevel.ADMIN: + return True + profile_count = user.channel_profiles.count() + if profile_count == 0: + return True + return ( + type(channel).objects.filter( + id=channel.id, + channelprofilemembership__enabled=True, + channelprofilemembership__channel_profile__in=user.channel_profiles.all(), + ) + .exists() + ) + + +# --------------------------------------------------------------------------- +# Stats integration (direct Redis writes, no ClientManager instance) +# --------------------------------------------------------------------------- + + +def _register_stats_client( + redis_client, + virtual_channel_id, + client_id, + client_ip, + user_agent, + user, + *, + channel_display_name, + timestamp_utc, + primary_url, + channel_logo_id=None, + m3u_profile_id=None, +): + """Write the same Redis keys the live proxy's ClientManager writes so the + catch-up viewer appears on `/stats`. `is_timeshift=1` toggles the badge.""" + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + now = str(time.time()) + client_payload = { + "user_agent": user_agent or "unknown", + "ip_address": client_ip, + "connected_at": now, + "last_active": now, + "user_id": str(user.id) if user is not None else "0", + "username": user.username if user is not None else "unknown", + } + metadata_payload = { + ChannelMetadataField.STATE: ChannelState.ACTIVE, + ChannelMetadataField.INIT_TIME: now, + ChannelMetadataField.OWNER: "timeshift", + ChannelMetadataField.CHANNEL_NAME: channel_display_name or "Timeshift", + ChannelMetadataField.STREAM_NAME: f"Catch-up @ {timestamp_utc} UTC" if timestamp_utc else "Catch-up", + ChannelMetadataField.URL: _redact_url(primary_url) if primary_url else "", + ChannelMetadataField.IS_TIMESHIFT: "1", + } + if channel_logo_id is not None: + metadata_payload[ChannelMetadataField.LOGO_ID] = str(channel_logo_id) + if m3u_profile_id is not None: + metadata_payload[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) + try: + pipe = redis_client.pipeline(transaction=False) + pipe.hset(client_key, mapping=client_payload) + pipe.expire(client_key, CLIENT_TTL_SECONDS) + pipe.sadd(client_set_key, client_id) + pipe.expire(client_set_key, CLIENT_TTL_SECONDS) + pipe.hset(metadata_key, mapping=metadata_payload) + pipe.expire(metadata_key, CLIENT_TTL_SECONDS) + pipe.execute() + except Exception as exc: + logger.warning("Timeshift stats register failed: %s", exc) + + +def _heartbeat_stats_client(redis_client, virtual_channel_id, client_id, bytes_delta=0): + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + try: + pipe = redis_client.pipeline(transaction=False) + pipe.hset(client_key, "last_active", str(time.time())) + pipe.expire(client_key, CLIENT_TTL_SECONDS) + pipe.expire(client_set_key, CLIENT_TTL_SECONDS) + if bytes_delta > 0: + pipe.hincrby(metadata_key, ChannelMetadataField.TOTAL_BYTES, bytes_delta) + pipe.expire(metadata_key, CLIENT_TTL_SECONDS) + pipe.execute() + except Exception: + pass + + +def _unregister_stats_client(redis_client, virtual_channel_id, client_id): + if redis_client is None: + return + client_set_key = RedisKeys.clients(virtual_channel_id) + client_key = RedisKeys.client_metadata(virtual_channel_id, client_id) + metadata_key = RedisKeys.channel_metadata(virtual_channel_id) + try: + redis_client.srem(client_set_key, client_id) + redis_client.delete(client_key) + if (redis_client.scard(client_set_key) or 0) == 0: + redis_client.delete(client_set_key) + redis_client.delete(metadata_key) + except Exception as exc: + logger.warning("Timeshift stats unregister failed: %s", exc) + + +# --------------------------------------------------------------------------- +# Provider streaming +# --------------------------------------------------------------------------- + + + +def _open_upstream(url, user_agent, range_header): + """Open the upstream HTTP request (status + headers known synchronously).""" + headers = {} + if user_agent: + headers["User-Agent"] = user_agent + if range_header: + headers["Range"] = range_header + return requests.get( + url, + headers=headers, + stream=True, + timeout=ConfigHelper.connection_timeout(), + ) + + +_format_cache_lock = threading.Lock() +_format_cache = {} + + +def _get_cached_format_index(account_id): + """Return the index of the URL shape that last worked for this account, + or None if we haven't seen one succeed yet. + + Caching the winning shape lets a fast-forward session — where the player + reissues GETs at one-second intervals as the user scrubs — skip the + cascade after the first successful candidate and go straight to the + working URL on every subsequent request. + """ + if account_id is None: + return None + with _format_cache_lock: + return _format_cache.get(account_id) + + +def _set_cached_format_index(account_id, index): + if account_id is None: + return + with _format_cache_lock: + _format_cache[account_id] = index + + +def _stream_from_provider( + *, + candidate_urls, + user_agent, + range_header, + virtual_channel_id, + client_id, + client_ip, + user, + channel_display_name, + timestamp_utc, + channel_logo_id, + m3u_profile_id, + debug, + account_id=None, +): + # Use 256 KB chunks: amortises per-yield uWSGI/gevent overhead. + chunk_size = max(ConfigHelper.chunk_size(), 262144) + + # Reorder the candidates so the cached winning shape is tried first. + # The cache is per-account; within a single fast-forward session every + # seek lands in the same archive entry and therefore the same parser + # path, so the first attempt almost always succeeds. + cached_index = _get_cached_format_index(account_id) + if cached_index is not None and 0 <= cached_index < len(candidate_urls): + ordered_urls = [candidate_urls[cached_index]] + [ + u for i, u in enumerate(candidate_urls) if i != cached_index + ] + original_indices = [cached_index] + [ + i for i in range(len(candidate_urls)) if i != cached_index + ] + else: + ordered_urls = list(candidate_urls) + original_indices = list(range(len(candidate_urls))) + + # Try each candidate URL until one returns a streamable status. We keep + # going on 400/404 (the two refusal modes XC servers use when the URL + # shape doesn't match their parser); a 403, 5xx or connection error is + # decisive and short-circuits the loop. + upstream = None + last_status = None + last_url = ordered_urls[0] + winning_index = None + for url, orig_idx in zip(ordered_urls, original_indices): + try: + response = _open_upstream(url, user_agent, range_header) + except requests.exceptions.RequestException as exc: + logger.error("Timeshift provider unreachable: %s", exc) + return HttpResponseBadRequest("Provider connection error") + last_status = response.status_code + last_url = url + if response.status_code in (200, 206): + upstream = response + winning_index = orig_idx + break + response.close() + if response.status_code not in (400, 404): + # Decisive failure (403, 5xx, …) — no point trying alternative URLs. + break + + if winning_index is not None: + _set_cached_format_index(account_id, winning_index) + + if upstream is None: + logger.error("Timeshift upstream rejected: status=%s url=%s", + last_status, _redact_url(last_url)) + # Mirror semantically meaningful upstream statuses so clients react + # correctly: 404 stops retry loops (catch-up not yet indexed), 403 + # surfaces auth problems. Everything else stays a generic 400. + if last_status == 404: + return HttpResponseNotFound("Catch-up not available yet") + if last_status == 403: + return HttpResponseForbidden("Provider denied access") + return HttpResponseBadRequest(f"Provider error: {last_status}") + + content_type = upstream.headers.get("Content-Type", "video/mp2t") + content_range = upstream.headers.get("Content-Range", "") + status = upstream.status_code + + redis_client = RedisClient.get_client() + _register_stats_client( + redis_client, + virtual_channel_id, + client_id, + client_ip, + user_agent, + user, + channel_display_name=channel_display_name, + timestamp_utc=timestamp_utc, + primary_url=last_url, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + ) + + # Use HTTPStreamReader (same component as live TV) with a 1 MB pipe + # buffer and TS preamble stripping. The producer thread reads from the + # provider and writes to the pipe; the WSGI greenlet reads from the + # pipe and yields to the client. + reader = HTTPStreamReader( + url=last_url, + chunk_size=chunk_size, + response=upstream, + strip_ts_preamble=True, + ) + pipe_read_fd = reader.start() + pipe_reader = os.fdopen(pipe_read_fd, "rb", buffering=0) + + def stream_generator(): + last_heartbeat = time.time() + bytes_since_heartbeat = 0 + total_yielded = 0 + loop_start = time.time() + try: + while True: + data = pipe_reader.read(chunk_size) + if not data: + break + yield data + bytes_since_heartbeat += len(data) + total_yielded += len(data) + + now = time.time() + if now - last_heartbeat >= 5: + _heartbeat_stats_client( + redis_client, virtual_channel_id, client_id, + bytes_delta=bytes_since_heartbeat, + ) + last_heartbeat = now + bytes_since_heartbeat = 0 + except GeneratorExit: + pass + except Exception: + logger.exception("Timeshift stream loop error") + finally: + elapsed = time.time() - loop_start + if debug and total_yielded > 0: + mbps = (total_yielded * 8) / elapsed / 1_000_000 if elapsed > 0 else 0 + logger.info( + "Timeshift disconnect: vid=%s client=%s yielded=%d bytes in %.1fs (%.2f Mbps avg)", + virtual_channel_id, client_id, total_yielded, elapsed, mbps, + ) + reader.stop() + try: + pipe_reader.close() + except Exception: + pass + _unregister_stats_client(redis_client, virtual_channel_id, client_id) + + response = StreamingHttpResponse( + stream_generator(), + content_type=content_type, + status=status, + ) + # Tell nginx not to buffer this streaming response. Without this, the + # default uwsgi_buffering=on under `location /` throttles the response + # roughly to half the upstream rate (back-pressure from buffer flushes + # propagates back into the generator). + response["X-Accel-Buffering"] = "no" + # Forward Content-Range so iPlayTV / TiviMate seek cursor stays correct. + # Content-Length is intentionally NOT forwarded: chunked transfer is the + # safer interchange format for long-lived streaming. + if content_range: + response["Content-Range"] = content_range + response["Accept-Ranges"] = "bytes" + return response + + +def _redact_url(url): + """Strip credentials from a URL for safe logging.""" + if not url or "://" not in url: + return url + scheme, rest = url.split("://", 1) + if "@" in rest: + rest = rest.split("@", 1)[1] + return f"{scheme}://{rest.split('?')[0]}/..." diff --git a/core/models.py b/core/models.py index 0ec4efda..40c17a7f 100644 --- a/core/models.py +++ b/core/models.py @@ -195,6 +195,14 @@ NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" USER_LIMITS_SETTINGS_KEY = "user_limit_settings" +TIMESHIFT_SETTINGS_KEY = "timeshift_settings" + +TIMESHIFT_DEFAULTS = { + "default_timezone": "UTC", + "default_language": "en", + "xmltv_prev_days_override": 0, + "debug_logging": False, +} class CoreSettings(models.Model): @@ -436,6 +444,22 @@ class CoreSettings(models.Model): "terminate_oldest": True, }) + # Timeshift Settings (XC catch-up) + @classmethod + def get_timeshift_settings(cls): + """Return all timeshift-related settings, falling back to neutral defaults.""" + stored = cls._get_group(TIMESHIFT_SETTINGS_KEY, {}) or {} + merged = dict(TIMESHIFT_DEFAULTS) + for key, value in stored.items(): + if key in merged and value is not None: + merged[key] = value + return merged + + @classmethod + def update_timeshift_settings(cls, updates): + clean = {k: v for k, v in (updates or {}).items() if k in TIMESHIFT_DEFAULTS} + return cls._update_group(TIMESHIFT_SETTINGS_KEY, "Timeshift Settings", clean) + class SystemEvent(models.Model): """ diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index e8818108..e95f91cd 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -97,6 +97,7 @@ INSTALLED_APPS = [ "django_filters", "django_celery_beat", "apps.plugins", + "apps.timeshift.apps.TimeshiftConfig", ] # EPG Processing optimization settings diff --git a/dispatcharr/urls.py b/dispatcharr/urls.py index 9123f3da..d8e462a5 100644 --- a/dispatcharr/urls.py +++ b/dispatcharr/urls.py @@ -7,6 +7,7 @@ from .routing import websocket_urlpatterns from apps.output.views import xc_player_api, xc_panel_api, xc_get, xc_xmltv from apps.proxy.live_proxy.views import stream_xc from apps.proxy.vod_proxy.views import stream_xc_movie, stream_xc_episode +from apps.timeshift.views import timeshift_proxy urlpatterns = [ # API Routes @@ -41,6 +42,13 @@ urlpatterns = [ stream_xc, name="xc_stream_endpoint", ), + # XC catch-up (timeshift). The "duration" slot is actually the provider's + # stream_id — matches what iPlayTV / TiviMate emit (see apps/timeshift/views). + path( + "timeshift/////", + timeshift_proxy, + name="timeshift_proxy", + ), # XC VOD endpoints path( "movie///.", diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 30509a86..794045f4 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -26,6 +26,7 @@ import { HardDriveDownload, HardDriveUpload, Radio, + Rewind, SquareX, Timer, Users, @@ -496,7 +497,7 @@ const StreamConnectionCard = ({ ); }, - mantineExpandButtonProps: ({ row, table }) => ({ + mantineExpandButtonProps: ({ row }) => ({ size: 'xs', style: { transform: row.getIsExpanded() ? 'rotate(180deg)' : 'rotate(-90deg)', @@ -533,7 +534,10 @@ const StreamConnectionCard = ({ }, [channel.name, channel.stream_id]); const channelName = - channel.name || previewedStream?.name || 'Unnamed Channel'; + channel.name || + channel.channel_name || + previewedStream?.name || + 'Unnamed Channel'; const uptime = channel.uptime || 0; const bitrates = channel.bitrates || []; const totalBytes = channel.total_bytes || 0; @@ -745,6 +749,18 @@ const StreamConnectionCard = ({ {/* Add stream information badges */} + {channel.is_timeshift && ( + + } + > + TIMESHIFT + + + )} {channel.resolution && ( diff --git a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx new file mode 100644 index 00000000..59db1e62 --- /dev/null +++ b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx @@ -0,0 +1,116 @@ +import useSettingsStore from '../../../store/settings.jsx'; +import React, { useEffect, useState, useMemo } from 'react'; +import { + getChangedSettings, + parseSettings, + saveChangedSettings, +} from '../../../utils/pages/SettingsUtils.js'; +import { + Button, + NumberInput, + Select, + Stack, + Switch, + Text, + TextInput, +} from '@mantine/core'; +import { useForm } from '@mantine/form'; +import { buildTimeZoneOptions } from '../../../utils/dateTimeUtils.js'; +import { + getTimeshiftSettingsFormInitialValues, + getTimeshiftSettingsFormValidation, +} from '../../../utils/forms/settings/TimeshiftSettingsFormUtils.js'; + +const TimeshiftSettingsForm = React.memo(({ active }) => { + const settings = useSettingsStore((s) => s.settings); + const [saved, setSaved] = useState(false); + + const form = useForm({ + mode: 'controlled', + initialValues: getTimeshiftSettingsFormInitialValues(), + validate: getTimeshiftSettingsFormValidation(), + }); + + const tzOptions = useMemo( + () => + buildTimeZoneOptions( + form.getValues().timeshift_default_timezone || 'UTC' + ), + [] + ); + + useEffect(() => { + if (!active) setSaved(false); + }, [active]); + + useEffect(() => { + if (settings) { + const parsed = parseSettings(settings); + // Fall back explicitly to the neutral defaults so the Select shows + // "UTC (now UTC+00:00)" when no setting has been saved yet. + form.setValues({ + timeshift_default_timezone: parsed.timeshift_default_timezone || 'UTC', + timeshift_default_language: parsed.timeshift_default_language || 'en', + xmltv_prev_days_override: parsed.xmltv_prev_days_override ?? 0, + timeshift_debug_logging: !!parsed.timeshift_debug_logging, + }); + } + }, [settings]); + + const onSubmit = async () => { + setSaved(false); + const changed = getChangedSettings(form.getValues(), settings); + try { + await saveChangedSettings(settings, changed); + setSaved(true); + } catch (error) { + console.error('Error saving timeshift settings:', error); + } + }; + + return ( +
+ + + XC catch-up uses these settings to convert UTC EPG timestamps into the + provider's local time and to set the XMLTV lookback window. The + defaults are neutral (UTC, en, off) — adjust to match your provider. + + - { step={1} {...form.getInputProps('xmltv_prev_days_override')} /> - diff --git a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js index 3b5813f5..880c32b0 100644 --- a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js @@ -1,17 +1,9 @@ export const getTimeshiftSettingsFormInitialValues = () => { return { - timeshift_default_timezone: 'UTC', - timeshift_default_language: 'en', xmltv_prev_days_override: 0, - timeshift_debug_logging: false, }; }; export const getTimeshiftSettingsFormValidation = () => { - return { - timeshift_default_language: (value) => - value && !/^[a-z]{2}$/i.test(value) - ? 'Must be a 2-letter ISO 639-1 code' - : null, - }; + return {}; }; diff --git a/frontend/src/utils/pages/SettingsUtils.js b/frontend/src/utils/pages/SettingsUtils.js index 28a5764c..e796af96 100644 --- a/frontend/src/utils/pages/SettingsUtils.js +++ b/frontend/src/utils/pages/SettingsUtils.js @@ -70,12 +70,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'auto_import_mapped_files', 'enable_ip_lookup', ]; - const timeshiftFields = [ - 'timeshift_default_timezone', - 'timeshift_default_language', - 'xmltv_prev_days_override', - 'timeshift_debug_logging', - ]; + const timeshiftFields = ['xmltv_prev_days_override']; for (const formKey in changedSettings) { let value = changedSettings[formKey]; @@ -142,23 +137,12 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'comskip_enabled', 'schedule_enabled', 'auto_import_mapped_files', - 'timeshift_debug_logging', 'enable_ip_lookup', ]; if (booleanFields.includes(formKey) && value != null) { value = typeof value === 'boolean' ? value : Boolean(value); } - // Map UI form keys (with timeshift_ prefix) to the storage subkeys used in - // CoreSettings.value. The prefix lets us share form keys across groups - // without ambiguity but the JSON storage uses short keys. - const timeshiftFormKeyToStorage = { - timeshift_default_timezone: 'default_timezone', - timeshift_default_language: 'default_language', - xmltv_prev_days_override: 'xmltv_prev_days_override', - timeshift_debug_logging: 'debug_logging', - }; - // Route to appropriate group if (streamFields.includes(formKey)) { groupedChanges.stream_settings[formKey] = value; @@ -171,8 +155,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { } else if (systemFields.includes(formKey)) { groupedChanges.system_settings[formKey] = value; } else if (timeshiftFields.includes(formKey)) { - const storageKey = timeshiftFormKeyToStorage[formKey]; - groupedChanges.timeshift_settings[storageKey] = value; + groupedChanges.timeshift_settings[formKey] = value; } } @@ -388,26 +371,16 @@ export const parseSettings = (settings) => { : true; } - // Timeshift settings - direct mapping with timeshift_ prefixed form keys + // Timeshift settings - direct key mapping. + // The only key is the XMLTV lookback override: the XC API surface is strictly + // UTC (no timezone setting) and verbose logging follows the standard logger level. const timeshiftSettings = settings['timeshift_settings']?.value; - parsed.timeshift_default_timezone = - timeshiftSettings && timeshiftSettings.default_timezone - ? String(timeshiftSettings.default_timezone) - : 'UTC'; - parsed.timeshift_default_language = - timeshiftSettings && timeshiftSettings.default_language - ? String(timeshiftSettings.default_language) - : 'en'; parsed.xmltv_prev_days_override = timeshiftSettings && timeshiftSettings.xmltv_prev_days_override != null ? typeof timeshiftSettings.xmltv_prev_days_override === 'number' ? timeshiftSettings.xmltv_prev_days_override : parseInt(timeshiftSettings.xmltv_prev_days_override, 10) || 0 : 0; - parsed.timeshift_debug_logging = - timeshiftSettings && timeshiftSettings.debug_logging != null - ? Boolean(timeshiftSettings.debug_logging) - : false; // Proxy and network access are already grouped objects if (settings['proxy_settings']?.value) { From 2909c9b339b6b41dfec1b296055b1dff161fb42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Wed, 10 Jun 2026 22:05:01 +0200 Subject: [PATCH 10/31] chore(timeshift): pre-review hardening and dead-code cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - http_streamer.py: restore HTTPStreamReader to its upstream form and keep only the find_ts_sync() addition. The response=/extra_headers=/ strip_ts_preamble= extensions had no remaining callers since the timeshift view moved to direct iter_content streaming (delta shrinks from +235/-83 lines to +28). - Unify the catchup_days semantics everywhere: a channel's archive depth is MAX(catchup_days) over its CATCH-UP streams only. The SQL rollup now uses a FILTER (WHERE s.is_catchup) aggregate and the ChannelStream signal uses the same MAX aggregation (it previously took the first stream by order, and the rollup aggregated over all streams including non-catchup ones). - Migration backfill: also accept the lowercase 'true' that ->> extraction yields for JSON booleans. - get_channel_catchup_info(): drop the tv_archive_duration key — its only caller never used it. - Document why timeshift termination fails closed when Redis is unavailable (denying the new stream is what protects the provider connection limit), and update the stale stop-key comment (5 s cadence, not 100 chunks). --- .../migrations/0038_add_catchup_fields.py | 4 + apps/channels/signals.py | 15 +- apps/channels/utils.py | 1 - apps/m3u/tasks.py | 4 +- apps/proxy/live_proxy/input/http_streamer.py | 431 +++++++----------- apps/proxy/utils.py | 8 +- 6 files changed, 183 insertions(+), 280 deletions(-) diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py index 50e1ad01..d6cdc335 100644 --- a/apps/channels/migrations/0038_add_catchup_fields.py +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -26,6 +26,10 @@ def backfill_stream_catchup(apps, schema_editor): AND custom_properties != 'null'::jsonb AND ( custom_properties->>'tv_archive' = '1' + -- JSON booleans extract as lowercase 'true' via ->>; the + -- 'True' spelling covers Python-str values stored by + -- older import code. + OR custom_properties->>'tv_archive' = 'true' OR custom_properties->>'tv_archive' = 'True' ) """) diff --git a/apps/channels/signals.py b/apps/channels/signals.py index f1f3f0ca..c896bd13 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -382,16 +382,19 @@ def update_channel_catchup_fields(sender, instance, **kwargs): Covers the UI/API path (admin adds/removes a stream from a channel). The bulk-import path bypasses signals and uses explicit SQL instead — - see apps/m3u/tasks.py after ChannelStream.objects.bulk_create(). + see rollup_channel_catchup_fields() in apps/m3u/tasks.py. Same semantics + as that rollup: is_catchup = any catch-up stream, catchup_days = MAX of + the catch-up streams' archive depth. """ + from django.db.models import Max + channel = instance.channel - best = ( + max_days = ( channel.streams .filter(is_catchup=True) - .order_by("channelstream__order") - .first() + .aggregate(max_days=Max("catchup_days"))["max_days"] ) Channel.objects.filter(pk=channel.pk).update( - is_catchup=best is not None, - catchup_days=best.catchup_days if best else 0, + is_catchup=max_days is not None, + catchup_days=max_days or 0, ) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index d5dea651..043a8a36 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -41,7 +41,6 @@ def get_channel_catchup_info(channel): return { "stream": stream, "props": stream.custom_properties or {}, - "tv_archive_duration": channel.catchup_days or 7, } diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 2e3cde6d..7ebe2ad0 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1845,7 +1845,9 @@ def rollup_channel_catchup_fields(account_id): SELECT cs.channel_id, bool_or(s.is_catchup) AS any_catchup, - MAX(s.catchup_days) AS max_days + -- Only catch-up streams contribute their archive depth: + -- a non-catchup stream may carry a stale catchup_days value. + MAX(s.catchup_days) FILTER (WHERE s.is_catchup) AS max_days FROM dispatcharr_channels_channelstream cs JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id WHERE cs.channel_id IN ( diff --git a/apps/proxy/live_proxy/input/http_streamer.py b/apps/proxy/live_proxy/input/http_streamer.py index cedb09d9..d90297c5 100644 --- a/apps/proxy/live_proxy/input/http_streamer.py +++ b/apps/proxy/live_proxy/input/http_streamer.py @@ -1,48 +1,182 @@ """ HTTP Stream Reader - Thread-based HTTP stream reader that writes to a pipe. -This allows us to use the same fetch_chunk() path for both transcode and HTTP -streams (live TV) and for timeshift/catch-up archives. - -When *response* is supplied the reader skips connection setup and streams the -already-open ``requests.Response`` directly. This is used by the timeshift -view which needs to cascade through multiple candidate URLs before handing -the winning response to the reader. - -When *strip_ts_preamble* is True the first bytes written to the pipe are -aligned to the first MPEG-TS sync chain (0x47 at 188-byte intervals). Some -XC servers emit PHP warnings or HTML error text before the binary stream; -strict demuxers (ExoPlayer / TiviMate) raise ``UnrecognizedInputFormat`` on -those bytes. The live path typically receives clean TS so the flag defaults -to False, but the timeshift path enables it. +This allows us to use the same fetch_chunk() path for both transcode and HTTP streams. """ -import fcntl -import os -import select as _select import threading - +import os import requests from requests.adapters import HTTPAdapter - from ..utils import get_logger logger = get_logger() -# MPEG-TS sync constants (shared with timeshift views) + +class HTTPStreamReader: + """Thread-based HTTP stream reader that writes to a pipe""" + + def __init__(self, url, user_agent=None, chunk_size=8192): + self.url = url + self.user_agent = user_agent + self.chunk_size = chunk_size + self.session = None + self.response = None + self.thread = None + self.pipe_read = None + self.pipe_write = None + self.running = False + + def start(self): + """Start the HTTP stream reader thread""" + self.pipe_read, self.pipe_write = os.pipe() + + # Make the write end non-blocking so that os.write() raises BlockingIOError + # instead of stalling the OS thread when the pipe buffer is full. Without + # this, a full pipe blocks the entire gevent worker (all greenlets freeze) + # because gevent does not patch os.write() on pipes. + import fcntl + flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) + fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + self.running = True + self.thread = threading.Thread(target=self._read_stream, daemon=True) + self.thread.start() + + logger.info(f"Started HTTP stream reader thread for {self.url}") + return self.pipe_read + + def _read_stream(self): + """Thread worker that reads HTTP stream and writes to pipe""" + try: + # Build headers + headers = {} + if self.user_agent: + headers['User-Agent'] = self.user_agent + + logger.info(f"HTTP reader connecting to {self.url}") + + # Create session + self.session = requests.Session() + + # Disable retries for faster failure detection + adapter = HTTPAdapter(max_retries=0, pool_connections=1, pool_maxsize=1) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) + + # Stream the URL + self.response = self.session.get( + self.url, + headers=headers, + stream=True, + timeout=(5, 30) # 5s connect, 30s read + ) + + if self.response.status_code != 200: + logger.error(f"HTTP {self.response.status_code} from {self.url}") + return + + logger.info(f"HTTP reader connected successfully, streaming data...") + + import select as _select + + # Stream chunks to pipe + chunk_count = 0 + for chunk in self.response.iter_content(chunk_size=self.chunk_size): + if not self.running: + break + + if chunk: + # Write the chunk in a non-blocking loop. The pipe write end is + # set O_NONBLOCK in start(), so os.write() raises BlockingIOError + # instead of stalling the OS thread. We use select.select on the + # write fd (gevent-patched - yields to hub) to wait for space, + # then retry. Partial writes are handled by advancing the offset. + offset = 0 + write_error = False + while offset < len(chunk) and self.running: + try: + n = os.write(self.pipe_write, chunk[offset:]) + offset += n + except BlockingIOError: + _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) + if not writable and not self.running: + write_error = True + break + except OSError as e: + logger.error(f"Pipe write error: {e}") + write_error = True + break + if write_error: + break + + chunk_count += 1 + if chunk_count % 1000 == 0: + logger.debug(f"HTTP reader streamed {chunk_count} chunks") + + logger.info("HTTP stream ended") + + except requests.exceptions.RequestException as e: + logger.error(f"HTTP reader request error: {e}") + except Exception as e: + logger.error(f"HTTP reader unexpected error: {e}", exc_info=True) + finally: + self.running = False + # Close write end of pipe to signal EOF + try: + if self.pipe_write is not None: + os.close(self.pipe_write) + self.pipe_write = None + except: + pass + + def stop(self): + """Stop the HTTP stream reader""" + logger.info("Stopping HTTP stream reader") + self.running = False + + # Close response + if self.response: + try: + self.response.close() + except: + pass + + # Close session + if self.session: + try: + self.session.close() + except: + pass + + # Close write end of pipe + if self.pipe_write is not None: + try: + os.close(self.pipe_write) + self.pipe_write = None + except: + pass + + # Wait for thread + if self.thread and self.thread.is_alive(): + self.thread.join(timeout=2.0) + + +# --------------------------------------------------------------------------- +# MPEG-TS sync detection (used by the timeshift catch-up proxy) +# --------------------------------------------------------------------------- + _TS_PACKET_SIZE = 188 _TS_SYNC_BYTE = 0x47 -_TS_SYNC_SEARCH_LIMIT = 65536 - -# Linux fcntl constant — increase pipe buffer beyond the 64 KB default. -_F_SETPIPE_SZ = 1031 -_PIPE_TARGET_SIZE = 1 << 20 # 1 MB def find_ts_sync(buf): """Offset of the first MPEG-TS sync chain in *buf*, or -1. A valid chain needs 0x47 at offsets i, i+188 and i+376 — three sync - bytes one packet apart (the standard demuxer probe). + bytes one packet apart (the standard demuxer probe). The timeshift + proxy peeks at the first upstream bytes with this to reject HTTP-200 + PHP error pages and to strip any pre-sync preamble before the bytes + reach a strict demuxer (ExoPlayer). """ end = len(buf) - 2 * _TS_PACKET_SIZE for i in range(0, end): @@ -53,248 +187,3 @@ def find_ts_sync(buf): ): return i return -1 - - -class HTTPStreamReader: - """Thread-based HTTP stream reader that writes to an OS pipe. - - Parameters - ---------- - url : str - Provider URL (used when *response* is None to open a new connection). - user_agent : str, optional - User-Agent header for the upstream request. - chunk_size : int - Chunk size for ``iter_content`` and ``pipe_reader.read``. - response : requests.Response, optional - An already-open streaming response. When provided the reader skips - connection setup and streams this response directly. The caller is - responsible for having opened it with ``stream=True``. - extra_headers : dict, optional - Additional HTTP headers (e.g. Range) forwarded to the provider when - *response* is None. - strip_ts_preamble : bool - Strip non-MPEG-TS bytes before the first 0x47 sync chain. - """ - - def __init__( - self, - url, - user_agent=None, - chunk_size=8192, - *, - response=None, - extra_headers=None, - strip_ts_preamble=False, - ): - self.url = url - self.user_agent = user_agent - self.chunk_size = chunk_size - self._provided_response = response - self._extra_headers = extra_headers or {} - self._strip_ts_preamble = strip_ts_preamble - self.session = None - self.response = response # may be pre-set - self.thread = None - self.pipe_read = None - self.pipe_write = None - self.running = False - - def start(self): - """Start the HTTP stream reader thread.""" - self.pipe_read, self.pipe_write = os.pipe() - - # Grow the kernel pipe buffer from 64 KB to 1 MB so the producer - # thread is not blocked while the consumer yields to uWSGI. This - # is the single biggest throughput improvement for the timeshift - # path: it turns the producer/consumer from ping-pong alternation - # into genuinely parallel operation. - try: - fcntl.fcntl(self.pipe_write, _F_SETPIPE_SZ, _PIPE_TARGET_SIZE) - except (OSError, ValueError): - pass # non-Linux or unprivileged — keep 64 KB default - - # Make the write end non-blocking so that os.write() raises - # BlockingIOError instead of stalling the OS thread when the pipe - # buffer is full. Without this, a full pipe blocks the entire gevent - # worker (all greenlets freeze) because gevent does not patch - # os.write() on pipes. - # Skip for pre-opened responses (timeshift path) — those run in a - # real OS thread where blocking writes are fine and the 1 MB pipe - # buffer provides sufficient decoupling. - if self._provided_response is None: - flags = fcntl.fcntl(self.pipe_write, fcntl.F_GETFL) - fcntl.fcntl(self.pipe_write, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - self.running = True - self.thread = threading.Thread(target=self._read_stream, daemon=True) - self.thread.start() - - logger.info("Started HTTP stream reader thread for %s", self.url) - return self.pipe_read - - # ------------------------------------------------------------------ - # Non-blocking pipe write helper - # ------------------------------------------------------------------ - - def _pipe_write_all(self, data): - """Write *data* to the pipe, handling O_NONBLOCK + partial writes. - - Uses select() on the write fd (gevent-patched — yields to hub) to - wait for space when the pipe is full, instead of blocking the OS - thread. - """ - offset = 0 - while offset < len(data) and self.running: - try: - n = os.write(self.pipe_write, data[offset:]) - offset += n - except BlockingIOError: - _, writable, _ = _select.select([], [self.pipe_write], [], 1.0) - if not writable and not self.running: - return False - except (BrokenPipeError, OSError): - return False - return True - - # ------------------------------------------------------------------ - # Producer thread - # ------------------------------------------------------------------ - - def _read_stream(self): - """Thread worker: read HTTP stream → write to pipe.""" - try: - if self._provided_response is not None: - # Re-use the already-open response (timeshift cascade path). - self.response = self._provided_response - else: - # Open a fresh connection (live TV path). - headers = dict(self._extra_headers) - if self.user_agent: - headers["User-Agent"] = self.user_agent - - logger.info("HTTP reader connecting to %s", self.url) - - self.session = requests.Session() - adapter = HTTPAdapter( - max_retries=0, pool_connections=1, pool_maxsize=1, - ) - self.session.mount("http://", adapter) - self.session.mount("https://", adapter) - - self.response = self.session.get( - self.url, - headers=headers, - stream=True, - timeout=(5, 30), - ) - - if self.response.status_code not in (200, 206): - logger.error( - "HTTP %d from %s", self.response.status_code, self.url, - ) - return - - logger.info("HTTP reader connected, streaming data…") - - # Optional TS preamble stripping — find the first MPEG-TS sync - # chain and discard everything before it. - synced = not self._strip_ts_preamble - sync_buf = bytearray() if not synced else None - - # If the caller peeked at the response body (e.g. to validate TS - # sync before accepting the response), prepend those bytes so they - # aren't lost. They've already been consumed from the raw socket. - peek_data = getattr(self.response, "_peek_data", None) - - chunk_count = 0 - chunks_iter = self.response.iter_content(chunk_size=self.chunk_size) - if peek_data: - import itertools - chunks_iter = itertools.chain([peek_data], chunks_iter) - - for chunk in chunks_iter: - if not self.running: - break - if not chunk: - continue - - # ---- TS preamble strip logic ---- - if not synced: - sync_buf.extend(chunk) - if len(sync_buf) < 2 * _TS_PACKET_SIZE + 1: - continue - offset = find_ts_sync(sync_buf) - if offset >= 0: - chunk = bytes(sync_buf[offset:]) - synced = True - sync_buf = None - elif len(sync_buf) >= _TS_SYNC_SEARCH_LIMIT: - logger.warning( - "Upstream produced %d bytes without TS sync " - "chain; passing through unmodified", - len(sync_buf), - ) - chunk = bytes(sync_buf) - synced = True - sync_buf = None - else: - continue - - # ---- Non-blocking write to pipe ---- - if not self._pipe_write_all(chunk): - break - - chunk_count += 1 - if chunk_count % 1000 == 0: - logger.debug("HTTP reader streamed %d chunks", chunk_count) - - # Flush any remaining sync_buf that never found a chain. - if not synced and sync_buf: - self._pipe_write_all(bytes(sync_buf)) - - logger.info("HTTP stream ended") - - except requests.exceptions.RequestException as exc: - logger.error("HTTP reader request error: %s", exc) - except Exception as exc: - logger.error("HTTP reader unexpected error: %s", exc, exc_info=True) - finally: - self.running = False - try: - if self.pipe_write is not None: - os.close(self.pipe_write) - self.pipe_write = None - except OSError: - pass - - # ------------------------------------------------------------------ - # Shutdown - # ------------------------------------------------------------------ - - def stop(self): - """Stop the HTTP stream reader.""" - logger.info("Stopping HTTP stream reader") - self.running = False - - if self.response: - try: - self.response.close() - except Exception: - pass - - if self.session: - try: - self.session.close() - except Exception: - pass - - if self.pipe_write is not None: - try: - os.close(self.pipe_write) - self.pipe_write = None - except OSError: - pass - - if self.thread and self.thread.is_alive(): - self.thread.join(timeout=2.0) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 5c54c7b1..32e2b674 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -65,9 +65,15 @@ def attempt_stream_termination(user_id, requesting_client_id, active_connections elif t['type'] == 'timeshift': # Timeshift uses the same Redis key pattern as live # (RedisKeys.client_stop). The stream_generator in - # apps/timeshift/views.py checks this key every 100 chunks. + # apps/timeshift/views.py polls this key on its 5-second + # heartbeat cadence. redis_client = RedisClient.get_client() if not redis_client: + # Without Redis the stop key cannot be set, so the old + # stream cannot be terminated. Failing the whole attempt + # is the safe outcome: the caller then denies the NEW + # stream instead of letting the user exceed the provider's + # connection limit (which escalates to an IP ban). return False stop_key = RedisKeys.client_stop(t['media_id'], t['client_id']) redis_client.setex(stop_key, 60, "true") From 1703d6a70371f1cab202276766c4ff068ac3f3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Thu, 11 Jun 2026 20:08:16 +0200 Subject: [PATCH 11/31] =?UTF-8?q?feat(timeshift):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20network=20gate,=20catch-up=20failover,=20live=5Fpro?= =?UTF-8?q?xy=20revert,=20setting=20moved=20to=20proxy=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements all four points from the latest review, plus hardening from a pre-submission audit pass. 1. Access control: timeshift_proxy now enforces network_access_allowed(request, "STREAMS", user) — same key and placement as the live XC stream endpoint. 2. Catch-up failover: the proxy walks the channel's catch-up streams in channelstream order (get_channel_catchup_streams), mirroring live playback. Each attempt carries its own provider context: account credentials, provider stream id, reported server_info timezone (the UTC->provider conversion is recomputed per attempt), user-agent, and the per-account URL-format cache. The first streamable response wins; if all providers fail the last failure is returned. Ban-safety is per account: a decisive auth/ban-class failure (401/403/406) marks the account and skips its remaining streams (e.g. FHD/HD variants of the same channel) instead of hammering a banning provider, while other accounts — different hosts — are still tried. Streams from disabled M3U accounts are excluded, same as live dispatch. Redirects stay enabled on purpose (XC providers legitimately 302 to load-balanced streaming nodes); the 3xx decisive branch is kept as defense-in-depth and documented as such. 3. apps/proxy/live_proxy/views.py restored byte-identical to upstream — the leftover channel-id wrapper from the removed provider-stream-id fallback is gone (zero-line diff). 4. Single remaining setting relocated: xmltv_prev_days_override now lives in proxy_settings (backend default in get_proxy_settings, consumed by the XMLTV prev_days resolution). The timeshift_settings group, TIMESHIFT_DEFAULTS, get_timeshift_settings, the Settings → Timeshift form and tab are all removed; the field appears under Settings → Proxy Settings (0 = auto-detect, capped at 30). Audit hardening in the same pass: - Updated the proxy-settings defaults unit test for the new key (would have failed CI otherwise). - Migration backfills use schema_editor.connection instead of the global connection (multi-database correctness). - CHANGELOG and module docstring brought in line with the final architecture (PATH-first cascade, failover, setting under Proxy Settings). - Tests grown to 69 backend tests: failover success/exhaustion/skip semantics, decisive-account skip vs soft-failure retry, per-stream timezone conversion (different zones per provider), 406/connection-error cascade paths, stream-limit and no-eligible-stream outcomes, network-gate 403, server_info strict-UTC guarantee, EPG duration window resolution, and DB-backed coverage of xc_password auth, user_level access and the failover stream ordering (catch-up-only, active accounts, channelstream order). The format-cache test now runs on an isolated locmem cache. --- CHANGELOG.md | 3 +- .../migrations/0038_add_catchup_fields.py | 8 +- apps/channels/utils.py | 26 +- apps/output/views.py | 6 +- apps/proxy/live_proxy/views.py | 13 +- apps/timeshift/tests/test_helpers.py | 50 +++ apps/timeshift/tests/test_views.py | 413 +++++++++++++++++- apps/timeshift/views.py | 240 +++++----- core/models.py | 29 +- .../components/cards/StreamConnectionCard.jsx | 2 +- .../forms/settings/ProxySettingsForm.jsx | 5 +- .../forms/settings/TimeshiftSettingsForm.jsx | 74 ---- frontend/src/constants.js | 5 + frontend/src/pages/Settings.jsx | 16 - .../forms/settings/ProxySettingsFormUtils.js | 1 + .../settings/TimeshiftSettingsFormUtils.js | 9 - .../__tests__/ProxySettingsFormUtils.test.js | 2 + frontend/src/utils/pages/SettingsUtils.js | 17 +- 18 files changed, 624 insertions(+), 295 deletions(-) delete mode 100644 frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx delete mode 100644 frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc264f7..1b26cd77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,7 +156,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - - **`Settings → Timeshift` panel** with a single knob: `xmltv_prev_days_override` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. + - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. + - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. - **HDHR output profile URL support.** HDHomeRun lineup URLs now support an `output_profile` path segment so HDHR clients (Plex, Channels DVR, Emby, etc.) can request a specific transcode profile without any query-parameter support. URL formats accepted: diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py index d6cdc335..2ec76333 100644 --- a/apps/channels/migrations/0038_add_catchup_fields.py +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -10,9 +10,7 @@ from django.db import migrations, models def backfill_stream_catchup(apps, schema_editor): """Derive is_catchup/catchup_days from Stream.custom_properties JSON.""" - from django.db import connection - - with connection.cursor() as cursor: + with schema_editor.connection.cursor() as cursor: cursor.execute(""" UPDATE dispatcharr_channels_stream SET is_catchup = TRUE, @@ -37,9 +35,7 @@ def backfill_stream_catchup(apps, schema_editor): def backfill_channel_catchup(apps, schema_editor): """Roll up catch-up fields from streams to channels.""" - from django.db import connection - - with connection.cursor() as cursor: + with schema_editor.connection.cursor() as cursor: cursor.execute(""" UPDATE dispatcharr_channels_channel c SET is_catchup = EXISTS ( diff --git a/apps/channels/utils.py b/apps/channels/utils.py index 043a8a36..2efcef62 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -24,24 +24,22 @@ def format_channel_number(value, empty=""): return value -def get_channel_catchup_info(channel): - """Return catch-up info for a Channel, or None if catch-up is unavailable.""" - if not getattr(channel, "is_catchup", False): - return None +def get_channel_catchup_streams(channel): + """Ordered catch-up streams for a Channel (empty list if unavailable). - stream = ( - channel.streams.filter(is_catchup=True) + Returned in ``channelstream__order`` so the timeshift proxy can fail over: + it tries each stream's provider in turn until one serves the archive — + mirroring how live playback walks a channel's stream list. Streams from + disabled M3U accounts are excluded, same as live dispatch. + """ + if not getattr(channel, "is_catchup", False): + return [] + + return list( + channel.streams.filter(is_catchup=True, m3u_account__is_active=True) .order_by("channelstream__order") .select_related("m3u_account") - .first() ) - if stream is None: - return None - - return { - "stream": stream, - "props": stream.custom_properties or {}, - } def increment_stream_count(account): diff --git a/apps/output/views.py b/apps/output/views.py index 0cc20e3f..6ba06e9f 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -1323,7 +1323,7 @@ def generate_epg(request, profile_name=None, user=None): # Timeshift: prev_days resolution order: # 1. URL ?prev_days= (explicit, even 0 means "no past") # 2. user.custom_properties.epg_prev_days - # 3. CoreSettings.timeshift_settings.xmltv_prev_days_override (>0) + # 3. CoreSettings.proxy_settings.xmltv_prev_days_override (>0) # 4. Auto-detect: max provider tv_archive_duration capped at 30 url_prev = request.GET.get('prev_days') user_prev = user_custom.get('epg_prev_days') if user_custom else None @@ -1339,9 +1339,9 @@ def generate_epg(request, profile_name=None, user=None): prev_days = 0 else: from apps.timeshift.helpers import compute_provider_archive_days_capped - timeshift_settings = CoreSettings.get_timeshift_settings() + proxy_settings = CoreSettings.get_proxy_settings() try: - override = int(timeshift_settings.get("xmltv_prev_days_override", 0) or 0) + override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) except (TypeError, ValueError): override = 0 prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 15a4a8ab..d60d9703 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -609,13 +609,6 @@ def stream_xc(request, username, password, channel_id): if custom_properties["xc_password"] != password: return Response({"error": "Invalid credentials"}, status=401) - # Clients always receive channel.id from get_live_streams — straightforward - # int lookup, no provider stream_id fallback needed. - try: - resolved_internal_id = int(channel_id) - except (TypeError, ValueError): - return JsonResponse({"error": "Not found"}, status=404) - if user.user_level < 10: user_profile_count = user.channel_profiles.count() @@ -623,14 +616,14 @@ def stream_xc(request, username, password, channel_id): if user_profile_count == 0: # No profile filtering - user sees all channels based on user_level filters = { - "id": resolved_internal_id, + "id": int(channel_id), "user_level__lte": user.user_level } channel = Channel.objects.filter(**filters).first() else: # User has specific limited profiles assigned filters = { - "id": resolved_internal_id, + "id": int(channel_id), "channelprofilemembership__enabled": True, "user_level__lte": user.user_level, "channelprofilemembership__channel_profile__in": user.channel_profiles.all() @@ -640,7 +633,7 @@ def stream_xc(request, username, password, channel_id): if not channel: return JsonResponse({"error": "Not found"}, status=404) else: - channel = get_object_or_404(Channel, id=resolved_internal_id) + channel = get_object_or_404(Channel, id=channel_id) if extension.lower() == '.mp4': force_format = 'fmp4' diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index 3f4b10ea..abd3089f 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -171,3 +171,53 @@ class ConvertTimestampToProviderTzTests(TestCase): convert_timestamp_to_provider_tz("garbage", "Europe/Brussels"), "garbage", ) + + +class GetProgrammeDurationTests(TestCase): + """Duration window resolution: programme length + buffer, capped, with a + safe default whenever the EPG lookup cannot resolve.""" + + def _channel_with_programme(self, minutes): + from datetime import datetime, timedelta, timezone as dt_timezone + from unittest.mock import MagicMock + + start = datetime(2026, 6, 8, 17, 0, tzinfo=dt_timezone.utc) + programme = MagicMock( + start_time=start, end_time=start + timedelta(minutes=minutes) + ) + channel = MagicMock() + channel.epg_data.programs.filter.return_value.first.return_value = programme + return channel + + def test_duration_is_programme_length_plus_buffer(self): + from apps.timeshift.helpers import get_programme_duration + # 40-minute programme + 5-minute buffer. + self.assertEqual( + get_programme_duration(self._channel_with_programme(40), "2026-06-08:17-00"), + 45, + ) + + def test_duration_capped_at_max(self): + from apps.timeshift.helpers import get_programme_duration + self.assertEqual( + get_programme_duration(self._channel_with_programme(1000), "2026-06-08:17-00"), + 480, + ) + + def test_no_epg_data_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + channel = MagicMock(epg_data=None) + self.assertEqual(get_programme_duration(channel, "2026-06-08:17-00"), 120) + + def test_no_matching_programme_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + channel = MagicMock() + channel.epg_data.programs.filter.return_value.first.return_value = None + self.assertEqual(get_programme_duration(channel, "2026-06-08:17-00"), 120) + + def test_garbage_timestamp_falls_back_to_default(self): + from unittest.mock import MagicMock + from apps.timeshift.helpers import get_programme_duration + self.assertEqual(get_programme_duration(MagicMock(), "garbage"), 120) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 4db30a66..2f2b6fbb 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch -from django.test import RequestFactory, TestCase +from django.test import RequestFactory, TestCase, override_settings from apps.timeshift import views from apps.proxy.live_proxy.input.http_streamer import find_ts_sync as _find_ts_sync @@ -171,14 +171,16 @@ class StreamFromProviderStatusMappingTests(TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(mocked_open.call_count, 3) + @override_settings(CACHES={ + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"} + }) @patch.object(views, "_open_upstream") def test_cache_promotes_winning_index_to_first(self, mocked_open): """Once a candidate succeeds for an account, the next request reorders the list so the cached winner is tried first — saving cascade overhead on fast-forward.""" - # The format cache lives in django.core.cache (Redis-backed), which - # persists across test runs — clear this account's key so the first - # request starts from the unordered candidate list. + # locmem cache: isolates this test from the shared Redis-backed django + # cache (which persists across runs and parallel test sessions). from django.core.cache import cache as django_cache django_cache.delete(views._FORMAT_CACHE_KEY.format(999)) @@ -252,6 +254,22 @@ class RedactUrlTests(TestCase): self.assertIsNone(views._redact_url(None)) +def _make_catchup_stream(provider_tz="Europe/Brussels", *, account_id=9, + stream_id="22372", account_type="XC", profile_id=31): + """Build a mocked catch-up Stream with its own provider context.""" + profile = MagicMock() + profile.id = profile_id + profile.custom_properties = {"server_info": {"timezone": provider_tz}} + m3u_account = MagicMock() + m3u_account.account_type = account_type + m3u_account.id = account_id + m3u_account.profiles.filter.return_value.first.return_value = profile + stream = MagicMock() + stream.m3u_account = m3u_account + stream.custom_properties = {"stream_id": stream_id} if stream_id else {} + return stream + + class TimeshiftProxyTimestampWiringTests(TestCase): """`timeshift_proxy` must convert the client's UTC timestamp to the serving provider's zone for the upstream URL, while keeping the ORIGINAL @@ -261,26 +279,15 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def setUp(self): self.factory = RequestFactory() - def _make_catchup(self, provider_tz): - profile = MagicMock() - profile.id = 31 - profile.custom_properties = {"server_info": {"timezone": provider_tz}} - m3u_account = MagicMock() - m3u_account.account_type = "XC" - m3u_account.id = 9 - m3u_account.profiles.filter.return_value.first.return_value = profile - stream = MagicMock() - stream.m3u_account = m3u_account - return {"stream": stream, "props": {"stream_id": "22372"}} - def _call(self, timestamp, provider_tz="Europe/Brussels"): request = self.factory.get(f"/timeshift/u/p/8/{timestamp}/8.ts") - sentinel = MagicMock() + sentinel = MagicMock(status_code=200) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ - patch.object(views, "get_channel_catchup_info", - return_value=self._make_catchup(provider_tz)), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream(provider_tz)]), \ patch.object(views, "get_programme_duration", return_value=40) as duration_mock, \ patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ @@ -309,12 +316,378 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def test_invalid_timestamp_rejected_before_upstream(self): request = self.factory.get("/timeshift/u/p/8/garbage/8.ts") with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ - patch.object(views, "get_channel_catchup_info") as catchup_mock, \ + patch.object(views, "get_channel_catchup_streams") as catchup_mock, \ patch.object(views, "_stream_from_provider") as stream_mock: channel_cls.objects.get.return_value = MagicMock(id=8) response = views.timeshift_proxy(request, "u", "p", "8", "garbage", "8.ts") self.assertEqual(response.status_code, 400) catchup_mock.assert_not_called() stream_mock.assert_not_called() + + def test_network_access_denied_returns_403(self): + # Same network-level gate as the live XC endpoint: when the request's + # network is not allowed for STREAMS, nothing else runs. + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=False) as gate, \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_stream_from_provider") as stream_mock: + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(gate.call_args[0][1], "STREAMS") + channel_cls.objects.get.assert_not_called() + stream_mock.assert_not_called() + + +class TimeshiftProxyFailoverTests(TestCase): + """When the first catch-up stream's provider cannot serve the archive, + the proxy must fail over to the channel's next catch-up stream — each + attempt with its own provider context.""" + + def setUp(self): + self.factory = RequestFactory() + + def _call(self, streams, provider_responses): + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "build_timeshift_candidate_urls", + return_value=["http://example.test/x.ts"]) as build_mock, \ + patch.object(views, "check_user_stream_limits", return_value=True) as limits_mock, \ + patch.object(views, "_stream_from_provider", + side_effect=provider_responses) as stream_mock: + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + return response, stream_mock, build_mock, limits_mock + + def test_second_stream_serves_after_first_fails(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + ok = MagicMock(status_code=200) + response, stream_mock, build_mock, _ = self._call( + streams, [MagicMock(status_code=404), ok] + ) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + # Each attempt used its own provider context. + self.assertEqual( + [c.args[0] for c in build_mock.call_args_list], + [streams[0].m3u_account, streams[1].m3u_account], + ) + self.assertEqual( + [c.args[1] for c in build_mock.call_args_list], ["111", "222"] + ) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 2] + ) + + def test_all_streams_fail_returns_last_failure(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + last = MagicMock(status_code=404) + response, stream_mock, _, _ = self._call( + streams, [MagicMock(status_code=400), last] + ) + self.assertIs(response, last) + self.assertEqual(stream_mock.call_count, 2) + + def test_non_xc_and_missing_stream_id_are_skipped(self): + streams = [ + _make_catchup_stream(account_id=1, account_type="M3U"), + _make_catchup_stream(account_id=2, stream_id=None), + _make_catchup_stream(account_id=3, stream_id="333"), + ] + ok = MagicMock(status_code=200) + response, stream_mock, _, _ = self._call(streams, [ok]) + self.assertIs(response, ok) + # Only the eligible third stream produced an upstream attempt. + self.assertEqual(stream_mock.call_count, 1) + self.assertEqual(stream_mock.call_args.kwargs["account_id"], 3) + + def test_stream_limits_checked_once_for_the_request(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + _, _, _, limits_mock = self._call( + streams, [MagicMock(status_code=404), MagicMock(status_code=200)] + ) + self.assertEqual(limits_mock.call_count, 1) + + +class TimeshiftProxyFailoverHardeningTests(TestCase): + """Ban-safety and per-provider context guarantees of the failover loop.""" + + def setUp(self): + self.factory = RequestFactory() + + def _call(self, streams, provider_responses, limits=True): + request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "build_timeshift_candidate_urls", + return_value=["http://example.test/x.ts"]) as build_mock, \ + patch.object(views, "check_user_stream_limits", return_value=limits), \ + patch.object(views, "_stream_from_provider", + side_effect=provider_responses) as stream_mock: + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + return response, stream_mock, build_mock + + def test_decisive_failure_skips_same_accounts_other_streams(self): + # Account 1 carries two variants (e.g. FHD + HD). A decisive + # (auth/ban-class) failure on the first must NOT retry account 1's + # second stream — that would hammer a banning provider — but a + # DIFFERENT account stays fair game. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=1, stream_id="112"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + decisive = MagicMock(status_code=403, timeshift_decisive=True) + ok = MagicMock(status_code=200) + response, stream_mock, _ = self._call(streams, [decisive, ok]) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 2] + ) + + def test_soft_failure_still_tries_same_accounts_other_streams(self): + # A soft failure (404: this stream's archive missing) is stream- + # specific — the same account's other variant may still have it. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=1, stream_id="112"), + ] + soft = MagicMock(status_code=404, timeshift_decisive=False) + ok = MagicMock(status_code=200) + response, stream_mock, _ = self._call(streams, [soft, ok]) + self.assertIs(response, ok) + self.assertEqual(stream_mock.call_count, 2) + self.assertEqual( + [c.kwargs["account_id"] for c in stream_mock.call_args_list], [1, 1] + ) + + def test_each_stream_uses_its_own_provider_timezone(self): + # June: 17:00 UTC = 19:00 Brussels (CEST) but 13:00 New York (EDT). + # The converted timestamp must be recomputed per attempt. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", + provider_tz="Europe/Brussels"), + _make_catchup_stream(account_id=2, stream_id="222", + provider_tz="America/New_York"), + ] + response, _, build_mock = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False), + MagicMock(status_code=200)], + ) + self.assertEqual(response.status_code, 200) + self.assertEqual( + [c.args[2] for c in build_mock.call_args_list], + ["2026-06-08:19-00", "2026-06-08:13-00"], + ) + + def test_stream_limit_exceeded_returns_403_before_upstream(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111")] + response, stream_mock, _ = self._call(streams, [], limits=False) + self.assertEqual(response.status_code, 403) + stream_mock.assert_not_called() + + def test_no_catchup_streams_returns_400(self): + response, stream_mock, _ = self._call([], []) + self.assertEqual(response.status_code, 400) + stream_mock.assert_not_called() + + def test_all_streams_ineligible_returns_400(self): + streams = [ + _make_catchup_stream(account_id=1, account_type="M3U"), + _make_catchup_stream(account_id=2, stream_id=None), + ] + response, stream_mock, _ = self._call(streams, []) + self.assertEqual(response.status_code, 400) + stream_mock.assert_not_called() + + +class XcServerInfoUtcTests(TestCase): + """The XC server_info 'timezone triple' guarantee the timeshift chain + relies on: server_info.timezone is always UTC and time_now is UTC + wall-clock. (Tested here because catch-up seek correctness depends on + it: clients build the timeshift URL from this declared zone.)""" + + def test_server_info_is_strictly_utc(self): + from datetime import datetime, timezone as dt_timezone + from apps.output.views import _build_xc_server_info + + request = MagicMock(scheme="http") + info = _build_xc_server_info(request, "example.test", "9191") + self.assertEqual(info["timezone"], "UTC") + reported = datetime.strptime(info["time_now"], "%Y-%m-%d %H:%M:%S") + now_utc = datetime.now(dt_timezone.utc).replace(tzinfo=None) + self.assertLess(abs((now_utc - reported).total_seconds()), 60) + self.assertIsInstance(info["timestamp_now"], int) + + +class StreamFromProviderDecisiveEdgeTests(TestCase): + """Remaining decisive-status and transport-error paths of the cascade.""" + + def setUp(self): + self.kwargs = dict( + candidate_urls=[ + "http://example.test/timeshift/u/p/60/2026-05-12:17-00/1.ts", + "http://example.test/streaming/timeshift.php?stream=1&start=2026-05-12_17-00", + ], + user_agent="test-agent", + range_header=None, + virtual_channel_id="timeshift_1_2026-05-12-17-00_1", + client_id="timeshift_test456", + client_ip="127.0.0.1", + user=None, + channel_display_name="Test", + timestamp_utc="2026-05-12:17-00", + channel_logo_id=None, + m3u_profile_id=None, + debug=False, + ) + + @patch.object(views, "_open_upstream") + def test_406_is_decisive_and_marks_response(self, mocked_open): + # 406 = IP-wide block in the XC ban escalation — single attempt, + # generic 400 to the client, and the failover loop must see the + # decisive marker so it skips this account's other streams. + mocked_open.return_value = _fake_upstream(406) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + self.assertTrue(response.timeshift_decisive) + + @patch.object(views, "_open_upstream") + def test_404_failure_is_not_decisive(self, mocked_open): + mocked_open.return_value = _fake_upstream(404) + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 404) + self.assertFalse(response.timeshift_decisive) + + @patch.object(views, "_open_upstream") + def test_connection_error_returns_400_after_single_attempt(self, mocked_open): + import requests as _requests + mocked_open.side_effect = _requests.exceptions.ConnectionError("boom") + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 400) + self.assertEqual(mocked_open.call_count, 1) + # Transport errors are host-level, not auth/ban-class: the failover + # loop may still try a different account. + self.assertFalse(getattr(response, "timeshift_decisive", False)) + + +class CatchupStreamsDbTests(TestCase): + """get_channel_catchup_streams: the function that defines the failover + order — channelstream order, catch-up streams only, active accounts only.""" + + @classmethod + def setUpTestData(cls): + from apps.channels.models import Channel, ChannelStream, Stream + from apps.m3u.models import M3UAccount + + cls.active = M3UAccount.objects.create( + name="ts-test-active", server_url="http://example.test", + account_type="XC", is_active=True, + ) + cls.inactive = M3UAccount.objects.create( + name="ts-test-inactive", server_url="http://example.test", + account_type="XC", is_active=False, + ) + cls.channel = Channel.objects.create(name="ts-test-channel", is_catchup=True) + + def add(name, account, *, catchup, order): + s = Stream.objects.create( + name=name, url=f"http://example.test/{name}", + m3u_account=account, is_catchup=catchup, + ) + ChannelStream.objects.create(channel=cls.channel, stream=s, order=order) + return s + + cls.s_inactive = add("s-inactive", cls.inactive, catchup=True, order=0) + cls.s_second = add("s-second", cls.active, catchup=True, order=2) + cls.s_first = add("s-first", cls.active, catchup=True, order=1) + cls.s_live_only = add("s-live-only", cls.active, catchup=False, order=3) + + def test_ordered_active_catchup_streams_only(self): + from apps.channels.utils import get_channel_catchup_streams + + result = get_channel_catchup_streams(self.channel) + # Inactive-account and non-catchup streams excluded; channelstream order. + self.assertEqual([s.id for s in result], [self.s_first.id, self.s_second.id]) + + def test_channel_without_catchup_flag_returns_empty(self): + from apps.channels.models import Channel + from apps.channels.utils import get_channel_catchup_streams + + ch = Channel.objects.create(name="ts-test-nocatchup", is_catchup=False) + self.assertEqual(get_channel_catchup_streams(ch), []) + + +class AuthHelpersDbTests(TestCase): + """_authenticate_user (xc_password custom property) and + _user_can_access_channel (user_level gate) — exercised against real models + instead of being mocked away.""" + + @classmethod + def setUpTestData(cls): + from apps.accounts.models import User + from apps.channels.models import Channel + + cls.viewer = User.objects.create( + username="ts-test-viewer", user_level=0, + custom_properties={"xc_password": "right-pass"}, + ) + cls.no_xc = User.objects.create( + username="ts-test-noxc", user_level=10, + custom_properties={}, + ) + cls.basic_channel = Channel.objects.create(name="ts-test-basic", user_level=0) + cls.admin_channel = Channel.objects.create(name="ts-test-adult", user_level=10) + + def test_valid_xc_password_authenticates(self): + user = views._authenticate_user("ts-test-viewer", "right-pass") + self.assertIsNotNone(user) + self.assertEqual(user.id, self.viewer.id) + + def test_wrong_xc_password_rejected(self): + self.assertIsNone(views._authenticate_user("ts-test-viewer", "wrong")) + + def test_user_without_xc_password_rejected(self): + # Accounts with no xc_password set (e.g. admins) must be denied even + # if the caller guesses any string — there is nothing to compare to. + self.assertIsNone(views._authenticate_user("ts-test-noxc", "")) + self.assertIsNone(views._authenticate_user("ts-test-noxc", "anything")) + + def test_unknown_username_rejected(self): + self.assertIsNone(views._authenticate_user("ts-test-ghost", "x")) + + def test_user_level_gate(self): + # Level-0 viewer with no profiles: allowed on level-0, denied on level-10. + self.assertTrue(views._user_can_access_channel(self.viewer, self.basic_channel)) + self.assertFalse(views._user_can_access_channel(self.viewer, self.admin_channel)) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 43e0e80a..651ede15 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -1,5 +1,7 @@ -"""XC catch-up (timeshift) HTTP view — proxies the provider's -`/streaming/timeshift.php` endpoint to clients like iPlayTV and TiviMate. +"""XC catch-up (timeshift) HTTP view — proxies the provider's catch-up +archive (PATH `/timeshift/.../{id}.ts` form first, `/streaming/timeshift.php` +query form as fallback) to clients like iPlayTV and TiviMate, with +multi-provider failover across the channel's catch-up streams. URL parameter quirk matching what iPlayTV / TiviMate emit: Position "stream_id" -> EPG channel number, IGNORED here. @@ -25,7 +27,7 @@ from django.http import ( from apps.accounts.models import User from apps.channels.models import Channel -from apps.channels.utils import get_channel_catchup_info +from apps.channels.utils import get_channel_catchup_streams from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState from apps.proxy.live_proxy.redis_keys import RedisKeys @@ -33,6 +35,7 @@ from apps.proxy.live_proxy.utils import get_client_ip from apps.proxy.live_proxy.input.http_streamer import find_ts_sync from apps.proxy.utils import check_user_stream_limits from core.utils import RedisClient +from dispatcharr.utils import network_access_allowed from .helpers import ( build_timeshift_candidate_urls, @@ -56,6 +59,10 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if user is None: return HttpResponseForbidden("Invalid credentials") + # Same network-level gate as the live XC stream endpoint (stream_xc). + if not network_access_allowed(request, "STREAMS", user): + return HttpResponseForbidden("Access denied") + try: channel = Channel.objects.get(id=int(raw_id)) except (Channel.DoesNotExist, ValueError, TypeError): @@ -71,115 +78,132 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if parse_catchup_timestamp(timestamp) is None: return HttpResponseBadRequest("Invalid timestamp") - catchup = get_channel_catchup_info(channel) - if catchup is None: + catchup_streams = get_channel_catchup_streams(channel) + if not catchup_streams: return HttpResponseBadRequest("Timeshift not supported for this channel") - catchup_stream = catchup["stream"] - props = catchup["props"] - m3u_account = catchup_stream.m3u_account - if m3u_account is None or m3u_account.account_type != "XC": - return HttpResponseBadRequest("Channel not from Xtream Codes provider") - # Verbose timeshift logging follows the standard logger level (set via # DISPATCHARR_LOG_LEVEL / the logging config), not a per-feature toggle. debug = logger.isEnabledFor(logging.DEBUG) - stream_id_value = (props or {}).get("stream_id") - if stream_id_value is None: - return HttpResponseBadRequest("Cannot build timeshift URL") - # The client supplies the catch-up timestamp in UTC — Dispatcharr's XC API # surface is strictly UTC (server_info.timezone="UTC", EPG start/end in UTC), - # and the client builds the URL from those UTC strings. Real XC providers, - # however, index their archive in their OWN local zone (the - # server_info.timezone they report on auth). So convert the UTC instant to - # the serving provider's zone before building the upstream URL — empirically - # a provider reads `17-00` as 17:00 LOCAL, not UTC, so skipping this would - # seek ~1-2h off. The EPG duration lookup stays on the original UTC timestamp - # (the EPG is UTC too). This is the ONLY timezone conversion in the chain. + # and the client builds the URL from those UTC strings. The EPG duration + # lookup therefore uses the original UTC value (programmes are stored in + # UTC); the per-provider conversion happens inside the failover loop below. duration_minutes = get_programme_duration(channel, timestamp) - # Default profile carries the provider's server_info (its reported timezone) - # and is reused below for the Stats card's M3U profile name. - default_profile = m3u_account.profiles.filter(is_default=True).first() - provider_tz_name = None - if default_profile is not None: - _server_info = (default_profile.custom_properties or {}).get("server_info") or {} - if isinstance(_server_info, dict): - provider_tz_name = _server_info.get("timezone") - provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) - if debug and provider_timestamp != timestamp: - logger.debug( - "Timeshift tz convert: %s UTC -> %s (provider tz=%s)", - timestamp, provider_timestamp, provider_tz_name, - ) - - # Ordered upstream candidates, PATH form first — see - # build_timeshift_candidate_urls() for the full rationale (the QUERY form is - # a last-resort fallback because some providers return LIVE on it, ignoring - # the requested timestamp). We try them in order until one returns a - # streamable MPEG-TS response — see `_stream_from_provider`. - candidate_urls = build_timeshift_candidate_urls( - m3u_account, stream_id_value, provider_timestamp, duration_minutes - ) - - try: - user_agent = m3u_account.get_user_agent().user_agent - except AttributeError: - user_agent = "" - safe_ts = timestamp.replace(":", "-").replace("/", "-") - virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" client_id = f"timeshift_{uuid.uuid4().hex[:16]}" client_ip = get_client_ip(request) range_header = request.META.get("HTTP_RANGE") - - # Channel logo + M3U default-profile id for the Stats card (default_profile - # was already resolved above for the provider timezone). channel_logo_id = getattr(channel, "logo_id", None) - m3u_profile_id = default_profile.id if default_profile is not None else None - # Free the provider slot before connecting upstream. - # 1. Enforce user stream limits — this terminates the oldest active stream - # (live, timeshift, or VOD) via the Redis stop-key mechanism, same as - # the live and VOD proxy entry points. - if not check_user_stream_limits(user, client_id, media_id=virtual_channel_id): + # Enforce user stream limits once, before connecting upstream — this + # terminates the oldest active stream (live, timeshift, or VOD) via the + # Redis stop-key mechanism, same as the live and VOD proxy entry points. + # (No explicit ChannelService.stop_channel(): that would kill other users' + # live streams on the same channel. The media_id only matters for the + # live same-channel exemption, so a channel-scoped id is enough.) + if not check_user_stream_limits( + user, client_id, media_id=f"timeshift_{channel.id}_{safe_ts}" + ): return HttpResponseForbidden("Stream limit exceeded") - # Note: no explicit ChannelService.stop_channel() here. The - # check_user_stream_limits() call above already terminates the oldest - # active stream (live, timeshift, or VOD) via the Redis stop-key - # mechanism — same pattern as the live and VOD proxy entry points. - # Calling stop_channel() unconditionally would kill other users' live - # streams on the same channel. + # Failover: try each catch-up stream in channelstream order — mirroring how + # live playback walks a channel's stream list. Every attempt carries ITS + # OWN provider context (account, provider stream id, reported timezone, + # user-agent, per-account format cache). Ban safety is per ACCOUNT: when an + # account fails decisively (401/403/406 — auth or ban-class status), its + # other catch-up streams (e.g. FHD/HD variants of the same channel) are + # skipped too, so the cascade never hammers a banning provider; a DIFFERENT + # account is a different host and remains safe to try. + last_response = None + decisive_accounts = set() + for catchup_stream in catchup_streams: + m3u_account = catchup_stream.m3u_account + if m3u_account is None or m3u_account.account_type != "XC": + continue + if m3u_account.id in decisive_accounts: + continue - if debug: - logger.debug( - "Timeshift request: channel=%s ts=%s provider_sid=%s vid=%s client=%s range=%s", - channel.name, - timestamp, - stream_id_value, - virtual_channel_id, - client_id, - range_header or "(none)", + stream_id_value = (catchup_stream.custom_properties or {}).get("stream_id") + if stream_id_value is None: + continue + + # Real XC providers index their archive in their OWN local zone (the + # server_info.timezone they report on auth — stored on the default + # profile). Convert the UTC instant to that zone for the upstream URL — + # empirically a provider reads `17-00` as 17:00 LOCAL, not UTC, so + # skipping this would seek 1-2h off. This is the ONLY timezone + # conversion in the chain, and it is per-provider. + default_profile = m3u_account.profiles.filter(is_default=True).first() + provider_tz_name = None + if default_profile is not None: + _server_info = (default_profile.custom_properties or {}).get("server_info") or {} + if isinstance(_server_info, dict): + provider_tz_name = _server_info.get("timezone") + provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) + + # Ordered upstream candidates, PATH form first — see + # build_timeshift_candidate_urls() for the full rationale (the QUERY + # form is a last-resort fallback because some providers return LIVE on + # it, ignoring the requested timestamp). + candidate_urls = build_timeshift_candidate_urls( + m3u_account, stream_id_value, provider_timestamp, duration_minutes ) - return _stream_from_provider( - candidate_urls=candidate_urls, - user_agent=user_agent, - range_header=range_header, - virtual_channel_id=virtual_channel_id, - client_id=client_id, - client_ip=client_ip, - user=user, - channel_display_name=channel.name, - timestamp_utc=timestamp, - channel_logo_id=channel_logo_id, - m3u_profile_id=m3u_profile_id, - debug=debug, - account_id=m3u_account.id, - ) + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" + m3u_profile_id = default_profile.id if default_profile is not None else None + + if debug: + logger.debug( + "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " + "account=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, timestamp, provider_tz_name, provider_timestamp, + m3u_account.id, stream_id_value, virtual_channel_id, client_id, + range_header or "(none)", + ) + + response = _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=m3u_profile_id, + debug=debug, + account_id=m3u_account.id, + ) + if response.status_code < 400: + return response + + last_response = response + if getattr(response, "timeshift_decisive", False): + decisive_accounts.add(m3u_account.id) + logger.warning( + "Timeshift attempt failed (HTTP %d%s) on account %s for channel %s — " + "trying next catch-up stream", + response.status_code, + ", decisive: skipping this account's other streams" + if m3u_account.id in decisive_accounts else "", + m3u_account.id, channel.name, + ) + + if last_response is not None: + return last_response + # Streams existed but none was usable (non-XC accounts / missing stream_id). + return HttpResponseBadRequest("Cannot build timeshift URL") # --------------------------------------------------------------------------- @@ -320,7 +344,15 @@ def _unregister_stats_client(redis_client, virtual_channel_id, client_id): def _open_upstream(url, user_agent, range_header): - """Open the upstream HTTP request (status + headers known synchronously).""" + """Open the upstream HTTP request (status + headers known synchronously). + + Redirects are followed on purpose: XC providers routinely 302 from the + main host to a load-balanced streaming node carrying a session token, so + disabling redirects would break normal catch-up. The cascade therefore + observes the FINAL status; a ban-onset 302 that redirects somewhere + non-streamable surfaces as a failed TS-sync peek (soft reject) or an + error status. + """ # identity: the TS-sync peek reads raw bytes (response.raw), which are NOT # transparently decompressed — a gzip-encoded body would fail the sync check. headers = {"Accept-Encoding": "identity"} @@ -402,6 +434,7 @@ def _stream_from_provider( last_status = None last_url = ordered_urls[0] winning_index = None + decisive_failure = False for url, orig_idx in zip(ordered_urls, original_indices): try: response = _open_upstream(url, user_agent, range_header) @@ -451,8 +484,12 @@ def _stream_from_provider( continue response.close() # Decisive statuses where trying other URL shapes can't help and may - # escalate an IP ban: auth failures (401/403), IP-level block (406), and - # any redirect (3xx) — for XC providers a 302 is the first sign of a ban. + # escalate an IP ban: auth failures (401/403) and IP-level block (406). + # The 3xx case is defense-in-depth only: requests follows redirects + # transparently (XC providers legitimately 302 to load-balanced + # streaming nodes, so redirects MUST be followed), meaning a 3xx can + # only surface here if that ever changes — but if one does, it is the + # documented first sign of an IP ban and must stop the cascade. # A 5xx is different: it is usually format-specific — some XC servers run # PHP with display_errors off, so the "Undefined array key" warning that # another server emits as 200+text becomes a hard 500. The very next @@ -460,6 +497,7 @@ def _stream_from_provider( # giving up (this is why catch-up appeared broken on providers that 500). code = response.status_code if code in (401, 403, 406) or 300 <= code < 400: + decisive_failure = True break if winning_index is not None: @@ -472,10 +510,16 @@ def _stream_from_provider( # correctly: 404 stops retry loops (catch-up not yet indexed), 403 # surfaces auth problems. Everything else stays a generic 400. if last_status == 404: - return HttpResponseNotFound("Catch-up not available yet") - if last_status == 403: - return HttpResponseForbidden("Provider denied access") - return HttpResponseBadRequest("Provider error") + failure = HttpResponseNotFound("Catch-up not available yet") + elif last_status == 403: + failure = HttpResponseForbidden("Provider denied access") + else: + failure = HttpResponseBadRequest("Provider error") + # Tell the failover loop whether this account failed DECISIVELY + # (auth/ban-class status): its other catch-up streams must be skipped, + # while a different provider remains safe to try. + failure.timeshift_decisive = decisive_failure + return failure content_type = upstream.headers.get("Content-Type", "video/mp2t") content_range = upstream.headers.get("Content-Range", "") diff --git a/core/models.py b/core/models.py index 922a1e3e..ffb5d2c7 100644 --- a/core/models.py +++ b/core/models.py @@ -195,16 +195,6 @@ NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" USER_LIMITS_SETTINGS_KEY = "user_limit_settings" -TIMESHIFT_SETTINGS_KEY = "timeshift_settings" - -# The XC API surface is strictly UTC, so there is no timezone setting here: -# server_info.timezone, the EPG start/end strings and time_now are all UTC, and -# any provider-local catch-up conversion happens at proxy time against the -# serving provider's own server_info.timezone (apps/timeshift/views). -# Verbose timeshift logging follows the standard logger DEBUG level, not a toggle. -TIMESHIFT_DEFAULTS = { - "xmltv_prev_days_override": 0, -} class CoreSettings(models.Model): @@ -406,6 +396,9 @@ class CoreSettings(models.Model): "channel_shutdown_delay": 0, "channel_init_grace_period": 5, "new_client_behind_seconds": 5, + # XC catch-up: forced XMLTV lookback window in days (0 = auto-detect + # from the providers' largest tv_archive_duration, capped at 30). + "xmltv_prev_days_override": 0, }) # System Settings @@ -447,22 +440,6 @@ class CoreSettings(models.Model): "terminate_oldest": True, }) - # Timeshift Settings (XC catch-up) - @classmethod - def get_timeshift_settings(cls): - """Return all timeshift-related settings, falling back to neutral defaults. - - Keys not in TIMESHIFT_DEFAULTS are ignored, so stale stored values from - removed settings are filtered out automatically. Writes go through the - generic settings API (same path as every other settings group). - """ - stored = cls._get_group(TIMESHIFT_SETTINGS_KEY, {}) or {} - merged = dict(TIMESHIFT_DEFAULTS) - for key, value in stored.items(): - if key in merged and value is not None: - merged[key] = value - return merged - class SystemEvent(models.Model): """ diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index f2ae0b8f..ae85b410 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -450,7 +450,7 @@ const StreamConnectionCard = ({ ); }, - mantineExpandButtonProps: ({ row }) => ({ + mantineExpandButtonProps: ({ row, table }) => ({ size: 'xs', style: { transform: row.getIsExpanded() ? 'rotate(180deg)' : 'rotate(-90deg)', diff --git a/frontend/src/components/forms/settings/ProxySettingsForm.jsx b/frontend/src/components/forms/settings/ProxySettingsForm.jsx index 52440769..9b2c2c62 100644 --- a/frontend/src/components/forms/settings/ProxySettingsForm.jsx +++ b/frontend/src/components/forms/settings/ProxySettingsForm.jsx @@ -25,6 +25,7 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { 'channel_shutdown_delay', 'channel_init_grace_period', 'new_client_behind_seconds', + 'xmltv_prev_days_override', ].includes(key); }; const isFloatField = (key) => { @@ -39,7 +40,9 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { ? 300 : key === 'new_client_behind_seconds' ? 120 - : 60; + : key === 'xmltv_prev_days_override' + ? 30 + : 60; }; return ( <> diff --git a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx b/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx deleted file mode 100644 index 93d0652c..00000000 --- a/frontend/src/components/forms/settings/TimeshiftSettingsForm.jsx +++ /dev/null @@ -1,74 +0,0 @@ -import useSettingsStore from '../../../store/settings.jsx'; -import React, { useEffect, useState } from 'react'; -import { - getChangedSettings, - parseSettings, - saveChangedSettings, -} from '../../../utils/pages/SettingsUtils.js'; -import { Button, NumberInput, Stack, Text } from '@mantine/core'; -import { useForm } from '@mantine/form'; -import { - getTimeshiftSettingsFormInitialValues, - getTimeshiftSettingsFormValidation, -} from '../../../utils/forms/settings/TimeshiftSettingsFormUtils.js'; - -const TimeshiftSettingsForm = React.memo(({ active }) => { - const settings = useSettingsStore((s) => s.settings); - const [saved, setSaved] = useState(false); - - const form = useForm({ - mode: 'controlled', - initialValues: getTimeshiftSettingsFormInitialValues(), - validate: getTimeshiftSettingsFormValidation(), - }); - - useEffect(() => { - if (!active) setSaved(false); - }, [active]); - - useEffect(() => { - if (settings) { - const parsed = parseSettings(settings); - form.setValues({ - xmltv_prev_days_override: parsed.xmltv_prev_days_override ?? 0, - }); - } - }, [settings]); - - const onSubmit = async () => { - setSaved(false); - const changed = getChangedSettings(form.getValues(), settings); - try { - await saveChangedSettings(settings, changed); - setSaved(true); - } catch (error) { - console.error('Error saving timeshift settings:', error); - } - }; - - return ( - - - - XC catch-up serves EPG and archive timestamps in UTC. Each provider's - own timezone is applied automatically at request time, so there is no - timezone to configure here. The only setting is the XMLTV lookback - window. - - - - - - ); -}); - -export default TimeshiftSettingsForm; diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 92aba531..32aa4bed 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -60,6 +60,11 @@ export const PROXY_SETTINGS_OPTIONS = { description: 'Seconds of received buffer to start behind live when a new client connects (0 = start at live). Note: this is chunk receive time, not video duration.', }, + xmltv_prev_days_override: { + label: 'XMLTV prev_days Override (catch-up)', + description: + 'Days of past programmes in the XC EPG output. 0 = auto-detect from the providers’ tv_archive_duration (capped at 30).', + }, }; export const USER_LIMITS_OPTIONS = { diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index ab4792e4..29d46dd8 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -48,9 +48,6 @@ const SystemSettingsForm = React.lazy( const NavOrderForm = React.lazy( () => import('../components/forms/settings/NavOrderForm.jsx') ); -const TimeshiftSettingsForm = React.lazy( - () => import('../components/forms/settings/TimeshiftSettingsForm.jsx') -); const SettingsPage = () => { const authUser = useAuthStore((s) => s.user); @@ -235,19 +232,6 @@ const SettingsPage = () => { - - - Timeshift - - - }> - - - - - )} diff --git a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js index eddbba91..93cccc9b 100644 --- a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js @@ -15,5 +15,6 @@ export const getProxySettingDefaults = () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, + xmltv_prev_days_override: 0, }; }; diff --git a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js b/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js deleted file mode 100644 index 880c32b0..00000000 --- a/frontend/src/utils/forms/settings/TimeshiftSettingsFormUtils.js +++ /dev/null @@ -1,9 +0,0 @@ -export const getTimeshiftSettingsFormInitialValues = () => { - return { - xmltv_prev_days_override: 0, - }; -}; - -export const getTimeshiftSettingsFormValidation = () => { - return {}; -}; diff --git a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js index 21c48269..c1fd0908 100644 --- a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js +++ b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js @@ -62,6 +62,7 @@ describe('ProxySettingsFormUtils', () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, + xmltv_prev_days_override: 0, }); }); @@ -82,6 +83,7 @@ describe('ProxySettingsFormUtils', () => { expect(typeof result.channel_shutdown_delay).toBe('number'); expect(typeof result.channel_init_grace_period).toBe('number'); expect(typeof result.new_client_behind_seconds).toBe('number'); + expect(typeof result.xmltv_prev_days_override).toBe('number'); }); }); }); diff --git a/frontend/src/utils/pages/SettingsUtils.js b/frontend/src/utils/pages/SettingsUtils.js index e796af96..acdc6cae 100644 --- a/frontend/src/utils/pages/SettingsUtils.js +++ b/frontend/src/utils/pages/SettingsUtils.js @@ -24,7 +24,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { dvr_settings: {}, backup_settings: {}, system_settings: {}, - timeshift_settings: {}, }; // Map of field prefixes to their groups @@ -70,7 +69,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'auto_import_mapped_files', 'enable_ip_lookup', ]; - const timeshiftFields = ['xmltv_prev_days_override']; for (const formKey in changedSettings) { let value = changedSettings[formKey]; @@ -127,7 +125,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'retention_count', 'schedule_day_of_week', 'max_system_events', - 'xmltv_prev_days_override', ]; if (numericFields.includes(formKey) && value != null) { value = typeof value === 'number' ? value : parseInt(value, 10); @@ -154,8 +151,6 @@ export const saveChangedSettings = async (settings, changedSettings) => { groupedChanges.backup_settings[formKey] = value; } else if (systemFields.includes(formKey)) { groupedChanges.system_settings[formKey] = value; - } else if (timeshiftFields.includes(formKey)) { - groupedChanges.timeshift_settings[formKey] = value; } } @@ -371,18 +366,8 @@ export const parseSettings = (settings) => { : true; } - // Timeshift settings - direct key mapping. - // The only key is the XMLTV lookback override: the XC API surface is strictly - // UTC (no timezone setting) and verbose logging follows the standard logger level. - const timeshiftSettings = settings['timeshift_settings']?.value; - parsed.xmltv_prev_days_override = - timeshiftSettings && timeshiftSettings.xmltv_prev_days_override != null - ? typeof timeshiftSettings.xmltv_prev_days_override === 'number' - ? timeshiftSettings.xmltv_prev_days_override - : parseInt(timeshiftSettings.xmltv_prev_days_override, 10) || 0 - : 0; - // Proxy and network access are already grouped objects + // (xmltv_prev_days_override lives inside proxy_settings) if (settings['proxy_settings']?.value) { parsed.proxy_settings = settings['proxy_settings'].value; } From c91f4f9a6e606e9676728d257f20d0181ac3df94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Marcoux?= Date: Fri, 12 Jun 2026 16:36:02 +0200 Subject: [PATCH 12/31] feat(timeshift): provider pool accounting, per-channel session takeover, rollup self-heal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns catch-up with how live and VOD manage provider capacity: - Reserve a provider profile slot (connection_pool) before every upstream connect, walking the account's active profiles default-first when the default is at capacity (profile_full/credential_full are transient and never mark the account decisive). Credentials for the reserved profile are resolved via get_transformed_credentials — the same credential extraction live playback uses — so pool accounting and real upstream usage always agree. All eligible streams blocked on capacity alone returns 503 (the VOD pool-exhausted precedent). - Release exactly once via a one-shot Redis ownership token consumed with a transactional GET+DEL: the generator finally, the response-close wrapper, failed failover attempts and session takeover all share it, so no path can double-decrement and a client disconnecting before the first chunk still releases (Django registers the iterator's close() as a resource closer). Failures between reservation and the streaming response owning the slot release before propagating; an ownership-token write failure releases directly and reports transient unavailability. - One catch-up session per user and channel: a new request (programme jump or seek) displaces the user's previous session on that channel — its slot is released synchronously, its stats are unregistered, and its generator is stopped through the standard stop-key mechanism — so rapid seeking cannot stack upstream provider connections. - rollup_channel_catchup_fields self-heals channels left flagged is_catchup with no remaining catch-up stream (outside the account-scoped CTE), covering bulk removals on manual/multi-provider channels regardless of how the link rows disappeared. A regression test locks the ChannelStream post_delete signal firing on queryset bulk deletes. Backend test suite grows to 94 (slot reservation/release on every failover outcome, profile walk, mixed capacity-vs-upstream precedence, exception-path release, token exactly-once semantics, takeover scoping and ordering, never-started-generator release, rollup self-heal). --- CHANGELOG.md | 4 +- apps/m3u/tasks.py | 25 +- apps/timeshift/helpers.py | 39 +- apps/timeshift/tests/test_helpers.py | 26 +- apps/timeshift/tests/test_views.py | 533 ++++++++++++++++++++++++++- apps/timeshift/views.py | 347 ++++++++++++++--- 6 files changed, 893 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c070643b..4e66fcf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,9 +159,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **NVIDIA NVDEC (`--cuvid`)**: requires the NVIDIA container toolkit and a supported GPU inside the container. - **Intel Quick Sync (`--qsv`)**: requires an Intel iGPU or ARC GPU with the i915 driver exposed to the container. - **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. - - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries. + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. + - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. + - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 7eb6443b..8dcc8b68 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1818,7 +1818,8 @@ def rollup_channel_catchup_fields(account_id): Updates ``is_catchup`` and ``catchup_days`` on every Channel that has at least one Stream belonging to *account_id* (auto-created or manually - assigned). + assigned), then self-heals any channel left flagged with no catch-up + stream at all. Called from the account-refresh pipeline after streams have been written so that both new and existing channels reflect the current @@ -1859,6 +1860,28 @@ def rollup_channel_catchup_fields(account_id): WHERE c.id = agg.channel_id """, [account_id]) + # Self-heal: reset channels still flagged as catch-up that no longer + # have ANY catch-up stream. The CTE above only reaches channels that + # still hold at least one stream from this account, so a channel whose + # last stream was just removed (stale cleanup, manual unassignment) + # falls outside its scope. The ChannelStream post_delete signal + # normally resets those rows already — this pass guarantees the + # invariant regardless of how the link rows disappeared (raw SQL, + # signal-less bulk operations, historic staleness). Idempotent and + # cheap: ``is_catchup`` is indexed and TRUE on a small minority of + # channels. + cur.execute(""" + UPDATE dispatcharr_channels_channel c + SET is_catchup = FALSE, catchup_days = 0 + WHERE c.is_catchup = TRUE + AND NOT EXISTS ( + SELECT 1 + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + ) + """) + @shared_task def sync_auto_channels(account_id, scan_start_time=None): diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py index 041b6ce5..1075946a 100644 --- a/apps/timeshift/helpers.py +++ b/apps/timeshift/helpers.py @@ -1,6 +1,7 @@ """URL builders + timestamp conversion + archive-days probe for XC catch-up.""" import logging +from collections import namedtuple from datetime import datetime, timezone from urllib.parse import quote from zoneinfo import ZoneInfo @@ -9,6 +10,16 @@ from django.core.cache import cache logger = logging.getLogger(__name__) +#: Resolved upstream credentials for one catch-up attempt. Produced from +#: ``get_transformed_credentials(account, reserved_profile)`` so the URL is +#: built with the credentials of the profile whose pool slot was actually +#: reserved — alternate profiles express different logins as URL regex +#: transforms, and using the raw account fields for them would desynchronize +#: pool accounting from real upstream usage. +TimeshiftCredentials = namedtuple( + "TimeshiftCredentials", ("server_url", "username", "password") +) + DEFAULT_DURATION_MINUTES = 120 DURATION_BUFFER_MINUTES = 5 MAX_DURATION_MINUTES = 480 @@ -129,33 +140,33 @@ def get_programme_duration(channel, timestamp_str): return DEFAULT_DURATION_MINUTES -def build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes): """Format A: `/streaming/timeshift.php?username=&password=&stream=&start=&duration=`.""" # Credentials are URL-encoded: a `&`, `/` or `#` in the password would # otherwise corrupt the URL structure. return ( - f"{m3u_account.server_url.rstrip('/')}/streaming/timeshift.php" - f"?username={quote(str(m3u_account.username), safe='')}" - f"&password={quote(str(m3u_account.password), safe='')}" + f"{creds.server_url.rstrip('/')}/streaming/timeshift.php" + f"?username={quote(str(creds.username), safe='')}" + f"&password={quote(str(creds.password), safe='')}" f"&stream={stream_id}" f"&start={timestamp}" f"&duration={duration_minutes}" ) -def build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes): """Format B: `/timeshift/{user}/{pass}/{duration}/{timestamp}/{stream_id}.ts`.""" return ( - f"{m3u_account.server_url.rstrip('/')}/timeshift" - f"/{quote(str(m3u_account.username), safe='')}" - f"/{quote(str(m3u_account.password), safe='')}" + f"{creds.server_url.rstrip('/')}/timeshift" + f"/{quote(str(creds.username), safe='')}" + f"/{quote(str(creds.password), safe='')}" f"/{duration_minutes}" f"/{timestamp}" f"/{stream_id}.ts" ) -def build_timeshift_candidate_urls(m3u_account, stream_id, timestamp, duration_minutes): +def build_timeshift_candidate_urls(creds, stream_id, timestamp, duration_minutes): """Ordered upstream catch-up candidates — PATH form first. Two URL layouts exist on XC servers and they do NOT behave the same: @@ -180,12 +191,12 @@ def build_timeshift_candidate_urls(m3u_account, stream_id, timestamp, duration_m sql_ts = format_timestamp_as_sql_datetime(timestamp) return [ # PATH form first — it seeks the archive correctly. - build_timeshift_url_format_b(m3u_account, stream_id, timestamp, duration_minutes), - build_timeshift_url_format_b(m3u_account, stream_id, underscore_ts, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, underscore_ts, duration_minutes), # QUERY form fallback — may return LIVE on some providers (see above). - build_timeshift_url_format_a(m3u_account, stream_id, underscore_ts, duration_minutes), - build_timeshift_url_format_a(m3u_account, stream_id, sql_ts, duration_minutes), - build_timeshift_url_format_a(m3u_account, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, underscore_ts, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, sql_ts, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes), ] diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index abd3089f..2c29e337 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -3,6 +3,7 @@ from django.test import TestCase from apps.timeshift.helpers import ( + TimeshiftCredentials, build_timeshift_candidate_urls, build_timeshift_url_format_a, build_timeshift_url_format_b, @@ -12,11 +13,10 @@ from apps.timeshift.helpers import ( ) -class _FakeAccount: - def __init__(self): - self.server_url = "http://example.test" - self.username = "user" - self.password = "pass" +def _make_creds(): + # The builders consume resolved per-profile credentials, never an account + # object — get_transformed_credentials() produces these in the view. + return TimeshiftCredentials("http://example.test", "user", "pass") class TimestampFormatTests(TestCase): @@ -56,11 +56,11 @@ class TimestampFormatTests(TestCase): class BuildTimeshiftUrlTests(TestCase): def setUp(self): - self.account = _FakeAccount() + self.creds = _make_creds() def test_format_a_passes_dash_shape_unchanged(self): url = build_timeshift_url_format_a( - self.account, "22372", "2026-05-12:19-00", 40 + self.creds, "22372", "2026-05-12:19-00", 40 ) self.assertIn("start=2026-05-12:19-00", url) self.assertIn("stream=22372", url) @@ -68,13 +68,13 @@ class BuildTimeshiftUrlTests(TestCase): def test_format_a_passes_sql_shape_unchanged(self): url = build_timeshift_url_format_a( - self.account, "22372", "2026-05-12 19:00:00", 40 + self.creds, "22372", "2026-05-12 19:00:00", 40 ) self.assertIn("start=2026-05-12 19:00:00", url) def test_format_b_path_with_dash_shape(self): url = build_timeshift_url_format_b( - self.account, "22372", "2026-05-12:19-00", 40 + self.creds, "22372", "2026-05-12:19-00", 40 ) self.assertIn("/40/2026-05-12:19-00/22372.ts", url) @@ -86,7 +86,7 @@ class CandidateOrderingTests(TestCase): "catch-up plays the live stream instead of the requested programme" bug.""" def setUp(self): - self.account = _FakeAccount() + self.creds = _make_creds() def _is_path_form(self, url): return "/timeshift/" in url and url.endswith(".ts") and "timeshift.php" not in url @@ -95,7 +95,7 @@ class CandidateOrderingTests(TestCase): return "timeshift.php?" in url def test_every_path_candidate_precedes_every_query_candidate(self): - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12:19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12:19-00", 40) path_indices = [i for i, u in enumerate(urls) if self._is_path_form(u)] query_indices = [i for i, u in enumerate(urls) if self._is_query_form(u)] # Each URL is classified as exactly one form. @@ -105,14 +105,14 @@ class CandidateOrderingTests(TestCase): self.assertLess(max(path_indices), min(query_indices)) def test_first_candidate_is_path_form_with_canonical_dash_timestamp(self): - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12:19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12:19-00", 40) self.assertTrue(self._is_path_form(urls[0])) # Canonical colon-dash timestamp, passed through unchanged. self.assertIn("/40/2026-05-12:19-00/22372.ts", urls[0]) def test_accepts_underscore_input_timestamp(self): # Client may send the underscore shape; PATH form still leads. - urls = build_timeshift_candidate_urls(self.account, "22372", "2026-05-12_19-00", 40) + urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12_19-00", 40) self.assertTrue(self._is_path_form(urls[0])) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 2f2b6fbb..59b8fd0b 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -255,21 +255,91 @@ class RedactUrlTests(TestCase): def _make_catchup_stream(provider_tz="Europe/Brussels", *, account_id=9, - stream_id="22372", account_type="XC", profile_id=31): - """Build a mocked catch-up Stream with its own provider context.""" + stream_id="22372", account_type="XC", profile_id=31, + extra_profiles=()): + """Build a mocked catch-up Stream with its own provider context. + + The default (tz-bearing) profile leads the active-profile list the view + walks; ``extra_profiles`` appends alternate (non-default) profiles for + capacity-walk tests. + """ profile = MagicMock() profile.id = profile_id + profile.is_default = True profile.custom_properties = {"server_info": {"timezone": provider_tz}} m3u_account = MagicMock() m3u_account.account_type = account_type m3u_account.id = account_id - m3u_account.profiles.filter.return_value.first.return_value = profile + m3u_account.profiles.filter.return_value = [profile, *extra_profiles] stream = MagicMock() stream.m3u_account = m3u_account stream.custom_properties = {"stream_id": stream_id} if stream_id else {} return stream +def _make_alt_profile(profile_id): + """A non-default active profile for the capacity walk.""" + profile = MagicMock() + profile.id = profile_id + profile.is_default = False + profile.custom_properties = {} + return profile + + +class _FakeRedis: + """Just enough of the redis-py surface for the slot-token protocol: + setex/get/delete plus a transactional pipeline doing GET+DEL.""" + + def __init__(self): + self.store = {} + + def setex(self, key, ttl, value): + self.store[key] = str(value) + + def set(self, key, value): + self.store[key] = str(value) + + def get(self, key): + return self.store.get(key) + + def delete(self, *keys): + return sum(1 for k in keys if self.store.pop(k, None) is not None) + + def exists(self, key): + return 1 if key in self.store else 0 + + def pipeline(self, transaction=False): + return _FakeRedisPipeline(self) + + +class _FakeRedisPipeline: + def __init__(self, redis): + self._redis = redis + self._ops = [] + + def get(self, key): + self._ops.append(("get", key)) + + def delete(self, key): + self._ops.append(("delete", key)) + + def execute(self): + results = [] + for op, key in self._ops: + if op == "get": + results.append(self._redis.get(key)) + else: + results.append(self._redis.delete(key)) + self._ops = [] + return results + + +def _fake_creds(acc, prof): + """Distinguishable per-account credentials, mirroring what + get_transformed_credentials returns for the reserved profile.""" + return (f"http://a{acc.id}.test", f"u{acc.id}", "p") + + class TimeshiftProxyTimestampWiringTests(TestCase): """`timeshift_proxy` must convert the client's UTC timestamp to the serving provider's zone for the upstream URL, while keeping the ORIGINAL @@ -292,7 +362,13 @@ class TimeshiftProxyTimestampWiringTests(TestCase): patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", return_value=sentinel) as stream_mock: + redis_cls.get_client.return_value = _FakeRedis() channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) response = views.timeshift_proxy(request, "u", "p", "8", timestamp, "8.ts") return response, sentinel, build_mock, duration_mock, stream_mock @@ -363,12 +439,20 @@ class TimeshiftProxyFailoverTests(TestCase): patch.object(views, "build_timeshift_candidate_urls", return_value=["http://example.test/x.ts"]) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=True) as limits_mock, \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", + side_effect=_fake_creds) as creds_mock, \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", side_effect=provider_responses) as stream_mock: + redis_cls.get_client.return_value = _FakeRedis() channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) response = views.timeshift_proxy( request, "u", "p", "8", "2026-06-08:17-00", "8.ts" ) + self.creds_mock = creds_mock return response, stream_mock, build_mock, limits_mock def test_second_stream_serves_after_first_fails(self): @@ -382,9 +466,14 @@ class TimeshiftProxyFailoverTests(TestCase): ) self.assertIs(response, ok) self.assertEqual(stream_mock.call_count, 2) - # Each attempt used its own provider context. + # Each attempt used its own provider context: credentials resolved per + # account/profile (via get_transformed_credentials) and its stream id. self.assertEqual( [c.args[0] for c in build_mock.call_args_list], + [("http://a1.test", "u1", "p"), ("http://a2.test", "u2", "p")], + ) + self.assertEqual( + [c.args[0] for c in self.creds_mock.call_args_list], [streams[0].m3u_account, streams[1].m3u_account], ) self.assertEqual( @@ -430,14 +519,27 @@ class TimeshiftProxyFailoverTests(TestCase): self.assertEqual(limits_mock.call_count, 1) -class TimeshiftProxyFailoverHardeningTests(TestCase): - """Ban-safety and per-provider context guarantees of the failover loop.""" +class _ProxyLoopTestMixin: + """Shared driver for tests exercising the failover loop end to end — + pool reservation, credential resolution and Redis are all controlled.""" def setUp(self): self.factory = RequestFactory() - def _call(self, streams, provider_responses, limits=True): + def _call(self, streams, provider_responses, limits=True, reserve_results=None, + build_side_effect=None): request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + self.fake_redis = _FakeRedis() + reserve_kwargs = ( + {"side_effect": reserve_results} + if reserve_results is not None + else {"return_value": (True, 1, None)} + ) + build_kwargs = ( + {"side_effect": build_side_effect} + if build_side_effect is not None + else {"return_value": ["http://example.test/x.ts"]} + ) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ @@ -445,16 +547,32 @@ class TimeshiftProxyFailoverHardeningTests(TestCase): patch.object(views, "get_channel_catchup_streams", return_value=streams), \ patch.object(views, "get_programme_duration", return_value=40), \ patch.object(views, "build_timeshift_candidate_urls", - return_value=["http://example.test/x.ts"]) as build_mock, \ + **build_kwargs) as build_mock, \ patch.object(views, "check_user_stream_limits", return_value=limits), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", **reserve_kwargs) as reserve_mock, \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "get_transformed_credentials", + side_effect=_fake_creds) as creds_mock, \ + patch.object(views, "get_user_active_connections", return_value=[]), \ patch.object(views, "_stream_from_provider", side_effect=provider_responses) as stream_mock: + redis_cls.get_client.return_value = self.fake_redis channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + # Exposed before the call so raising tests can still assert on them. + self.reserve_mock = reserve_mock + self.release_mock = release_mock + self.creds_mock = creds_mock + self.stream_mock = stream_mock response = views.timeshift_proxy( request, "u", "p", "8", "2026-06-08:17-00", "8.ts" ) return response, stream_mock, build_mock + +class TimeshiftProxyFailoverHardeningTests(_ProxyLoopTestMixin, TestCase): + """Ban-safety and per-provider context guarantees of the failover loop.""" + def test_decisive_failure_skips_same_accounts_other_streams(self): # Account 1 carries two variants (e.g. FHD + HD). A decisive # (auth/ban-class) failure on the first must NOT retry account 1's @@ -691,3 +809,402 @@ class AuthHelpersDbTests(TestCase): # Level-0 viewer with no profiles: allowed on level-0, denied on level-10. self.assertTrue(views._user_can_access_channel(self.viewer, self.basic_channel)) self.assertFalse(views._user_can_access_channel(self.viewer, self.admin_channel)) + + +class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): + """Provider pool participation: a profile slot is reserved before any + upstream attempt and released exactly once afterwards — the same + accounting contract live (Channel.get_stream) and VOD follow.""" + + def _slot_token_keys(self): + return [k for k in self.fake_redis.store if k.startswith("timeshift_slot:")] + + def test_reserve_called_with_default_profile_before_upstream(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, stream_mock, _ = self._call(streams, [MagicMock(status_code=200)]) + self.assertEqual(response.status_code, 200) + self.reserve_mock.assert_called_once() + reserved_profile = self.reserve_mock.call_args.args[0] + self.assertEqual(reserved_profile.id, 31) + # The reserved profile's id is what reaches the stats metadata. + self.assertEqual(stream_mock.call_args.kwargs["m3u_profile_id"], 31) + + def test_slot_released_after_failed_attempt(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, _, _ = self._call( + streams, [MagicMock(status_code=404, timeshift_decisive=False)] + ) + self.assertEqual(response.status_code, 404) + # The one-shot token was consumed and the pool counter decremented. + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_slot_kept_on_success_for_the_streaming_session(self): + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + response, _, _ = self._call(streams, [MagicMock(status_code=200)]) + self.assertEqual(response.status_code, 200) + # Slot still owned by the (mocked) streaming session: token present, + # nothing released yet. + self.release_mock.assert_not_called() + self.assertEqual(len(self._slot_token_keys()), 1) + + def test_decisive_failure_releases_slot_and_skips_account(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=1, stream_id="112", profile_id=31), + ] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=403, timeshift_decisive=True)] + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(stream_mock.call_count, 1) + self.release_mock.assert_called_once_with(31, self.fake_redis) + # Decisive skip means the second stream never reserved a slot. + self.assertEqual(self.reserve_mock.call_count, 1) + + def test_profile_full_walks_to_next_profile_same_account(self): + alt = _make_alt_profile(32) + streams = [_make_catchup_stream( + account_id=1, stream_id="111", profile_id=31, extra_profiles=(alt,) + )] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=200)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 200) + # Default profile full -> alternate profile reserved and used. + self.assertEqual( + [c.args[0].id for c in self.reserve_mock.call_args_list], [31, 32] + ) + self.assertEqual(stream_mock.call_args.kwargs["m3u_profile_id"], 32) + # Credentials were resolved for the RESERVED (alternate) profile. + self.assertIs(self.creds_mock.call_args.args[1], alt) + + def test_all_profiles_full_returns_503_without_upstream_attempt(self): + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, stream_mock, _ = self._call( + streams, [], + reserve_results=[ + (False, 1, "profile_full"), + (False, 1, "credential_full"), + ], + ) + # Pool capacity exhausted everywhere: 503 (VOD's pool-exhausted + # status), and crucially the provider was never contacted. + self.assertEqual(response.status_code, 503) + stream_mock.assert_not_called() + self.release_mock.assert_not_called() + + def test_capacity_failure_is_not_decisive_for_the_account(self): + # profile_full on account 1's first stream must NOT mark account 1 + # decisive — capacity is transient, unlike a ban-class status. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=1, stream_id="112", profile_id=31), + ] + response, stream_mock, _ = self._call( + streams, [MagicMock(status_code=200)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 200) + # Second stream of the SAME account still got its reservation attempt. + self.assertEqual(self.reserve_mock.call_count, 2) + self.assertEqual(stream_mock.call_count, 1) + + def test_account_without_active_default_profile_is_skipped(self): + # Mirrors live dispatch: no active default profile -> skip the account + # without reserving anything. + stream = _make_catchup_stream(account_id=1, stream_id="111") + stream.m3u_account.profiles.filter.return_value = [_make_alt_profile(32)] + response, stream_mock, _ = self._call([stream], []) + self.assertEqual(response.status_code, 400) + self.reserve_mock.assert_not_called() + stream_mock.assert_not_called() + + def test_exception_from_provider_releases_slot(self): + # An unexpected exception between reserve and response construction + # must release the slot before propagating — otherwise the counter + # (no TTL) leaks until the next Redis flush. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with self.assertRaises(RuntimeError): + self._call(streams, RuntimeError("boom")) + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_exception_before_upstream_releases_slot(self): + # Same guarantee for failures BEFORE the upstream call (URL building, + # credential resolution, user-agent lookup) — the guarded window + # starts right after the reservation. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with self.assertRaises(RuntimeError): + self._call(streams, [], build_side_effect=RuntimeError("boom")) + self.stream_mock.assert_not_called() + self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._slot_token_keys(), []) + + def test_token_store_failure_releases_directly(self): + # If the ownership token cannot be written, no release path could + # ever free the slot — the view must release it directly and report + # transient unavailability instead of streaming unaccounted. + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with patch.object(views, "_store_slot_token", return_value=False): + response, stream_mock, _ = self._call(streams, []) + self.assertEqual(response.status_code, 503) + stream_mock.assert_not_called() + self.release_mock.assert_called_once_with(31, self.fake_redis) + + def test_mixed_capacity_then_upstream_failure_returns_failure(self): + # Mixed outcome: one stream capacity-blocked, another actually tried + # upstream and failed -> the REAL upstream failure wins over 503 + # (capacity was not the sole blocker). + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, _, _ = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False)], + reserve_results=[(False, 1, "profile_full"), (True, 1, None)], + ) + self.assertEqual(response.status_code, 404) + + def test_mixed_upstream_failure_then_capacity_returns_failure(self): + # Same in the opposite order. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111", profile_id=31), + _make_catchup_stream(account_id=2, stream_id="222", profile_id=41), + ] + response, _, _ = self._call( + streams, + [MagicMock(status_code=404, timeshift_decisive=False)], + reserve_results=[(True, 1, None), (False, 1, "profile_full")], + ) + self.assertEqual(response.status_code, 404) + + +class TimeshiftSlotTokenTests(TestCase): + """The one-shot release token: exactly-once semantics across all release + paths (generator finally, response close, takeover, failed attempt).""" + + def setUp(self): + self.redis = _FakeRedis() + + def test_release_happens_exactly_once(self): + views._store_slot_token(self.redis, "timeshift_abc", 31) + with patch.object(views, "release_profile_slot") as release_mock: + self.assertTrue(views._release_slot_token(self.redis, "timeshift_abc")) + self.assertFalse(views._release_slot_token(self.redis, "timeshift_abc")) + release_mock.assert_called_once_with(31, self.redis) + + def test_release_without_token_is_noop(self): + with patch.object(views, "release_profile_slot") as release_mock: + self.assertFalse(views._release_slot_token(self.redis, "timeshift_ghost")) + release_mock.assert_not_called() + + def test_release_without_redis_is_noop(self): + with patch.object(views, "release_profile_slot") as release_mock: + self.assertFalse(views._release_slot_token(None, "timeshift_abc")) + release_mock.assert_not_called() + + def test_wrapper_close_releases_even_when_generator_never_started(self): + # The WSGI layer can close the response before the first chunk is + # pulled; closing a never-started generator runs NO body code, so the + # generator's own finally cannot be the only release point. + finally_ran = [] + + def gen(): + try: + yield b"x" + finally: + finally_ran.append(True) + + on_close = MagicMock() + wrapper = views._SlotReleasingStream(gen(), on_close) + wrapper.close() + on_close.assert_called_once() + self.assertEqual(finally_ran, []) # proves the leak this wrapper fixes + + def test_streaming_response_close_invokes_wrapper_close(self): + # Locks the Django contract the wrapper relies on: an iterator with a + # close() method is registered as a resource closer of the response. + from django.http import StreamingHttpResponse + + on_close = MagicMock() + wrapper = views._SlotReleasingStream(iter([b"x"]), on_close) + response = StreamingHttpResponse(wrapper, content_type="video/mp2t") + response.close() + on_close.assert_called_once() + + +class TimeshiftTakeoverTests(TestCase): + """One catch-up session per user+channel: a new request displaces the + user's previous session on the same channel — synchronous slot release + + stats unregister + stop key — and never touches other users, other + channels, or live sessions.""" + + def setUp(self): + self.redis = _FakeRedis() + self.user = MagicMock(id=5) + + def _conn(self, media_id, client_id, conn_type="timeshift"): + return { + "media_id": media_id, + "client_id": client_id, + "connected_at": 0.0, + "type": conn_type, + } + + def test_displaces_only_same_channel_timeshift_sessions(self): + views._store_slot_token(self.redis, "timeshift_old1", 31) + views._store_slot_token(self.redis, "timeshift_other", 41) + connections = [ + self._conn("timeshift_8_2026-06-08-17-00_111", "timeshift_old1"), + self._conn("timeshift_9_2026-06-08-17-00_222", "timeshift_other"), + self._conn("42", "live_client", conn_type="live"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections) as conns_mock, \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + conns_mock.assert_called_once_with(5) + # Channel 8's old session: slot released, stats dropped, stop key set. + release_mock.assert_called_once_with(31, self.redis) + unregister_mock.assert_called_once_with( + self.redis, "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" + ) + from apps.proxy.live_proxy.redis_keys import RedisKeys + stop_key = RedisKeys.client_stop( + "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" + ) + self.assertIn(stop_key, self.redis.store) + # Channel 9's session untouched: token still present, no stop key. + self.assertIn("timeshift_slot:timeshift_other", self.redis.store) + + def test_channel_id_prefix_cannot_match_other_channels(self): + # Channel 8 must not displace channel 80/81 sessions (prefix ends + # with an underscore). + connections = [ + self._conn("timeshift_80_2026-06-08-17-00_111", "timeshift_c80"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections), \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + release_mock.assert_not_called() + unregister_mock.assert_not_called() + + def test_noop_without_redis_or_user(self): + with patch.object(views, "get_user_active_connections") as conns_mock: + views._terminate_previous_timeshift_sessions(None, self.user, 8) + views._terminate_previous_timeshift_sessions(self.redis, None, 8) + conns_mock.assert_not_called() + + def test_proxy_runs_takeover_before_stream_limit_check(self): + # Order matters: with terminate_on_limit_exceeded=False a seek must + # displace its own predecessor BEFORE the limit check counts it, or + # the user's own seek gets denied. + call_order = [] + request = RequestFactory().get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "_terminate_previous_timeshift_sessions", + side_effect=lambda *a: call_order.append("takeover")) as takeover_mock, \ + patch.object(views, "check_user_stream_limits", + side_effect=lambda *a, **k: call_order.append("limits") or False): + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts" + ) + self.assertEqual(response.status_code, 403) + self.assertEqual(call_order, ["takeover", "limits"]) + self.assertEqual(takeover_mock.call_args.args[2], 8) + + +class RollupSelfHealDbTests(TestCase): + """Catch-up flag consistency after stream removal — the review-#4 point 1 + guarantees: the ChannelStream signal handles bulk deletes (locked by a + regression test) and the rollup self-heals any channel left flagged with + no catch-up stream, regardless of how the link rows disappeared.""" + + @classmethod + def setUpTestData(cls): + from apps.m3u.models import M3UAccount + + cls.account = M3UAccount.objects.create( + name="ts-rollup-account", server_url="http://example.test", + account_type="XC", is_active=True, + ) + + def _make_channel_with_catchup_stream(self, name, days=5): + from apps.channels.models import Channel, ChannelStream, Stream + + channel = Channel.objects.create(name=name) + stream = Stream.objects.create( + name=f"{name}-stream", url=f"http://example.test/{name}", + m3u_account=self.account, is_catchup=True, catchup_days=days, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + return channel, stream + + def test_bulk_stream_delete_resets_channel_flags_via_signal(self): + # cleanup_streams() removes stale streams with a queryset bulk delete; + # the cascaded ChannelStream rows still fire post_delete (signal + # listeners disable Django's fast-delete path), which must reset the + # channel's denormalized catch-up fields. + from apps.channels.models import Stream + + channel, stream = self._make_channel_with_catchup_stream("ts-rollup-bulk") + channel.refresh_from_db() + self.assertTrue(channel.is_catchup) + self.assertEqual(channel.catchup_days, 5) + + Stream.objects.filter(id=stream.id).delete() + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) + + def test_rollup_self_heals_stale_channel_without_streams(self): + # Simulate staleness no signal caught (raw SQL delete, historic data): + # a flagged channel with zero catch-up streams must be reset by the + # rollup even though no remaining stream ties it to the account. + from apps.channels.models import Channel + from apps.m3u.tasks import rollup_channel_catchup_fields + + channel = Channel.objects.create(name="ts-rollup-stale") + Channel.objects.filter(pk=channel.pk).update(is_catchup=True, catchup_days=9) + + rollup_channel_catchup_fields(self.account.id) + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) + + def test_rollup_keeps_and_corrects_channels_with_catchup_streams(self): + # The self-heal pass must not touch channels that legitimately have + # catch-up streams — and the account-scoped pass still corrects their + # values. + from apps.channels.models import Channel + from apps.m3u.tasks import rollup_channel_catchup_fields + + channel, _ = self._make_channel_with_catchup_stream("ts-rollup-valid", days=7) + # Knock the denormalized values out of sync (bypasses signals). + Channel.objects.filter(pk=channel.pk).update(is_catchup=False, catchup_days=0) + + rollup_channel_catchup_fields(self.account.id) + + channel.refresh_from_db() + self.assertTrue(channel.is_catchup) + self.assertEqual(channel.catchup_days, 7) diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 651ede15..f158df4b 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -3,6 +3,13 @@ archive (PATH `/timeshift/.../{id}.ts` form first, `/streaming/timeshift.php` query form as fallback) to clients like iPlayTV and TiviMate, with multi-provider failover across the channel's catch-up streams. +Provider capacity accounting follows the same contract as live and VOD: a +profile slot is reserved through ``apps.m3u.connection_pool`` before any +upstream connect and released exactly once when the session ends (one-shot +Redis token), so Dispatcharr's pool counters always match real upstream usage. +A user gets ONE active catch-up session per channel — a seek displaces the +previous one. + URL parameter quirk matching what iPlayTV / TiviMate emit: Position "stream_id" -> EPG channel number, IGNORED here. Position "duration" -> Dispatcharr's Channel.id (the XC API emits @@ -19,6 +26,7 @@ import requests from django.core.cache import cache from django.http import ( Http404, + HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseNotFound, @@ -28,16 +36,19 @@ from django.http import ( from apps.accounts.models import User from apps.channels.models import Channel from apps.channels.utils import get_channel_catchup_streams +from apps.m3u.connection_pool import release_profile_slot, reserve_profile_slot +from apps.m3u.tasks import get_transformed_credentials from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.utils import get_client_ip from apps.proxy.live_proxy.input.http_streamer import find_ts_sync -from apps.proxy.utils import check_user_stream_limits +from apps.proxy.utils import check_user_stream_limits, get_user_active_connections from core.utils import RedisClient from dispatcharr.utils import network_access_allowed from .helpers import ( + TimeshiftCredentials, build_timeshift_candidate_urls, convert_timestamp_to_provider_tz, get_programme_duration, @@ -99,6 +110,18 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) range_header = request.META.get("HTTP_RANGE") channel_logo_id = getattr(channel, "logo_id", None) + redis_client = RedisClient.get_client() + + # One active catch-up session per user+channel: a new request (programme + # jump, or a seek re-request on the same programme) DISPLACES the user's + # previous session on this channel. Its provider slot is released + # synchronously — so the reservation below cannot collide with the dying + # session on max_connections=1 providers — and its generator is stopped + # via the same Redis stop-key the stream-limit path uses. Runs BEFORE the + # stream-limit check so a seek displaces its own predecessor instead of + # being denied when terminate_on_limit_exceeded is off. + _terminate_previous_timeshift_sessions(redis_client, user, channel.id) + # Enforce user stream limits once, before connecting upstream — this # terminates the oldest active stream (live, timeshift, or VOD) via the # Redis stop-key mechanism, same as the live and VOD proxy entry points. @@ -120,6 +143,7 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) # account is a different host and remains safe to try. last_response = None decisive_accounts = set() + capacity_blocked = False for catchup_stream in catchup_streams: m3u_account = catchup_stream.m3u_account if m3u_account is None or m3u_account.account_type != "XC": @@ -131,63 +155,158 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if stream_id_value is None: continue + # Provider profile walk — same ordering as live's Channel.get_stream(): + # active profiles only, default first; an account without an active + # default profile is skipped entirely, mirroring live dispatch. + m3u_profiles = list(m3u_account.profiles.filter(is_active=True)) + default_profile = next((p for p in m3u_profiles if p.is_default), None) + if default_profile is None: + logger.debug( + "Timeshift: account %s has no active default profile, skipping", + m3u_account.id, + ) + continue + profile_walk = [default_profile] + [ + p for p in m3u_profiles if not p.is_default + ] + # Real XC providers index their archive in their OWN local zone (the # server_info.timezone they report on auth — stored on the default # profile). Convert the UTC instant to that zone for the upstream URL — # empirically a provider reads `17-00` as 17:00 LOCAL, not UTC, so # skipping this would seek 1-2h off. This is the ONLY timezone - # conversion in the chain, and it is per-provider. - default_profile = m3u_account.profiles.filter(is_default=True).first() + # conversion in the chain, and it is per-provider (account-level: all + # profiles log into the same server, so the default profile's zone + # applies to every profile of the walk). provider_tz_name = None - if default_profile is not None: - _server_info = (default_profile.custom_properties or {}).get("server_info") or {} - if isinstance(_server_info, dict): - provider_tz_name = _server_info.get("timezone") + _server_info = (default_profile.custom_properties or {}).get("server_info") or {} + if isinstance(_server_info, dict): + provider_tz_name = _server_info.get("timezone") provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) - # Ordered upstream candidates, PATH form first — see - # build_timeshift_candidate_urls() for the full rationale (the QUERY - # form is a last-resort fallback because some providers return LIVE on - # it, ignoring the requested timestamp). - candidate_urls = build_timeshift_candidate_urls( - m3u_account, stream_id_value, provider_timestamp, duration_minutes - ) + # Reserve a provider profile slot BEFORE connecting upstream — the + # same accounting contract live (Channel.get_stream) and VOD follow, + # so the Redis pool counters always match real upstream usage. Walk + # to the next profile only on RESERVATION failure (profile_full / + # credential_full — transient capacity, never ban-class); an upstream + # failure instead moves to the next catch-up STREAM, because probing + # the same server again through an alternate profile is wasteful and + # ban-adjacent. + reserved_profile = None + for profile in profile_walk: + if redis_client is None: + # Without Redis no accounting is possible — proceed unpooled + # rather than blocking playback (stats writes are equally + # fail-open). Deployments always run Redis; this is a dev-only + # path. + reserved_profile = profile + break + reserved, _count, reason = reserve_profile_slot(profile, redis_client) + if reserved: + reserved_profile = profile + break + logger.info( + "Timeshift: profile %s %s on account %s — trying next profile", + profile.id, reason or "unavailable", m3u_account.id, + ) + if reserved_profile is None: + capacity_blocked = True + logger.warning( + "Timeshift: all profiles at capacity on account %s for channel %s", + m3u_account.id, channel.name, + ) + continue + token_stored = False + if redis_client is not None: + token_stored = _store_slot_token( + redis_client, client_id, reserved_profile.id + ) + if not token_stored: + # Redis hiccup right after the INCR: without a token no + # release path could ever free this slot, so release it + # directly NOW (race-free: the session is not yet visible to + # the takeover scan) and report transient unavailability + # instead of streaming unaccounted. + try: + release_profile_slot(reserved_profile.id, redis_client) + except Exception as exc: + logger.error( + "Timeshift: could not release slot for profile %s " + "after token-store failure: %s", reserved_profile.id, exc, + ) + capacity_blocked = True + continue + + # From here until the streaming response owns the slot, ANY failure + # must release the reservation before propagating — otherwise the + # pool counter (which has no TTL) leaks until the next Redis flush. try: - user_agent = m3u_account.get_user_agent().user_agent - except AttributeError: - user_agent = "" + # Build the upstream URLs with the RESERVED profile's credentials, + # resolved the same way live playback does (credential extraction + # via the profile's URL transform). Using the raw account fields + # for a non-default profile would consume that profile's slot + # while authenticating with the default login — double-occupying + # the provider connection the pool thinks is free. + server_url, xc_username, xc_password = get_transformed_credentials( + m3u_account, reserved_profile + ) + creds = TimeshiftCredentials(server_url, xc_username, xc_password) - virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" - m3u_profile_id = default_profile.id if default_profile is not None else None - - if debug: - logger.debug( - "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " - "account=%s provider_sid=%s vid=%s client=%s range=%s", - channel.name, timestamp, provider_tz_name, provider_timestamp, - m3u_account.id, stream_id_value, virtual_channel_id, client_id, - range_header or "(none)", + # Ordered upstream candidates, PATH form first — see + # build_timeshift_candidate_urls() for the full rationale (the + # QUERY form is a last-resort fallback because some providers + # return LIVE on it, ignoring the requested timestamp). + candidate_urls = build_timeshift_candidate_urls( + creds, stream_id_value, provider_timestamp, duration_minutes ) - response = _stream_from_provider( - candidate_urls=candidate_urls, - user_agent=user_agent, - range_header=range_header, - virtual_channel_id=virtual_channel_id, - client_id=client_id, - client_ip=client_ip, - user=user, - channel_display_name=channel.name, - timestamp_utc=timestamp, - channel_logo_id=channel_logo_id, - m3u_profile_id=m3u_profile_id, - debug=debug, - account_id=m3u_account.id, - ) + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" + + if debug: + logger.debug( + "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " + "account=%s profile=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, timestamp, provider_tz_name, provider_timestamp, + m3u_account.id, reserved_profile.id, stream_id_value, + virtual_channel_id, client_id, range_header or "(none)", + ) + + response = _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=reserved_profile.id, + debug=debug, + account_id=m3u_account.id, + redis_client=redis_client, + ) + except Exception: + if token_stored: + # A False return here means someone else (a takeover) already + # consumed the token and released the slot — nothing to do. + _release_slot_token(redis_client, client_id) + raise if response.status_code < 400: + # The reserved slot is now owned by the streaming response: it is + # released (exactly once, via the one-shot token) when the + # generator finishes or the WSGI layer closes the response. return response + # Failed attempt: free the slot before trying the next stream. + _release_slot_token(redis_client, client_id) last_response = response if getattr(response, "timeshift_decisive", False): decisive_accounts.add(m3u_account.id) @@ -202,6 +321,11 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if last_response is not None: return last_response + if capacity_blocked: + # Every eligible stream failed on pool capacity alone (no upstream + # attempt was made) — mirror the VOD proxy's pool-exhausted status so + # clients back off instead of treating it as a permanent failure. + return HttpResponse("No available stream slot", status=503) # Streams existed but none was usable (non-XC accounts / missing stream_id). return HttpResponseBadRequest("Cannot build timeshift URL") @@ -242,6 +366,127 @@ def _user_can_access_channel(user, channel): ) +# --------------------------------------------------------------------------- +# Provider pool slot ownership (one-shot release token) + session takeover +# --------------------------------------------------------------------------- + +# Maps a timeshift client to the profile slot it reserved. Consumed (GET+DEL, +# atomically) by whichever release path runs first — the generator's finally, +# the response-close wrapper, a failed failover attempt, or a takeover by the +# user's next request on the same channel — so the slot is released exactly +# once even across uWSGI workers. Note the TTL does NOT recover the slot: if a +# worker dies mid-session the token eventually expires while the counter (no +# TTL) stays consumed — like the rest of the pool, the recovery for +# crash-leaked counters is the Redis flush at container start +# (scripts/wait_for_redis.py). +_SLOT_TOKEN_KEY = "timeshift_slot:{client_id}" +_SLOT_TOKEN_TTL = 24 * 3600 + + +def _store_slot_token(redis_client, client_id, profile_id): + """Record slot ownership; returns False when the token could not be + written (the caller must then release the reservation itself — without a + token no other path ever could).""" + if redis_client is None: + return False + try: + redis_client.setex( + _SLOT_TOKEN_KEY.format(client_id=client_id), + _SLOT_TOKEN_TTL, + str(profile_id), + ) + return True + except Exception as exc: + logger.warning("Timeshift slot token store failed for %s: %s", client_id, exc) + return False + + +def _release_slot_token(redis_client, client_id): + """Release the profile slot owned by *client_id*, exactly once. + + GET+DEL inside a MULTI/EXEC transaction: concurrent release attempts are + serialized by Redis, so only the one that actually deletes a live token + decrements the pool counters. Returns True when this call performed the + release. + """ + if redis_client is None: + return False + key = _SLOT_TOKEN_KEY.format(client_id=client_id) + try: + pipe = redis_client.pipeline(transaction=True) + pipe.get(key) + pipe.delete(key) + token_value, deleted = pipe.execute() + if not token_value or not deleted: + return False + release_profile_slot(int(token_value), redis_client) + return True + except Exception as exc: + logger.warning("Timeshift slot release failed for %s: %s", client_id, exc) + return False + + +def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): + """Displace the user's previous catch-up session(s) on this channel. + + For each of the user's active timeshift sessions whose virtual channel id + belongs to *channel_id*: release its provider slot NOW (synchronously — + the dying generator's own release becomes a no-op thanks to the one-shot + token), drop its stats keys so the stream-limit count no longer includes + it, and set the stop key its generator polls on the 5-second heartbeat. + The accounting is therefore correct immediately; only the old TCP socket + closes lazily (≤ ~5 s). + """ + if redis_client is None or user is None: + return + prefix = f"timeshift_{channel_id}_" + try: + for conn in get_user_active_connections(user.id): + if conn.get("type") != "timeshift": + continue + media_id = str(conn.get("media_id") or "") + if not media_id.startswith(prefix): + continue + old_client_id = conn.get("client_id") + logger.info( + "Timeshift takeover: displacing session %s on %s", + old_client_id, media_id, + ) + _release_slot_token(redis_client, old_client_id) + _unregister_stats_client(redis_client, media_id, old_client_id) + stop_key = RedisKeys.client_stop(media_id, old_client_id) + redis_client.setex(stop_key, 60, "true") + except Exception as exc: + logger.warning("Timeshift takeover check failed: %s", exc) + + +class _SlotReleasingStream: + """Iterator wrapper whose close() always releases the reserved slot. + + Django registers ``streaming_content.close`` in the response's resource + closers, and the WSGI layer guarantees close() runs — including when the + client disconnects before the first chunk, in which case the generator + never starts and its ``finally`` would never execute. The one-shot token + makes the duplicate call from an already-finished generator a no-op. + """ + + def __init__(self, generator, on_close): + self._generator = generator + self._on_close = on_close + + def __iter__(self): + return self + + def __next__(self): + return next(self._generator) + + def close(self): + try: + self._generator.close() + finally: + self._on_close() + + # --------------------------------------------------------------------------- # Stats integration (direct Redis writes, no ClientManager instance) # --------------------------------------------------------------------------- @@ -406,6 +651,7 @@ def _stream_from_provider( m3u_profile_id, debug, account_id=None, + redis_client=None, ): # Use 256 KB chunks: amortises per-yield uWSGI/gevent overhead. chunk_size = max(ConfigHelper.chunk_size(), 262144) @@ -525,7 +771,6 @@ def _stream_from_provider( content_range = upstream.headers.get("Content-Range", "") status = upstream.status_code - redis_client = RedisClient.get_client() _register_stats_client( redis_client, virtual_channel_id, @@ -604,9 +849,23 @@ def _stream_from_provider( except Exception: pass _unregister_stats_client(redis_client, virtual_channel_id, client_id) + # Free the provider pool slot this session reserved. One-shot + # token: a no-op if a takeover (the user's next request on this + # channel) already released it. + _release_slot_token(redis_client, client_id) + # The wrapper guarantees session teardown even when the generator never + # starts (client gone before the first chunk — its finally would then + # never run): Django registers the iterator's close() as a resource + # closer, which WSGI always invokes. Both calls are idempotent, so the + # duplicate from a normally-finished generator is harmless. + def _close_session(): + _unregister_stats_client(redis_client, virtual_channel_id, client_id) + _release_slot_token(redis_client, client_id) + + stream_iter = _SlotReleasingStream(stream_generator(), _close_session) response = StreamingHttpResponse( - stream_generator(), + stream_iter, content_type=content_type, status=status, ) From 6b4453e9ca7d439730f20f1dea086a49c4b5e9c3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 21:51:24 -0500 Subject: [PATCH 13/31] fix: Readd prev_days logic after dev merge. --- apps/output/epg.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/apps/output/epg.py b/apps/output/epg.py index 626b21aa..b017b8e5 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1100,11 +1100,32 @@ def generate_epg(request, profile_name=None, user=None): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 + # prev_days resolution order: + # 1. URL ?prev_days= (explicit, even 0 means "no past") + # 2. user.custom_properties.epg_prev_days + # 3. CoreSettings.proxy_settings.xmltv_prev_days_override (>0) + # 4. Auto-detect: max provider tv_archive_duration capped at 30 + url_prev = request.GET.get('prev_days') + user_prev = user_custom.get('epg_prev_days') if user_custom else None + if url_prev is not None: + try: + prev_days = max(0, min(int(url_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + elif user_prev not in (None, ""): + try: + prev_days = max(0, min(int(user_prev), 30)) + except (ValueError, TypeError): + prev_days = 0 + else: + from apps.timeshift.helpers import compute_provider_archive_days_capped + from core.models import CoreSettings + proxy_settings = CoreSettings.get_proxy_settings() + try: + override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + override = 0 + prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() cache_params = ( From e596a508c71cb5dc750562889890c1d4970286b8 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:39:54 -0500 Subject: [PATCH 14/31] refactor(models): Simplify comments for catch-up fields in Stream and Channel models, clarifying their population process. Update migration docstring for consistency. Remove outdated comments in ProxyServer and URLs related to timeshift handling. --- apps/channels/migrations/0038_add_catchup_fields.py | 7 +------ apps/channels/models.py | 8 ++------ apps/proxy/live_proxy/server.py | 6 ------ apps/timeshift/tests/test_helpers.py | 4 ++-- core/models.py | 3 +-- dispatcharr/urls.py | 3 --- 6 files changed, 6 insertions(+), 25 deletions(-) diff --git a/apps/channels/migrations/0038_add_catchup_fields.py b/apps/channels/migrations/0038_add_catchup_fields.py index 2ec76333..8f653a3c 100644 --- a/apps/channels/migrations/0038_add_catchup_fields.py +++ b/apps/channels/migrations/0038_add_catchup_fields.py @@ -1,9 +1,4 @@ -"""Add denormalized catch-up fields to Stream and Channel. - -Populated at M3U/XC import time so _xc_channel_entry() can read them as -zero-cost column reads instead of introspecting custom_properties JSON -per channel on every xc_get_live_streams call. -""" +"""Add denormalized catch-up fields to Stream and Channel.""" from django.db import migrations, models diff --git a/apps/channels/models.py b/apps/channels/models.py index 2ba10ad7..fb421080 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -135,9 +135,7 @@ class Stream(models.Model): db_index=True ) - # Denormalized catch-up fields — populated at M3U/XC import time from - # custom_properties["tv_archive"] and ["tv_archive_duration"]. Avoids - # per-channel JSON introspection on every xc_get_live_streams call. + # Populated at import from tv_archive / tv_archive_duration. is_catchup = models.BooleanField( default=False, db_index=True, @@ -377,9 +375,7 @@ class Channel(models.Model): help_text="The M3U account that auto-created this channel" ) - # Denormalized catch-up fields — rolled up from the channel's streams at - # import time and via post_save/post_delete signals on ChannelStream. - # Eliminates per-channel DB queries in _xc_channel_entry(). + # Populated at import; rolled up via ChannelStream signal / m3u refresh. is_catchup = models.BooleanField( default=False, db_index=True, diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 4f3efc57..6d6a97bc 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -2079,12 +2079,6 @@ class ProxyServer: if not channel_id: continue - # Skip channel ids that don't follow Dispatcharr's UUID - # convention. Those are managed by features outside of - # ProxyServer (e.g. timeshift catch-up sessions use a - # `timeshift___` virtual id and are - # owned by the WSGI request that opened them — they - # expire naturally via the clients-set TTL). if channel_id.startswith("timeshift_"): continue diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index 2c29e337..20f922d9 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -1,4 +1,4 @@ -"""Tests for `apps.timeshift.helpers` — timestamp shape conversion and URL build.""" +"""Tests for `apps.timeshift.helpers`: timestamp shape conversion and URL build.""" from django.test import TestCase @@ -20,7 +20,7 @@ def _make_creds(): class TimestampFormatTests(TestCase): - """Timestamp reshape functions change format only — no timezone conversion.""" + """Timestamp reshape functions change format only; no timezone conversion.""" def test_format_sql_reshapes_without_tz_conversion(self): self.assertEqual( diff --git a/core/models.py b/core/models.py index ffb5d2c7..9c5a5795 100644 --- a/core/models.py +++ b/core/models.py @@ -396,8 +396,7 @@ class CoreSettings(models.Model): "channel_shutdown_delay": 0, "channel_init_grace_period": 5, "new_client_behind_seconds": 5, - # XC catch-up: forced XMLTV lookback window in days (0 = auto-detect - # from the providers' largest tv_archive_duration, capped at 30). + # XC catch-up: forced XMLTV lookback (0 = auto-detect, capped at 30). "xmltv_prev_days_override": 0, }) diff --git a/dispatcharr/urls.py b/dispatcharr/urls.py index 0816bfd1..cdedb0c4 100644 --- a/dispatcharr/urls.py +++ b/dispatcharr/urls.py @@ -42,9 +42,6 @@ urlpatterns = [ stream_xc, name="xc_stream_endpoint", ), - # XC catch-up (timeshift). The "duration" slot carries Dispatcharr's internal - # Channel.id (the XC API emits channel.id as stream_id to clients), see - # apps/timeshift/views. path( "timeshift/////", timeshift_proxy, From 8d990ebf9363506f9605e8fcda1ed89a3b4f305d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:40:57 -0500 Subject: [PATCH 15/31] refactor(signals): Simplify catch-up field update logic in ChannelStream signal, enhancing clarity and performance by filtering active streams directly. Updated docstring for improved understanding of the functionality. --- apps/channels/signals.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/apps/channels/signals.py b/apps/channels/signals.py index 521b9d66..1c3d00ef 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -373,23 +373,16 @@ def revoke_task_on_delete(sender, instance, **kwargs): @receiver([post_save, post_delete], sender=ChannelStream) def update_channel_catchup_fields(sender, instance, **kwargs): - """Roll up denormalized catch-up fields from streams to the channel. - - Covers the UI/API path (admin adds/removes a stream from a channel). - The bulk-import path bypasses signals and uses explicit SQL instead — - see rollup_channel_catchup_fields() in apps/m3u/tasks.py. Same semantics - as that rollup: is_catchup = any catch-up stream, catchup_days = MAX of - the catch-up streams' archive depth. - """ + """Roll up catch-up flags from active streams (UI path; import uses SQL rollup).""" from django.db.models import Max channel = instance.channel - max_days = ( - channel.streams - .filter(is_catchup=True) - .aggregate(max_days=Max("catchup_days"))["max_days"] + catchup_qs = channel.streams.filter( + is_catchup=True, + m3u_account__is_active=True, ) + max_days = catchup_qs.aggregate(max_days=Max("catchup_days"))["max_days"] Channel.objects.filter(pk=channel.pk).update( - is_catchup=max_days is not None, + is_catchup=catchup_qs.exists(), catchup_days=max_days or 0, ) From f2dfe6deb634c8c93139887b7cab1b955e375eb7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:45:13 -0500 Subject: [PATCH 16/31] feat(channels): Add caching and resolution logic for provider archive days in XC catch-up streams. Implemented `compute_provider_archive_days_capped` and `resolve_xc_epg_prev_days` functions to enhance performance and flexibility in determining catch-up days. Updated docstrings for clarity. --- apps/channels/utils.py | 84 +++++++++++++++++++-- apps/timeshift/helpers.py | 155 ++++++++++---------------------------- 2 files changed, 118 insertions(+), 121 deletions(-) diff --git a/apps/channels/utils.py b/apps/channels/utils.py index 2efcef62..b2cceab6 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -1,8 +1,13 @@ import logging import threading +from django.core.cache import cache + logger = logging.getLogger(__name__) +PROVIDER_ARCHIVE_CACHE_TTL_SECONDS = 300 +MAX_AUTO_PREV_DAYS = 30 + # Bound memory/DB work per chunk for large libraries (20k+ channels). EPG_LOGO_APPLY_BATCH_SIZE = 500 EPG_LOGO_APPLY_MAX_ERRORS = 100 @@ -24,13 +29,80 @@ def format_channel_number(value, empty=""): return value -def get_channel_catchup_streams(channel): - """Ordered catch-up streams for a Channel (empty list if unavailable). +def compute_provider_archive_days_capped(): + """Max ``catchup_days`` across active XC catch-up streams (capped, cached). - Returned in ``channelstream__order`` so the timeshift proxy can fail over: - it tries each stream's provider in turn until one serves the archive — - mirroring how live playback walks a channel's stream list. Streams from - disabled M3U accounts are excluded, same as live dispatch. + Cached briefly so XC XMLTV exports without an explicit ``prev_days`` do not + repeat the aggregate query on every request. + """ + def _scan(): + from django.db.models import Max + + from apps.channels.models import Stream + + result = Stream.objects.filter( + m3u_account__account_type="XC", + m3u_account__is_active=True, + is_catchup=True, + ).aggregate(max_days=Max("catchup_days")) + return min(result["max_days"] or 0, MAX_AUTO_PREV_DAYS) + + return cache.get_or_set( + "channels:provider_archive_days_capped", + _scan, + PROVIDER_ARCHIVE_CACHE_TTL_SECONDS, + ) + + +def resolve_xc_epg_prev_days(request, user, *, auto_detect_fallback=True): + """Resolve ``prev_days`` for XC XMLTV and player_api EPG. + + Args: + request: HTTP request (reads ``?prev_days=``). + user: Authenticated user (reads ``custom_properties.epg_prev_days``). + auto_detect_fallback: When True (XC XMLTV), fall back to the largest + provider archive depth. When False (per-channel EPG), return 0 so + ``xc_get_epg`` can expand to each channel's ``catchup_days``. + + Resolution order: + 1. URL ``?prev_days=`` (explicit; 0 means no past programmes) + 2. ``user.custom_properties.epg_prev_days`` + 3. ``CoreSettings.proxy_settings.xmltv_prev_days_override`` when > 0 + 4. Auto-detect (only when *auto_detect_fallback* is True) + """ + user_custom = (user.custom_properties or {}) if user else {} + url_prev = request.GET.get("prev_days") + user_prev = user_custom.get("epg_prev_days") if user_custom else None + + if url_prev is not None: + try: + return max(0, min(int(url_prev), MAX_AUTO_PREV_DAYS)) + except (ValueError, TypeError): + return 0 + if user_prev not in (None, ""): + try: + return max(0, min(int(user_prev), MAX_AUTO_PREV_DAYS)) + except (ValueError, TypeError): + return 0 + + from core.models import CoreSettings + + proxy_settings = CoreSettings.get_proxy_settings() + try: + override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + override = 0 + if override > 0: + return max(0, min(override, MAX_AUTO_PREV_DAYS)) + if auto_detect_fallback: + return compute_provider_archive_days_capped() + return 0 + + +def get_channel_catchup_streams(channel): + """Active catch-up streams for a channel, in ``channelstream`` order. + + Inactive M3U accounts are excluded, matching live dispatch. """ if not getattr(channel, "is_catchup", False): return [] diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py index 1075946a..7f26cb73 100644 --- a/apps/timeshift/helpers.py +++ b/apps/timeshift/helpers.py @@ -1,4 +1,4 @@ -"""URL builders + timestamp conversion + archive-days probe for XC catch-up.""" +"""URL builders and timestamp helpers for XC catch-up.""" import logging from collections import namedtuple @@ -6,16 +6,9 @@ from datetime import datetime, timezone from urllib.parse import quote from zoneinfo import ZoneInfo -from django.core.cache import cache - logger = logging.getLogger(__name__) -#: Resolved upstream credentials for one catch-up attempt. Produced from -#: ``get_transformed_credentials(account, reserved_profile)`` so the URL is -#: built with the credentials of the profile whose pool slot was actually -#: reserved — alternate profiles express different logins as URL regex -#: transforms, and using the raw account fields for them would desynchronize -#: pool accounting from real upstream usage. +# Credentials for the profile whose pool slot was reserved (not raw account fields). TimeshiftCredentials = namedtuple( "TimeshiftCredentials", ("server_url", "username", "password") ) @@ -24,43 +17,16 @@ DEFAULT_DURATION_MINUTES = 120 DURATION_BUFFER_MINUTES = 5 MAX_DURATION_MINUTES = 480 -PROVIDER_ARCHIVE_CACHE_TTL_SECONDS = 300 -MAX_AUTO_PREV_DAYS = 30 - - -def compute_provider_archive_days_capped(): - """Largest `catchup_days` across all XC streams with catch-up (capped, cached). - - Uses the denormalized ``Stream.catchup_days`` field instead of iterating - JSON blobs — one aggregate query, no Python loop. - Returns 0 when no XC stream advertises catch-up. - """ - def _scan(): - from apps.channels.models import Stream - from django.db.models import Max - - result = ( - Stream.objects.filter( - m3u_account__account_type="XC", - is_catchup=True, - ) - .aggregate(max_days=Max("catchup_days")) - ) - return min(result["max_days"] or 0, MAX_AUTO_PREV_DAYS) - - return cache.get_or_set( - "timeshift:provider_archive_days_capped", - _scan, - PROVIDER_ARCHIVE_CACHE_TTL_SECONDS, - ) - def parse_catchup_timestamp(timestamp_str): - """Parse a timestamp string into a datetime, accepting colon-dash or underscore shapes. + """Parse a catch-up timestamp string. - Accepts: ``YYYY-MM-DD:HH-MM`` (iPlayTV/TiviMate native), - ``YYYY-MM-DD_HH-MM`` (XC underscore shape). - Returns None on failure. + Args: + timestamp_str: ``YYYY-MM-DD:HH-MM`` (iPlayTV/TiviMate) or + ``YYYY-MM-DD_HH-MM`` (XC underscore form). + + Returns: + A naive datetime on success, or None. """ for fmt in ("%Y-%m-%d:%H-%M", "%Y-%m-%d_%H-%M"): try: @@ -71,21 +37,15 @@ def parse_catchup_timestamp(timestamp_str): def convert_timestamp_to_provider_tz(timestamp_str, provider_tz_name): - """Convert a UTC catch-up timestamp to the serving provider's local zone. + """Convert a UTC catch-up timestamp to the provider's local zone. - The XC API surface is strictly UTC, so the client builds the catch-up URL - from a UTC wall-clock. Real XC providers, however, index their archive in - their OWN local zone (the ``server_info.timezone`` they report on auth). This - shifts the UTC instant into that zone so the upstream seek lands on the right - hour — empirically a provider reading ``17-00`` as its local time returns the - 17:00-local programme, not 17:00 UTC. + Args: + timestamp_str: UTC wall-clock in ``YYYY-MM-DD:HH-MM`` or underscore form. + provider_tz_name: IANA zone from the provider's ``server_info.timezone`` + (e.g. ``Europe/Brussels``). Falsy, ``UTC``, or unknown: no conversion. - ``timestamp_str`` is ``YYYY-MM-DD:HH-MM`` or ``YYYY-MM-DD_HH-MM`` (UTC). - ``provider_tz_name`` is an IANA name (e.g. ``Europe/Brussels``). When it is - falsy, ``"UTC"``, or unknown, the timestamp is returned unchanged — providers - that index in UTC (or whose zone we don't know) need no shift. Returns the - colon-dash shape ``YYYY-MM-DD:HH-MM`` (the canonical PATH shape). On any parse - failure the input is returned unchanged. + Returns: + ``YYYY-MM-DD:HH-MM`` in the provider zone, or the input unchanged on skip/failure. """ if not provider_tz_name or provider_tz_name == "UTC": return timestamp_str @@ -96,33 +56,31 @@ def convert_timestamp_to_provider_tz(timestamp_str, provider_tz_name): target = ZoneInfo(provider_tz_name) except Exception: logger.warning( - "Timeshift: unknown provider timezone %r — no conversion applied", + "Timeshift: unknown provider timezone %r, no conversion applied", provider_tz_name, ) return timestamp_str - # datetime.timezone.utc (not ZoneInfo("UTC")) for the source side — immune to - # a mis-set host /etc/timezone. astimezone() is DST-correct for the date. + # timezone.utc, not ZoneInfo("UTC"): avoids mis-set Docker /etc/timezone. local_dt = dt.replace(tzinfo=timezone.utc).astimezone(target) return local_dt.strftime("%Y-%m-%d:%H-%M") def get_programme_duration(channel, timestamp_str): - """Duration in minutes of the EPG programme starting at `timestamp_str`. + """Look up catch-up duration in minutes from EPG. - `timestamp_str` is `YYYY-MM-DD:HH-MM` or `YYYY-MM-DD_HH-MM` in UTC — the - client's original value, built from the strictly-UTC EPG output. The - UTC→provider-zone conversion happens only for the upstream URL (in - ``timeshift_proxy``), never for this EPG lookup: programmes are stored in - UTC, so the lookup must stay in UTC too. - Falls back to a 120-minute default if EPG lookup fails. + Args: + channel: Channel with optional ``epg_data`` relation loaded. + timestamp_str: Programme start in UTC (same shape as the client URL). + + Returns: + Programme length plus a small buffer, capped at ``MAX_DURATION_MINUTES``, + or ``DEFAULT_DURATION_MINUTES`` when EPG lookup fails. """ try: dt = parse_catchup_timestamp(timestamp_str) if dt is None: return DEFAULT_DURATION_MINUTES - # EPG start_time/end_time are timezone-aware (USE_TZ=True), so the - # parsed datetime must also be aware to avoid a TypeError in the ORM - # filter. + # EPG times are timezone-aware; parsed value must be too. dt = dt.replace(tzinfo=timezone.utc) if not channel.epg_data: return DEFAULT_DURATION_MINUTES @@ -141,9 +99,7 @@ def get_programme_duration(channel, timestamp_str): def build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes): - """Format A: `/streaming/timeshift.php?username=&password=&stream=&start=&duration=`.""" - # Credentials are URL-encoded: a `&`, `/` or `#` in the password would - # otherwise corrupt the URL structure. + """QUERY layout: ``/streaming/timeshift.php?username=...&start=...``""" return ( f"{creds.server_url.rstrip('/')}/streaming/timeshift.php" f"?username={quote(str(creds.username), safe='')}" @@ -155,7 +111,7 @@ def build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes): def build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes): - """Format B: `/timeshift/{user}/{pass}/{duration}/{timestamp}/{stream_id}.ts`.""" + """PATH layout: ``/timeshift/{user}/{pass}/{dur}/{start}/{id}.ts``""" return ( f"{creds.server_url.rstrip('/')}/timeshift" f"/{quote(str(creds.username), safe='')}" @@ -167,33 +123,23 @@ def build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes): def build_timeshift_candidate_urls(creds, stream_id, timestamp, duration_minutes): - """Ordered upstream catch-up candidates — PATH form first. + """Build ordered upstream URL candidates (PATH forms first, QUERY last). - Two URL layouts exist on XC servers and they do NOT behave the same: + Args: + creds: ``TimeshiftCredentials`` for the reserved profile. + stream_id: Provider stream id from the catch-up stream's custom properties. + timestamp: Already converted to the serving provider's local zone. + duration_minutes: Archive window length passed to the provider. - • Format B — PATH layout: ``/timeshift/{user}/{pass}/{dur}/{START}/{id}.ts`` - The canonical XC catch-up form (what TiviMate emits natively). It actually - SEEKS the requested archive instant. Tried FIRST. - • Format A — QUERY layout: ``/streaming/timeshift.php?...&start={START}`` - A non-standard variant. Some providers implement it incorrectly and return - the LIVE stream (HTTP 200, ignoring ``start``) — indistinguishable from a - real archive at the byte level, so it would masquerade as a successful - catch-up. Kept only as a fallback for providers that expose ONLY - timeshift.php, and therefore tried AFTER every PATH candidate. - - Within each layout we vary the timestamp shape, because different servers' - parsers accept different ones (colon-dash is the canonical PATH shape; - underscore and SQL-datetime cover other servers' parsers). No timezone - conversion happens here — the caller (``timeshift_proxy``) has already - converted the value to the serving provider's zone; only the *shape* varies. + Returns: + List of URL strings to try in order. QUERY forms are last because some + providers return live TV even when ``start`` is set. """ underscore_ts = format_timestamp_as_underscore(timestamp) sql_ts = format_timestamp_as_sql_datetime(timestamp) return [ - # PATH form first — it seeks the archive correctly. build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes), build_timeshift_url_format_b(creds, stream_id, underscore_ts, duration_minutes), - # QUERY form fallback — may return LIVE on some providers (see above). build_timeshift_url_format_a(creds, stream_id, underscore_ts, duration_minutes), build_timeshift_url_format_a(creds, stream_id, sql_ts, duration_minutes), build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes), @@ -201,18 +147,7 @@ def build_timeshift_candidate_urls(creds, stream_id, timestamp, duration_minutes def format_timestamp_as_underscore(timestamp): - """Reshape ``YYYY-MM-DD:HH-MM`` to ``YYYY-MM-DD_HH-MM`` without any - timezone conversion. - - Many XC servers use the underscore shape as their - canonical catch-up URL format, especially for recently-indexed archives - (< 5–6 hours old). The colon-dash and SQL shapes only resolve against the - legacy catch-up parser, which covers archives older than roughly half a day. - - Shape-only, by design: the single timezone conversion in the chain happens - upstream in ``timeshift_proxy`` (UTC → serving provider's zone), so the - value received here is already in the provider's zone. - """ + """Reshape to ``YYYY-MM-DD_HH-MM`` without timezone conversion.""" dt = parse_catchup_timestamp(timestamp) if dt is None: logger.error("Timeshift underscore reshape failed for %r: unrecognised format", timestamp) @@ -221,17 +156,7 @@ def format_timestamp_as_underscore(timestamp): def format_timestamp_as_sql_datetime(timestamp): - """Reshape ``YYYY-MM-DD:HH-MM`` (or underscore variant) to ``YYYY-MM-DD HH:MM:SS`` - without any timezone conversion. - - Some XC servers refuse the dash-only shape for archives whose recording - is still being finalised and only resolve the SQL-datetime shape. - - Shape-only, by design: the single timezone conversion in the chain happens - upstream in ``timeshift_proxy`` (UTC → serving provider's zone via - ``convert_timestamp_to_provider_tz``), so the value received here is - already in the provider's zone — this function must not convert again. - """ + """Reshape to ``YYYY-MM-DD HH:MM:SS`` without timezone conversion.""" dt = parse_catchup_timestamp(timestamp) if dt is None: logger.error("Timeshift SQL timestamp reshape failed for %r: unrecognised format", timestamp) From a8f48a1cbbf170c2e776b7f7d7c405c852c24f14 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:47:38 -0500 Subject: [PATCH 17/31] refactor(tasks): Remove outdated comments and simplify the rollup_channel_catchup_fields function. Enhance clarity by focusing on active account streams and self-healing logic for catch-up flags. --- apps/m3u/tasks.py | 51 +++++++++-------------------------------------- 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 0f36d4fc..34a72316 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1053,9 +1053,6 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, account_type='XC', stream_id=provider_stream_id ) - # Derive denormalized catch-up fields from the XC - # API response at import time so output views can - # read them as zero-cost column reads. _tv_archive = str(stream.get("tv_archive", "0")) _is_catchup = _tv_archive in ("1", "True") try: @@ -1291,8 +1288,6 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): account_type=account_type_for_hash, stream_id=provider_stream_id ) - # Derive catch-up fields from stream attributes (M3U files - # may carry tv_archive via custom attributes). _attrs = stream_info["attributes"] _tv_archive_m3u = str(_attrs.get("tv_archive", "0")) _is_catchup_m3u = _tv_archive_m3u in ("1", "True") @@ -1890,23 +1885,7 @@ def _classify_sync_failure(exc): def rollup_channel_catchup_fields(account_id): - """Roll up denormalized catch-up fields from streams to channels. - - Updates ``is_catchup`` and ``catchup_days`` on every Channel that has - at least one Stream belonging to *account_id* (auto-created or manually - assigned), then self-heals any channel left flagged with no catch-up - stream at all. - - Called from the account-refresh pipeline after streams have been - written so that both new and existing channels reflect the current - ``tv_archive`` flags. Intentionally separate from - ``sync_auto_channels`` because it applies to manual channels too and - has no dependency on auto-channel logic. - - Uses a single-pass CTE (``bool_or`` + ``MAX``) instead of correlated - subqueries so the work stays O(channelstream rows for this account) - rather than O(channels * subqueries). - """ + """Roll up catch-up flags from streams to channels (active accounts only).""" from django.db import connection with connection.cursor() as cur: @@ -1914,12 +1893,11 @@ def rollup_channel_catchup_fields(account_id): WITH agg AS ( SELECT cs.channel_id, - bool_or(s.is_catchup) AS any_catchup, - -- Only catch-up streams contribute their archive depth: - -- a non-catchup stream may carry a stale catchup_days value. - MAX(s.catchup_days) FILTER (WHERE s.is_catchup) AS max_days + bool_or(s.is_catchup AND a.is_active) AS any_catchup, + MAX(s.catchup_days) FILTER (WHERE s.is_catchup AND a.is_active) AS max_days FROM dispatcharr_channels_channelstream cs JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + JOIN m3u_m3uaccount a ON a.id = s.m3u_account_id WHERE cs.channel_id IN ( SELECT DISTINCT cs2.channel_id FROM dispatcharr_channels_channelstream cs2 @@ -1936,16 +1914,7 @@ def rollup_channel_catchup_fields(account_id): WHERE c.id = agg.channel_id """, [account_id]) - # Self-heal: reset channels still flagged as catch-up that no longer - # have ANY catch-up stream. The CTE above only reaches channels that - # still hold at least one stream from this account, so a channel whose - # last stream was just removed (stale cleanup, manual unassignment) - # falls outside its scope. The ChannelStream post_delete signal - # normally resets those rows already — this pass guarantees the - # invariant regardless of how the link rows disappeared (raw SQL, - # signal-less bulk operations, historic staleness). Idempotent and - # cheap: ``is_catchup`` is indexed and TRUE on a small minority of - # channels. + # Self-heal stale is_catchup flags. cur.execute(""" UPDATE dispatcharr_channels_channel c SET is_catchup = FALSE, catchup_days = 0 @@ -1954,7 +1923,10 @@ def rollup_channel_catchup_fields(account_id): SELECT 1 FROM dispatcharr_channels_channelstream cs JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id - WHERE cs.channel_id = c.id AND s.is_catchup = TRUE + JOIN m3u_m3uaccount a ON a.id = s.m3u_account_id + WHERE cs.channel_id = c.id + AND s.is_catchup = TRUE + AND a.is_active = TRUE ) """) @@ -3728,11 +3700,6 @@ def _refresh_single_m3u_account_impl(account_id): f"Error running auto channel sync for account {account_id}: {str(e)}" ) - # Roll up catch-up fields from streams to channels. Runs after - # sync_auto_channels so new and existing channels both reflect the - # current tv_archive flags from the just-completed stream refresh. - # Covers manual channels too, so it lives here rather than inside - # sync_auto_channels which only manages auto-created channels. try: rollup_channel_catchup_fields(account_id) logger.debug(f"Catch-up field rollup complete for account {account_id}") From fa39a594be246d2f3234f0d9f37eeeb3b1c2decd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:53:02 -0500 Subject: [PATCH 18/31] feat(epg): Enhance generate_epg function to support xc_catchup_prev_days parameter for improved catch-up day resolution. Update related views and tests to ensure consistent behavior across EPG generation and catch-up functionality. --- apps/output/epg.py | 34 +++++++----------------- apps/output/tests.py | 19 ++++++++++++++ apps/output/views.py | 62 +++++++------------------------------------- 3 files changed, 37 insertions(+), 78 deletions(-) diff --git a/apps/output/epg.py b/apps/output/epg.py index b017b8e5..d34d209e 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1088,7 +1088,7 @@ def generate_dummy_epg( return xml_lines -def generate_epg(request, profile_name=None, user=None): +def generate_epg(request, profile_name=None, user=None, *, xc_catchup_prev_days=False): """ Dynamically generate an XMLTV (EPG) file using a streaming response. Since the EPG data is stored independently of Channels, we group programmes @@ -1100,32 +1100,16 @@ def generate_epg(request, profile_name=None, user=None): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - # prev_days resolution order: - # 1. URL ?prev_days= (explicit, even 0 means "no past") - # 2. user.custom_properties.epg_prev_days - # 3. CoreSettings.proxy_settings.xmltv_prev_days_override (>0) - # 4. Auto-detect: max provider tv_archive_duration capped at 30 - url_prev = request.GET.get('prev_days') - user_prev = user_custom.get('epg_prev_days') if user_custom else None - if url_prev is not None: - try: - prev_days = max(0, min(int(url_prev), 30)) - except (ValueError, TypeError): - prev_days = 0 - elif user_prev not in (None, ""): - try: - prev_days = max(0, min(int(user_prev), 30)) - except (ValueError, TypeError): - prev_days = 0 + if xc_catchup_prev_days: + from apps.channels.utils import resolve_xc_epg_prev_days + + prev_days = resolve_xc_epg_prev_days(request, user) else: - from apps.timeshift.helpers import compute_provider_archive_days_capped - from core.models import CoreSettings - proxy_settings = CoreSettings.get_proxy_settings() try: - override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) - except (TypeError, ValueError): - override = 0 - prev_days = max(0, min(override, 30)) if override > 0 else compute_provider_archive_days_capped() + prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) + prev_days = max(0, min(prev_days, 30)) + except (ValueError, TypeError): + prev_days = 0 use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() cache_params = ( diff --git a/apps/output/tests.py b/apps/output/tests.py index 30b22a18..25a38d18 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -896,3 +896,22 @@ class XcVodSeriesRegressionTests(TestCase): stream = xc_get_vod_streams(self.request, self.user)[0] self.assertEqual(stream["container_extension"], first.container_extension) + + +class GenerateEpgPrevDaysTests(SimpleTestCase): + """Profile EPG keeps legacy prev_days=0 unless URL or user setting says otherwise.""" + + def setUp(self): + self.factory = RequestFactory() + + @patch("apps.output.epg.stream_cached_response") + @patch("apps.output.epg.Channel.objects") + def test_non_xc_epg_defaults_prev_days_to_zero(self, _channels, mock_cache): + from apps.output.epg import generate_epg + + mock_cache.side_effect = lambda cache_key, _source, **_kwargs: cache_key + request = self.factory.get("/epg/") + + cache_key = generate_epg(request, profile_name="test", user=None) + + self.assertIn(":p=0:", cache_key) diff --git a/apps/output/views.py b/apps/output/views.py index eee288cd..a7e34c31 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -369,20 +369,12 @@ def _xc_allowed_output_formats(user): def _build_xc_server_info(request, hostname, port): - """Build the server_info dict for XC API responses. + """Build XC ``server_info``; keep timezone, ``time_now``, and EPG times in UTC. - The timezone, time_now, and xc_get_epg start/end fields form a "timezone - triple" that MUST all use the same zone — XC clients (iPlayTV, TiviMate) - use server_info.timezone to interpret start/end strings and calculate seek - offsets into catch-up archives. We keep the whole triple in **UTC**: the EPG - surface stays timezone-neutral and any provider-local conversion happens at - catch-up request time, against the serving provider's own zone (see - apps/timeshift/views.timeshift_proxy). This is what keeps multi-provider - setups consistent. Learned from plugin v1.1.4 → v1.2.6 (the earlier - per-instance timezone shifting caused 6 iterations of wrong-programme bugs). + XC clients use ``server_info.timezone`` to interpret EPG start/end strings. + Provider-local conversion happens in the timeshift proxy at request time. """ - # datetime.timezone.utc, not ZoneInfo("UTC"): the latter can read a mis-set - # host /etc/timezone in some Docker setups. + # datetime.timezone.utc, not ZoneInfo("UTC"); avoids mis-set Docker /etc/timezone. return { "url": hostname, "server_protocol": request.scheme, @@ -552,10 +544,7 @@ def xc_xmltv(request): ) return JsonResponse({'error': 'Unauthorized'}, status=401) - # XMLTV is emitted in UTC — the XC API surface is strictly UTC. Catch-up - # clients build their seek from the UTC wall-clock; the timeshift proxy - # applies any provider-local offset at request time. No per-instance rewrite. - return generate_epg(request, None, user) + return generate_epg(request, None, user, xc_catchup_prev_days=True) def xc_get_live_categories(user): @@ -705,11 +694,6 @@ def _xc_channel_entry(channel, channel_num_map, _get_default_group_id, _logo_url effective_group = channel.effective_channel_group_obj group_id = effective_group.id if effective_group else _get_default_group_id() - # Denormalized catch-up fields — populated at M3U/XC import time and - # rolled up via ChannelStream signal. Zero DB queries here. - # Always emit channel.id as stream_id — clients must only see - # Dispatcharr's internal IDs. The provider's stream_id is a backend - # detail looked up internally by the timeshift endpoint. if channel.is_catchup: tv_archive = 1 tv_archive_duration = channel.catchup_days @@ -767,17 +751,12 @@ def xc_get_epg(request, user, short=False): if not channel_id: raise Http404() - # Clients always receive channel.id from get_live_streams, so this is - # a straightforward int lookup — no provider stream_id fallback needed. try: resolved_channel_id = int(channel_id) except (TypeError, ValueError): raise Http404() channel = None - # Apply effective-value annotation + hidden-exclusion at every channel - # resolution path so a single channel lookup honors the same visibility - # rules as xc_get_live_streams. def _annotate(qs): return with_effective_values(qs, select_related_fks=True).exclude(hidden_from_output=True) @@ -859,6 +838,8 @@ def xc_get_epg(request, user, short=False): int(channel.effective_channel_number) if channel.effective_channel_number is not None else 0, ) + from apps.channels.utils import resolve_xc_epg_prev_days + limit = int(request.GET.get('limit', 4)) user_custom = user.custom_properties or {} try: @@ -866,21 +847,10 @@ def xc_get_epg(request, user, short=False): num_days = max(0, min(num_days, 365)) except (ValueError, TypeError): num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 + prev_days = resolve_xc_epg_prev_days(request, user, auto_detect_fallback=False) now = django_timezone.now() - # When a channel supports catch-up, automatically include past programmes - # within the archive window even if the client did not request prev_days. - # XC clients (iPlayTV, TiviMate) call get_simple_data_table without - # prev_days and expect the response to already contain archived entries - # marked with has_archive=1 — they use that list to populate the catch-up - # programme menu. - # Use denormalized catch-up fields (zero DB queries). Clamp to the same - # 30-day ceiling applied to explicit prev_days values. + # XC catch-up clients expect past programmes when prev_days was not set. _channel_is_catchup = getattr(channel, "is_catchup", False) _channel_catchup_days = min(getattr(channel, "catchup_days", 0) or 0, 30) if _channel_is_catchup and prev_days == 0: @@ -931,21 +901,11 @@ def xc_get_epg(request, user, short=False): output = {"epg_listings": []} - # Reuse the denormalized catch-up fields for the has_archive flag. if _channel_is_catchup: archive_window = timedelta(days=_channel_catchup_days) else: archive_window = None - # start/end are emitted in UTC — the whole XC API surface is strictly UTC, - # so the "timezone triple" (server_info.timezone, these start/end strings, - # and time_now) is consistently UTC. XC clients display these strings and - # use server_info.timezone (also UTC) to build the catch-up URL; the proxy - # then converts that UTC instant to the SERVING provider's own local zone at - # request time (see apps/timeshift/views.timeshift_proxy). Keeping the EPG - # UTC is what makes multi-provider setups consistent — we cannot know at EPG - # time which provider will serve a given catch-up. - # start_timestamp/stop_timestamp (epoch) are inherently timezone-agnostic. _epg_utc = dt_timezone.utc for program in programs: @@ -981,10 +941,6 @@ def xc_get_epg(request, user, short=False): "stream_id": f"{channel_id}", } - # has_archive tells XC clients which past programmes are available - # for catch-up playback. Always emitted (not gated by short) because - # both get_simple_data_table and get_short_epg callers need it to - # populate the catch-up programme menu. if archive_window is not None and end < now and end > now - archive_window: program_output["has_archive"] = 1 else: From 9e941c7011c1788aec610d522caaa8135b5b6493 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 12:53:22 -0500 Subject: [PATCH 19/31] refactor(proxy): Consolidate MPEG-TS sync detection logic into utils.py and streamline comments in timeshift views. Update network access checks for consistency across XC API endpoints. Enhance clarity in timeshift proxy documentation and tests. --- apps/channels/tests/test_catchup_utils.py | 100 ++++++ apps/proxy/live_proxy/input/http_streamer.py | 28 -- apps/proxy/utils.py | 37 ++- apps/timeshift/tests/test_views.py | 7 +- apps/timeshift/views.py | 305 ++++--------------- 5 files changed, 181 insertions(+), 296 deletions(-) create mode 100644 apps/channels/tests/test_catchup_utils.py diff --git a/apps/channels/tests/test_catchup_utils.py b/apps/channels/tests/test_catchup_utils.py new file mode 100644 index 00000000..68ac20c3 --- /dev/null +++ b/apps/channels/tests/test_catchup_utils.py @@ -0,0 +1,100 @@ +from unittest.mock import patch + +from django.test import RequestFactory, SimpleTestCase, TestCase + +from apps.accounts.models import User +from apps.channels.models import Channel, ChannelStream, Stream +from apps.channels.utils import resolve_xc_epg_prev_days +from apps.m3u.models import M3UAccount + + +class ResolveXcEpgPrevDaysTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + self.user = User(username="xc-prev", custom_properties={}) + + def test_url_prev_days_zero_is_explicit(self): + request = self.factory.get("/xmltv.php?prev_days=0") + self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 0) + + def test_user_epg_prev_days_used_when_url_omitted(self): + self.user.custom_properties = {"epg_prev_days": 5} + request = self.factory.get("/xmltv.php") + self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 5) + + @patch("apps.channels.utils.compute_provider_archive_days_capped", return_value=14) + def test_auto_detect_only_when_no_url_or_user_default(self, mock_compute): + request = self.factory.get("/xmltv.php") + self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 14) + mock_compute.assert_called_once() + + @patch("apps.channels.utils.compute_provider_archive_days_capped", return_value=14) + def test_per_channel_epg_skips_global_auto_detect(self, mock_compute): + request = self.factory.get("/player_api.php") + self.assertEqual( + resolve_xc_epg_prev_days(request, self.user, auto_detect_fallback=False), + 0, + ) + mock_compute.assert_not_called() + + @patch("apps.channels.utils.compute_provider_archive_days_capped", return_value=14) + def test_user_default_prevents_auto_detect(self, mock_compute): + self.user.custom_properties = {"epg_prev_days": 3} + request = self.factory.get("/xmltv.php") + self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 3) + mock_compute.assert_not_called() + + @patch("apps.channels.utils.compute_provider_archive_days_capped", return_value=14) + @patch("core.models.CoreSettings.get_proxy_settings", return_value={"xmltv_prev_days_override": 7}) + def test_proxy_override_prevents_auto_detect(self, _settings, mock_compute): + request = self.factory.get("/xmltv.php") + self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 7) + mock_compute.assert_not_called() + + +class CatchupRollupActiveAccountTests(TestCase): + """Denormalized catch-up flags ignore disabled M3U accounts.""" + + @classmethod + def setUpTestData(cls): + cls.inactive = M3UAccount.objects.create( + name="catchup-inactive", + server_url="http://example.test", + account_type="XC", + is_active=False, + ) + + def test_channelstream_signal_ignores_inactive_catchup_stream(self): + channel = Channel.objects.create(name="inactive-only") + stream = Stream.objects.create( + name="inactive-catchup", + url="http://example.test/inactive", + m3u_account=self.inactive, + is_catchup=True, + catchup_days=9, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) + + def test_rollup_ignores_inactive_catchup_stream(self): + from apps.m3u.tasks import rollup_channel_catchup_fields + + channel = Channel.objects.create(name="rollup-inactive-only") + stream = Stream.objects.create( + name="rollup-inactive-catchup", + url="http://example.test/rollup-inactive", + m3u_account=self.inactive, + is_catchup=True, + catchup_days=9, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + Channel.objects.filter(pk=channel.pk).update(is_catchup=True, catchup_days=9) + + rollup_channel_catchup_fields(self.inactive.id) + + channel.refresh_from_db() + self.assertFalse(channel.is_catchup) + self.assertEqual(channel.catchup_days, 0) diff --git a/apps/proxy/live_proxy/input/http_streamer.py b/apps/proxy/live_proxy/input/http_streamer.py index d90297c5..7efb5d33 100644 --- a/apps/proxy/live_proxy/input/http_streamer.py +++ b/apps/proxy/live_proxy/input/http_streamer.py @@ -159,31 +159,3 @@ class HTTPStreamReader: # Wait for thread if self.thread and self.thread.is_alive(): self.thread.join(timeout=2.0) - - -# --------------------------------------------------------------------------- -# MPEG-TS sync detection (used by the timeshift catch-up proxy) -# --------------------------------------------------------------------------- - -_TS_PACKET_SIZE = 188 -_TS_SYNC_BYTE = 0x47 - - -def find_ts_sync(buf): - """Offset of the first MPEG-TS sync chain in *buf*, or -1. - - A valid chain needs 0x47 at offsets i, i+188 and i+376 — three sync - bytes one packet apart (the standard demuxer probe). The timeshift - proxy peeks at the first upstream bytes with this to reject HTTP-200 - PHP error pages and to strip any pre-sync preamble before the bytes - reach a strict demuxer (ExoPlayer). - """ - end = len(buf) - 2 * _TS_PACKET_SIZE - for i in range(0, end): - if ( - buf[i] == _TS_SYNC_BYTE - and buf[i + _TS_PACKET_SIZE] == _TS_SYNC_BYTE - and buf[i + 2 * _TS_PACKET_SIZE] == _TS_SYNC_BYTE - ): - return i - return -1 diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 32e2b674..d3720801 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -63,17 +63,10 @@ def attempt_stream_termination(user_id, requesting_client_id, active_connections if result.get("status") == "error": logger.warning(f"[stream limits][{requesting_client_id}] Failed to stop client {t['client_id']} on channel {t['media_id']}") elif t['type'] == 'timeshift': - # Timeshift uses the same Redis key pattern as live - # (RedisKeys.client_stop). The stream_generator in - # apps/timeshift/views.py polls this key on its 5-second - # heartbeat cadence. + # Same Redis stop key as live; timeshift generator polls it every 5s. redis_client = RedisClient.get_client() if not redis_client: - # Without Redis the stop key cannot be set, so the old - # stream cannot be terminated. Failing the whole attempt - # is the safe outcome: the caller then denies the NEW - # stream instead of letting the user exceed the provider's - # connection limit (which escalates to an IP ban). + # Deny the new stream if we cannot stop the old one. return False stop_key = RedisKeys.client_stop(t['media_id'], t['client_id']) redis_client.setex(stop_key, 60, "true") @@ -123,7 +116,6 @@ def get_user_active_connections(user_id): if user_id is None or (client_user_id and int(client_user_id) == user_id): try: - # Timeshift virtual_channel_ids start with "timeshift_" conn_type = 'timeshift' if channel_id.startswith('timeshift_') else 'live' logger.debug(f"[stream limits] Found {conn_type.upper()} connection for user {user_id} on channel {channel_id} with client ID {client_id}") connected_at = float(connected_at) if connected_at else 0 @@ -205,3 +197,28 @@ def check_user_stream_limits(user, client_id, media_id=None): return False return True + + +_TS_PACKET_SIZE = 188 +_TS_SYNC_BYTE = 0x47 + + +def find_ts_sync(buf): + """Return byte offset of the first valid MPEG-TS sync chain in *buf*, or -1. + + Args: + buf: Raw bytes from an upstream HTTP response (typically the first 1 KB). + + Returns: + Offset of the first 0x47 byte that starts three consecutive 188-byte + packets, or -1. Used to strip PHP/HTML preamble before streaming. + """ + end = len(buf) - 2 * _TS_PACKET_SIZE + for i in range(0, end): + if ( + buf[i] == _TS_SYNC_BYTE + and buf[i + _TS_PACKET_SIZE] == _TS_SYNC_BYTE + and buf[i + 2 * _TS_PACKET_SIZE] == _TS_SYNC_BYTE + ): + return i + return -1 diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 59b8fd0b..7d741784 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch from django.test import RequestFactory, TestCase, override_settings from apps.timeshift import views -from apps.proxy.live_proxy.input.http_streamer import find_ts_sync as _find_ts_sync +from apps.proxy.utils import find_ts_sync as _find_ts_sync class FindTsSyncTests(TestCase): @@ -404,8 +404,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): stream_mock.assert_not_called() def test_network_access_denied_returns_403(self): - # Same network-level gate as the live XC endpoint: when the request's - # network is not allowed for STREAMS, nothing else runs. + # Same network gate as other XC API endpoints (player_api, xmltv, etc.). request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=False) as gate, \ @@ -415,7 +414,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): request, "u", "p", "8", "2026-06-08:17-00", "8.ts" ) self.assertEqual(response.status_code, 403) - self.assertEqual(gate.call_args[0][1], "STREAMS") + self.assertEqual(gate.call_args[0][1], "XC_API") channel_cls.objects.get.assert_not_called() stream_mock.assert_not_called() diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index f158df4b..e93777d8 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -1,20 +1,4 @@ -"""XC catch-up (timeshift) HTTP view — proxies the provider's catch-up -archive (PATH `/timeshift/.../{id}.ts` form first, `/streaming/timeshift.php` -query form as fallback) to clients like iPlayTV and TiviMate, with -multi-provider failover across the channel's catch-up streams. - -Provider capacity accounting follows the same contract as live and VOD: a -profile slot is reserved through ``apps.m3u.connection_pool`` before any -upstream connect and released exactly once when the session ends (one-shot -Redis token), so Dispatcharr's pool counters always match real upstream usage. -A user gets ONE active catch-up session per channel — a seek displaces the -previous one. - -URL parameter quirk matching what iPlayTV / TiviMate emit: - Position "stream_id" -> EPG channel number, IGNORED here. - Position "duration" -> Dispatcharr's Channel.id (the XC API emits - channel.id as stream_id to clients). -""" +"""XC catch-up (timeshift) proxy with multi-provider failover.""" import hmac import itertools @@ -42,8 +26,11 @@ from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.utils import get_client_ip -from apps.proxy.live_proxy.input.http_streamer import find_ts_sync -from apps.proxy.utils import check_user_stream_limits, get_user_active_connections +from apps.proxy.utils import ( + check_user_stream_limits, + find_ts_sync, + get_user_active_connections, +) from core.utils import RedisClient from dispatcharr.utils import network_access_allowed @@ -61,17 +48,20 @@ CLIENT_TTL_SECONDS = 60 def timeshift_proxy(request, username, password, stream_id, timestamp, duration): # noqa: ARG001 stream_id - # The "duration" URL slot carries Dispatcharr's internal Channel.id — the - # XC API emits channel.id as the stream_id to clients, never the provider's - # stream_id. Resolve by Channel.id; 404 if it doesn't exist. + """Proxy an XC catch-up request to the provider with multi-stream failover. + + URL shape (iPlayTV / TiviMate): + ``stream_id``: EPG channel number (ignored here). + ``duration``: Dispatcharr ``Channel.id`` (XC API exposes channel.id as stream_id). + ``timestamp``: UTC programme start (``YYYY-MM-DD:HH-MM``). + """ raw_id = duration[:-3] if duration.endswith(".ts") else duration user = _authenticate_user(username, password) if user is None: return HttpResponseForbidden("Invalid credentials") - # Same network-level gate as the live XC stream endpoint (stream_xc). - if not network_access_allowed(request, "STREAMS", user): + if not network_access_allowed(request, "XC_API", user): return HttpResponseForbidden("Access denied") try: @@ -82,10 +72,7 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if not _user_can_access_channel(user, channel): return HttpResponseForbidden("Access denied") - # Reject malformed timestamps up front: every shape helper falls back to - # pass-through on parse failure, so an unvalidated value would be forwarded - # verbatim into the upstream URL (query-injection vector + a pointless - # provider request). + # Shape helpers pass through on parse failure; reject bad input before upstream. if parse_catchup_timestamp(timestamp) is None: return HttpResponseBadRequest("Invalid timestamp") @@ -93,15 +80,9 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if not catchup_streams: return HttpResponseBadRequest("Timeshift not supported for this channel") - # Verbose timeshift logging follows the standard logger level (set via - # DISPATCHARR_LOG_LEVEL / the logging config), not a per-feature toggle. debug = logger.isEnabledFor(logging.DEBUG) - # The client supplies the catch-up timestamp in UTC — Dispatcharr's XC API - # surface is strictly UTC (server_info.timezone="UTC", EPG start/end in UTC), - # and the client builds the URL from those UTC strings. The EPG duration - # lookup therefore uses the original UTC value (programmes are stored in - # UTC); the per-provider conversion happens inside the failover loop below. + # EPG duration lookup stays in UTC; provider TZ conversion is per-attempt below. duration_minutes = get_programme_duration(channel, timestamp) safe_ts = timestamp.replace(":", "-").replace("/", "-") @@ -112,35 +93,14 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) redis_client = RedisClient.get_client() - # One active catch-up session per user+channel: a new request (programme - # jump, or a seek re-request on the same programme) DISPLACES the user's - # previous session on this channel. Its provider slot is released - # synchronously — so the reservation below cannot collide with the dying - # session on max_connections=1 providers — and its generator is stopped - # via the same Redis stop-key the stream-limit path uses. Runs BEFORE the - # stream-limit check so a seek displaces its own predecessor instead of - # being denied when terminate_on_limit_exceeded is off. + # Displace any prior catch-up session on this channel before reserving a slot. _terminate_previous_timeshift_sessions(redis_client, user, channel.id) - # Enforce user stream limits once, before connecting upstream — this - # terminates the oldest active stream (live, timeshift, or VOD) via the - # Redis stop-key mechanism, same as the live and VOD proxy entry points. - # (No explicit ChannelService.stop_channel(): that would kill other users' - # live streams on the same channel. The media_id only matters for the - # live same-channel exemption, so a channel-scoped id is enough.) if not check_user_stream_limits( user, client_id, media_id=f"timeshift_{channel.id}_{safe_ts}" ): return HttpResponseForbidden("Stream limit exceeded") - # Failover: try each catch-up stream in channelstream order — mirroring how - # live playback walks a channel's stream list. Every attempt carries ITS - # OWN provider context (account, provider stream id, reported timezone, - # user-agent, per-account format cache). Ban safety is per ACCOUNT: when an - # account fails decisively (401/403/406 — auth or ban-class status), its - # other catch-up streams (e.g. FHD/HD variants of the same channel) are - # skipped too, so the cascade never hammers a banning provider; a DIFFERENT - # account is a different host and remains safe to try. last_response = None decisive_accounts = set() capacity_blocked = False @@ -155,9 +115,6 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if stream_id_value is None: continue - # Provider profile walk — same ordering as live's Channel.get_stream(): - # active profiles only, default first; an account without an active - # default profile is skipped entirely, mirroring live dispatch. m3u_profiles = list(m3u_account.profiles.filter(is_active=True)) default_profile = next((p for p in m3u_profiles if p.is_default), None) if default_profile is None: @@ -170,35 +127,17 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) p for p in m3u_profiles if not p.is_default ] - # Real XC providers index their archive in their OWN local zone (the - # server_info.timezone they report on auth — stored on the default - # profile). Convert the UTC instant to that zone for the upstream URL — - # empirically a provider reads `17-00` as 17:00 LOCAL, not UTC, so - # skipping this would seek 1-2h off. This is the ONLY timezone - # conversion in the chain, and it is per-provider (account-level: all - # profiles log into the same server, so the default profile's zone - # applies to every profile of the walk). + # Providers index archives in their own timezone (from server_info on auth). provider_tz_name = None _server_info = (default_profile.custom_properties or {}).get("server_info") or {} if isinstance(_server_info, dict): provider_tz_name = _server_info.get("timezone") provider_timestamp = convert_timestamp_to_provider_tz(timestamp, provider_tz_name) - # Reserve a provider profile slot BEFORE connecting upstream — the - # same accounting contract live (Channel.get_stream) and VOD follow, - # so the Redis pool counters always match real upstream usage. Walk - # to the next profile only on RESERVATION failure (profile_full / - # credential_full — transient capacity, never ban-class); an upstream - # failure instead moves to the next catch-up STREAM, because probing - # the same server again through an alternate profile is wasteful and - # ban-adjacent. + # Reserve a provider profile slot before connecting (same contract as live/VOD). reserved_profile = None for profile in profile_walk: if redis_client is None: - # Without Redis no accounting is possible — proceed unpooled - # rather than blocking playback (stats writes are equally - # fail-open). Deployments always run Redis; this is a dev-only - # path. reserved_profile = profile break reserved, _count, reason = reserve_profile_slot(profile, redis_client) @@ -206,7 +145,7 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) reserved_profile = profile break logger.info( - "Timeshift: profile %s %s on account %s — trying next profile", + "Timeshift: profile %s %s on account %s, trying next profile", profile.id, reason or "unavailable", m3u_account.id, ) if reserved_profile is None: @@ -223,11 +162,7 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) redis_client, client_id, reserved_profile.id ) if not token_stored: - # Redis hiccup right after the INCR: without a token no - # release path could ever free this slot, so release it - # directly NOW (race-free: the session is not yet visible to - # the takeover scan) and report transient unavailability - # instead of streaming unaccounted. + # Release immediately if we cannot record ownership for later cleanup. try: release_profile_slot(reserved_profile.id, redis_client) except Exception as exc: @@ -238,25 +173,14 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) capacity_blocked = True continue - # From here until the streaming response owns the slot, ANY failure - # must release the reservation before propagating — otherwise the - # pool counter (which has no TTL) leaks until the next Redis flush. try: - # Build the upstream URLs with the RESERVED profile's credentials, - # resolved the same way live playback does (credential extraction - # via the profile's URL transform). Using the raw account fields - # for a non-default profile would consume that profile's slot - # while authenticating with the default login — double-occupying - # the provider connection the pool thinks is free. + # Failures before streaming starts must release the reserved slot. + # Use credentials for the profile whose slot was reserved. server_url, xc_username, xc_password = get_transformed_credentials( m3u_account, reserved_profile ) creds = TimeshiftCredentials(server_url, xc_username, xc_password) - # Ordered upstream candidates, PATH form first — see - # build_timeshift_candidate_urls() for the full rationale (the - # QUERY form is a last-resort fallback because some providers - # return LIVE on it, ignoring the requested timestamp). candidate_urls = build_timeshift_candidate_urls( creds, stream_id_value, provider_timestamp, duration_minutes ) @@ -295,23 +219,17 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) ) except Exception: if token_stored: - # A False return here means someone else (a takeover) already - # consumed the token and released the slot — nothing to do. _release_slot_token(redis_client, client_id) raise if response.status_code < 400: - # The reserved slot is now owned by the streaming response: it is - # released (exactly once, via the one-shot token) when the - # generator finishes or the WSGI layer closes the response. return response - # Failed attempt: free the slot before trying the next stream. _release_slot_token(redis_client, client_id) last_response = response if getattr(response, "timeshift_decisive", False): decisive_accounts.add(m3u_account.id) logger.warning( - "Timeshift attempt failed (HTTP %d%s) on account %s for channel %s — " + "Timeshift attempt failed (HTTP %d%s) on account %s for channel %s, " "trying next catch-up stream", response.status_code, ", decisive: skipping this account's other streams" @@ -322,19 +240,10 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) if last_response is not None: return last_response if capacity_blocked: - # Every eligible stream failed on pool capacity alone (no upstream - # attempt was made) — mirror the VOD proxy's pool-exhausted status so - # clients back off instead of treating it as a permanent failure. return HttpResponse("No available stream slot", status=503) - # Streams existed but none was usable (non-XC accounts / missing stream_id). return HttpResponseBadRequest("Cannot build timeshift URL") -# --------------------------------------------------------------------------- -# Authentication, lookup, access control -# --------------------------------------------------------------------------- - - def _authenticate_user(username, password): try: user = User.objects.get(username=username) @@ -366,27 +275,17 @@ def _user_can_access_channel(user, channel): ) -# --------------------------------------------------------------------------- -# Provider pool slot ownership (one-shot release token) + session takeover -# --------------------------------------------------------------------------- - -# Maps a timeshift client to the profile slot it reserved. Consumed (GET+DEL, -# atomically) by whichever release path runs first — the generator's finally, -# the response-close wrapper, a failed failover attempt, or a takeover by the -# user's next request on the same channel — so the slot is released exactly -# once even across uWSGI workers. Note the TTL does NOT recover the slot: if a -# worker dies mid-session the token eventually expires while the counter (no -# TTL) stays consumed — like the rest of the pool, the recovery for -# crash-leaked counters is the Redis flush at container start -# (scripts/wait_for_redis.py). +# One-shot Redis token: whichever release path runs first frees the pool slot. _SLOT_TOKEN_KEY = "timeshift_slot:{client_id}" _SLOT_TOKEN_TTL = 24 * 3600 def _store_slot_token(redis_client, client_id, profile_id): - """Record slot ownership; returns False when the token could not be - written (the caller must then release the reservation itself — without a - token no other path ever could).""" + """Store a one-shot Redis token mapping *client_id* to *profile_id*. + + Returns False if the token could not be written; the caller must release + the reserved slot directly in that case. + """ if redis_client is None: return False try: @@ -402,13 +301,7 @@ def _store_slot_token(redis_client, client_id, profile_id): def _release_slot_token(redis_client, client_id): - """Release the profile slot owned by *client_id*, exactly once. - - GET+DEL inside a MULTI/EXEC transaction: concurrent release attempts are - serialized by Redis, so only the one that actually deletes a live token - decrements the pool counters. Returns True when this call performed the - release. - """ + """Release the profile slot owned by *client_id* (at most once via GET+DEL).""" if redis_client is None: return False key = _SLOT_TOKEN_KEY.format(client_id=client_id) @@ -427,15 +320,10 @@ def _release_slot_token(redis_client, client_id): def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): - """Displace the user's previous catch-up session(s) on this channel. + """Displace the user's prior catch-up session(s) on this channel. - For each of the user's active timeshift sessions whose virtual channel id - belongs to *channel_id*: release its provider slot NOW (synchronously — - the dying generator's own release becomes a no-op thanks to the one-shot - token), drop its stats keys so the stream-limit count no longer includes - it, and set the stop key its generator polls on the 5-second heartbeat. - The accounting is therefore correct immediately; only the old TCP socket - closes lazily (≤ ~5 s). + Releases the provider slot, unregisters stats keys, and sets the live + stop key so the old generator exits on its next heartbeat. """ if redis_client is None or user is None: return @@ -461,14 +349,7 @@ def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): class _SlotReleasingStream: - """Iterator wrapper whose close() always releases the reserved slot. - - Django registers ``streaming_content.close`` in the response's resource - closers, and the WSGI layer guarantees close() runs — including when the - client disconnects before the first chunk, in which case the generator - never starts and its ``finally`` would never execute. The one-shot token - makes the duplicate call from an already-finished generator a no-op. - """ + """Iterator wrapper that releases the pool slot when WSGI closes the response.""" def __init__(self, generator, on_close): self._generator = generator @@ -487,11 +368,6 @@ class _SlotReleasingStream: self._on_close() -# --------------------------------------------------------------------------- -# Stats integration (direct Redis writes, no ClientManager instance) -# --------------------------------------------------------------------------- - - def _register_stats_client( redis_client, virtual_channel_id, @@ -506,8 +382,7 @@ def _register_stats_client( channel_logo_id=None, m3u_profile_id=None, ): - """Write the same Redis keys the live proxy's ClientManager writes so the - catch-up viewer appears on `/stats`. `is_timeshift=1` toggles the badge.""" + """Write Redis keys so catch-up viewers appear on ``/stats``.""" if redis_client is None: return client_set_key = RedisKeys.clients(virtual_channel_id) @@ -583,23 +458,9 @@ def _unregister_stats_client(redis_client, virtual_channel_id, client_id): logger.warning("Timeshift stats unregister failed: %s", exc) -# --------------------------------------------------------------------------- -# Provider streaming -# --------------------------------------------------------------------------- - - def _open_upstream(url, user_agent, range_header): - """Open the upstream HTTP request (status + headers known synchronously). - - Redirects are followed on purpose: XC providers routinely 302 from the - main host to a load-balanced streaming node carrying a session token, so - disabling redirects would break normal catch-up. The cascade therefore - observes the FINAL status; a ban-onset 302 that redirects somewhere - non-streamable surfaces as a failed TS-sync peek (soft reject) or an - error status. - """ - # identity: the TS-sync peek reads raw bytes (response.raw), which are NOT - # transparently decompressed — a gzip-encoded body would fail the sync check. + """Open upstream HTTP; redirects are followed (XC load-balancer nodes).""" + # identity: raw peek bytes are not gzip-transparent. headers = {"Accept-Encoding": "identity"} if user_agent: headers["User-Agent"] = user_agent @@ -618,13 +479,7 @@ _FORMAT_CACHE_TTL = 3600 # 1 hour def _get_cached_format_index(account_id): - """Return the index of the URL shape that last worked for this account, - or None if we haven't seen one succeed yet. - - Uses ``django.core.cache`` (Redis-backed) so every uWSGI worker shares - the same format discovery — the first worker that finds the winning URL - shape caches it for all others. - """ + """Index of the URL shape that last worked for this account, or None.""" if account_id is None: return None return cache.get(_FORMAT_CACHE_KEY.format(account_id)) @@ -653,13 +508,13 @@ def _stream_from_provider( account_id=None, redis_client=None, ): - # Use 256 KB chunks: amortises per-yield uWSGI/gevent overhead. + """Try each upstream URL until one returns streamable MPEG-TS. + + Sets ``timeshift_decisive`` on auth/ban-class failures (401/403/406) so the + failover loop skips the rest of that account's streams. + """ chunk_size = max(ConfigHelper.chunk_size(), 262144) - # Reorder the candidates so the cached winning shape is tried first. - # The cache is per-account; within a single fast-forward session every - # seek lands in the same archive entry and therefore the same parser - # path, so the first attempt almost always succeeds. cached_index = _get_cached_format_index(account_id) if cached_index is not None and 0 <= cached_index < len(candidate_urls): ordered_urls = [candidate_urls[cached_index]] + [ @@ -672,10 +527,7 @@ def _stream_from_provider( ordered_urls = list(candidate_urls) original_indices = list(range(len(candidate_urls))) - # Try each candidate URL until one returns a streamable MPEG-TS response. - # Some XC servers return HTTP 200 with PHP error text instead of 404 when - # the timestamp format doesn't match their parser. We peek at the first - # bytes to confirm TS sync before accepting the response. + # Peek for MPEG-TS sync; some providers return HTTP 200 with PHP/HTML errors. upstream = None last_status = None last_url = ordered_urls[0] @@ -685,9 +537,6 @@ def _stream_from_provider( try: response = _open_upstream(url, user_agent, range_header) except requests.exceptions.RequestException as exc: - # Log only the exception class and a redacted URL: requests - # exceptions embed the full URL, which carries the XC credentials - # (path segments in format B, query params in format A). logger.error( "Timeshift provider unreachable (%s): %s", _redact_url(url), type(exc).__name__, @@ -703,24 +552,18 @@ def _stream_from_provider( _redact_url(url), ) if response.status_code in (200, 206): - # Peek at the first bytes to confirm we're getting MPEG-TS, not - # a PHP error page disguised as 200. Read up to 1 KB — enough - # to find a TS sync chain or detect HTML/PHP text. peek = response.raw.read(1024) sync_offset = find_ts_sync(peek) if peek else -1 if sync_offset >= 0: - # Valid TS — strip any pre-sync garbage (PHP warnings, BOM) - # and prepend the clean bytes into the iter_content chain. response._peek_data = peek[sync_offset:] upstream = response winning_index = orig_idx break else: - # No TS sync found — likely a PHP error. Log and try next. snippet = peek[:200].decode("utf-8", errors="replace") if peek else "(empty)" logger.warning( "Timeshift upstream returned 200 but no TS sync in first %d " - "bytes (likely PHP error): %s — url=%s", + "bytes (likely PHP error): %s, url=%s", len(peek) if peek else 0, snippet.replace("\n", " ")[:120], _redact_url(url), @@ -729,18 +572,7 @@ def _stream_from_provider( last_status = 404 # Treat as soft rejection for cascade continue response.close() - # Decisive statuses where trying other URL shapes can't help and may - # escalate an IP ban: auth failures (401/403) and IP-level block (406). - # The 3xx case is defense-in-depth only: requests follows redirects - # transparently (XC providers legitimately 302 to load-balanced - # streaming nodes, so redirects MUST be followed), meaning a 3xx can - # only surface here if that ever changes — but if one does, it is the - # documented first sign of an IP ban and must stop the cascade. - # A 5xx is different: it is usually format-specific — some XC servers run - # PHP with display_errors off, so the "Undefined array key" warning that - # another server emits as 200+text becomes a hard 500. The very next - # candidate timestamp shape often succeeds, so keep trying instead of - # giving up (this is why catch-up appeared broken on providers that 500). + # Auth/ban-class statuses stop trying more shapes on this account; 5xx does not. code = response.status_code if code in (401, 403, 406) or 300 <= code < 400: decisive_failure = True @@ -752,18 +584,13 @@ def _stream_from_provider( if upstream is None: logger.error("Timeshift upstream rejected: status=%s url=%s", last_status, _redact_url(last_url)) - # Mirror semantically meaningful upstream statuses so clients react - # correctly: 404 stops retry loops (catch-up not yet indexed), 403 - # surfaces auth problems. Everything else stays a generic 400. + # Map 404/403 to meaningful client responses; other failures stay 400. if last_status == 404: failure = HttpResponseNotFound("Catch-up not available yet") elif last_status == 403: failure = HttpResponseForbidden("Provider denied access") else: failure = HttpResponseBadRequest("Provider error") - # Tell the failover loop whether this account failed DECISIVELY - # (auth/ban-class status): its other catch-up streams must be skipped, - # while a different provider remains safe to try. failure.timeshift_decisive = decisive_failure return failure @@ -785,8 +612,6 @@ def _stream_from_provider( m3u_profile_id=m3u_profile_id, ) - # Stream directly via iter_content+yield (same pattern as VOD proxy). - # The peek already validated TS sync and stripped any preamble. peek_data = getattr(upstream, "_peek_data", None) chunks_iter = upstream.iter_content(chunk_size=chunk_size) if peek_data: @@ -798,8 +623,6 @@ def _stream_from_provider( total_yielded = 0 chunk_count = 0 loop_start = time.time() - # Same stop-key pattern as live (generator.py:395) and VOD - # (multi_worker_connection_manager.py:1095). stop_key = RedisKeys.client_stop(virtual_channel_id, client_id) try: for data in chunks_iter: @@ -810,12 +633,8 @@ def _stream_from_provider( total_yielded += len(data) chunk_count += 1 - # Stop-signal + stats heartbeat on the same 5-second cadence. - # Time-based (not chunk-modulo) so the provider slot is freed - # within seconds of a stream-limit termination regardless of - # bitrate — at 256 KB chunks a 100-chunk interval would mean - # ~25 MB (~27 s at typical FHD rates) before noticing the stop. now = time.time() + # Poll stop key and refresh stats every 5 seconds. if now - last_heartbeat >= 5: if redis_client and redis_client.exists(stop_key): logger.info("Timeshift client %s received stop signal", client_id) @@ -849,16 +668,8 @@ def _stream_from_provider( except Exception: pass _unregister_stats_client(redis_client, virtual_channel_id, client_id) - # Free the provider pool slot this session reserved. One-shot - # token: a no-op if a takeover (the user's next request on this - # channel) already released it. _release_slot_token(redis_client, client_id) - # The wrapper guarantees session teardown even when the generator never - # starts (client gone before the first chunk — its finally would then - # never run): Django registers the iterator's close() as a resource - # closer, which WSGI always invokes. Both calls are idempotent, so the - # duplicate from a normally-finished generator is harmless. def _close_session(): _unregister_stats_client(redis_client, virtual_channel_id, client_id) _release_slot_token(redis_client, client_id) @@ -869,16 +680,7 @@ def _stream_from_provider( content_type=content_type, status=status, ) - # Tell nginx not to buffer this streaming response. Without this, the - # default uwsgi_buffering=on under `location /` throttles the response - # roughly to half the upstream rate (back-pressure from buffer flushes - # propagates back into the generator). - response["X-Accel-Buffering"] = "no" - # Forward Content-Range so iPlayTV / TiviMate seek cursor stays correct. - # Content-Length is NOT forwarded: Django's StreamingHttpResponse uses - # chunked transfer encoding, which is incompatible with Content-Length. - # Sending both causes clients to wait for the full file instead of - # playing progressively. + response["X-Accel-Buffering"] = "no" # avoid nginx throttling the stream if content_range: response["Content-Range"] = content_range response["Accept-Ranges"] = "bytes" @@ -886,12 +688,7 @@ def _stream_from_provider( def _redact_url(url): - """Strip credentials from a URL for safe logging. - - Handles both ``user:pass@host`` and XC path-based credentials - (``/username/password/...``) by truncating to ``scheme://host/...``. - Query-string parameters are always stripped. - """ + """Truncate *url* to ``scheme://host/...`` for safe logging (drops credentials).""" if not url or "://" not in url: return url scheme, rest = url.split("://", 1) From 8e0a7b44f5fa8a4c0abecf90098740a749204d6a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 13:33:50 -0500 Subject: [PATCH 20/31] feat(timestamps): Introduce comprehensive timestamp normalization and parsing for catch-up functionality. Support various input formats including colon-dash, underscore, and Unix epoch. Enhance related tests to ensure robust handling of timestamp shapes in timeshift operations. --- apps/timeshift/helpers.py | 136 ++++++++++++++++++++++----- apps/timeshift/tests/test_helpers.py | 102 ++++++++++++++++++++ apps/timeshift/tests/test_views.py | 7 ++ apps/timeshift/views.py | 3 +- 4 files changed, 223 insertions(+), 25 deletions(-) diff --git a/apps/timeshift/helpers.py b/apps/timeshift/helpers.py index 7f26cb73..98d10c36 100644 --- a/apps/timeshift/helpers.py +++ b/apps/timeshift/helpers.py @@ -1,6 +1,7 @@ """URL builders and timestamp helpers for XC catch-up.""" import logging +import re from collections import namedtuple from datetime import datetime, timezone from urllib.parse import quote @@ -17,23 +18,97 @@ DEFAULT_DURATION_MINUTES = 120 DURATION_BUFFER_MINUTES = 5 MAX_DURATION_MINUTES = 480 +# Wall-clock shapes seen from XC / iPlayTV / TiviMate clients. Compiled once. +_CATCHUP_WALL_CLOCK_RE = re.compile( + r"^" + r"(?P\d{4}-\d{2}-\d{2})" + r"(?P[:_]| )" + r"(?P\d{2})" + r"(?P[-:])" + r"(?P\d{2})" + r"(?:" + r":" + r"(?P\d{2})" + r")?" + r"$" +) + + +def normalize_catchup_timestamp_input(timestamp_str): + """Map a client catch-up timestamp to an ISO-8601 string for ``fromisoformat``. + + Supported inputs: + - ``YYYY-MM-DD:HH-MM`` (iPlayTV/TiviMate colon-dash) + - ``YYYY-MM-DD_HH-MM`` (XC underscore) + - ``YYYY-MM-DD:HH:MM[:SS]`` (XC colon time in catch-up URLs) + - ``YYYY-MM-DD HH:MM[:SS]`` (EPG / SQL datetime) + - Unix epoch seconds (10 digits) or milliseconds (13 digits) + + Returns: + An ISO-8601 date-time string (``YYYY-MM-DDTHH:MM:SS``), or None if + the value does not match a known catch-up shape. + """ + if timestamp_str is None: + return None + if not isinstance(timestamp_str, str): + timestamp_str = str(timestamp_str) + value = timestamp_str.strip() + if not value: + return None + + if value.isdigit(): + length = len(value) + if length == 10: + dt = datetime.fromtimestamp(int(value), tz=timezone.utc) + return dt.replace(tzinfo=None).isoformat(timespec="seconds") + if length == 13: + dt = datetime.fromtimestamp(int(value) / 1000, tz=timezone.utc) + return dt.replace(tzinfo=None).isoformat(timespec="seconds") + return None + + match = _CATCHUP_WALL_CLOCK_RE.match(value) + if not match: + return None + + parts = match.groupdict() + second = parts["second"] or "00" + return f"{parts['date']}T{parts['hour']}:{parts['minute']}:{second}" + def parse_catchup_timestamp(timestamp_str): - """Parse a catch-up timestamp string. + """Parse a catch-up timestamp string into a naive UTC wall-clock datetime. - Args: - timestamp_str: ``YYYY-MM-DD:HH-MM`` (iPlayTV/TiviMate) or - ``YYYY-MM-DD_HH-MM`` (XC underscore form). + See ``normalize_catchup_timestamp_input`` for supported input shapes. Returns: A naive datetime on success, or None. """ - for fmt in ("%Y-%m-%d:%H-%M", "%Y-%m-%d_%H-%M"): - try: - return datetime.strptime(timestamp_str, fmt) - except ValueError: - continue - return None + iso_value = normalize_catchup_timestamp_input(timestamp_str) + if iso_value is None: + if timestamp_str is not None and str(timestamp_str).strip(): + logger.debug( + "Timeshift: unrecognised catch-up timestamp: %r", timestamp_str + ) + return None + try: + return datetime.fromisoformat(iso_value) + except ValueError: + logger.debug( + "Timeshift: invalid catch-up timestamp after normalize: %r -> %r", + timestamp_str, + iso_value, + ) + return None + + +def _reshape_timestamp(timestamp, strftime_fmt, label): + dt = parse_catchup_timestamp(timestamp) + if dt is None: + logger.error( + "Timeshift %s reshape failed for %r: unrecognised format", label, timestamp + ) + return timestamp + return dt.strftime(strftime_fmt) def convert_timestamp_to_provider_tz(timestamp_str, provider_tz_name): @@ -135,30 +210,43 @@ def build_timeshift_candidate_urls(creds, stream_id, timestamp, duration_minutes List of URL strings to try in order. QUERY forms are last because some providers return live TV even when ``start`` is set. """ - underscore_ts = format_timestamp_as_underscore(timestamp) - sql_ts = format_timestamp_as_sql_datetime(timestamp) + dt = parse_catchup_timestamp(timestamp) + if dt is None: + colon_dash_ts = timestamp + underscore_ts = timestamp + colon_seconds_ts = timestamp + sql_ts = timestamp + else: + colon_dash_ts = dt.strftime("%Y-%m-%d:%H-%M") + underscore_ts = dt.strftime("%Y-%m-%d_%H-%M") + colon_seconds_ts = dt.strftime("%Y-%m-%d:%H:%M:%S") + sql_ts = dt.strftime("%Y-%m-%d %H:%M:%S") return [ - build_timeshift_url_format_b(creds, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, colon_dash_ts, duration_minutes), build_timeshift_url_format_b(creds, stream_id, underscore_ts, duration_minutes), + build_timeshift_url_format_b(creds, stream_id, colon_seconds_ts, duration_minutes), build_timeshift_url_format_a(creds, stream_id, underscore_ts, duration_minutes), build_timeshift_url_format_a(creds, stream_id, sql_ts, duration_minutes), - build_timeshift_url_format_a(creds, stream_id, timestamp, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, colon_dash_ts, duration_minutes), + build_timeshift_url_format_a(creds, stream_id, colon_seconds_ts, duration_minutes), ] +def format_timestamp_as_colon_dash(timestamp): + """Reshape to ``YYYY-MM-DD:HH-MM`` without timezone conversion.""" + return _reshape_timestamp(timestamp, "%Y-%m-%d:%H-%M", "colon-dash") + + +def format_timestamp_as_colon_seconds(timestamp): + """Reshape to ``YYYY-MM-DD:HH:MM:SS`` without timezone conversion.""" + return _reshape_timestamp(timestamp, "%Y-%m-%d:%H:%M:%S", "colon-seconds") + + def format_timestamp_as_underscore(timestamp): """Reshape to ``YYYY-MM-DD_HH-MM`` without timezone conversion.""" - dt = parse_catchup_timestamp(timestamp) - if dt is None: - logger.error("Timeshift underscore reshape failed for %r: unrecognised format", timestamp) - return timestamp - return dt.strftime("%Y-%m-%d_%H-%M") + return _reshape_timestamp(timestamp, "%Y-%m-%d_%H-%M", "underscore") def format_timestamp_as_sql_datetime(timestamp): """Reshape to ``YYYY-MM-DD HH:MM:SS`` without timezone conversion.""" - dt = parse_catchup_timestamp(timestamp) - if dt is None: - logger.error("Timeshift SQL timestamp reshape failed for %r: unrecognised format", timestamp) - return timestamp - return dt.strftime("%Y-%m-%d %H:%M:%S") + return _reshape_timestamp(timestamp, "%Y-%m-%d %H:%M:%S", "SQL") diff --git a/apps/timeshift/tests/test_helpers.py b/apps/timeshift/tests/test_helpers.py index 20f922d9..4bd7e0e3 100644 --- a/apps/timeshift/tests/test_helpers.py +++ b/apps/timeshift/tests/test_helpers.py @@ -1,5 +1,7 @@ """Tests for `apps.timeshift.helpers`: timestamp shape conversion and URL build.""" +from datetime import datetime, timezone + from django.test import TestCase from apps.timeshift.helpers import ( @@ -8,8 +10,12 @@ from apps.timeshift.helpers import ( build_timeshift_url_format_a, build_timeshift_url_format_b, convert_timestamp_to_provider_tz, + format_timestamp_as_colon_dash, + format_timestamp_as_colon_seconds, format_timestamp_as_sql_datetime, format_timestamp_as_underscore, + normalize_catchup_timestamp_input, + parse_catchup_timestamp, ) @@ -22,6 +28,87 @@ def _make_creds(): class TimestampFormatTests(TestCase): """Timestamp reshape functions change format only; no timezone conversion.""" + def test_normalize_colon_dash_shape(self): + self.assertEqual( + normalize_catchup_timestamp_input("2026-05-21:12-55"), + "2026-05-21T12:55:00", + ) + + def test_normalize_colon_seconds_xc_format(self): + self.assertEqual( + normalize_catchup_timestamp_input("2026-06-23:04:00:00"), + "2026-06-23T04:00:00", + ) + + def test_normalize_epg_sql_format(self): + self.assertEqual( + normalize_catchup_timestamp_input("2026-06-23 04:00:00"), + "2026-06-23T04:00:00", + ) + + def test_normalize_unix_epoch_seconds(self): + epoch = str(int(datetime(2026, 6, 23, 4, 0, 0, tzinfo=timezone.utc).timestamp())) + self.assertEqual( + normalize_catchup_timestamp_input(epoch), + "2026-06-23T04:00:00", + ) + + def test_normalize_unix_epoch_milliseconds(self): + epoch_ms = str( + int(datetime(2026, 6, 23, 4, 0, 0, tzinfo=timezone.utc).timestamp() * 1000) + ) + self.assertEqual( + normalize_catchup_timestamp_input(epoch_ms), + "2026-06-23T04:00:00", + ) + + def test_normalize_rejects_garbage(self): + self.assertIsNone(normalize_catchup_timestamp_input("garbage")) + self.assertIsNone(normalize_catchup_timestamp_input("")) + self.assertIsNone(normalize_catchup_timestamp_input("12345")) + + def test_parse_rejects_invalid_calendar_date(self): + self.assertIsNone(parse_catchup_timestamp("2026-13-45:04-00")) + + def test_parse_colon_dash_format(self): + dt = parse_catchup_timestamp("2026-05-21:12-55") + self.assertEqual(dt, datetime(2026, 5, 21, 12, 55, 0)) + + def test_parse_underscore_format(self): + dt = parse_catchup_timestamp("2026-05-21_12-55") + self.assertEqual(dt, datetime(2026, 5, 21, 12, 55, 0)) + + def test_parse_colon_minutes_without_seconds(self): + dt = parse_catchup_timestamp("2026-06-23:04:00") + self.assertEqual(dt, datetime(2026, 6, 23, 4, 0, 0)) + + def test_parse_colon_seconds_xc_format(self): + dt = parse_catchup_timestamp("2026-06-23:04:00:00") + self.assertEqual(dt, datetime(2026, 6, 23, 4, 0, 0)) + + def test_parse_epg_sql_format(self): + dt = parse_catchup_timestamp("2026-06-23 04:00:00") + self.assertEqual(dt, datetime(2026, 6, 23, 4, 0, 0)) + + def test_format_colon_dash_from_colon_seconds(self): + self.assertEqual( + format_timestamp_as_colon_dash("2026-06-23:04:00:00"), + "2026-06-23:04-00", + ) + + def test_format_colon_seconds_from_colon_dash(self): + self.assertEqual( + format_timestamp_as_colon_seconds("2026-06-23:04-00"), + "2026-06-23:04:00:00", + ) + + def test_format_colon_seconds_from_unix_epoch(self): + epoch = str(int(datetime(2026, 6, 23, 4, 0, 0, tzinfo=timezone.utc).timestamp())) + self.assertEqual( + format_timestamp_as_colon_dash(epoch), + "2026-06-23:04-00", + ) + def test_format_sql_reshapes_without_tz_conversion(self): self.assertEqual( format_timestamp_as_sql_datetime("2026-05-12:17-00"), @@ -110,6 +197,14 @@ class CandidateOrderingTests(TestCase): # Canonical colon-dash timestamp, passed through unchanged. self.assertIn("/40/2026-05-12:19-00/22372.ts", urls[0]) + def test_accepts_colon_seconds_input_timestamp(self): + urls = build_timeshift_candidate_urls( + self.creds, "22372", "2026-06-23:04:00:00", 40 + ) + self.assertTrue(self._is_path_form(urls[0])) + self.assertIn("/40/2026-06-23:04-00/22372.ts", urls[0]) + self.assertIn("/40/2026-06-23:04:00:00/22372.ts", urls[2]) + def test_accepts_underscore_input_timestamp(self): # Client may send the underscore shape; PATH form still leads. urls = build_timeshift_candidate_urls(self.creds, "22372", "2026-05-12_19-00", 40) @@ -166,6 +261,13 @@ class ConvertTimestampToProviderTzTests(TestCase): "2026-06-08:17-00", ) + def test_utc_to_brussels_from_unix_epoch(self): + epoch = str(int(datetime(2026, 6, 8, 17, 0, 0, tzinfo=timezone.utc).timestamp())) + self.assertEqual( + convert_timestamp_to_provider_tz(epoch, "Europe/Brussels"), + "2026-06-08:19-00", + ) + def test_garbage_timestamp_passthrough(self): self.assertEqual( convert_timestamp_to_provider_tz("garbage", "Europe/Brussels"), diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 7d741784..de18238e 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -389,6 +389,13 @@ class TimeshiftProxyTimestampWiringTests(TestCase): _, _, build_mock, _, _ = self._call("2026-06-08:17-00", provider_tz="UTC") self.assertEqual(build_mock.call_args[0][2], "2026-06-08:17-00") + def test_colon_seconds_timestamp_accepted(self): + response, sentinel, build_mock, duration_mock, _ = self._call( + "2026-06-23:04:00:00" + ) + self.assertIs(response, sentinel) + self.assertEqual(duration_mock.call_args[0][1], "2026-06-23:04:00:00") + def test_invalid_timestamp_rejected_before_upstream(self): request = self.factory.get("/timeshift/u/p/8/garbage/8.ts") with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index e93777d8..6393cef0 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -53,7 +53,8 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) URL shape (iPlayTV / TiviMate): ``stream_id``: EPG channel number (ignored here). ``duration``: Dispatcharr ``Channel.id`` (XC API exposes channel.id as stream_id). - ``timestamp``: UTC programme start (``YYYY-MM-DD:HH-MM``). + ``timestamp``: UTC programme start (``YYYY-MM-DD:HH-MM`` or XC colon form + ``YYYY-MM-DD:HH:MM:SS``). """ raw_id = duration[:-3] if duration.endswith(".ts") else duration From 7289e60aa4a6424d7ab16dd7c1929b2aeb6b3d57 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 13:34:55 -0500 Subject: [PATCH 21/31] changelog: move to unreleased. --- CHANGELOG.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa5b402d..4890e407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). + - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. + - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. + - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. + - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. + - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. + - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. + - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. + ## [0.27.1] - 2026-06-25 ### Security @@ -241,15 +253,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **None** (default): software decode. - **NVIDIA NVDEC (`--cuvid`)**: requires the NVIDIA container toolkit and a supported GPU inside the container. - **Intel Quick Sync (`--qsv`)**: requires an Intel iGPU or ARC GPU with the i915 driver exposed to the container. -- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. - - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). - - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. - - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. - - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. - - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. - **HDHR output profile URL support.** HDHomeRun lineup URLs now support an `output_profile` path segment so HDHR clients (Plex, Channels DVR, Emby, etc.) can request a specific transcode profile without any query-parameter support. URL formats accepted: - `/hdhr/output_profile//lineup.json` - output profile only - `/hdhr//output_profile//lineup.json` - channel profile + output profile From ca3d0eb9e1b61dc1d7968dee32c842df0f92aeb1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 17:36:23 -0500 Subject: [PATCH 22/31] feat(timeshift): Implement session-based media ID handling and improve stream limit checks for timeshift proxy. Enhance session management by allowing distinct client/session identification for timeshift requests, ensuring better resource allocation and user experience. Update tests to validate new session handling logic. --- apps/proxy/utils.py | 16 + apps/timeshift/tests/test_views.py | 874 ++++++++++++++++++++++++++-- apps/timeshift/views.py | 896 ++++++++++++++++++++++++----- 3 files changed, 1595 insertions(+), 191 deletions(-) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index d3720801..fc157791 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -183,6 +183,22 @@ def check_user_stream_limits(user, client_id, media_id=None): logger.debug(f"[stream limits][{client_id}] Same-channel reconnect for {media_id} allowed (ignore_same_channel=True)") return True + # Timeshift sibling range/probe requests share one provider slot per + # session_id. Each distinct client/session still consumes its own slot. + if ignore_same_channel and media_id: + media_id_str = str(media_id) + for conn in active_connections: + if conn.get('type') != 'timeshift': + continue + if conn.get('client_id') != client_id: + continue + conn_media_id = str(conn.get('media_id') or '') + if conn_media_id == media_id_str or conn_media_id.startswith(f"{media_id_str}_"): + logger.debug( + f"[stream limits][{client_id}] Same timeshift session probe for {media_id} allowed (ignore_same_channel=True)" + ) + return True + if user_stream_count >= user.stream_limit: if user_limit_settings.get("terminate_on_limit_exceeded", True) == False: return False diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index de18238e..310649cb 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -1,16 +1,56 @@ """Tests for the timeshift proxy view, focused on upstream status mapping.""" +import fnmatch +import time from unittest.mock import MagicMock, patch from django.test import RequestFactory, TestCase, override_settings from apps.timeshift import views +from apps.proxy.utils import check_user_stream_limits as _check_user_stream_limits from apps.proxy.utils import find_ts_sync as _find_ts_sync +TEST_SESSION_ID = "timeshift_testsession1" +TEST_MEDIA_ID = "timeshift_8_2026-06-08-17-00" + + +def _proxy_url(session_id=TEST_SESSION_ID): + base = "/timeshift/u/p/8/2026-06-08:17-00/8.ts" + return f"{base}?session_id={session_id}" if session_id else base + + +def _seed_pool_session( + redis, + session_id=TEST_SESSION_ID, + media_id=TEST_MEDIA_ID, + *, + busy="1", + serving_range=None, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", +): + views._create_pool_session( + redis, + session_id=session_id, + media_id=media_id, + user_id=user_id, + client_ip=client_ip, + client_user_agent=client_user_agent, + account_id=1, + profile_id=31, + stream_id="111", + provider_timestamp="2026-06-08:19-00", + ) + if serving_range is not None: + redis.hset(f"timeshift_pool:{session_id}", "serving_range", serving_range) + if busy is not None: + redis.hset(f"timeshift_pool:{session_id}", "busy", busy) + class FindTsSyncTests(TestCase): """Locate the first MPEG-TS sync chain so a leading HTML/PHP preamble - can be skipped before the bytes reach a strict demuxer (ExoPlayer).""" + can be skipped before the bytes reach the strict demuxer (ExoPlayer).""" def test_returns_zero_when_buffer_already_aligned(self): buf = b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 + b"\x47" + b"\x00" * 187 @@ -231,6 +271,76 @@ class StreamFromProviderStatusMappingTests(TestCase): # PHP response was rejected, second candidate accepted self.assertEqual(mocked_open.call_count, 2) + @patch.object(views, "_open_upstream") + def test_416_range_not_satisfiable_passes_through(self, mocked_open): + # A tail/seek probe past EOF must go back to the client verbatim, + # never cascaded to other URL shapes (byte offsets are file-specific, + # so cascading only multiplies upstream connections). + resp = _fake_upstream(416) + resp.headers = {"Content-Type": "video/mp2t", "Content-Range": "bytes */1000"} + mocked_open.return_value = resp + kwargs = dict(self.kwargs, range_header="bytes=999999-") + response = views._stream_from_provider(**kwargs) + self.assertEqual(response.status_code, 416) + self.assertEqual(response["Content-Range"], "bytes */1000") + self.assertTrue(getattr(response, "timeshift_passthrough", False)) + # No cascade: the first (and only) candidate decided the outcome. + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_partial_206_to_range_request_accepted_mid_packet(self, mocked_open): + # A 206 answering a Range request legitimately starts mid-TS-packet + # (no 0x47 sync at offset 0). It must be served, not rejected as a + # PHP error and cascaded across every URL shape and provider account. + mid_packet = b"\x00" * 300 + self.assertEqual(_find_ts_sync(mid_packet), -1) + resp = _fake_upstream(206, body=mid_packet) + resp.raw = MagicMock() + resp.raw.read = MagicMock(return_value=mid_packet) + mocked_open.return_value = resp + kwargs = dict(self.kwargs, range_header="bytes=1000-") + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "_unregister_stats_client"): + response = views._stream_from_provider(**kwargs) + self.assertEqual(response.status_code, 206) + self.assertEqual(mocked_open.call_count, 1) + + @patch.object(views, "_open_upstream") + def test_partial_206_html_error_still_rejected(self, mocked_open): + # The mid-packet allowance is gated on content type: a 206 whose body + # is an HTML/PHP error page must still be rejected and cascaded. + html = b"error" + bad = _fake_upstream(206, content_type="text/html", body=html) + bad.raw = MagicMock() + bad.raw.read = MagicMock(return_value=html) + good = _fake_upstream(206, body=_make_ts_payload()) + mocked_open.side_effect = [bad, good] + kwargs = dict(self.kwargs, range_header="bytes=0-") + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "_unregister_stats_client"): + response = views._stream_from_provider(**kwargs) + self.assertEqual(response.status_code, 206) + self.assertEqual(mocked_open.call_count, 2) + + @patch.object(views, "_open_upstream") + def test_206_without_range_header_still_requires_sync(self, mocked_open): + # Without a Range header a 206 is unexpected; it must still pass the + # TS-sync probe (the mid-packet allowance is range-only). + mid_packet = b"\x00" * 300 + bad = _fake_upstream(206, body=mid_packet) + bad.raw = MagicMock() + bad.raw.read = MagicMock(return_value=mid_packet) + good = _fake_upstream(206, body=_make_ts_payload()) + mocked_open.side_effect = [bad, good] + with patch.object(views, "RedisClient"), \ + patch.object(views, "_register_stats_client"), \ + patch.object(views, "_unregister_stats_client"): + response = views._stream_from_provider(**self.kwargs) + self.assertEqual(response.status_code, 206) + self.assertEqual(mocked_open.call_count, 2) + class RedactUrlTests(TestCase): """`_redact_url` is the guard that keeps XC credentials out of logs — @@ -287,8 +397,9 @@ def _make_alt_profile(profile_id): class _FakeRedis: - """Just enough of the redis-py surface for the slot-token protocol: - setex/get/delete plus a transactional pipeline doing GET+DEL.""" + """Just enough of the redis-py surface for the idle-session pool: setex/get/ + delete plus a transactional pipeline doing GET+DEL, and the hash, set and + lock primitives the pool entries rely on.""" def __init__(self): self.store = {} @@ -311,6 +422,93 @@ class _FakeRedis: def pipeline(self, transaction=False): return _FakeRedisPipeline(self) + # --- hash + lock surface for the session slot --- + def hgetall(self, key): + value = self.store.get(key) + return dict(value) if isinstance(value, dict) else {} + + def hset(self, key, field=None, value=None, mapping=None, **kwargs): + hash_value = self.store.get(key) + if not isinstance(hash_value, dict): + hash_value = {} + self.store[key] = hash_value + if field is not None and value is not None: + hash_value[str(field)] = str(value) + for f, v in (mapping or {}).items(): + hash_value[str(f)] = str(v) + for f, v in kwargs.items(): + hash_value[str(f)] = str(v) + return len(hash_value) + + def hincrby(self, key, field, amount=1): + hash_value = self.store.get(key) + if not isinstance(hash_value, dict): + hash_value = {} + self.store[key] = hash_value + new_value = int(hash_value.get(field, 0)) + amount + hash_value[field] = str(new_value) + return new_value + + def hget(self, key, field): + hash_value = self.store.get(key) + return hash_value.get(field) if isinstance(hash_value, dict) else None + + def expire(self, key, ttl): + return 1 if key in self.store else 0 + + # --- set surface for the idle-session pool --- + def sadd(self, key, *members): + existing = self.store.get(key) + if not isinstance(existing, set): + existing = set() + self.store[key] = existing + before = len(existing) + existing.update(str(m) for m in members) + return len(existing) - before + + def srem(self, key, *members): + existing = self.store.get(key) + if not isinstance(existing, set): + return 0 + removed = 0 + for member in members: + if str(member) in existing: + existing.discard(str(member)) + removed += 1 + return removed + + def smembers(self, key): + existing = self.store.get(key) + return set(existing) if isinstance(existing, set) else set() + + def scard(self, key): + existing = self.store.get(key) + return len(existing) if isinstance(existing, set) else 0 + + def lock(self, name, timeout=None, blocking_timeout=None): + return _FakeRedisLock() + + def scan(self, cursor=0, match=None, count=100): + keys = sorted( + k for k in self.store + if match is None or fnmatch.fnmatch(k, match) + ) + if cursor >= len(keys): + return 0, [] + batch = keys[cursor:cursor + count] + next_cursor = cursor + len(batch) + if next_cursor >= len(keys): + return 0, batch + return next_cursor, batch + + +class _FakeRedisLock: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + class _FakeRedisPipeline: def __init__(self, redis): @@ -350,7 +548,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): self.factory = RequestFactory() def _call(self, timestamp, provider_tz="Europe/Brussels"): - request = self.factory.get(f"/timeshift/u/p/8/{timestamp}/8.ts") + request = self.factory.get(f"/timeshift/u/p/8/{timestamp}/8.ts?session_id={TEST_SESSION_ID}") sentinel = MagicMock(status_code=200) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=True), \ @@ -412,7 +610,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def test_network_access_denied_returns_403(self): # Same network gate as other XC API endpoints (player_api, xmltv, etc.). - request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + request = self.factory.get(_proxy_url()) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=False) as gate, \ patch.object(views, "Channel") as channel_cls, \ @@ -435,7 +633,7 @@ class TimeshiftProxyFailoverTests(TestCase): self.factory = RequestFactory() def _call(self, streams, provider_responses): - request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + request = self.factory.get(_proxy_url()) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ @@ -524,6 +722,20 @@ class TimeshiftProxyFailoverTests(TestCase): ) self.assertEqual(limits_mock.call_count, 1) + def test_passthrough_is_not_failed_over_to_other_accounts(self): + # A terminal range answer (e.g. 416 past EOF) must be returned as-is; + # the loop must NOT try the next account, whose byte offsets would not + # match this file and which would just burn another provider slot. + streams = [ + _make_catchup_stream(account_id=1, stream_id="111"), + _make_catchup_stream(account_id=2, stream_id="222"), + ] + passthrough = MagicMock(status_code=416) + passthrough.timeshift_passthrough = True + response, stream_mock, _, _ = self._call(streams, [passthrough]) + self.assertIs(response, passthrough) + self.assertEqual(stream_mock.call_count, 1) + class _ProxyLoopTestMixin: """Shared driver for tests exercising the failover loop end to end — @@ -534,7 +746,7 @@ class _ProxyLoopTestMixin: def _call(self, streams, provider_responses, limits=True, reserve_results=None, build_side_effect=None): - request = self.factory.get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + request = self.factory.get(_proxy_url()) self.fake_redis = _FakeRedis() reserve_kwargs = ( {"side_effect": reserve_results} @@ -819,11 +1031,15 @@ class AuthHelpersDbTests(TestCase): class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): """Provider pool participation: a profile slot is reserved before any - upstream attempt and released exactly once afterwards — the same - accounting contract live (Channel.get_stream) and VOD follow.""" + upstream attempt and released exactly once afterwards, the same accounting + contract live (Channel.get_stream) and VOD follow. Each active stream + reserves its own slot so concurrent provider connections stay capped by + max_streams.""" - def _slot_token_keys(self): - return [k for k in self.fake_redis.store if k.startswith("timeshift_slot:")] + POOL_KEY = f"timeshift_pool:{TEST_SESSION_ID}" + + def _pool_entry_ids(self): + return [k for k in self.fake_redis.store if k.startswith("timeshift_pool:")] def test_reserve_called_with_default_profile_before_upstream(self): streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] @@ -841,18 +1057,18 @@ class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): streams, [MagicMock(status_code=404, timeshift_decisive=False)] ) self.assertEqual(response.status_code, 404) - # The one-shot token was consumed and the pool counter decremented. + # The failed attempt's slot was released and its pool entry removed. self.release_mock.assert_called_once_with(31, self.fake_redis) - self.assertEqual(self._slot_token_keys(), []) + self.assertEqual(self._pool_entry_ids(), []) def test_slot_kept_on_success_for_the_streaming_session(self): streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] response, _, _ = self._call(streams, [MagicMock(status_code=200)]) self.assertEqual(response.status_code, 200) - # Slot still owned by the (mocked) streaming session: token present, - # nothing released yet. + # Slot still owned by the (mocked) streaming session: a busy pool entry + # remains for the next request to reuse, nothing released yet. self.release_mock.assert_not_called() - self.assertEqual(len(self._slot_token_keys()), 1) + self.assertEqual(len(self._pool_entry_ids()), 1) def test_decisive_failure_releases_slot_and_skips_account(self): streams = [ @@ -938,7 +1154,7 @@ class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): with self.assertRaises(RuntimeError): self._call(streams, RuntimeError("boom")) self.release_mock.assert_called_once_with(31, self.fake_redis) - self.assertEqual(self._slot_token_keys(), []) + self.assertEqual(self._pool_entry_ids(), []) def test_exception_before_upstream_releases_slot(self): # Same guarantee for failures BEFORE the upstream call (URL building, @@ -949,18 +1165,7 @@ class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): self._call(streams, [], build_side_effect=RuntimeError("boom")) self.stream_mock.assert_not_called() self.release_mock.assert_called_once_with(31, self.fake_redis) - self.assertEqual(self._slot_token_keys(), []) - - def test_token_store_failure_releases_directly(self): - # If the ownership token cannot be written, no release path could - # ever free the slot — the view must release it directly and report - # transient unavailability instead of streaming unaccounted. - streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] - with patch.object(views, "_store_slot_token", return_value=False): - response, stream_mock, _ = self._call(streams, []) - self.assertEqual(response.status_code, 503) - stream_mock.assert_not_called() - self.release_mock.assert_called_once_with(31, self.fake_redis) + self.assertEqual(self._pool_entry_ids(), []) def test_mixed_capacity_then_upstream_failure_returns_failure(self): # Mixed outcome: one stream capacity-blocked, another actually tried @@ -991,28 +1196,37 @@ class TimeshiftSlotPoolTests(_ProxyLoopTestMixin, TestCase): self.assertEqual(response.status_code, 404) -class TimeshiftSlotTokenTests(TestCase): - """The one-shot release token: exactly-once semantics across all release - paths (generator finally, response close, takeover, failed attempt).""" +class TimeshiftPoolReleaseTests(TestCase): + """Pool slot release and response close paths for a pooled session.""" def setUp(self): self.redis = _FakeRedis() + self.session_id = TEST_SESSION_ID - def test_release_happens_exactly_once(self): - views._store_slot_token(self.redis, "timeshift_abc", 31) + def _pool_key(self): + return f"timeshift_pool:{self.session_id}" + + def test_release_callback_frees_slot_exactly_once(self): + _seed_pool_session(self.redis, session_id=self.session_id) + release = views._make_release_once(self.redis, self.session_id, 31) with patch.object(views, "release_profile_slot") as release_mock: - self.assertTrue(views._release_slot_token(self.redis, "timeshift_abc")) - self.assertFalse(views._release_slot_token(self.redis, "timeshift_abc")) + release() + release() release_mock.assert_called_once_with(31, self.redis) + self.assertEqual(self.redis.hget(self._pool_key(), "busy"), "0") + self.assertTrue(self.redis.exists(self._pool_key())) - def test_release_without_token_is_noop(self): + def test_discard_frees_slot_and_removes_entry(self): + _seed_pool_session(self.redis, session_id=self.session_id) with patch.object(views, "release_profile_slot") as release_mock: - self.assertFalse(views._release_slot_token(self.redis, "timeshift_ghost")) - release_mock.assert_not_called() + views._discard_pool_session(self.redis, self.session_id, 31) + release_mock.assert_called_once_with(31, self.redis) + self.assertFalse(self.redis.exists(self._pool_key())) def test_release_without_redis_is_noop(self): + release = views._make_release_once(None, self.session_id, 31) with patch.object(views, "release_profile_slot") as release_mock: - self.assertFalse(views._release_slot_token(None, "timeshift_abc")) + release() release_mock.assert_not_called() def test_wrapper_close_releases_even_when_generator_never_started(self): @@ -1046,10 +1260,11 @@ class TimeshiftSlotTokenTests(TestCase): class TimeshiftTakeoverTests(TestCase): - """One catch-up session per user+channel: a new request displaces the - user's previous session on the same channel — synchronous slot release + - stats unregister + stop key — and never touches other users, other - channels, or live sessions.""" + """A new request displaces the user's previous catch-up session(s) on the + same channel at a DIFFERENT position (stats unregister + stop key, leaving + the displaced generator to free its own slot), while leaving sibling range + requests of the same playback alone, and never touching other users, + channels, or live.""" def setUp(self): self.redis = _FakeRedis() @@ -1063,9 +1278,7 @@ class TimeshiftTakeoverTests(TestCase): "type": conn_type, } - def test_displaces_only_same_channel_timeshift_sessions(self): - views._store_slot_token(self.redis, "timeshift_old1", 31) - views._store_slot_token(self.redis, "timeshift_other", 41) + def test_displaces_other_positions_on_same_channel(self): connections = [ self._conn("timeshift_8_2026-06-08-17-00_111", "timeshift_old1"), self._conn("timeshift_9_2026-06-08-17-00_222", "timeshift_other"), @@ -1075,10 +1288,13 @@ class TimeshiftTakeoverTests(TestCase): return_value=connections) as conns_mock, \ patch.object(views, "release_profile_slot") as release_mock, \ patch.object(views, "_unregister_stats_client") as unregister_mock: - views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + views._terminate_previous_timeshift_sessions( + self.redis, self.user, 8, "timeshift_8_2026-06-09-20-00", "timeshift_current", + ) conns_mock.assert_called_once_with(5) - # Channel 8's old session: slot released, stats dropped, stop key set. - release_mock.assert_called_once_with(31, self.redis) + # Takeover defers slot release to the displaced generator's stop path; + # it only drops stats and signals the stop key. + release_mock.assert_not_called() unregister_mock.assert_called_once_with( self.redis, "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" ) @@ -1087,8 +1303,28 @@ class TimeshiftTakeoverTests(TestCase): "timeshift_8_2026-06-08-17-00_111", "timeshift_old1" ) self.assertIn(stop_key, self.redis.store) - # Channel 9's session untouched: token still present, no stop key. - self.assertIn("timeshift_slot:timeshift_other", self.redis.store) + # Channel 9's session untouched: no stop key set for it. + other_stop = RedisKeys.client_stop( + "timeshift_9_2026-06-08-17-00_222", "timeshift_other" + ) + self.assertNotIn(other_stop, self.redis.store) + + def test_leaves_sibling_requests_of_current_playback(self): + # Concurrent range/probe requests of the SAME playback must not + # displace one another. + connections = [ + self._conn("timeshift_8_2026-06-08-17-00_111", "timeshift_sibling"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections), \ + patch.object(views, "release_profile_slot") as release_mock, \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._terminate_previous_timeshift_sessions( + self.redis, self.user, 8, "timeshift_8_2026-06-08-17-00", + "timeshift_sibling", + ) + release_mock.assert_not_called() + unregister_mock.assert_not_called() def test_channel_id_prefix_cannot_match_other_channels(self): # Channel 8 must not displace channel 80/81 sessions (prefix ends @@ -1100,14 +1336,21 @@ class TimeshiftTakeoverTests(TestCase): return_value=connections), \ patch.object(views, "release_profile_slot") as release_mock, \ patch.object(views, "_unregister_stats_client") as unregister_mock: - views._terminate_previous_timeshift_sessions(self.redis, self.user, 8) + views._terminate_previous_timeshift_sessions( + self.redis, self.user, 8, "timeshift_8_2026-06-08-17-00", + "timeshift_new", + ) release_mock.assert_not_called() unregister_mock.assert_not_called() def test_noop_without_redis_or_user(self): with patch.object(views, "get_user_active_connections") as conns_mock: - views._terminate_previous_timeshift_sessions(None, self.user, 8) - views._terminate_previous_timeshift_sessions(self.redis, None, 8) + views._terminate_previous_timeshift_sessions( + None, self.user, 8, "timeshift_8_ts", "timeshift_s" + ) + views._terminate_previous_timeshift_sessions( + self.redis, None, 8, "timeshift_8_ts", "timeshift_s" + ) conns_mock.assert_not_called() def test_proxy_runs_takeover_before_stream_limit_check(self): @@ -1115,7 +1358,7 @@ class TimeshiftTakeoverTests(TestCase): # displace its own predecessor BEFORE the limit check counts it, or # the user's own seek gets denied. call_order = [] - request = RequestFactory().get("/timeshift/u/p/8/2026-06-08:17-00/8.ts") + request = RequestFactory().get(_proxy_url()) with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ @@ -1138,6 +1381,525 @@ class TimeshiftTakeoverTests(TestCase): self.assertEqual(takeover_mock.call_args.args[2], 8) +class TimeshiftSessionReuseTests(TestCase): + """Per-client session pool acquire/reuse paths.""" + + SESSION = TEST_SESSION_ID + + def setUp(self): + self.redis = _FakeRedis() + self.factory = RequestFactory() + self.channel = MagicMock(id=8, name="Test") + self.user = MagicMock(id=5) + + def _pool_key(self): + return f"timeshift_pool:{self.SESSION}" + + def _make_idle_entry(self): + _seed_pool_session(self.redis, session_id=self.SESSION) + with patch.object(views, "release_profile_slot"): + views._release_pool_session(self.redis, self.SESSION, 31) + + def test_wait_returns_none_without_blocking_when_pool_empty(self): + start = time.monotonic() + acquired = views._wait_for_idle_pool_session(self.redis, self.SESSION) + self.assertIsNone(acquired) + self.assertLess(time.monotonic() - start, 0.5) + + def test_acquire_reuses_idle_entry_and_reserves_slot(self): + self._make_idle_entry() + profile = MagicMock(id=31) + with patch.object(views.M3UAccountProfile.objects, "get", + return_value=profile), \ + patch.object(views, "reserve_profile_slot", + return_value=(True, 1, None)) as reserve_mock: + acquired = views._acquire_idle_pool_session(self.redis, self.SESSION) + self.assertIsNotNone(acquired) + descriptor, got_profile = acquired + self.assertEqual(descriptor["stream_id"], "111") + self.assertIs(got_profile, profile) + reserve_mock.assert_called_once_with(profile, self.redis) + self.assertEqual(self.redis.hget(self._pool_key(), "busy"), "1") + + def test_acquire_skips_busy_entry(self): + _seed_pool_session(self.redis, session_id=self.SESSION, busy="1") + with patch.object(views.M3UAccountProfile.objects, "get") as prof_mock, \ + patch.object(views, "reserve_profile_slot") as reserve_mock: + acquired = views._acquire_idle_pool_session(self.redis, self.SESSION) + self.assertIsNone(acquired) + prof_mock.assert_not_called() + reserve_mock.assert_not_called() + + def test_find_matching_idle_session_requires_ip_and_user_agent(self): + _seed_pool_session( + self.redis, session_id="timeshift_other", + user_id=5, client_ip="1.2.3.4", client_user_agent="test-agent", + ) + with patch.object(views, "release_profile_slot"): + views._release_pool_session(self.redis, "timeshift_other", 31) + matched = views._find_matching_idle_session( + self.redis, + media_id=TEST_MEDIA_ID, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", + ) + self.assertEqual(matched, "timeshift_other") + + def test_find_matching_idle_session_rejects_ip_only_partial_fingerprint(self): + _seed_pool_session( + self.redis, session_id="timeshift_other", + user_id=5, client_ip="1.2.3.4", client_user_agent="other-agent", + ) + with patch.object(views, "release_profile_slot"): + views._release_pool_session(self.redis, "timeshift_other", 31) + matched = views._find_matching_idle_session( + self.redis, + media_id=TEST_MEDIA_ID, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", + ) + self.assertIsNone(matched) + + def test_find_matching_idle_session_rejects_different_user(self): + _seed_pool_session( + self.redis, session_id="timeshift_other", + user_id=99, client_ip="1.2.3.4", client_user_agent="test-agent", + ) + with patch.object(views, "release_profile_slot"): + views._release_pool_session(self.redis, "timeshift_other", 31) + matched = views._find_matching_idle_session( + self.redis, + media_id=TEST_MEDIA_ID, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", + ) + self.assertIsNone(matched) + + def test_legacy_pool_entry_exists_helper_removed(self): + self.assertFalse(hasattr(views, "_pool_entry_exists")) + + def test_new_session_uses_single_hgetall_before_pool_create(self): + redis = _FakeRedis() + request = self.factory.get(_proxy_url("timeshift_newsession1")) + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "_find_matching_idle_session", return_value=None), \ + patch.object(views, "_attempt_timeshift_stream", + return_value=MagicMock(status_code=200)), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]): + redis_cls.get_client.return_value = redis + channel_cls.objects.get.return_value = MagicMock(id=8, name="Test", logo_id=None) + with patch.object(redis, "hgetall", wraps=redis.hgetall) as hgetall_mock: + views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + pool_key = "timeshift_pool:timeshift_newsession1" + self.assertEqual( + sum(1 for c in hgetall_mock.call_args_list if c.args == (pool_key,)), + 1, + ) + + +class TimeshiftSessionRedirectTests(TestCase): + """First request must establish a session via 301 redirect (VOD-style).""" + + def setUp(self): + self.factory = RequestFactory() + + def test_missing_session_id_redirects(self): + request = self.factory.get(_proxy_url(session_id=None)) + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=1)), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "parse_catchup_timestamp", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls: + redis_cls.get_client.return_value = _FakeRedis() + channel_cls.objects.get.return_value = MagicMock(id=8) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 301) + self.assertIn("session_id=timeshift_", response["Location"]) + + def test_redirect_preserves_existing_query_params(self): + request = self.factory.get( + "/timeshift/u/p/8/2026-06-08:17-00/8.ts?foo=bar&baz=1", + ) + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=1)), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "parse_catchup_timestamp", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls: + redis_cls.get_client.return_value = _FakeRedis() + channel_cls.objects.get.return_value = MagicMock(id=8) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 301) + location = response["Location"] + self.assertIn("session_id=timeshift_", location) + self.assertIn("foo=bar", location) + self.assertIn("baz=1", location) + + +class TimeshiftStreamLimitExemptionTests(TestCase): + """Timeshift stream-limit bypass requires the same client session.""" + + MEDIA = TEST_MEDIA_ID + + def setUp(self): + self.user = MagicMock(id=5, username="viewer", stream_limit=1) + + def _limits_settings(self, ignore_same_channel=True): + return { + "ignore_same_channel_connections": ignore_same_channel, + "terminate_on_limit_exceeded": False, + } + + def test_same_session_probe_allowed_at_limit(self): + connections = [{ + "media_id": f"{self.MEDIA}_111", + "client_id": TEST_SESSION_ID, + "connected_at": 0.0, + "type": "timeshift", + }] + with patch("apps.proxy.utils.get_user_active_connections", + return_value=connections), \ + patch("apps.proxy.utils.CoreSettings.get_user_limits_settings", + return_value=self._limits_settings()): + allowed = _check_user_stream_limits( + self.user, TEST_SESSION_ID, media_id=self.MEDIA, + ) + self.assertTrue(allowed) + + def test_different_session_same_programme_counts_against_limit(self): + connections = [{ + "media_id": f"{self.MEDIA}_111", + "client_id": "timeshift_other_session", + "connected_at": 0.0, + "type": "timeshift", + }] + with patch("apps.proxy.utils.get_user_active_connections", + return_value=connections), \ + patch("apps.proxy.utils.CoreSettings.get_user_limits_settings", + return_value=self._limits_settings()): + allowed = _check_user_stream_limits( + self.user, TEST_SESSION_ID, media_id=self.MEDIA, + ) + self.assertFalse(allowed) + + +class FakeRedisScanTests(TestCase): + """FakeRedis SCAN matches redis-py glob semantics used by the pool scanner.""" + + def setUp(self): + self.redis = _FakeRedis() + self.redis.store["timeshift_pool:timeshift_a"] = {"busy": "0"} + self.redis.store["timeshift_pool:timeshift_b"] = {"busy": "0"} + self.redis.store["timeshift_pool:other_c"] = {"busy": "0"} + self.redis.store["vod_persistent_connection:x"] = {} + + def test_scan_glob_filters_pool_keys(self): + cursor = 0 + seen = [] + while True: + cursor, keys = self.redis.scan( + cursor, match="timeshift_pool:timeshift_*", count=1, + ) + seen.extend(keys) + if cursor == 0: + break + self.assertEqual( + seen, + ["timeshift_pool:timeshift_a", "timeshift_pool:timeshift_b"], + ) + + +class TimeshiftRangeClassificationTests(TestCase): + """Startup probes must not be treated as scrubs.""" + + def test_full_file_request_is_not_displacing(self): + self.assertFalse(views._should_displace_busy_playback(None)) + + def test_bytes_zero_displaces_full_file_probe(self): + self.assertTrue( + views._should_displace_busy_playback("bytes=0-", busy_serving_range="none") + ) + + def test_bytes_zero_does_not_displace_active_start_stream(self): + self.assertFalse( + views._should_displace_busy_playback("bytes=0-", busy_serving_range="start") + ) + + def test_bytes_zero_without_busy_context_is_not_displacing(self): + self.assertFalse(views._should_displace_busy_playback("bytes=0-")) + + def test_near_eof_probe_is_not_displacing(self): + self.assertTrue(views._is_near_eof_probe("bytes=2527702896-")) + self.assertFalse(views._should_displace_busy_playback("bytes=2527702896-")) + + def test_near_eof_probe_uses_cached_content_length(self): + # 5 MB into a 10 MB file is a scrub, not a tail probe. + self.assertFalse( + views._is_near_eof_probe("bytes=5000000-", content_length="10000000") + ) + self.assertTrue( + views._should_displace_busy_playback("bytes=5000000-", content_length="10000000") + ) + # Within 512 KB of EOF is a tail probe once length is known. + self.assertTrue( + views._is_near_eof_probe("bytes=9990000-", content_length="10000000") + ) + + def test_midfile_seek_is_displacing(self): + self.assertTrue(views._should_displace_busy_playback("bytes=5000000-")) + + def test_small_nonzero_range_is_displacing(self): + self.assertTrue(views._should_displace_busy_playback("bytes=1000-")) + + +class TimeshiftScrubPreemptTests(TestCase): + """Scrub/range requests must stop the in-flight stream and reuse the pooled + provider slot instead of opening parallel upstream connections.""" + + def setUp(self): + self.redis = _FakeRedis() + self.user = MagicMock(id=5) + self.factory = RequestFactory() + + def _conn(self, media_id, client_id): + return { + "media_id": media_id, + "client_id": client_id, + "connected_at": 0.0, + "type": "timeshift", + } + + def test_preempt_stops_sibling_clients_of_same_playback(self): + connections = [ + self._conn(f"{TEST_MEDIA_ID}_111", TEST_SESSION_ID), + self._conn("timeshift_9_2026-06-08-17-00_222", "timeshift_other"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections), \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._preempt_playback_streams(self.redis, TEST_SESSION_ID, self.user) + unregister_mock.assert_called_once_with( + self.redis, f"{TEST_MEDIA_ID}_111", TEST_SESSION_ID, + ) + from apps.proxy.live_proxy.redis_keys import RedisKeys + stop_key = RedisKeys.client_stop(f"{TEST_MEDIA_ID}_111", TEST_SESSION_ID) + self.assertIn(stop_key, self.redis.store) + + def test_preempt_leaves_other_playbacks_alone(self): + connections = [ + self._conn("timeshift_8_2026-06-09-20-00_111", "timeshift_other_pos"), + ] + with patch.object(views, "get_user_active_connections", + return_value=connections), \ + patch.object(views, "_unregister_stats_client") as unregister_mock: + views._preempt_playback_streams(self.redis, TEST_SESSION_ID, self.user) + unregister_mock.assert_not_called() + + def test_busy_pool_returns_503_instead_of_second_provider_connection(self): + _seed_pool_session(self.redis, session_id=TEST_SESSION_ID) + request = self.factory.get( + _proxy_url(TEST_SESSION_ID), + HTTP_RANGE="bytes=1000-", + ) + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ + patch.object(views, "_preempt_playback_streams") as preempt_mock, \ + patch.object(views, "_wait_for_idle_pool_session", return_value=None), \ + patch.object(views, "_attempt_timeshift_stream") as attempt_mock: + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock( + id=8, name="Test", logo_id=None, + ) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 503) + preempt_mock.assert_called_once() + attempt_mock.assert_not_called() + self.assertEqual(len(self._pool_entry_ids()), 1) + + def _pool_entry_ids(self): + return [k for k in self.redis.store if k.startswith("timeshift_pool:")] + + def test_startup_bytes_zero_deferred_without_preempt(self): + _seed_pool_session( + self.redis, session_id=TEST_SESSION_ID, serving_range="start", + ) + request = self.factory.get( + _proxy_url(TEST_SESSION_ID), + HTTP_RANGE="bytes=0-", + ) + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ + patch.object(views, "_preempt_playback_streams") as preempt_mock, \ + patch.object(views, "_attempt_timeshift_stream") as attempt_mock: + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock( + id=8, name="Test", logo_id=None, + ) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 503) + preempt_mock.assert_not_called() + attempt_mock.assert_not_called() + + def test_eof_probe_deferred_without_preempt(self): + _seed_pool_session(self.redis, session_id=TEST_SESSION_ID) + request = self.factory.get( + _proxy_url(TEST_SESSION_ID), + HTTP_RANGE="bytes=2527702896-", + ) + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)), \ + patch.object(views, "release_profile_slot"), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ + patch.object(views, "_preempt_playback_streams") as preempt_mock, \ + patch.object(views, "_attempt_timeshift_stream") as attempt_mock: + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock( + id=8, name="Test", logo_id=None, + ) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 503) + preempt_mock.assert_not_called() + attempt_mock.assert_not_called() + + def test_create_pool_session_rejects_duplicate_entry(self): + first = views._create_pool_session( + self.redis, + session_id=TEST_SESSION_ID, + media_id=TEST_MEDIA_ID, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", + account_id=1, + profile_id=31, + stream_id="111", + provider_timestamp="2026", + ) + second = views._create_pool_session( + self.redis, + session_id=TEST_SESSION_ID, + media_id=TEST_MEDIA_ID, + user_id=5, + client_ip="1.2.3.4", + client_user_agent="test-agent", + account_id=2, + profile_id=41, + stream_id="222", + provider_timestamp="2026", + ) + self.assertTrue(first) + self.assertFalse(second) + self.assertTrue(self.redis.exists(f"timeshift_pool:{TEST_SESSION_ID}")) + + def test_scrub_reuses_idle_pool_without_opening_failover(self): + _seed_pool_session(self.redis, session_id=TEST_SESSION_ID) + with patch.object(views, "release_profile_slot"): + views._release_pool_session(self.redis, TEST_SESSION_ID, 31) + + request = self.factory.get( + _proxy_url(TEST_SESSION_ID), + HTTP_RANGE="bytes=5000-", + ) + streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] + profile = MagicMock(id=31) + ok = MagicMock(status_code=206) + with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", return_value=streams), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "check_user_stream_limits", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "reserve_profile_slot", + return_value=(True, 1, None)) as reserve_mock, \ + patch.object(views, "release_profile_slot"), \ + patch.object(views.M3UAccountProfile.objects, "get", + return_value=profile), \ + patch.object(views, "get_transformed_credentials", side_effect=_fake_creds), \ + patch.object(views, "get_user_active_connections", return_value=[]), \ + patch.object(views, "_preempt_playback_streams") as preempt_mock, \ + patch.object(views, "_stream_reused_session", return_value=ok) as reuse_mock, \ + patch.object(views, "_attempt_timeshift_stream") as attempt_mock: + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock( + id=8, name="Test", logo_id=None, + ) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertIs(response, ok) + preempt_mock.assert_not_called() + reuse_mock.assert_called_once() + attempt_mock.assert_not_called() + # Pool acquire re-reserves the idle slot once; failover must not add another. + reserve_mock.assert_called_once_with(profile, self.redis) + + + class RollupSelfHealDbTests(TestCase): """Catch-up flag consistency after stream removal — the review-#4 point 1 guarantees: the ChannelStream signal handles bulk deletes (locked by a diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 6393cef0..113cd460 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -3,8 +3,9 @@ import hmac import itertools import logging +import random import time -import uuid +from urllib.parse import urlencode import requests from django.core.cache import cache @@ -21,6 +22,7 @@ from apps.accounts.models import User from apps.channels.models import Channel from apps.channels.utils import get_channel_catchup_streams from apps.m3u.connection_pool import release_profile_slot, reserve_profile_slot +from apps.m3u.models import M3UAccount, M3UAccountProfile from apps.m3u.tasks import get_transformed_credentials from apps.proxy.live_proxy.config_helper import ConfigHelper from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState @@ -45,6 +47,7 @@ from .helpers import ( logger = logging.getLogger(__name__) CLIENT_TTL_SECONDS = 60 +_MATCH_SCORE_THRESHOLD = 8 # client_ip (5) + client_user_agent (3) def timeshift_proxy(request, username, password, stream_id, timestamp, duration): # noqa: ARG001 stream_id @@ -87,21 +90,138 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) duration_minutes = get_programme_duration(channel, timestamp) safe_ts = timestamp.replace(":", "-").replace("/", "-") - client_id = f"timeshift_{uuid.uuid4().hex[:16]}" client_ip = get_client_ip(request) + client_user_agent = request.META.get("HTTP_USER_AGENT", "") or "" range_header = request.META.get("HTTP_RANGE") channel_logo_id = getattr(channel, "logo_id", None) redis_client = RedisClient.get_client() - # Displace any prior catch-up session on this channel before reserving a slot. - _terminate_previous_timeshift_sessions(redis_client, user, channel.id) + # Content identity (channel + catch-up position). Provider slot sharing is + # scoped per client session; never assume all requests for the same + # programme belong to one viewer. + media_id = f"timeshift_{channel.id}_{safe_ts}" - if not check_user_stream_limits( - user, client_id, media_id=f"timeshift_{channel.id}_{safe_ts}" - ): + session_id = request.GET.get("session_id") + if not session_id: + session_id = f"timeshift_{int(time.time() * 1000)}_{random.randint(1000, 9999)}" + query_params = {k: request.GET.getlist(k) for k in request.GET} + query_params["session_id"] = [session_id] + redirect_url = f"{request.path}?{urlencode(query_params, doseq=True)}" + logger.debug("Timeshift session redirect: %s -> %s", request.path, session_id) + return HttpResponse(status=301, headers={"Location": redirect_url}) + + # Stable client identity for stats, stop keys, and the provider pool. + effective_session_id = session_id + client_id = session_id + + session_entry = _get_pool_entry(redis_client, session_id) + + # Reuse an idle pool owned by this session, or fingerprint-match a prior + # idle session from the same client (VOD-style) before opening upstream. + if not session_entry: + matched = _find_matching_idle_session( + redis_client, + media_id=media_id, + user_id=user.id, + client_ip=client_ip, + client_user_agent=client_user_agent, + ) + if matched: + logger.info( + "Timeshift fingerprint matched idle session %s for %s", + matched, session_id, + ) + effective_session_id = matched + client_id = matched + + if debug: + if effective_session_id != session_id: + logger.debug( + "Timeshift request: channel=%s media=%s session=%s " + "effective=%s user=%s range=%s ip=%s", + channel.name, media_id, session_id, effective_session_id, + user.id, range_header or "(none)", client_ip, + ) + else: + logger.debug( + "Timeshift request: channel=%s media=%s session=%s " + "user=%s range=%s ip=%s", + channel.name, media_id, effective_session_id, user.id, + range_header or "(none)", client_ip, + ) + + # Displace this user's prior catch-up on other positions of this channel. + _terminate_previous_timeshift_sessions( + redis_client, user, channel.id, media_id, effective_session_id, + ) + + if not check_user_stream_limits(user, client_id, media_id=media_id): return HttpResponseForbidden("Stream limit exceeded") + if effective_session_id == session_id: + pool = _snapshot_from_entry(session_entry) + else: + pool = _pool_snapshot(redis_client, effective_session_id) + pool_exists = pool is not None + pool_busy = pool["busy"] if pool else False + pool_content_length = pool["content_length"] if pool else None + busy_serving_range = pool["serving_range"] if pool else None + + acquired = None + if pool_exists: + if pool_busy: + if _should_displace_busy_playback( + range_header, pool_content_length, busy_serving_range, + ): + _preempt_playback_streams(redis_client, effective_session_id, user) + acquired = _wait_for_idle_pool_session( + redis_client, + effective_session_id, + wait_seconds=_POOL_PREEMPT_WAIT_SECONDS, + ) + else: + acquired = _acquire_idle_pool_session( + redis_client, effective_session_id, + ) + + if acquired is not None: + descriptor, profile = acquired + reuse_response = _stream_reused_session( + redis_client, + session_id=effective_session_id, + descriptor=descriptor, + profile=profile, + channel=channel, + safe_ts=safe_ts, + timestamp=timestamp, + duration_minutes=duration_minutes, + client_id=client_id, + client_ip=client_ip, + range_header=range_header, + channel_logo_id=channel_logo_id, + user=user, + debug=debug, + ) + if reuse_response is not None: + return reuse_response + + if pool_exists and pool_busy and not _should_displace_busy_playback( + range_header, pool_content_length, busy_serving_range, + ): + logger.debug( + "Timeshift: deferring non-displacing request for session %s range=%s", + effective_session_id, range_header or "(none)", + ) + return HttpResponse("Stream slot busy", status=503) + + if pool_exists and pool_busy: + logger.warning( + "Timeshift: session %s did not become idle in time", + effective_session_id, + ) + return HttpResponse("Stream slot busy", status=503) + last_response = None decisive_accounts = set() capacity_blocked = False @@ -157,75 +277,71 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) ) continue - token_stored = False - if redis_client is not None: - token_stored = _store_slot_token( - redis_client, client_id, reserved_profile.id + if not _create_pool_session( + redis_client, + session_id=effective_session_id, + media_id=media_id, + user_id=user.id, + client_ip=client_ip, + client_user_agent=client_user_agent, + account_id=m3u_account.id, + profile_id=reserved_profile.id, + stream_id=stream_id_value, + provider_timestamp=provider_timestamp, + ): + try: + release_profile_slot(reserved_profile.id, redis_client) + except Exception as exc: + logger.warning( + "Timeshift slot release failed after pool race on profile %s: %s", + reserved_profile.id, exc, + ) + logger.debug( + "Timeshift: pool entry already exists for session %s, deferring", + effective_session_id, ) - if not token_stored: - # Release immediately if we cannot record ownership for later cleanup. - try: - release_profile_slot(reserved_profile.id, redis_client) - except Exception as exc: - logger.error( - "Timeshift: could not release slot for profile %s " - "after token-store failure: %s", reserved_profile.id, exc, - ) - capacity_blocked = True - continue + return HttpResponse("Stream slot busy", status=503) + release_cb = _make_release_once( + redis_client, effective_session_id, reserved_profile.id + ) try: - # Failures before streaming starts must release the reserved slot. - # Use credentials for the profile whose slot was reserved. - server_url, xc_username, xc_password = get_transformed_credentials( - m3u_account, reserved_profile - ) - creds = TimeshiftCredentials(server_url, xc_username, xc_password) - - candidate_urls = build_timeshift_candidate_urls( - creds, stream_id_value, provider_timestamp, duration_minutes - ) - - try: - user_agent = m3u_account.get_user_agent().user_agent - except AttributeError: - user_agent = "" - - virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" - - if debug: - logger.debug( - "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " - "account=%s profile=%s provider_sid=%s vid=%s client=%s range=%s", - channel.name, timestamp, provider_tz_name, provider_timestamp, - m3u_account.id, reserved_profile.id, stream_id_value, - virtual_channel_id, client_id, range_header or "(none)", - ) - - response = _stream_from_provider( - candidate_urls=candidate_urls, - user_agent=user_agent, - range_header=range_header, - virtual_channel_id=virtual_channel_id, + response = _attempt_timeshift_stream( + m3u_account=m3u_account, + profile=reserved_profile, + stream_id_value=stream_id_value, + provider_timestamp=provider_timestamp, + provider_tz_name=provider_tz_name, + duration_minutes=duration_minutes, + channel=channel, + safe_ts=safe_ts, + timestamp=timestamp, client_id=client_id, client_ip=client_ip, - user=user, - channel_display_name=channel.name, - timestamp_utc=timestamp, + range_header=range_header, channel_logo_id=channel_logo_id, - m3u_profile_id=reserved_profile.id, - debug=debug, - account_id=m3u_account.id, + user=user, redis_client=redis_client, + debug=debug, + release_cb=release_cb, + pool_session_id=effective_session_id, ) except Exception: - if token_stored: - _release_slot_token(redis_client, client_id) + _discard_pool_session(redis_client, effective_session_id, reserved_profile.id) raise if response.status_code < 400: + # Streaming: the generator's close path frees the slot via release_cb. return response - _release_slot_token(redis_client, client_id) + if getattr(response, "timeshift_passthrough", False) is True: + # Terminal range answer (e.g. 416 past EOF): the upstream session is + # healthy, so free the slot but keep the entry idle for the next + # probe, and return verbatim without failing over to other accounts. + release_cb() + return response + + # Real failure: drop this session entirely and fail over. + _discard_pool_session(redis_client, effective_session_id, reserved_profile.id) last_response = response if getattr(response, "timeshift_decisive", False): decisive_accounts.add(m3u_account.id) @@ -276,56 +392,367 @@ def _user_can_access_channel(user, channel): ) -# One-shot Redis token: whichever release path runs first frees the pool slot. -_SLOT_TOKEN_KEY = "timeshift_slot:{client_id}" -_SLOT_TOKEN_TTL = 24 * 3600 +# Per-client session pool (keyed by session_id from the 301 redirect). Each +# viewer gets their own provider slot even when watching the same catch-up +# programme. Idle sessions can be fingerprint-matched (VOD-style) when a client +# returns without its prior session_id. +_POOL_KEY = "timeshift_pool:{session_id}" +_POOL_LOCK_KEY = "timeshift_pool_lock:{session_id}" +_POOL_ENTRY_TTL = 6 * 3600 +_POOL_IDLE_TTL = 30 +_POOL_WAIT_SECONDS = 1.0 +_POOL_PREEMPT_WAIT_SECONDS = 5.0 +_POOL_POLL_INTERVAL = 0.05 +_EOF_PROBE_TAIL_BYTES = 512_000 +_EOF_PROBE_UNKNOWN_LENGTH_MIN = 100_000_000 -def _store_slot_token(redis_client, client_id, profile_id): - """Store a one-shot Redis token mapping *client_id* to *profile_id*. +def _pool_key(session_id): + return _POOL_KEY.format(session_id=session_id) - Returns False if the token could not be written; the caller must release - the reserved slot directly in that case. - """ - if redis_client is None: - return False + +def _parse_range_start(range_header): + """Return the byte offset from a ``Range: bytes=START-`` header, or None.""" + if not range_header or not range_header.startswith("bytes="): + return None + start_part = range_header[6:].split("-", 1)[0] + if not start_part: + return None try: - redis_client.setex( - _SLOT_TOKEN_KEY.format(client_id=client_id), - _SLOT_TOKEN_TTL, - str(profile_id), + return int(start_part) + except (TypeError, ValueError): + return None + + +def _is_near_eof_probe(range_header, content_length=None): + """True for tail/duration probes IPTV clients fire during startup.""" + start = _parse_range_start(range_header) + if start is None: + return False + if content_length is not None: + try: + total = int(content_length) + except (TypeError, ValueError): + total = None + else: + return start >= max(0, total - _EOF_PROBE_TAIL_BYTES) + return start >= _EOF_PROBE_UNKNOWN_LENGTH_MIN + + +def _should_displace_busy_playback( + range_header, content_length=None, busy_serving_range=None, +): + """True when this request should stop the in-flight stream (actual scrub).""" + if not range_header: + return False + start = _parse_range_start(range_header) + if start is None: + return False + if _is_near_eof_probe(range_header, content_length): + return False + if start == 0: + # Only displace a known full-file probe; unknown busy context is not a scrub. + return busy_serving_range == "none" + return True + + +def _score_pool_fingerprint(entry, client_ip, client_user_agent): + """Score IP/UA overlap for fingerprint adoption (user and media pre-filtered).""" + score = 0 + if entry.get("client_ip") and entry.get("client_ip") == client_ip: + score += 5 + if entry.get("client_user_agent") and entry.get("client_user_agent") == client_user_agent: + score += 3 + return score + + +def _find_matching_idle_session( + redis_client, *, media_id, user_id, client_ip, client_user_agent, +): + """Find an idle pooled session that likely belongs to the same client.""" + if redis_client is None: + return None + matches = [] + try: + cursor = 0 + while True: + cursor, keys = redis_client.scan( + cursor, match="timeshift_pool:timeshift_*", count=100, + ) + for key in keys: + try: + data = redis_client.hgetall(key) + if not data or data.get("busy") == "1": + continue + if str(data.get("user_id") or "") != str(user_id): + continue + if str(data.get("media_id") or "") != str(media_id): + continue + session_id = key.rsplit(":", 1)[-1] + score = _score_pool_fingerprint( + data, client_ip, client_user_agent, + ) + if score >= _MATCH_SCORE_THRESHOLD: + last_activity = float(data.get("last_activity") or "0") + matches.append((session_id, score, last_activity)) + except Exception as exc: + logger.debug("Timeshift pool scan skip %s: %s", key, exc) + if cursor == 0: + break + except Exception as exc: + logger.warning("Timeshift idle session search failed: %s", exc) + return None + + if not matches: + return None + matches.sort(key=lambda item: (item[1], item[2]), reverse=True) + best = matches[0][0] + logger.debug( + "Timeshift idle match: session=%s score=%s media=%s", + best, matches[0][1], media_id, + ) + return best + + +def _get_pool_entry(redis_client, session_id): + if redis_client is None or not session_id: + return {} + try: + return redis_client.hgetall(_pool_key(session_id)) or {} + except Exception: + return {} + + +def _snapshot_from_entry(entry): + if not entry: + return None + busy = entry.get("busy") == "1" + return { + "entry": entry, + "busy": busy, + "serving_range": (entry.get("serving_range") or "none") if busy else None, + "content_length": entry.get("content_length"), + } + + +def _pool_snapshot(redis_client, session_id): + """Single HGETALL view of pool state for request handling.""" + return _snapshot_from_entry(_get_pool_entry(redis_client, session_id)) + + +def _store_pool_serving_range(redis_client, session_id, range_header): + if redis_client is None or not session_id: + return + start = _parse_range_start(range_header) + if not range_header: + serving_range = "none" + elif start == 0: + serving_range = "start" + else: + serving_range = "range" + try: + redis_client.hset(_pool_key(session_id), "serving_range", serving_range) + except Exception as exc: + logger.debug("Timeshift pool serving_range store failed: %s", exc) + + +def _store_pool_content_length(redis_client, session_id, upstream_response): + if redis_client is None or not session_id or upstream_response is None: + return + content_length = upstream_response.headers.get("Content-Length") + content_range = upstream_response.headers.get("Content-Range", "") + if content_range and "/" in content_range: + total = content_range.rsplit("/", 1)[-1] + if total != "*": + content_length = total + if not content_length: + return + try: + redis_client.hset( + _pool_key(session_id), "content_length", str(content_length), ) - return True except Exception as exc: - logger.warning("Timeshift slot token store failed for %s: %s", client_id, exc) - return False + logger.debug("Timeshift pool content_length store failed: %s", exc) -def _release_slot_token(redis_client, client_id): - """Release the profile slot owned by *client_id* (at most once via GET+DEL).""" - if redis_client is None: - return False - key = _SLOT_TOKEN_KEY.format(client_id=client_id) +def _pool_lock(redis_client, session_id): + return redis_client.lock( + _POOL_LOCK_KEY.format(session_id=session_id), + timeout=10, + blocking_timeout=5, + ) + + +def _acquire_idle_pool_session(redis_client, session_id): + """Re-reserve an idle session's profile slot and mark it busy.""" + if redis_client is None or not session_id: + return None + key = _pool_key(session_id) try: - pipe = redis_client.pipeline(transaction=True) - pipe.get(key) - pipe.delete(key) - token_value, deleted = pipe.execute() - if not token_value or not deleted: - return False - release_profile_slot(int(token_value), redis_client) + with _pool_lock(redis_client, session_id): + data = redis_client.hgetall(key) + if not data or not data.get("profile_id"): + return None + if data.get("busy") == "1": + return None + try: + profile = M3UAccountProfile.objects.get(id=int(data["profile_id"])) + except M3UAccountProfile.DoesNotExist: + redis_client.delete(key) + return None + reserved, _count, _reason = reserve_profile_slot(profile, redis_client) + if not reserved: + return None + redis_client.hset(key, mapping={ + "busy": "1", + "last_activity": str(time.time()), + }) + redis_client.expire(key, _POOL_ENTRY_TTL) + return dict(data), profile + except Exception as exc: + logger.warning("Timeshift pool acquire failed for %s: %s", session_id, exc) + return None + + +def _wait_for_idle_pool_session(redis_client, session_id, wait_seconds=_POOL_WAIT_SECONDS): + if redis_client is None or not session_id: + return None + deadline = time.time() + wait_seconds + while True: + acquired = _acquire_idle_pool_session(redis_client, session_id) + if acquired is not None: + return acquired + if not _get_pool_entry(redis_client, session_id): + return None + if time.time() >= deadline: + return None + time.sleep(_POOL_POLL_INTERVAL) + + +def _create_pool_session( + redis_client, + *, + session_id, + media_id, + user_id, + client_ip, + client_user_agent, + account_id, + profile_id, + stream_id, + provider_timestamp, +): + """Register an already-reserved slot for this client session.""" + if redis_client is None or not session_id: + return False + key = _pool_key(session_id) + now = str(time.time()) + try: + with _pool_lock(redis_client, session_id): + if redis_client.exists(key): + return False + redis_client.hset(key, mapping={ + "media_id": str(media_id), + "user_id": str(user_id), + "client_ip": str(client_ip or ""), + "client_user_agent": str(client_user_agent or ""), + "account_id": str(account_id), + "profile_id": str(profile_id), + "stream_id": str(stream_id), + "provider_timestamp": str(provider_timestamp), + "busy": "1", + "last_activity": now, + }) + redis_client.expire(key, _POOL_ENTRY_TTL) return True except Exception as exc: - logger.warning("Timeshift slot release failed for %s: %s", client_id, exc) + logger.warning("Timeshift pool create failed for %s: %s", session_id, exc) return False -def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): - """Displace the user's prior catch-up session(s) on this channel. +def _release_pool_session(redis_client, session_id, profile_id): + if redis_client is None: + return + if profile_id is not None: + try: + release_profile_slot(int(profile_id), redis_client) + except Exception as exc: + logger.warning( + "Timeshift slot release failed for profile %s: %s", profile_id, exc + ) + if not session_id: + return + key = _pool_key(session_id) + try: + with _pool_lock(redis_client, session_id): + if redis_client.exists(key): + redis_client.hset(key, mapping={ + "busy": "0", + "last_activity": str(time.time()), + }) + redis_client.expire(key, _POOL_IDLE_TTL) + except Exception as exc: + logger.warning("Timeshift pool release failed for %s: %s", session_id, exc) - Releases the provider slot, unregisters stats keys, and sets the live - stop key so the old generator exits on its next heartbeat. - """ + +def _discard_pool_session(redis_client, session_id, profile_id): + if redis_client is None: + return + if profile_id is not None: + try: + release_profile_slot(int(profile_id), redis_client) + except Exception as exc: + logger.warning( + "Timeshift slot release failed for profile %s: %s", profile_id, exc + ) + if not session_id: + return + try: + with _pool_lock(redis_client, session_id): + redis_client.delete(_pool_key(session_id)) + except Exception as exc: + logger.warning("Timeshift pool discard failed for %s: %s", session_id, exc) + + +def _make_release_once(redis_client, session_id, profile_id): + state = {"done": False} + + def _release(): + if state["done"]: + return + state["done"] = True + _release_pool_session(redis_client, session_id, profile_id) + + return _release + + +def _preempt_playback_streams(redis_client, session_id, user): + """Stop in-flight streams for this client session only.""" + if redis_client is None or not session_id or user is None: + return + try: + for conn in get_user_active_connections(user.id): + if conn.get("type") != "timeshift": + continue + if conn.get("client_id") != session_id: + continue + conn_media_id = str(conn.get("media_id") or "") + old_client_id = conn.get("client_id") + logger.debug( + "Timeshift preempt: stopping client %s on %s for reuse", + old_client_id, conn_media_id, + ) + _unregister_stats_client(redis_client, conn_media_id, old_client_id) + stop_key = RedisKeys.client_stop(conn_media_id, old_client_id) + redis_client.setex(stop_key, 60, "true") + except Exception as exc: + logger.warning("Timeshift preempt failed: %s", exc) + + +def _terminate_previous_timeshift_sessions( + redis_client, user, channel_id, current_media_id, current_session_id, +): + """Displace this user's other catch-up positions on the same channel.""" if redis_client is None or user is None: return prefix = f"timeshift_{channel_id}_" @@ -333,22 +760,162 @@ def _terminate_previous_timeshift_sessions(redis_client, user, channel_id): for conn in get_user_active_connections(user.id): if conn.get("type") != "timeshift": continue - media_id = str(conn.get("media_id") or "") - if not media_id.startswith(prefix): + if conn.get("client_id") == current_session_id: + continue + conn_media_id = str(conn.get("media_id") or "") + if not conn_media_id.startswith(prefix): + continue + if conn_media_id.startswith(f"{current_media_id}_") or conn_media_id == current_media_id: continue old_client_id = conn.get("client_id") logger.info( "Timeshift takeover: displacing session %s on %s", - old_client_id, media_id, + old_client_id, conn_media_id, ) - _release_slot_token(redis_client, old_client_id) - _unregister_stats_client(redis_client, media_id, old_client_id) - stop_key = RedisKeys.client_stop(media_id, old_client_id) + _unregister_stats_client(redis_client, conn_media_id, old_client_id) + stop_key = RedisKeys.client_stop(conn_media_id, old_client_id) redis_client.setex(stop_key, 60, "true") except Exception as exc: logger.warning("Timeshift takeover check failed: %s", exc) +def _attempt_timeshift_stream( + *, + m3u_account, + profile, + stream_id_value, + provider_timestamp, + provider_tz_name, + duration_minutes, + channel, + safe_ts, + timestamp, + client_id, + client_ip, + range_header, + channel_logo_id, + user, + redis_client, + debug, + release_cb=None, + pool_session_id=None, +): + """Build the provider URL set for one (account, profile, stream) and stream it.""" + server_url, xc_username, xc_password = get_transformed_credentials( + m3u_account, profile + ) + creds = TimeshiftCredentials(server_url, xc_username, xc_password) + candidate_urls = build_timeshift_candidate_urls( + creds, stream_id_value, provider_timestamp, duration_minutes + ) + + try: + user_agent = m3u_account.get_user_agent().user_agent + except AttributeError: + user_agent = "" + + virtual_channel_id = f"timeshift_{channel.id}_{safe_ts}_{stream_id_value}" + + if debug: + logger.debug( + "Timeshift attempt: channel=%s ts=%s (provider tz=%s -> %s) " + "account=%s profile=%s provider_sid=%s vid=%s client=%s range=%s", + channel.name, timestamp, provider_tz_name, provider_timestamp, + m3u_account.id, profile.id, stream_id_value, + virtual_channel_id, client_id, range_header or "(none)", + ) + + return _stream_from_provider( + candidate_urls=candidate_urls, + user_agent=user_agent, + range_header=range_header, + virtual_channel_id=virtual_channel_id, + client_id=client_id, + client_ip=client_ip, + user=user, + channel_display_name=channel.name, + timestamp_utc=timestamp, + channel_logo_id=channel_logo_id, + m3u_profile_id=profile.id, + debug=debug, + account_id=m3u_account.id, + redis_client=redis_client, + release_cb=release_cb, + pool_session_id=pool_session_id, + ) + + +def _stream_reused_session( + redis_client, + *, + session_id, + descriptor, + profile, + channel, + safe_ts, + timestamp, + duration_minutes, + client_id, + client_ip, + range_header, + channel_logo_id, + user, + debug, +): + """Stream an idle pooled session that was just re-reserved for this request.""" + try: + m3u_account = M3UAccount.objects.get(id=int(descriptor["account_id"])) + except (M3UAccount.DoesNotExist, ValueError, TypeError): + _discard_pool_session(redis_client, session_id, profile.id) + return None + + provider_timestamp = descriptor.get("provider_timestamp") + if not provider_timestamp: + provider_tz_name = None + server_info = (profile.custom_properties or {}).get("server_info") or {} + if isinstance(server_info, dict): + provider_tz_name = server_info.get("timezone") + provider_timestamp = convert_timestamp_to_provider_tz( + timestamp, provider_tz_name + ) + + release_cb = _make_release_once(redis_client, session_id, profile.id) + try: + response = _attempt_timeshift_stream( + m3u_account=m3u_account, + profile=profile, + stream_id_value=descriptor["stream_id"], + provider_timestamp=provider_timestamp, + provider_tz_name=None, + duration_minutes=duration_minutes, + channel=channel, + safe_ts=safe_ts, + timestamp=timestamp, + client_id=client_id, + client_ip=client_ip, + range_header=range_header, + channel_logo_id=channel_logo_id, + user=user, + redis_client=redis_client, + debug=debug, + release_cb=release_cb, + pool_session_id=session_id, + ) + except Exception: + _discard_pool_session(redis_client, session_id, profile.id) + raise + + if response.status_code < 400: + return response + + if getattr(response, "timeshift_passthrough", False) is True: + release_cb() + return response + + _discard_pool_session(redis_client, session_id, profile.id) + return None + + class _SlotReleasingStream: """Iterator wrapper that releases the pool slot when WSGI closes the response.""" @@ -492,6 +1059,19 @@ def _set_cached_format_index(account_id, index): cache.set(_FORMAT_CACHE_KEY.format(account_id), index, _FORMAT_CACHE_TTL) +def _passthrough_response(status, content_range=None): + """A terminal response handed straight to the client (no streaming). + + Marked so the failover loop and reuse path return it verbatim instead of + cascading other URL shapes or failing over to another provider. + """ + response = HttpResponse(status=status) + if content_range: + response["Content-Range"] = content_range + response.timeshift_passthrough = True + return response + + def _stream_from_provider( *, candidate_urls, @@ -508,13 +1088,18 @@ def _stream_from_provider( debug, account_id=None, redis_client=None, + release_cb=None, + pool_session_id=None, ): """Try each upstream URL until one returns streamable MPEG-TS. Sets ``timeshift_decisive`` on auth/ban-class failures (401/403/406) so the - failover loop skips the rest of that account's streams. + failover loop skips the rest of that account's streams. ``release_cb`` frees + the provider slot when the streaming response is closed. """ chunk_size = max(ConfigHelper.chunk_size(), 262144) + if release_cb is None: + release_cb = lambda: None # noqa: E731 cached_index = _get_cached_format_index(account_id) if cached_index is not None and 0 <= cached_index < len(candidate_urls): @@ -552,6 +1137,14 @@ def _stream_from_provider( response.headers.get("Content-Type", "?"), _redact_url(url), ) + if response.status_code == 416: + # Range Not Satisfiable: a seek/tail probe past EOF. Hand it back to + # the client verbatim. Byte offsets are file-specific, so trying + # other URL shapes or failing over to another provider is pointless + # and only multiplies upstream connections. + content_range = response.headers.get("Content-Range") + response.close() + return _passthrough_response(416, content_range) if response.status_code in (200, 206): peek = response.raw.read(1024) sync_offset = find_ts_sync(peek) if peek else -1 @@ -560,18 +1153,29 @@ def _stream_from_provider( upstream = response winning_index = orig_idx break - else: - snippet = peek[:200].decode("utf-8", errors="replace") if peek else "(empty)" - logger.warning( - "Timeshift upstream returned 200 but no TS sync in first %d " - "bytes (likely PHP error): %s, url=%s", - len(peek) if peek else 0, - snippet.replace("\n", " ")[:120], - _redact_url(url), - ) - response.close() - last_status = 404 # Treat as soft rejection for cascade - continue + # A 206 to a Range request legitimately starts mid-packet, so the + # sync byte rarely lands at offset 0. Trust the partial status and + # content type rather than the sync probe; only a full 200 carrying + # a PHP/HTML error page must be rejected here. + content_type = response.headers.get("Content-Type", "") + is_partial = response.status_code == 206 and bool(range_header) + if is_partial and peek and "html" not in content_type and "json" not in content_type: + response._peek_data = peek + upstream = response + winning_index = orig_idx + break + snippet = peek[:200].decode("utf-8", errors="replace") if peek else "(empty)" + logger.warning( + "Timeshift upstream returned %d but no TS sync in first %d " + "bytes (likely PHP error): %s, url=%s", + response.status_code, + len(peek) if peek else 0, + snippet.replace("\n", " ")[:120], + _redact_url(url), + ) + response.close() + last_status = 404 # Treat as soft rejection for cascade + continue response.close() # Auth/ban-class statuses stop trying more shapes on this account; 5xx does not. code = response.status_code @@ -599,6 +1203,9 @@ def _stream_from_provider( content_range = upstream.headers.get("Content-Range", "") status = upstream.status_code + _store_pool_content_length(redis_client, pool_session_id, upstream) + _store_pool_serving_range(redis_client, pool_session_id, range_header) + _register_stats_client( redis_client, virtual_channel_id, @@ -618,29 +1225,57 @@ def _stream_from_provider( if peek_data: chunks_iter = itertools.chain([peek_data], chunks_iter) + session_closed = {"done": False} + + def _finish_session(*, close_upstream=False): + if session_closed["done"]: + return + session_closed["done"] = True + if close_upstream: + try: + upstream.close() + except Exception: + pass + _unregister_stats_client(redis_client, virtual_channel_id, client_id) + release_cb() + def stream_generator(): last_heartbeat = time.time() bytes_since_heartbeat = 0 total_yielded = 0 - chunk_count = 0 loop_start = time.time() stop_key = RedisKeys.client_stop(virtual_channel_id, client_id) + stream_started_logged = False try: for data in chunks_iter: if not data: continue + if debug and not stream_started_logged: + stream_started_logged = True + logger.debug( + "Timeshift stream started: client=%s vid=%s range=%s status=%d", + client_id, virtual_channel_id, range_header or "(none)", status, + ) yield data bytes_since_heartbeat += len(data) total_yielded += len(data) - chunk_count += 1 now = time.time() - # Poll stop key and refresh stats every 5 seconds. + if redis_client and redis_client.exists(stop_key): + logger.info("Timeshift client %s received stop signal", client_id) + redis_client.delete(stop_key) + break + # Refresh stats every 5 seconds. if now - last_heartbeat >= 5: - if redis_client and redis_client.exists(stop_key): - logger.info("Timeshift client %s received stop signal", client_id) - redis_client.delete(stop_key) - break + if debug and total_yielded > 0: + elapsed = now - loop_start + mbps = (total_yielded * 8) / elapsed / 1_000_000 if elapsed > 0 else 0 + logger.debug( + "Timeshift streaming: client=%s range=%s total=%d bytes " + "in %.1fs (%.2f Mbps avg)", + client_id, range_header or "(none)", + total_yielded, elapsed, mbps, + ) _heartbeat_stats_client( redis_client, virtual_channel_id, client_id, bytes_delta=bytes_since_heartbeat, @@ -664,18 +1299,9 @@ def _stream_from_provider( "Timeshift disconnect: vid=%s client=%s yielded=%d bytes in %.1fs (%.2f Mbps avg)", virtual_channel_id, client_id, total_yielded, elapsed, mbps, ) - try: - upstream.close() - except Exception: - pass - _unregister_stats_client(redis_client, virtual_channel_id, client_id) - _release_slot_token(redis_client, client_id) + _finish_session(close_upstream=True) - def _close_session(): - _unregister_stats_client(redis_client, virtual_channel_id, client_id) - _release_slot_token(redis_client, client_id) - - stream_iter = _SlotReleasingStream(stream_generator(), _close_session) + stream_iter = _SlotReleasingStream(stream_generator(), _finish_session) response = StreamingHttpResponse( stream_iter, content_type=content_type, From 5a552cd5658cc5f53d7e58869f86bb91ace5f28e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 17:48:06 -0500 Subject: [PATCH 23/31] refactor(epg): Update xmltv_prev_days_override setting to EPG settings and adjust related logic. Refactor CoreSettings to include a dedicated method for retrieving the override value. Update tests and frontend components to reflect the new structure and ensure consistent behavior across settings management. --- CHANGELOG.md | 2 +- apps/channels/tests/test_catchup_utils.py | 4 +- apps/channels/utils.py | 5 +- core/models.py | 12 ++- .../forms/settings/EpgSettingsForm.jsx | 81 +++++++++++++++++++ .../forms/settings/ProxySettingsForm.jsx | 5 +- frontend/src/constants.js | 3 + frontend/src/pages/Settings.jsx | 16 ++++ .../forms/settings/EpgSettingsFormUtils.js | 3 + .../forms/settings/ProxySettingsFormUtils.js | 1 - .../__tests__/ProxySettingsFormUtils.test.js | 2 - frontend/src/utils/pages/SettingsUtils.js | 23 +++++- 12 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 frontend/src/components/forms/settings/EpgSettingsForm.jsx create mode 100644 frontend/src/utils/forms/settings/EpgSettingsFormUtils.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 4890e407..232b06df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. - - **Single setting**, `xmltv_prev_days_override`, under `Settings → Proxy Settings` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. + - **Single setting**, `xmltv_prev_days_override`, under `Settings → EPG` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. diff --git a/apps/channels/tests/test_catchup_utils.py b/apps/channels/tests/test_catchup_utils.py index 68ac20c3..d79eec11 100644 --- a/apps/channels/tests/test_catchup_utils.py +++ b/apps/channels/tests/test_catchup_utils.py @@ -45,8 +45,8 @@ class ResolveXcEpgPrevDaysTests(SimpleTestCase): mock_compute.assert_not_called() @patch("apps.channels.utils.compute_provider_archive_days_capped", return_value=14) - @patch("core.models.CoreSettings.get_proxy_settings", return_value={"xmltv_prev_days_override": 7}) - def test_proxy_override_prevents_auto_detect(self, _settings, mock_compute): + @patch("core.models.CoreSettings.get_xmltv_prev_days_override", return_value=7) + def test_epg_settings_override_prevents_auto_detect(self, _override, mock_compute): request = self.factory.get("/xmltv.php") self.assertEqual(resolve_xc_epg_prev_days(request, self.user), 7) mock_compute.assert_not_called() diff --git a/apps/channels/utils.py b/apps/channels/utils.py index b2cceab6..70b9eecd 100644 --- a/apps/channels/utils.py +++ b/apps/channels/utils.py @@ -67,7 +67,7 @@ def resolve_xc_epg_prev_days(request, user, *, auto_detect_fallback=True): Resolution order: 1. URL ``?prev_days=`` (explicit; 0 means no past programmes) 2. ``user.custom_properties.epg_prev_days`` - 3. ``CoreSettings.proxy_settings.xmltv_prev_days_override`` when > 0 + 3. ``CoreSettings.epg_settings.xmltv_prev_days_override`` when > 0 4. Auto-detect (only when *auto_detect_fallback* is True) """ user_custom = (user.custom_properties or {}) if user else {} @@ -87,9 +87,8 @@ def resolve_xc_epg_prev_days(request, user, *, auto_detect_fallback=True): from core.models import CoreSettings - proxy_settings = CoreSettings.get_proxy_settings() try: - override = int(proxy_settings.get("xmltv_prev_days_override", 0) or 0) + override = int(CoreSettings.get_xmltv_prev_days_override() or 0) except (TypeError, ValueError): override = 0 if override > 0: diff --git a/core/models.py b/core/models.py index 9c5a5795..58fd37da 100644 --- a/core/models.py +++ b/core/models.py @@ -280,8 +280,18 @@ class CoreSettings(models.Model): "epg_match_ignore_prefixes": [], "epg_match_ignore_suffixes": [], "epg_match_ignore_custom": [], + # XC catch-up: forced XMLTV lookback (0 = auto-detect, capped at 30). + "xmltv_prev_days_override": 0, }) + @classmethod + def get_xmltv_prev_days_override(cls): + """Global XC XMLTV prev_days default (0 = auto-detect from provider archives).""" + try: + return int(cls.get_epg_settings().get("xmltv_prev_days_override", 0) or 0) + except (TypeError, ValueError): + return 0 + @classmethod def _safe_string_list(cls, value): """Return a list of strings, filtering out non-list or non-string values.""" @@ -396,8 +406,6 @@ class CoreSettings(models.Model): "channel_shutdown_delay": 0, "channel_init_grace_period": 5, "new_client_behind_seconds": 5, - # XC catch-up: forced XMLTV lookback (0 = auto-detect, capped at 30). - "xmltv_prev_days_override": 0, }) # System Settings diff --git a/frontend/src/components/forms/settings/EpgSettingsForm.jsx b/frontend/src/components/forms/settings/EpgSettingsForm.jsx new file mode 100644 index 00000000..5668dbdd --- /dev/null +++ b/frontend/src/components/forms/settings/EpgSettingsForm.jsx @@ -0,0 +1,81 @@ +import useSettingsStore from '../../../store/settings.jsx'; +import React, { useEffect, useState } from 'react'; +import { useForm } from '@mantine/form'; +import { + Alert, + Button, + Flex, + NumberInput, + Stack, + Text, +} from '@mantine/core'; +import { EPG_SETTINGS_OPTIONS } from '../../../constants.js'; +import { + getChangedSettings, + parseSettings, + saveChangedSettings, +} from '../../../utils/pages/SettingsUtils.js'; +import { getEpgSettingsFormInitialValues } from '../../../utils/forms/settings/EpgSettingsFormUtils.js'; + +const EpgSettingsForm = React.memo(({ active }) => { + const settings = useSettingsStore((s) => s.settings); + const [saved, setSaved] = useState(false); + + const form = useForm({ + mode: 'controlled', + initialValues: getEpgSettingsFormInitialValues(), + }); + + useEffect(() => { + if (!active) setSaved(false); + }, [active]); + + useEffect(() => { + if (settings) { + const parsed = parseSettings(settings); + form.setFieldValue( + 'xmltv_prev_days_override', + parsed.xmltv_prev_days_override ?? 0, + ); + } + }, [settings]); + + const onSubmit = async () => { + setSaved(false); + const changedSettings = getChangedSettings(form.getValues(), settings); + try { + await saveChangedSettings(settings, changedSettings); + setSaved(true); + } catch (error) { + console.error('Error saving EPG settings:', error); + } + }; + + const prevDaysConfig = EPG_SETTINGS_OPTIONS.xmltv_prev_days_override; + + return ( +
+ + {saved && ( + + )} + + + Per-user defaults and URL parameters still override this global value. + EPG channel matching options are configured from the Channels page. + + + + + +
+ ); +}); + +export default EpgSettingsForm; diff --git a/frontend/src/components/forms/settings/ProxySettingsForm.jsx b/frontend/src/components/forms/settings/ProxySettingsForm.jsx index 9b2c2c62..52440769 100644 --- a/frontend/src/components/forms/settings/ProxySettingsForm.jsx +++ b/frontend/src/components/forms/settings/ProxySettingsForm.jsx @@ -25,7 +25,6 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { 'channel_shutdown_delay', 'channel_init_grace_period', 'new_client_behind_seconds', - 'xmltv_prev_days_override', ].includes(key); }; const isFloatField = (key) => { @@ -40,9 +39,7 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { ? 300 : key === 'new_client_behind_seconds' ? 120 - : key === 'xmltv_prev_days_override' - ? 30 - : 60; + : 60; }; return ( <> diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 00af156e..b4503dc6 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -61,6 +61,9 @@ export const PROXY_SETTINGS_OPTIONS = { description: 'Seconds of received buffer to start behind live when a new client connects (0 = start at live). Note: this is chunk receive time, not video duration.', }, +}; + +export const EPG_SETTINGS_OPTIONS = { xmltv_prev_days_override: { label: 'XMLTV prev_days Override (catch-up)', description: diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index 29d46dd8..1c7964cc 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -45,6 +45,9 @@ const DvrSettingsForm = React.lazy( const SystemSettingsForm = React.lazy( () => import('../components/forms/settings/SystemSettingsForm.jsx') ); +const EpgSettingsForm = React.lazy( + () => import('../components/forms/settings/EpgSettingsForm.jsx') +); const NavOrderForm = React.lazy( () => import('../components/forms/settings/NavOrderForm.jsx') ); @@ -122,6 +125,19 @@ const SettingsPage = () => { + + EPG + + + }> + + + + + + System Settings diff --git a/frontend/src/utils/forms/settings/EpgSettingsFormUtils.js b/frontend/src/utils/forms/settings/EpgSettingsFormUtils.js new file mode 100644 index 00000000..d2138b6a --- /dev/null +++ b/frontend/src/utils/forms/settings/EpgSettingsFormUtils.js @@ -0,0 +1,3 @@ +export const getEpgSettingsFormInitialValues = () => ({ + xmltv_prev_days_override: 0, +}); diff --git a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js index 93cccc9b..eddbba91 100644 --- a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js @@ -15,6 +15,5 @@ export const getProxySettingDefaults = () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, - xmltv_prev_days_override: 0, }; }; diff --git a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js index c1fd0908..21c48269 100644 --- a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js +++ b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js @@ -62,7 +62,6 @@ describe('ProxySettingsFormUtils', () => { channel_shutdown_delay: 0, channel_init_grace_period: 5, new_client_behind_seconds: 5, - xmltv_prev_days_override: 0, }); }); @@ -83,7 +82,6 @@ describe('ProxySettingsFormUtils', () => { expect(typeof result.channel_shutdown_delay).toBe('number'); expect(typeof result.channel_init_grace_period).toBe('number'); expect(typeof result.new_client_behind_seconds).toBe('number'); - expect(typeof result.xmltv_prev_days_override).toBe('number'); }); }); }); diff --git a/frontend/src/utils/pages/SettingsUtils.js b/frontend/src/utils/pages/SettingsUtils.js index acdc6cae..bd46ac52 100644 --- a/frontend/src/utils/pages/SettingsUtils.js +++ b/frontend/src/utils/pages/SettingsUtils.js @@ -39,6 +39,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'epg_match_ignore_prefixes', 'epg_match_ignore_suffixes', 'epg_match_ignore_custom', + 'xmltv_prev_days_override', ]; const dvrFields = [ 'tv_template', @@ -125,6 +126,7 @@ export const saveChangedSettings = async (settings, changedSettings) => { 'retention_count', 'schedule_day_of_week', 'max_system_events', + 'xmltv_prev_days_override', ]; if (numericFields.includes(formKey) && value != null) { value = typeof value === 'number' ? value : parseInt(value, 10); @@ -222,6 +224,20 @@ export const getChangedSettings = (values, settings) => { continue; } + if (settingKey === 'xmltv_prev_days_override') { + const baseline = Number( + settings['epg_settings']?.value?.xmltv_prev_days_override ?? 0, + ); + const nextVal = + typeof actualValue === 'number' + ? actualValue + : parseInt(actualValue, 10) || 0; + if (nextVal !== baseline) { + changedSettings[settingKey] = nextVal; + } + continue; + } + // Convert array values (like m3u_hash_key) to comma-separated strings for comparison if (Array.isArray(actualValue)) { actualValue = actualValue.join(','); @@ -296,6 +312,12 @@ export const parseSettings = (settings) => { epgSettings && Array.isArray(epgSettings.epg_match_ignore_custom) ? epgSettings.epg_match_ignore_custom : []; + parsed.xmltv_prev_days_override = + epgSettings && epgSettings.xmltv_prev_days_override != null + ? typeof epgSettings.xmltv_prev_days_override === 'number' + ? epgSettings.xmltv_prev_days_override + : parseInt(epgSettings.xmltv_prev_days_override, 10) || 0 + : 0; // DVR settings - direct mapping with underscore keys const dvrSettings = settings['dvr_settings']?.value; @@ -367,7 +389,6 @@ export const parseSettings = (settings) => { } // Proxy and network access are already grouped objects - // (xmltv_prev_days_override lives inside proxy_settings) if (settings['proxy_settings']?.value) { parsed.proxy_settings = settings['proxy_settings'].value; } From 88de2d9bf0b72bf14ac45e37d524030d1b37f8d6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 26 Jun 2026 17:51:06 -0500 Subject: [PATCH 24/31] changelog: update for accuracy --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 232b06df..1ad83500 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. +- **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. — Thanks [@cedric-marcoux](https://github.com/cedric-marcoux) (#1242) - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. - - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it exactly once when the session ends (one-shot Redis token, covering disconnects before the first chunk). When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. - - **One catch-up session per user and channel.** A seek or programme jump displaces the user's previous session on the same channel: its provider slot is released synchronously and the old stream is stopped through the standard stop-key mechanism, so rapid seeking can't stack upstream provider connections. - - **Single setting**, `xmltv_prev_days_override`, under `Settings → EPG` (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. + - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it when the session ends. When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. + - **Per-client session pool.** The first request without `session_id` receives a `301` redirect to the same URL with a stable `?session_id=` query parameter; that ID becomes the client identity for stats, stop keys, and pool coordination. Each viewer gets their own Redis-backed pool entry even when watching the same programme; idle sessions can be reclaimed via fingerprint matching (same user and programme, IP + user-agent score). Real scrubs within a session stop the in-flight stream through the standard stop-key mechanism; parallel startup probes (full-file, open-ended, and EOF tail `Range` requests) are deferred with `503` instead of opening extra upstream connections. Stream-limit exemptions for `ignore_same_channel_connections` apply only to sibling requests from the same `session_id`. + - **`xmltv_prev_days_override` under `Settings → EPG`** (default `0` = auto-detect). Verbose timeshift logging follows the standard logger DEBUG level. - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. From 931f4dc50f029e3e043af578d0a18f7a80d9a039 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 09:28:24 -0500 Subject: [PATCH 25/31] refactor(tasks): Enhance rollup_channel_catchup_fields function to limit updates to channels linked to the specified account. Update SQL queries for clarity and efficiency, and improve docstrings for better understanding of the self-heal logic related to catch-up flags. --- apps/m3u/tasks.py | 29 ++++++++++++-------- apps/timeshift/tests/test_views.py | 44 ++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 34a72316..65326c75 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1885,11 +1885,22 @@ def _classify_sync_failure(exc): def rollup_channel_catchup_fields(account_id): - """Roll up catch-up flags from streams to channels (active accounts only).""" + """Roll up catch-up flags from streams to channels (active accounts only). + + Both the aggregate update and the self-heal pass are limited to channels + that still have at least one stream from *account_id*. + """ from django.db import connection + account_channels = """ + SELECT DISTINCT cs.channel_id + FROM dispatcharr_channels_channelstream cs + JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id + WHERE s.m3u_account_id = %s + """ + with connection.cursor() as cur: - cur.execute(""" + cur.execute(f""" WITH agg AS ( SELECT cs.channel_id, @@ -1898,12 +1909,7 @@ def rollup_channel_catchup_fields(account_id): FROM dispatcharr_channels_channelstream cs JOIN dispatcharr_channels_stream s ON s.id = cs.stream_id JOIN m3u_m3uaccount a ON a.id = s.m3u_account_id - WHERE cs.channel_id IN ( - SELECT DISTINCT cs2.channel_id - FROM dispatcharr_channels_channelstream cs2 - JOIN dispatcharr_channels_stream s2 ON s2.id = cs2.stream_id - WHERE s2.m3u_account_id = %s - ) + WHERE cs.channel_id IN ({account_channels}) GROUP BY cs.channel_id ) UPDATE dispatcharr_channels_channel c @@ -1914,11 +1920,12 @@ def rollup_channel_catchup_fields(account_id): WHERE c.id = agg.channel_id """, [account_id]) - # Self-heal stale is_catchup flags. - cur.execute(""" + # Self-heal stale is_catchup flags on account-linked channels only. + cur.execute(f""" UPDATE dispatcharr_channels_channel c SET is_catchup = FALSE, catchup_days = 0 WHERE c.is_catchup = TRUE + AND c.id IN ({account_channels}) AND NOT EXISTS ( SELECT 1 FROM dispatcharr_channels_channelstream cs @@ -1928,7 +1935,7 @@ def rollup_channel_catchup_fields(account_id): AND s.is_catchup = TRUE AND a.is_active = TRUE ) - """) + """, [account_id]) @shared_task diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 310649cb..535e455e 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -1901,10 +1901,9 @@ class TimeshiftScrubPreemptTests(TestCase): class RollupSelfHealDbTests(TestCase): - """Catch-up flag consistency after stream removal — the review-#4 point 1 - guarantees: the ChannelStream signal handles bulk deletes (locked by a - regression test) and the rollup self-heals any channel left flagged with - no catch-up stream, regardless of how the link rows disappeared.""" + """Catch-up flag consistency after stream removal — the ChannelStream signal + handles bulk deletes (locked by a regression test) and the account-scoped + rollup self-heals stale flags on channels still linked to that account.""" @classmethod def setUpTestData(cls): @@ -1944,14 +1943,21 @@ class RollupSelfHealDbTests(TestCase): self.assertFalse(channel.is_catchup) self.assertEqual(channel.catchup_days, 0) - def test_rollup_self_heals_stale_channel_without_streams(self): - # Simulate staleness no signal caught (raw SQL delete, historic data): - # a flagged channel with zero catch-up streams must be reset by the - # rollup even though no remaining stream ties it to the account. - from apps.channels.models import Channel + def test_rollup_self_heals_stale_channel_with_non_catchup_stream(self): + # Channel still linked to the account but no active catch-up streams + # (e.g. catch-up flag cleared on import) — rollup must reset stale flags. + from apps.channels.models import Channel, ChannelStream, Stream from apps.m3u.tasks import rollup_channel_catchup_fields channel = Channel.objects.create(name="ts-rollup-stale") + stream = Stream.objects.create( + name="ts-rollup-stale-stream", + url="http://example.test/ts-rollup-stale", + m3u_account=self.account, + is_catchup=False, + catchup_days=0, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) Channel.objects.filter(pk=channel.pk).update(is_catchup=True, catchup_days=9) rollup_channel_catchup_fields(self.account.id) @@ -1960,6 +1966,26 @@ class RollupSelfHealDbTests(TestCase): self.assertFalse(channel.is_catchup) self.assertEqual(channel.catchup_days, 0) + def test_rollup_self_heal_skips_channels_not_linked_to_account(self): + from apps.channels.models import Channel + from apps.m3u.models import M3UAccount + from apps.m3u.tasks import rollup_channel_catchup_fields + + other_account = M3UAccount.objects.create( + name="ts-rollup-other", + server_url="http://example.test/other", + account_type="XC", + is_active=True, + ) + channel = Channel.objects.create(name="ts-rollup-unrelated") + Channel.objects.filter(pk=channel.pk).update(is_catchup=True, catchup_days=9) + + rollup_channel_catchup_fields(other_account.id) + + channel.refresh_from_db() + self.assertTrue(channel.is_catchup) + self.assertEqual(channel.catchup_days, 9) + def test_rollup_keeps_and_corrects_channels_with_catchup_streams(self): # The self-heal pass must not touch channels that legitimately have # catch-up streams — and the account-scoped pass still corrects their From ec8594cd0adcecb284a9bacfcc979680cc842286 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 10:11:49 -0500 Subject: [PATCH 26/31] refactor(tasks): Add is_catchup and catchup_days fields to Stream processing logic. Update existing stream queries and conditions to handle new catch-up properties, ensuring accurate updates and memory management for bulk operations. --- apps/m3u/tasks.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 65326c75..aa67335f 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1089,7 +1089,7 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): existing_streams = { s.stream_hash: s for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( - 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id' + 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id', 'is_catchup', 'catchup_days' ) } @@ -1106,9 +1106,15 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): obj.is_adult != stream_props["is_adult"] or obj.stream_id != stream_props["stream_id"] or obj.stream_chno != stream_props["stream_chno"] or - obj.channel_group_id != stream_props["channel_group_id"] + obj.channel_group_id != stream_props["channel_group_id"] or + obj.is_catchup != stream_props["is_catchup"] or + obj.catchup_days != stream_props["catchup_days"] ) + # Keep denormalized catch-up columns in memory (bulk_update reads them). + obj.is_catchup = stream_props["is_catchup"] + obj.catchup_days = stream_props["catchup_days"] + if changed: for key, value in stream_props.items(): setattr(obj, key, value) @@ -1322,7 +1328,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): existing_streams = { s.stream_hash: s for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( - 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id' + 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id', 'is_catchup', 'catchup_days' ) } @@ -1339,9 +1345,15 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): obj.is_adult != stream_props["is_adult"] or obj.stream_id != stream_props["stream_id"] or obj.stream_chno != stream_props["stream_chno"] or - obj.channel_group_id != stream_props["channel_group_id"] + obj.channel_group_id != stream_props["channel_group_id"] or + obj.is_catchup != stream_props["is_catchup"] or + obj.catchup_days != stream_props["catchup_days"] ) + # Keep denormalized catch-up columns in memory (bulk_update reads them). + obj.is_catchup = stream_props["is_catchup"] + obj.catchup_days = stream_props["catchup_days"] + # Always update last_seen obj.last_seen = timezone.now() From 96a39ce5d2dcb708397f3b24e4d2af54767fa1ea Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 11:13:17 -0500 Subject: [PATCH 27/31] refactor(tasks): Improve stream processing logic by adding bulk update functionality for unchanged streams. Enhance memory management and error handling in database queries, and update cleanup functions to optimize memory usage. --- CHANGELOG.md | 11 ++- apps/m3u/tasks.py | 105 ++++++++++++++++------------- apps/timeshift/tests/test_views.py | 14 ++-- core/utils.py | 9 ++- dispatcharr/celery.py | 3 +- 5 files changed, 84 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad83500..ce80b8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **XC catch-up (timeshift) support**. Adds a native `/timeshift/{user}/{pass}/{stream_id}/{timestamp}/{duration}.ts` endpoint that proxies replay sessions from an Xtream Codes provider to clients such as iPlayTV (Apple TV) and TiviMate. Auth reuses the same `xc_password` custom-property the live `/live/.../{id}.ts` endpoint already uses. Catch-up sessions appear on `/stats` with a violet `TIMESHIFT` badge alongside live sessions and respect per-channel access rules. — Thanks [@cedric-marcoux](https://github.com/cedric-marcoux) (#1242) - - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; the per-account rollup also self-heals channels left flagged with no remaining catch-up stream (e.g. after stale-stream cleanup on manual/multi-provider channels). + - **Catch-up flags** (`tv_archive`, `tv_archive_duration`) denormalized at import time for zero-cost output queries; end-of-refresh SQL rollup updates channels linked to the refreshed account and self-heals stale flags on those channels only (e.g. after catch-up streams are removed or an account is deactivated). - **Strictly-UTC API surface, automatic per-provider timezone.** `server_info.timezone` is always `UTC` and the XC EPG `start`/`end` strings are emitted in UTC; the proxy converts the requested instant to the serving provider's own reported timezone (`server_info.timezone` captured on account refresh) at request time. No timezone to configure. - **Multi-provider failover.** Catch-up walks the channel's catch-up streams in order (like live playback): if one provider cannot serve the archive the next is tried, each attempt with its own account context (credentials, reported timezone, user-agent). Accounts that fail with an auth/ban-class status are not retried via their other streams. - **Provider pool accounting.** Catch-up reserves a provider profile slot (`connection_pool`) before connecting upstream — same contract as live and VOD — and releases it when the session ends. When the default profile is at capacity it walks the account's alternate profiles with their own resolved credentials, and answers `503` when every profile is full. @@ -19,6 +19,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **EPG XMLTV `prev_days` auto-detection** from the provider's largest `tv_archive_duration` (capped at 30 days), overridable via setting or `?prev_days=` URL parameter. - **Byte path streams directly via `iter_content` + generator yield** (same pattern as the VOD proxy), with an upfront MPEG-TS sync peek that strips any PHP-warning preamble before bytes reach strict demuxers. Throughput stays comfortably above the typical 5 Mbps FHD bitrate so clients don't buffer. +### Performance + +- **M3U/XC stream refresh is faster on large accounts.** Steady-state refreshes split `bulk_update` into a lightweight touch pass (`last_seen` / `is_stale` only) for unchanged streams and a full column update only when provider metadata or catch-up fields actually change. +- **Celery workers return RSS after memory-intensive tasks.** `cleanup_memory()` accepts an optional `trim_heap` flag (glibc `malloc_trim`); Celery `task_postrun` enables it after `close_old_connections()` for M3U account/group refresh, EPG, VOD, and channel-matching tasks so worker memory drops back toward baseline instead of ratcheting across successive large jobs. + +### Fixed + +- **M3U group processing retries on poisoned DB connections.** `_db_query_with_retry` now treats psycopg desync errors (`DatabaseError`, e.g. `lost synchronization with server`) as transient; `process_groups` relationship loading uses it so a stale Celery worker connection resets once instead of failing the whole refresh. + ## [0.27.1] - 2026-06-25 ### Security diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index aa67335f..7ef71dda 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -52,9 +52,9 @@ def _db_query_with_retry(fn, *, label="DB query", max_retries=2): Poisoned Celery worker connections often surface as OperationalError or as ``IndexError: list index out of range`` inside Django's row converters. """ - from django.db import InterfaceError, OperationalError + from django.db import DatabaseError, InterfaceError, OperationalError - transient_errors = (OperationalError, InterfaceError, IndexError) + transient_errors = (OperationalError, InterfaceError, IndexError, DatabaseError) for attempt in range(max_retries): try: return fn() @@ -746,13 +746,16 @@ def process_groups(account, groups, scan_start_time=None): all_group_objs = existing_group_objs + newly_created_group_objs # Get existing relationships for this account - existing_relationships = { - rel.channel_group.name: rel - for rel in ChannelGroupM3UAccount.objects.filter( - m3u_account=account, - channel_group__name__in=groups.keys() - ).select_related('channel_group') - } + existing_relationships = _db_query_with_retry( + lambda: { + rel.channel_group.name: rel + for rel in ChannelGroupM3UAccount.objects.filter( + m3u_account=account, + channel_group__name__in=groups.keys(), + ).select_related("channel_group") + }, + label=f"process_groups relationships for account {account.id}", + ) relations_to_create = [] relations_to_update = [] @@ -971,6 +974,27 @@ def collect_xc_streams(account_id, enabled_groups): ) return all_streams + +_STREAM_TOUCH_FIELDS = ("last_seen", "is_stale") +_STREAM_CHANGED_FIELDS = ( + "name", "url", "logo_url", "tvg_id", "custom_properties", "is_adult", + "last_seen", "updated_at", "is_stale", "stream_id", "stream_chno", + "channel_group_id", "is_catchup", "catchup_days", +) + + +def _bulk_update_stream_refresh_batches(changed_streams, touch_streams, *, batch_size): + """Unchanged streams only need last_seen/is_stale; changed rows get the full set.""" + if touch_streams: + Stream.objects.bulk_update( + touch_streams, list(_STREAM_TOUCH_FIELDS), batch_size=batch_size, + ) + if changed_streams: + Stream.objects.bulk_update( + changed_streams, list(_STREAM_CHANGED_FIELDS), batch_size=batch_size, + ) + + def process_xc_category_direct(account_id, batch, groups, hash_keys): from django.db import connections @@ -981,6 +1005,7 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): streams_to_create = [] streams_to_update = [] + streams_to_touch = [] stream_hashes = {} try: @@ -1111,10 +1136,6 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): obj.catchup_days != stream_props["catchup_days"] ) - # Keep denormalized catch-up columns in memory (bulk_update reads them). - obj.is_catchup = stream_props["is_catchup"] - obj.catchup_days = stream_props["catchup_days"] - if changed: for key, value in stream_props.items(): setattr(obj, key, value) @@ -1123,11 +1144,9 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): obj.is_stale = False streams_to_update.append(obj) else: - # Always update last_seen, even if nothing else changed obj.last_seen = timezone.now() obj.is_stale = False - # Don't update updated_at for unchanged streams - streams_to_update.append(obj) + streams_to_touch.append(obj) # Remove from existing_streams since we've processed it del existing_streams[stream_hash] @@ -1144,13 +1163,9 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): if streams_to_create: Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) - if streams_to_update: - # Simplified bulk update for better performance - Stream.objects.bulk_update( - streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'channel_group_id', 'is_catchup', 'catchup_days'], - batch_size=150 # Smaller batch size for XC processing - ) + _bulk_update_stream_refresh_batches( + streams_to_update, streams_to_touch, batch_size=150, + ) # Update last_seen for any remaining existing streams that weren't processed if len(existing_streams.keys()) > 0: @@ -1158,7 +1173,10 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): except Exception as e: logger.error(f"Bulk operation failed for XC streams: {str(e)}") - retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + retval = ( + f"Batch processed: {len(streams_to_create)} created, " + f"{len(streams_to_update) + len(streams_to_touch)} updated." + ) except Exception as e: logger.error(f"XC category processing error: {str(e)}") @@ -1168,7 +1186,7 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): connections.close_all() # Aggressive garbage collection - del streams_to_create, streams_to_update, stream_hashes, existing_streams + del streams_to_create, streams_to_update, streams_to_touch, stream_hashes, existing_streams gc.collect() return retval @@ -1203,6 +1221,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): streams_to_create = [] streams_to_update = [] + streams_to_touch = [] stream_hashes = {} name_max_length = Stream._meta.get_field('name').max_length @@ -1350,15 +1369,10 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): obj.catchup_days != stream_props["catchup_days"] ) - # Keep denormalized catch-up columns in memory (bulk_update reads them). - obj.is_catchup = stream_props["is_catchup"] - obj.catchup_days = stream_props["catchup_days"] - - # Always update last_seen obj.last_seen = timezone.now() + obj.is_stale = False if changed: - # Only update fields that changed and set updated_at obj.name = stream_props["name"] obj.url = stream_props["url"] obj.logo_url = stream_props["logo_url"] @@ -1368,12 +1382,12 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): obj.stream_id = stream_props["stream_id"] obj.stream_chno = stream_props["stream_chno"] obj.channel_group_id = stream_props["channel_group_id"] + obj.is_catchup = stream_props["is_catchup"] + obj.catchup_days = stream_props["catchup_days"] obj.updated_at = timezone.now() - - # Always mark as not stale since we saw it in this refresh - obj.is_stale = False - - streams_to_update.append(obj) + streams_to_update.append(obj) + else: + streams_to_touch.append(obj) else: # New stream stream_props["last_seen"] = timezone.now() @@ -1386,23 +1400,22 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): if streams_to_create: Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) - if streams_to_update: - # Update all streams in a single bulk operation - Stream.objects.bulk_update( - streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'channel_group_id', 'is_catchup', 'catchup_days'], - batch_size=200 - ) + _bulk_update_stream_refresh_batches( + streams_to_update, streams_to_touch, batch_size=200, + ) except Exception as e: logger.error(f"Bulk operation failed: {str(e)}") - retval = f"M3U account: {account_id}, Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + retval = ( + f"M3U account: {account_id}, Batch processed: {len(streams_to_create)} created, " + f"{len(streams_to_update) + len(streams_to_touch)} updated." + ) # Clean up database connections for threading connections.close_all() # Free batch data structures (reference-counted deallocation) - del streams_to_create, streams_to_update, stream_hashes, existing_streams + del streams_to_create, streams_to_update, streams_to_touch, stream_hashes, existing_streams gc.collect() return retval @@ -3814,8 +3827,6 @@ def _refresh_single_m3u_account_impl(account_id): except OSError: pass - _release_task_db_connection() - return f"Dispatched jobs complete." diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 535e455e..270dfbe5 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -1901,9 +1901,12 @@ class TimeshiftScrubPreemptTests(TestCase): class RollupSelfHealDbTests(TestCase): - """Catch-up flag consistency after stream removal — the ChannelStream signal - handles bulk deletes (locked by a regression test) and the account-scoped - rollup self-heals stale flags on channels still linked to that account.""" + """Catch-up flag consistency after stream removal. + + The ChannelStream signal handles bulk deletes (locked by a regression test). + The account-scoped rollup self-heals stale flags on channels still linked + to that account. + """ @classmethod def setUpTestData(cls): @@ -1945,7 +1948,7 @@ class RollupSelfHealDbTests(TestCase): def test_rollup_self_heals_stale_channel_with_non_catchup_stream(self): # Channel still linked to the account but no active catch-up streams - # (e.g. catch-up flag cleared on import) — rollup must reset stale flags. + # (e.g. catch-up flag cleared on import). Rollup must reset stale flags. from apps.channels.models import Channel, ChannelStream, Stream from apps.m3u.tasks import rollup_channel_catchup_fields @@ -1988,8 +1991,7 @@ class RollupSelfHealDbTests(TestCase): def test_rollup_keeps_and_corrects_channels_with_catchup_streams(self): # The self-heal pass must not touch channels that legitimately have - # catch-up streams — and the account-scoped pass still corrects their - # values. + # catch-up streams. The account-scoped pass still corrects their values. from apps.channels.models import Channel from apps.m3u.tasks import rollup_channel_catchup_fields diff --git a/core/utils.py b/core/utils.py index 44504b6a..16d9ed85 100644 --- a/core/utils.py +++ b/core/utils.py @@ -565,13 +565,15 @@ def trim_c_allocator_heap(): return False -def cleanup_memory(log_usage=False, force_collection=True): +def cleanup_memory(log_usage=False, force_collection=True, trim_heap=False): """ Comprehensive memory cleanup function to reduce memory footprint Args: log_usage: Whether to log memory usage before and after cleanup force_collection: Whether to force garbage collection + trim_heap: Return freed C heap pages to the OS. Only use after DB + connections are closed (e.g. Celery task_postrun). """ logger.trace("Starting memory cleanup django memory cleanup") # Skip logging if log level is not set to debug or more verbose (like trace) @@ -606,6 +608,8 @@ def cleanup_memory(log_usage=False, force_collection=True): logger.debug(f"Memory after cleanup: {after_mem:.2f} MB (change: {after_mem-before_mem:.2f} MB)") except (ImportError, Exception): pass + if trim_heap: + trim_c_allocator_heap() logger.trace("Memory cleanup complete for django") @@ -618,8 +622,7 @@ def spawn_memory_trim(close_connections=False): so the pooled DB connection is released first. """ def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() + cleanup_memory(force_collection=True, trim_heap=True) if close_connections: from django.db import close_old_connections diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 18be81eb..16a393dc 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -99,6 +99,7 @@ def cleanup_task_memory(**kwargs): memory_intensive_tasks = [ 'apps.m3u.tasks.refresh_single_m3u_account', 'apps.m3u.tasks.refresh_m3u_accounts', + 'apps.m3u.tasks.refresh_m3u_groups', 'apps.m3u.tasks.process_m3u_batch', 'apps.m3u.tasks.process_xc_category', 'apps.m3u.tasks.sync_auto_channels', @@ -121,7 +122,7 @@ def cleanup_task_memory(**kwargs): from core.utils import cleanup_memory # Use the comprehensive cleanup function - cleanup_memory(log_usage=True, force_collection=True) + cleanup_memory(log_usage=True, force_collection=True, trim_heap=True) # Log memory usage if psutil is installed try: From f01c6563c1937267345491074fb61794b3ca3735 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 11:52:00 -0500 Subject: [PATCH 28/31] refactor(tasks): Optimize M3U stream processing by pre-compiling filters for batch workers, reducing redundant regex evaluations. Update related functions to improve performance and memory management during account refreshes. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 118 ++++++++++++------- apps/m3u/tests/test_memory_cleanup.py | 24 +++- apps/m3u/tests/test_stream_filters.py | 156 ++++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 43 deletions(-) create mode 100644 apps/m3u/tests/test_stream_filters.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ce80b8e5..82058d39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **M3U/XC stream refresh is faster on large accounts.** Steady-state refreshes split `bulk_update` into a lightweight touch pass (`last_seen` / `is_stale` only) for unchanged streams and a full column update only when provider metadata or catch-up fields actually change. +- **M3U stream filters compile once per refresh.** Account filters are regex-compiled before batch workers start; accounts with no filters skip per-stream filter checks entirely. - **Celery workers return RSS after memory-intensive tasks.** `cleanup_memory()` accepts an optional `trim_heap` flag (glibc `malloc_trim`); Celery `task_postrun` enables it after `close_old_connections()` for M3U account/group refresh, EPG, VOD, and channel-matching tasks so worker memory drops back toward baseline instead of ratcheting across successive large jobs. ### Fixed diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 7ef71dda..3818683a 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -975,6 +975,39 @@ def collect_xc_streams(account_id, enabled_groups): return all_streams +def _compile_m3u_stream_filters(filter_queryset): + """Compile account M3UFilter rows once per refresh for batch workers.""" + compiled = [] + for filter_obj in filter_queryset: + flags = ( + re.IGNORECASE + if (filter_obj.custom_properties or {}).get("case_sensitive", True) is False + else 0 + ) + compiled.append((re.compile(filter_obj.regex_pattern, flags), filter_obj)) + return compiled + + +def _stream_passes_m3u_filters(name, url, group_title, compiled_filters): + """Return False when the first matching filter excludes the stream.""" + for pattern, filter_obj in compiled_filters: + logger.trace("Checking filter pattern %s", pattern.pattern) + if filter_obj.filter_type == "url": + target = url + elif filter_obj.filter_type == "group": + target = group_title + else: + target = name + + if pattern.search(target or ""): + logger.debug( + "Stream %s - %s matches filter pattern %s", + name, url, filter_obj.regex_pattern, + ) + return not filter_obj.exclude + return True + + _STREAM_TOUCH_FIELDS = ("last_seen", "is_stale") _STREAM_CHANGED_FIELDS = ( "name", "url", "logo_url", "tvg_id", "custom_properties", "is_adult", @@ -1192,8 +1225,12 @@ def process_xc_category_direct(account_id, batch, groups, hash_keys): return retval -def process_m3u_batch_direct(account_id, batch, groups, hash_keys): - """Processes a batch of M3U streams using bulk operations with thread-safe DB connections.""" +def process_m3u_batch_direct(account_id, batch, groups, hash_keys, compiled_filters=None): + """Processes a batch of M3U streams using bulk operations with thread-safe DB connections. + + ``compiled_filters`` should be pre-built once per account refresh and shared + across batch workers. Pass an empty list when the account has no filters. + """ from django.db import connections # Ensure clean database connections for threading @@ -1201,23 +1238,8 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): account = M3UAccount.objects.get(id=account_id) - compiled_filters = [ - ( - re.compile( - f.regex_pattern, - ( - re.IGNORECASE - if (f.custom_properties or {}).get( - "case_sensitive", True - ) - == False - else 0 - ), - ), - f, - ) - for f in account.filters.order_by("order") - ] + if compiled_filters is None: + compiled_filters = _compile_m3u_stream_filters(account.filters.order_by("order")) streams_to_create = [] streams_to_update = [] @@ -1228,7 +1250,10 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): logger.debug(f"Processing batch of {len(batch)} for M3U account {account_id}") if compiled_filters: - logger.debug(f"Using compiled filters: {[f[1].regex_pattern for f in compiled_filters]}") + logger.debug( + "Using compiled filters: %s", + [filter_obj.regex_pattern for _, filter_obj in compiled_filters], + ) for stream_info in batch: try: name, url = stream_info["name"], stream_info["url"] @@ -1249,25 +1274,12 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): group_title = get_case_insensitive_attr( stream_info["attributes"], "group-title", "Default Group" ) - logger.debug(f"Processing stream: {name} - {url} in group {group_title}") - include = True - for pattern, filter in compiled_filters: - logger.trace(f"Checking filter pattern {pattern}") - target = name - if filter.filter_type == "url": - target = url - elif filter.filter_type == "group": - target = group_title + logger.trace("Processing stream: %s - %s in group %s", name, url, group_title) - if pattern.search(target or ""): - logger.debug( - f"Stream {name} - {url} matches filter pattern {filter.regex_pattern}" - ) - include = not filter.exclude - break - - if not include: - logger.debug(f"Stream excluded by filter, skipping.") + if compiled_filters and not _stream_passes_m3u_filters( + name, url, group_title, compiled_filters, + ): + logger.debug("Stream excluded by filter, skipping.") continue # Filter out disabled groups for this account @@ -3322,7 +3334,15 @@ def _refresh_single_m3u_account_impl(account_id): ) account = _get_active_m3u_account(account_id) - filters = list(account.filters.all()) + compiled_stream_filters = _compile_m3u_stream_filters( + account.filters.order_by("order") + ) + if compiled_stream_filters: + logger.debug( + "Account %s has %s stream filter(s) for this refresh", + account_id, + len(compiled_stream_filters), + ) # Check if VOD is enabled for this account vod_enabled = ensure_custom_properties_dict(account.custom_properties).get( @@ -3499,7 +3519,14 @@ def _refresh_single_m3u_account_impl(account_id): with ThreadPoolExecutor(max_workers=max_workers) as executor: # Submit batch processing tasks using direct functions (now thread-safe) future_to_batch = { - executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i + executor.submit( + process_m3u_batch_direct, + account_id, + batch, + existing_groups, + hash_keys, + compiled_stream_filters, + ): i for i, batch in enumerate(batches) } @@ -3612,7 +3639,14 @@ def _refresh_single_m3u_account_impl(account_id): with ThreadPoolExecutor(max_workers=max_workers) as executor: # Submit stream batch processing tasks (reuse standard M3U processing) future_to_batch = { - executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i + executor.submit( + process_m3u_batch_direct, + account_id, + batch, + existing_groups, + hash_keys, + compiled_stream_filters, + ): i for i, batch in enumerate(batches) } @@ -3819,6 +3853,8 @@ def _refresh_single_m3u_account_impl(account_id): del filtered_groups if 'channel_group_relationships' in locals(): del channel_group_relationships + if 'compiled_stream_filters' in locals(): + del compiled_stream_filters # Remove cache file after processing (success or failure) cache_path = os.path.join(m3u_dir, f"{account_id}.json") diff --git a/apps/m3u/tests/test_memory_cleanup.py b/apps/m3u/tests/test_memory_cleanup.py index bfc964d0..b29ca947 100644 --- a/apps/m3u/tests/test_memory_cleanup.py +++ b/apps/m3u/tests/test_memory_cleanup.py @@ -30,7 +30,7 @@ class ProcessM3UBatchCleanupTests(SimpleTestCase): mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") with patch("django.db.connections") as mock_connections: - process_m3u_batch_direct(1, [], {}, ["name", "url"]) + process_m3u_batch_direct(1, [], {}, ["name", "url"], compiled_filters=[]) mock_connections.close_all.assert_called() @patch("apps.m3u.tasks.Stream") @@ -48,9 +48,29 @@ class ProcessM3UBatchCleanupTests(SimpleTestCase): mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") with patch("gc.collect") as mock_gc, patch("django.db.connections"): - process_m3u_batch_direct(1, [], {}, ["name", "url"]) + process_m3u_batch_direct(1, [], {}, ["name", "url"], compiled_filters=[]) mock_gc.assert_called() + @patch("apps.m3u.tasks.Stream") + @patch("apps.m3u.tasks.M3UAccount") + def test_precompiled_empty_filters_skip_db_lookup( + self, mock_account_cls, mock_stream_cls, + ): + """When filters are precompiled as empty, batch workers must not query filters.""" + from apps.m3u.tasks import process_m3u_batch_direct + + mock_account = MagicMock() + mock_account_cls.objects.get.return_value = mock_account + mock_stream_cls.objects.filter.return_value.select_related.return_value.only.return_value = ( + [] + ) + mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") + + with patch("django.db.connections"): + process_m3u_batch_direct(1, [], {}, ["name", "url"], compiled_filters=[]) + + mock_account.filters.order_by.assert_not_called() + class LockReleaseTests(SimpleTestCase): """Verify task lock is released on all exit paths.""" diff --git a/apps/m3u/tests/test_stream_filters.py b/apps/m3u/tests/test_stream_filters.py new file mode 100644 index 00000000..aba1a262 --- /dev/null +++ b/apps/m3u/tests/test_stream_filters.py @@ -0,0 +1,156 @@ +"""Tests for M3U stream filter compilation and batch application.""" +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.m3u.tasks import ( + _compile_m3u_stream_filters, + _stream_passes_m3u_filters, + process_m3u_batch_direct, +) + + +class CompileM3UStreamFiltersTests(SimpleTestCase): + def test_compiles_case_insensitive_when_configured(self): + filter_obj = MagicMock() + filter_obj.regex_pattern = "news" + filter_obj.custom_properties = {"case_sensitive": False} + + compiled = _compile_m3u_stream_filters([filter_obj]) + + self.assertEqual(len(compiled), 1) + pattern, _ = compiled[0] + self.assertTrue(pattern.search("NEWS")) + + def test_compiles_case_sensitive_by_default(self): + filter_obj = MagicMock() + filter_obj.regex_pattern = "news" + filter_obj.custom_properties = {} + + compiled = _compile_m3u_stream_filters([filter_obj]) + + pattern, _ = compiled[0] + self.assertIsNone(pattern.search("NEWS")) + self.assertTrue(pattern.search("news")) + + +class StreamPassesM3UFiltersTests(SimpleTestCase): + def _compiled(self, *, filter_type="name", exclude=False, pattern="Adult"): + filter_obj = MagicMock() + filter_obj.filter_type = filter_type + filter_obj.exclude = exclude + filter_obj.regex_pattern = pattern + filter_obj.custom_properties = {} + return _compile_m3u_stream_filters([filter_obj]) + + def test_include_filter_passes_matching_stream(self): + compiled = self._compiled(exclude=False) + self.assertTrue( + _stream_passes_m3u_filters("Adult Channel", "http://x", "News", compiled) + ) + + def test_include_filter_passes_non_matching_stream(self): + """Non-matching streams still pass unless a matching exclude filter hits.""" + compiled = self._compiled(exclude=False, pattern="news") + self.assertTrue( + _stream_passes_m3u_filters("Sports", "http://x", "Sports", compiled) + ) + + def test_exclude_filter_rejects_matching_stream(self): + compiled = self._compiled(exclude=True, pattern="Adult") + self.assertFalse( + _stream_passes_m3u_filters("Adult Channel", "http://x", "News", compiled) + ) + + def test_url_filter_type_targets_url(self): + compiled = self._compiled(filter_type="url", exclude=True, pattern="blocked") + self.assertFalse( + _stream_passes_m3u_filters("OK", "http://blocked.example/live", "News", compiled) + ) + self.assertTrue( + _stream_passes_m3u_filters("blocked name", "http://ok.example/live", "News", compiled) + ) + + def test_group_filter_type_targets_group(self): + compiled = self._compiled(filter_type="group", exclude=True, pattern="Hidden") + self.assertFalse( + _stream_passes_m3u_filters("Channel", "http://x", "Hidden Group", compiled) + ) + + +class ProcessM3UBatchFilterTests(SimpleTestCase): + def _mock_stream_meta(self, mock_stream_cls, max_length=255): + mock_field = MagicMock() + mock_field.max_length = max_length + mock_stream_cls._meta.get_field.return_value = mock_field + + @patch("apps.m3u.tasks._bulk_update_stream_refresh_batches") + @patch("apps.m3u.tasks.Stream") + @patch("apps.m3u.tasks.M3UAccount") + def test_exclude_filter_skips_stream_import( + self, mock_account_cls, mock_stream_cls, mock_bulk_update, + ): + self._mock_stream_meta(mock_stream_cls) + mock_account = MagicMock() + mock_account.account_type = "STD" + mock_account_cls.objects.get.return_value = mock_account + mock_stream_cls.objects.filter.return_value.select_related.return_value.only.return_value = ( + [] + ) + mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") + + filter_obj = MagicMock() + filter_obj.regex_pattern = "skip-me" + filter_obj.filter_type = "name" + filter_obj.exclude = True + filter_obj.custom_properties = {} + compiled = _compile_m3u_stream_filters([filter_obj]) + + batch = [{ + "name": "skip-me channel", + "url": "http://example/live", + "attributes": {"group-title": "News"}, + "vlc_opts": {}, + }] + + with patch("django.db.connections"): + result = process_m3u_batch_direct( + 1, batch, {"News": 1}, ["name", "url"], compiled_filters=compiled, + ) + + self.assertIn("0 created", result) + mock_stream_cls.objects.bulk_create.assert_not_called() + mock_bulk_update.assert_called_once_with([], [], batch_size=200) + + @patch("apps.m3u.tasks._bulk_update_stream_refresh_batches") + @patch("apps.m3u.tasks.Stream") + @patch("apps.m3u.tasks.M3UAccount") + def test_no_filters_imports_matching_stream( + self, mock_account_cls, mock_stream_cls, mock_bulk_update, + ): + self._mock_stream_meta(mock_stream_cls) + mock_account = MagicMock() + mock_account.account_type = "STD" + mock_account_cls.objects.get.return_value = mock_account + mock_stream_cls.objects.filter.return_value.select_related.return_value.only.return_value = ( + [] + ) + mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") + mock_stream_cls.objects.bulk_create.return_value = [] + + batch = [{ + "name": "News One", + "url": "http://example/live", + "attributes": {"group-title": "News"}, + "vlc_opts": {}, + }] + + with patch("django.db.connections"), patch( + "apps.m3u.tasks.transaction.atomic", + ): + result = process_m3u_batch_direct( + 1, batch, {"News": 1}, ["name", "url"], compiled_filters=[], + ) + + self.assertIn("1 created", result) + mock_stream_cls.objects.bulk_create.assert_called_once() From 6456d1179554b7a7aa05f979c1c15514f1b241bf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 12:48:02 -0500 Subject: [PATCH 29/31] refactor(tasks): Enhance M3U refresh process by introducing a line-by-line parsing method for on-disk files, improving memory efficiency. Update error handling to return None for failures, and streamline the fetching logic for better clarity and performance. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 112 +++++++++++++++++++++++++++++----------------- 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82058d39..9991c83e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U/XC stream refresh is faster on large accounts.** Steady-state refreshes split `bulk_update` into a lightweight touch pass (`last_seen` / `is_stale` only) for unchanged streams and a full column update only when provider metadata or catch-up fields actually change. - **M3U stream filters compile once per refresh.** Account filters are regex-compiled before batch workers start; accounts with no filters skip per-stream filter checks entirely. +- **M3U refresh releases parse catalogs sooner.** Standard accounts stream-parse the on-disk M3U instead of loading the full file into RAM; `extinf_data` is dropped immediately after batch DB work, before stale cleanup and auto-sync. - **Celery workers return RSS after memory-intensive tasks.** `cleanup_memory()` accepts an optional `trim_heap` flag (glibc `malloc_trim`); Celery `task_postrun` enables it after `close_old_connections()` for M3U account/group refresh, EPG, VOD, and channel-matching tasks so worker memory drops back toward baseline instead of ratcheting across successive large jobs. ### Fixed diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 3818683a..b06532d6 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -134,11 +134,23 @@ def _ensure_m3u_refresh_terminal_status(account_id): _EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') +def _open_m3u_text_source(source_path): + """Open an on-disk M3U (or .m3u.gz) file for line-by-line parsing.""" + if source_path.endswith(".gz"): + return gzip.open(source_path, "rt", encoding="utf-8") + return open(source_path, "r", encoding="utf-8") + + def fetch_m3u_lines(account, use_cache=False): + """Fetch M3U source for parsing. + + On success returns ``(source, True)`` where *source* is either a filesystem + path (streamed during parse) or, for ZIP uploads only, an in-memory line + list. Failures return ``(None, False)``. + """ os.makedirs(m3u_dir, exist_ok=True) file_path = os.path.join(m3u_dir, f"{account.id}.m3u") - """Fetch M3U file lines efficiently.""" if account.server_url: if not use_cache or not os.path.exists(file_path): try: @@ -213,7 +225,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False # Only call raise_for_status if we have a success code (this should not raise now) response.raise_for_status() @@ -288,7 +300,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False # Validate the file by reading only the first portion from # disk — no need to load the entire file into memory just @@ -360,7 +372,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False except UnicodeDecodeError: with open(temp_path, "rb") as vf: @@ -378,7 +390,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False # Validation passed — promote temp file to final path os.replace(temp_path, file_path) @@ -433,7 +445,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False except requests.exceptions.RequestException as e: # Handle other request errors (connection, timeout, etc.) if "timeout" in str(e).lower(): @@ -454,7 +466,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False except Exception as e: # Handle any other unexpected errors error_msg = f"Unexpected error while fetching M3U file from URL: {account.server_url} - {str(e)}" @@ -469,7 +481,7 @@ def fetch_m3u_lines(account, use_cache=False): status="error", error=error_msg, ) - return [], False + return None, False # Check if the file exists and is not empty (fallback check - should not happen with new validation) if not os.path.exists(file_path) or os.path.getsize(file_path) == 0: @@ -481,27 +493,14 @@ def fetch_m3u_lines(account, use_cache=False): send_m3u_update( account.id, "downloading", 100, status="error", error=error_msg ) - return [], False # Return empty list and False for success + return None, False - try: - with open(file_path, "r", encoding="utf-8") as f: - return f.readlines(), True - except Exception as e: - error_msg = f"Error reading M3U file: {str(e)}" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account.id, "downloading", 100, status="error", error=error_msg - ) - return [], False + return file_path, True elif account.file_path: try: if account.file_path.endswith(".gz"): - with gzip.open(account.file_path, "rt", encoding="utf-8") as f: - return f.readlines(), True + return account.file_path, True elif account.file_path.endswith(".zip"): with zipfile.ZipFile(account.file_path, "r") as zip_file: @@ -522,11 +521,10 @@ def fetch_m3u_lines(account, use_cache=False): send_m3u_update( account.id, "downloading", 100, status="error", error=error_msg ) - return [], False + return None, False else: - with open(account.file_path, "r", encoding="utf-8") as f: - return f.readlines(), True + return account.file_path, True except (IOError, OSError, zipfile.BadZipFile, gzip.BadGzipFile) as e: error_msg = f"Error opening file {account.file_path}: {e}" @@ -537,7 +535,7 @@ def fetch_m3u_lines(account, use_cache=False): send_m3u_update( account.id, "downloading", 100, status="error", error=error_msg ) - return [], False + return None, False # Neither server_url nor uploaded_file is available error_msg = "No M3U source available (missing URL and file)" @@ -546,7 +544,7 @@ def fetch_m3u_lines(account, use_cache=False): account.last_message = error_msg account.save(update_fields=["status", "last_message"]) send_m3u_update(account.id, "downloading", 100, status="error", error=error_msg) - return [], False + return None, False def get_case_insensitive_attr(attributes, key, default=""): @@ -1714,28 +1712,49 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_sta release_task_lock("refresh_m3u_account_groups", account_id) return error_msg, None else: - lines, success = fetch_m3u_lines(account, use_cache) + source, success = fetch_m3u_lines(account, use_cache) if not success: # If fetch failed, don't continue processing lock_renewer.stop() release_task_lock("refresh_m3u_account_groups", account_id) return f"Failed to fetch M3U data for account_id={account_id}.", None - # Log basic file structure for debugging - logger.debug(f"Processing {len(lines)} lines from M3U file") - valid_stream_count = 0 - for entry in iter_m3u_entries(lines): - valid_stream_count += 1 - group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") - if group_title_attr and group_title_attr not in groups: - logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") - groups[group_title_attr] = {} - extinf_data.append(entry) + if isinstance(source, str): + logger.debug(f"Streaming M3U parse from {source}") + with _open_m3u_text_source(source) as m3u_file: + entry_iter = iter_m3u_entries(m3u_file) + for entry in entry_iter: + valid_stream_count += 1 + group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") + if group_title_attr and group_title_attr not in groups: + logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") + groups[group_title_attr] = {} + extinf_data.append(entry) - if valid_stream_count % 1000 == 0: - logger.debug(f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}") + if valid_stream_count % 1000 == 0: + logger.debug( + f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}" + ) + else: + logger.debug(f"Processing {len(source)} in-memory M3U lines (zip upload)") + try: + for entry in iter_m3u_entries(source): + valid_stream_count += 1 + group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") + if group_title_attr and group_title_attr not in groups: + logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") + groups[group_title_attr] = {} + extinf_data.append(entry) + + if valid_stream_count % 1000 == 0: + logger.debug( + f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}" + ) + finally: + del source + gc.collect() logger.info(f"M3U parsing complete - Valid streams: {valid_stream_count}") @@ -3581,6 +3600,10 @@ def _refresh_single_m3u_account_impl(account_id): batches[batch_idx] = None logger.info(f"Thread-based processing completed for account {account_id}") + + # Parsed catalog is no longer needed; drop before stale cleanup / auto-sync. + del extinf_data, batches + gc.collect() else: # For XC accounts, get the groups with their custom properties containing xc_id logger.debug(f"Processing XC account with groups: {existing_groups}") @@ -3702,6 +3725,9 @@ def _refresh_single_m3u_account_impl(account_id): logger.info(f"XC thread-based processing completed for account {account_id}") + del batches + gc.collect() + # Ensure all database transactions are committed before cleanup logger.info( f"All thread processing completed, ensuring DB transactions are committed before cleanup" @@ -3856,6 +3882,8 @@ def _refresh_single_m3u_account_impl(account_id): if 'compiled_stream_filters' in locals(): del compiled_stream_filters + gc.collect() + # Remove cache file after processing (success or failure) cache_path = os.path.join(m3u_dir, f"{account_id}.json") try: From 6d17663f84b52a96d6277dc0c7639fd2c38bf646 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 13:33:26 -0500 Subject: [PATCH 30/31] refactor(tasks): Optimize XC stream collection by avoiding redundant work during catalog filtering. Introduce a shared URL prefix for stream URLs and enhance memory management by releasing per-group caches after processing. Update changelog to reflect these improvements. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 101 +++++++++++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9991c83e..a20b9032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U/XC stream refresh is faster on large accounts.** Steady-state refreshes split `bulk_update` into a lightweight touch pass (`last_seen` / `is_stale` only) for unchanged streams and a full column update only when provider metadata or catch-up fields actually change. - **M3U stream filters compile once per refresh.** Account filters are regex-compiled before batch workers start; accounts with no filters skip per-stream filter checks entirely. - **M3U refresh releases parse catalogs sooner.** Standard accounts stream-parse the on-disk M3U instead of loading the full file into RAM; `extinf_data` is dropped immediately after batch DB work, before stale cleanup and auto-sync. +- **XC live refresh avoids redundant work during catalog filtering.** `collect_xc_streams` skips disabled categories before building entries and uses a shared URL prefix instead of formatting each stream URL separately. Auto-sync releases per-group logo/EPG caches after each group iteration. - **Celery workers return RSS after memory-intensive tasks.** `cleanup_memory()` accepts an optional `trim_heap` flag (glibc `malloc_trim`); Celery `task_postrun` enables it after `close_old_connections()` for M3U account/group refresh, EPG, VOD, and channel-matching tasks so worker memory drops back toward baseline instead of ratcheting across successive large jobs. ### Fixed diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index b06532d6..a5894892 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -906,6 +906,11 @@ def collect_xc_streams(account_id, enabled_groups): account.get_user_agent(), ) as xc_client: + stream_url_prefix = ( + f"{xc_client.server_url.rstrip('/')}/live/" + f"{xc_client.username}/{xc_client.password}/" + ) + # Fetch ALL live streams in a single API call (much more efficient) logger.info("Fetching ALL live streams from XC provider...") all_xc_streams = xc_client.get_all_live_streams() # Get all streams without category filter @@ -918,46 +923,49 @@ def collect_xc_streams(account_id, enabled_groups): # Filter streams based on enabled categories for stream in all_xc_streams: - # Fall back to a generated name if the provider returns null/empty - stream_name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" + category_id = str(stream.get("category_id", "")) + if category_id not in enabled_category_ids: + continue + + group_info = enabled_category_ids[category_id] + stream_name = stream.get("name") or ( + f"{account.name} - {stream.get('stream_id', 'Unknown')}" + ) if not stream.get("name"): logger.warning( - f"XC stream has null/empty name; using generated name '{stream_name}' " - f"(stream_id={stream.get('stream_id', 'unknown')})" + "XC stream has null/empty name; using generated name '%s' " + "(stream_id=%s)", + stream_name, stream.get("stream_id", "unknown"), ) - # Get the category_id for this stream - category_id = str(stream.get("category_id", "")) - - # Only include streams from enabled categories - if category_id in enabled_category_ids: - group_info = enabled_category_ids[category_id] - - # Convert XC stream to our standard format with all properties preserved - stream_data = { - "name": stream_name, - "url": xc_client.get_stream_url(stream["stream_id"]), - "attributes": { - "tvg-id": stream.get("epg_channel_id", ""), - "tvg-logo": stream.get("stream_icon", ""), - "group-title": group_info["name"], - # Preserve all XC stream properties as custom attributes - "stream_id": str(stream.get("stream_id", "")), - "num": stream.get("num"), - "category_id": category_id, - "stream_type": stream.get("stream_type", ""), - "added": stream.get("added", ""), - "is_adult": str(stream.get("is_adult", "0")), - "custom_sid": stream.get("custom_sid", ""), - # Include any other properties that might be present - **{k: str(v) for k, v in stream.items() if k not in [ - "name", "stream_id", "epg_channel_id", "stream_icon", - "category_id", "stream_type", "added", "is_adult", "custom_sid", "num" - ] and v is not None} - } - } - all_streams.append(stream_data) - filtered_count += 1 + stream_id = stream.get("stream_id") + attributes = { + "tvg-id": stream.get("epg_channel_id", ""), + "tvg-logo": stream.get("stream_icon", ""), + "group-title": group_info["name"], + "stream_id": str(stream.get("stream_id", "")), + "num": stream.get("num"), + "category_id": category_id, + "stream_type": stream.get("stream_type", ""), + "added": stream.get("added", ""), + "is_adult": str(stream.get("is_adult", "0")), + "custom_sid": stream.get("custom_sid", ""), + **{ + k: str(v) + for k, v in stream.items() + if k not in [ + "name", "stream_id", "epg_channel_id", "stream_icon", + "category_id", "stream_type", "added", "is_adult", + "custom_sid", "num", + ] and v is not None + }, + } + all_streams.append({ + "name": stream_name, + "url": f"{stream_url_prefix}{stream_id}.ts", + "attributes": attributes, + }) + filtered_count += 1 # Drop the full provider catalog before returning; only filtered rows are needed. del all_xc_streams @@ -2927,6 +2935,17 @@ def sync_auto_channels(account_id, scan_start_time=None): f"{pack_result['failed']} failed" ) + # Release per-group working sets before the next group iteration. + del ( + current_streams, + logo_cache_by_url, + epg_cache_by_tvg_id, + existing_channel_map, + existing_channels_by_id, + existing_channel_streams, + processed_stream_ids, + ) + # Cleanup mode read from account.custom_properties.orphan_channel_cleanup: # "always" (default; key absent) removes every orphan auto channel; # "preserve_customized" keeps those with a ChannelOverride row; @@ -2962,7 +2981,7 @@ def sync_auto_channels(account_id, scan_start_time=None): f"{channels_created} created, {channels_updated} updated, " f"{channels_deleted} deleted, {channels_failed} failed" ) - return { + result = { "status": "ok", "channels_created": channels_created, "channels_updated": channels_updated, @@ -2970,6 +2989,9 @@ def sync_auto_channels(account_id, scan_start_time=None): "channels_failed": channels_failed, "failed_stream_details": failed_stream_details, } + del failed_stream_details + gc.collect() + return result except Exception as e: logger.error(f"Error in auto channel sync for account {account_id}: {str(e)}") @@ -3641,6 +3663,8 @@ def _refresh_single_m3u_account_impl(account_id): logger.info("Fetching all XC streams from provider and filtering by enabled categories...") all_xc_streams = collect_xc_streams(account_id, filtered_groups) + del channel_group_relationships, filtered_groups + if not all_xc_streams: logger.warning("No streams collected from XC groups") else: @@ -3848,6 +3872,9 @@ def _refresh_single_m3u_account_impl(account_id): message=account.last_message, ) + del auto_sync_result + gc.collect() + # Trigger VOD refresh if enabled and account is XtreamCodes type if vod_enabled and account.account_type == M3UAccount.Types.XC: logger.info(f"VOD is enabled for account {account_id}, triggering VOD refresh") From 269acc2dda5a75f872368fdacd24d3256617edad Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 27 Jun 2026 16:50:44 -0500 Subject: [PATCH 31/31] refactor(timeshift): Replace random session ID generation with secure secrets for timeshift sessions. Introduce helper functions for session management and enhance session validation to prevent foreign session reuse. Update tests to cover new session handling logic and ensure proper user ownership checks. --- apps/timeshift/tests/test_views.py | 68 ++++++++++++++++++++++++------ apps/timeshift/views.py | 56 ++++++++++++++++++------ 2 files changed, 99 insertions(+), 25 deletions(-) diff --git a/apps/timeshift/tests/test_views.py b/apps/timeshift/tests/test_views.py index 270dfbe5..1e45f53b 100644 --- a/apps/timeshift/tests/test_views.py +++ b/apps/timeshift/tests/test_views.py @@ -550,7 +550,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def _call(self, timestamp, provider_tz="Europe/Brussels"): request = self.factory.get(f"/timeshift/u/p/8/{timestamp}/8.ts?session_id={TEST_SESSION_ID}") sentinel = MagicMock(status_code=200) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -596,7 +596,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def test_invalid_timestamp_rejected_before_upstream(self): request = self.factory.get("/timeshift/u/p/8/garbage/8.ts") - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -611,7 +611,7 @@ class TimeshiftProxyTimestampWiringTests(TestCase): def test_network_access_denied_returns_403(self): # Same network gate as other XC API endpoints (player_api, xmltv, etc.). request = self.factory.get(_proxy_url()) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=False) as gate, \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_stream_from_provider") as stream_mock: @@ -634,7 +634,7 @@ class TimeshiftProxyFailoverTests(TestCase): def _call(self, streams, provider_responses): request = self.factory.get(_proxy_url()) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -758,7 +758,7 @@ class _ProxyLoopTestMixin: if build_side_effect is not None else {"return_value": ["http://example.test/x.ts"]} ) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -1359,7 +1359,7 @@ class TimeshiftTakeoverTests(TestCase): # the user's own seek gets denied. call_order = [] request = RequestFactory().get(_proxy_url()) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -1413,7 +1413,9 @@ class TimeshiftSessionReuseTests(TestCase): return_value=profile), \ patch.object(views, "reserve_profile_slot", return_value=(True, 1, None)) as reserve_mock: - acquired = views._acquire_idle_pool_session(self.redis, self.SESSION) + acquired = views._acquire_idle_pool_session( + self.redis, self.SESSION, user_id=5, + ) self.assertIsNotNone(acquired) descriptor, got_profile = acquired self.assertEqual(descriptor["stream_id"], "111") @@ -1425,11 +1427,53 @@ class TimeshiftSessionReuseTests(TestCase): _seed_pool_session(self.redis, session_id=self.SESSION, busy="1") with patch.object(views.M3UAccountProfile.objects, "get") as prof_mock, \ patch.object(views, "reserve_profile_slot") as reserve_mock: - acquired = views._acquire_idle_pool_session(self.redis, self.SESSION) + acquired = views._acquire_idle_pool_session( + self.redis, self.SESSION, user_id=5, + ) self.assertIsNone(acquired) prof_mock.assert_not_called() reserve_mock.assert_not_called() + def test_acquire_rejects_foreign_user(self): + self._make_idle_entry() + profile = MagicMock(id=31) + with patch.object(views.M3UAccountProfile.objects, "get", + return_value=profile), \ + patch.object(views, "reserve_profile_slot", + return_value=(True, 1, None)) as reserve_mock: + acquired = views._acquire_idle_pool_session( + self.redis, self.SESSION, user_id=99, + ) + self.assertIsNone(acquired) + reserve_mock.assert_not_called() + + def test_foreign_session_id_redirects_instead_of_reusing_pool(self): + victim_session = "timeshift_victim_session" + _seed_pool_session(self.redis, session_id=victim_session, user_id=99) + request = self.factory.get(_proxy_url(victim_session)) + attacker = MagicMock(id=5) + with patch.object(views, "_authenticate_user", return_value=attacker), \ + patch.object(views, "network_access_allowed", return_value=True), \ + patch.object(views, "Channel") as channel_cls, \ + patch.object(views, "_user_can_access_channel", return_value=True), \ + patch.object(views, "get_channel_catchup_streams", + return_value=[_make_catchup_stream()]), \ + patch.object(views, "get_programme_duration", return_value=40), \ + patch.object(views, "parse_catchup_timestamp", return_value=True), \ + patch.object(views, "RedisClient") as redis_cls, \ + patch.object(views, "_acquire_idle_pool_session") as acquire_mock, \ + patch.object(views, "_attempt_timeshift_stream") as attempt_mock: + redis_cls.get_client.return_value = self.redis + channel_cls.objects.get.return_value = MagicMock(id=8) + response = views.timeshift_proxy( + request, "u", "p", "8", "2026-06-08:17-00", "8.ts", + ) + self.assertEqual(response.status_code, 301) + self.assertIn("session_id=timeshift_", response["Location"]) + self.assertNotIn(victim_session, response["Location"]) + acquire_mock.assert_not_called() + attempt_mock.assert_not_called() + def test_find_matching_idle_session_requires_ip_and_user_agent(self): _seed_pool_session( self.redis, session_id="timeshift_other", @@ -1729,7 +1773,7 @@ class TimeshiftScrubPreemptTests(TestCase): HTTP_RANGE="bytes=1000-", ) streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -1768,7 +1812,7 @@ class TimeshiftScrubPreemptTests(TestCase): HTTP_RANGE="bytes=0-", ) streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -1800,7 +1844,7 @@ class TimeshiftScrubPreemptTests(TestCase): HTTP_RANGE="bytes=2527702896-", ) streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ @@ -1866,7 +1910,7 @@ class TimeshiftScrubPreemptTests(TestCase): streams = [_make_catchup_stream(account_id=1, stream_id="111", profile_id=31)] profile = MagicMock(id=31) ok = MagicMock(status_code=206) - with patch.object(views, "_authenticate_user", return_value=MagicMock()), \ + with patch.object(views, "_authenticate_user", return_value=MagicMock(id=5)), \ patch.object(views, "network_access_allowed", return_value=True), \ patch.object(views, "Channel") as channel_cls, \ patch.object(views, "_user_can_access_channel", return_value=True), \ diff --git a/apps/timeshift/views.py b/apps/timeshift/views.py index 113cd460..0ba3a5a7 100644 --- a/apps/timeshift/views.py +++ b/apps/timeshift/views.py @@ -3,7 +3,7 @@ import hmac import itertools import logging -import random +import secrets import time from urllib.parse import urlencode @@ -104,19 +104,20 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) session_id = request.GET.get("session_id") if not session_id: - session_id = f"timeshift_{int(time.time() * 1000)}_{random.randint(1000, 9999)}" - query_params = {k: request.GET.getlist(k) for k in request.GET} - query_params["session_id"] = [session_id] - redirect_url = f"{request.path}?{urlencode(query_params, doseq=True)}" - logger.debug("Timeshift session redirect: %s -> %s", request.path, session_id) - return HttpResponse(status=301, headers={"Location": redirect_url}) + logger.debug("Timeshift session redirect: %s (new session)", request.path) + return _redirect_with_new_session(request) + + session_entry = _get_pool_entry(redis_client, session_id) + if session_entry and not _pool_entry_owned_by_user(session_entry, user.id): + logger.info( + "Timeshift: rejecting foreign session_id for user %s", user.id, + ) + return _redirect_with_new_session(request) # Stable client identity for stats, stop keys, and the provider pool. effective_session_id = session_id client_id = session_id - session_entry = _get_pool_entry(redis_client, session_id) - # Reuse an idle pool owned by this session, or fingerprint-match a prior # idle session from the same client (VOD-style) before opening upstream. if not session_entry: @@ -178,11 +179,12 @@ def timeshift_proxy(request, username, password, stream_id, timestamp, duration) acquired = _wait_for_idle_pool_session( redis_client, effective_session_id, + user_id=user.id, wait_seconds=_POOL_PREEMPT_WAIT_SECONDS, ) else: acquired = _acquire_idle_pool_session( - redis_client, effective_session_id, + redis_client, effective_session_id, user_id=user.id, ) if acquired is not None: @@ -466,6 +468,28 @@ def _score_pool_fingerprint(entry, client_ip, client_user_agent): return score +def _mint_timeshift_session_id(): + return f"timeshift_{secrets.token_urlsafe(16)}" + + +def _redirect_with_new_session(request): + session_id = _mint_timeshift_session_id() + query_params = {k: request.GET.getlist(k) for k in request.GET} + query_params["session_id"] = [session_id] + redirect_url = f"{request.path}?{urlencode(query_params, doseq=True)}" + return HttpResponse(status=301, headers={"Location": redirect_url}) + + +def _pool_entry_owned_by_user(entry, user_id): + """True when *entry* is unclaimed or owned by *user_id*.""" + if not entry or not entry.get("profile_id"): + return True + owner = entry.get("user_id") + if owner is None or owner == "": + return False + return str(owner) == str(user_id) + + def _find_matching_idle_session( redis_client, *, media_id, user_id, client_ip, client_user_agent, ): @@ -583,7 +607,7 @@ def _pool_lock(redis_client, session_id): ) -def _acquire_idle_pool_session(redis_client, session_id): +def _acquire_idle_pool_session(redis_client, session_id, *, user_id=None): """Re-reserve an idle session's profile slot and mark it busy.""" if redis_client is None or not session_id: return None @@ -593,6 +617,8 @@ def _acquire_idle_pool_session(redis_client, session_id): data = redis_client.hgetall(key) if not data or not data.get("profile_id"): return None + if user_id is not None and not _pool_entry_owned_by_user(data, user_id): + return None if data.get("busy") == "1": return None try: @@ -614,12 +640,16 @@ def _acquire_idle_pool_session(redis_client, session_id): return None -def _wait_for_idle_pool_session(redis_client, session_id, wait_seconds=_POOL_WAIT_SECONDS): +def _wait_for_idle_pool_session( + redis_client, session_id, *, user_id=None, wait_seconds=_POOL_WAIT_SECONDS, +): if redis_client is None or not session_id: return None deadline = time.time() + wait_seconds while True: - acquired = _acquire_idle_pool_session(redis_client, session_id) + acquired = _acquire_idle_pool_session( + redis_client, session_id, user_id=user_id, + ) if acquired is not None: return acquired if not _get_pool_entry(redis_client, session_id):