From d89f63421337e15bb31b58c027a62e0b2de275d7 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Wed, 20 May 2026 03:30:48 -0400 Subject: [PATCH 01/81] Enforce shared credential connection pools across M3U accounts (#1137) Group M3U/XC accounts and profiles that share the same provider login into auto-assigned ServerGroups keyed by credential fingerprint. Enforce combined Redis limits for live TV and VOD via apps/m3u/connection_pool.py. - Per-profile fingerprinting (XC transforms and STD stream URLs) - VOD profile selection tries alternates when default credential pool is full (fixes live then VOD failure) - Stats UI shows provider login from active stream URL - Tests: apps/m3u/tests/test_connection_pool.py (11 tests, all passing) Co-authored-by: Cursor --- apps/channels/models.py | 137 ++++---- apps/m3u/connection_pool.py | 299 ++++++++++++++++++ .../0020_servergroup_max_streams.py | 21 ++ ...0021_servergroup_credential_fingerprint.py | 102 ++++++ apps/m3u/models.py | 32 ++ apps/m3u/serializers.py | 2 +- apps/m3u/tests/test_connection_pool.py | 247 +++++++++++++++ apps/proxy/live_proxy/channel_status.py | 22 +- apps/proxy/live_proxy/url_utils.py | 79 +++-- .../multi_worker_connection_manager.py | 80 ++--- apps/proxy/vod_proxy/views.py | 34 +- .../components/cards/StreamConnectionCard.jsx | 22 +- .../components/cards/VodConnectionCard.jsx | 7 + 13 files changed, 917 insertions(+), 167 deletions(-) create mode 100644 apps/m3u/connection_pool.py create mode 100644 apps/m3u/migrations/0020_servergroup_max_streams.py create mode 100644 apps/m3u/migrations/0021_servergroup_credential_fingerprint.py create mode 100644 apps/m3u/tests/test_connection_pool.py diff --git a/apps/channels/models.py b/apps/channels/models.py index 59af3e00..59a2bc3d 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -4,7 +4,7 @@ from django.conf import settings from core.models import StreamProfile, CoreSettings from core.utils import RedisClient from apps.proxy.live_proxy.redis_keys import RedisKeys -from apps.proxy.live_proxy.constants import ChannelMetadataField +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState import logging import uuid from django.utils import timezone @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) # If you have an M3UAccount model in apps.m3u, you can still import it: from apps.m3u.models import M3UAccount +from apps.m3u.connection_pool import reserve_profile_slot, release_profile_slot # Add fallback functions if Redis isn't available @@ -231,17 +232,8 @@ class Stream(models.Model): if profile.is_active == False: continue - # Atomic slot reservation: INCR first, check, rollback if over capacity - if profile.max_streams == 0: - reserved = True - else: - profile_connections_key = f"profile_connections:{profile.id}" - new_count = redis_client.incr(profile_connections_key) - if new_count <= profile.max_streams: - reserved = True - else: - redis_client.decr(profile_connections_key) - reserved = False + # Atomic slot reservation via shared connection pool helper + reserved, _count = reserve_profile_slot(profile, redis_client) if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) @@ -277,12 +269,7 @@ class Stream(models.Model): f"Stream {stream_id}: found profile_id={profile_id}" ) - profile_connections_key = f"profile_connections:{profile_id}" - - # Only decrement if the profile had a max_connections limit - current_count = int(redis_client.get(profile_connections_key) or 0) - if current_count > 0: - redis_client.decr(profile_connections_key) + release_profile_slot(profile_id, redis_client) return True @@ -599,37 +586,29 @@ class Channel(models.Model): return victim_id - def _check_and_reserve_profile_slot(self, profile, redis_client): - """ - Atomically check and reserve a connection slot for the given profile. + def _channel_proxy_is_active(self, redis_client) -> bool: + """True when live proxy metadata shows this channel is still running.""" + metadata_key = RedisKeys.channel_metadata(str(self.uuid)) + if not redis_client.exists(metadata_key): + return False + state = redis_client.hget(metadata_key, ChannelMetadataField.STATE) + if state is None: + return False + if isinstance(state, bytes): + state = state.decode() + return state in ( + ChannelState.ACTIVE, + ChannelState.WAITING_FOR_CLIENTS, + ChannelState.BUFFERING, + ChannelState.INITIALIZING, + ChannelState.CONNECTING, + ) - Uses an INCR-first-then-check pattern to eliminate the TOCTOU race - condition where separate GET + check + INCR operations could allow - concurrent requests to both pass the capacity check. - - For profiles with max_streams=0 (unlimited), no reservation is needed. - - Args: - profile: M3UAccountProfile instance - redis_client: Redis client instance - - Returns: - tuple: (reserved: bool, current_count: int) - """ - if profile.max_streams == 0: - return (True, 0) - - profile_connections_key = f"profile_connections:{profile.id}" - - # Atomically increment first — this is a single Redis command - new_count = redis_client.incr(profile_connections_key) - - if new_count <= profile.max_streams: - return (True, new_count) - - # Over capacity — roll back the increment - redis_client.decr(profile_connections_key) - return (False, new_count - 1) + def _clear_stream_assignment_keys(self, redis_client, stream_id=None) -> None: + """Remove channel/stream profile assignment keys from Redis.""" + redis_client.delete(f"channel_stream:{self.id}") + if stream_id is not None: + redis_client.delete(f"stream_profile:{stream_id}") def get_stream(self, requester=None): """ @@ -646,24 +625,40 @@ class Channel(models.Model): error_reason = "No streams assigned to channel" return None, None, error_reason - # Check if a stream is already active for this channel + # Reuse assignment only when this channel is still active in the proxy. + # Stale channel_stream keys after stop/disconnect caused every tune to + # reuse the default profile without re-running rotation or INCR. stream_id_bytes = redis_client.get(f"channel_stream:{self.id}") if stream_id_bytes: try: stream_id = int(stream_id_bytes) - profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") - if profile_id_bytes: - try: - profile_id = int(profile_id_bytes) - return stream_id, profile_id, None - except (ValueError, TypeError): - logger.debug( - f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" - ) except (ValueError, TypeError): logger.debug( f"Invalid stream ID retrieved from Redis: {stream_id_bytes}" ) + stream_id = None + + if stream_id is not None: + if self._channel_proxy_is_active(redis_client): + profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") + if profile_id_bytes: + try: + profile_id = int(profile_id_bytes) + logger.debug( + f"Channel {self.uuid}: reusing active assignment " + f"stream={stream_id} profile={profile_id}" + ) + return stream_id, profile_id, None + except (ValueError, TypeError): + logger.debug( + f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" + ) + else: + logger.info( + f"Channel {self.uuid}: clearing stale stream assignment " + f"(stream={stream_id}, proxy not active)" + ) + self._clear_stream_assignment_keys(redis_client, stream_id) # No existing active stream, attempt to assign a new one has_streams_but_maxed_out = False @@ -697,14 +692,16 @@ class Channel(models.Model): has_active_profiles = True # Atomically check and reserve a slot (INCR-first pattern) - reserved, current_count = self._check_and_reserve_profile_slot( - profile, redis_client - ) + reserved, current_count = reserve_profile_slot(profile, redis_client) if reserved: # Slot reserved — assign stream to this channel redis_client.set(f"channel_stream:{self.id}", stream.id) redis_client.set(f"stream_profile:{stream.id}", profile.id) + logger.info( + f"Channel {self.uuid}: assigned stream {stream.id} " + f"profile {profile.id} ({profile.name})" + ) return ( stream.id, @@ -782,12 +779,11 @@ class Channel(models.Model): ChannelMetadataField.M3U_PROFILE, ) - profile_connections_key = f"profile_connections:{profile_id}" - current_count = int( - redis_client.get(profile_connections_key) or 0 - ) - if current_count > 0: - redis_client.decr(profile_connections_key) + stream = Stream.objects.select_related("m3u_account").filter( + id=stream_id + ).first() + m3u_account = stream.m3u_account if stream else None + release_profile_slot(profile_id, redis_client) return True logger.debug( @@ -832,12 +828,7 @@ class Channel(models.Model): f"stream {stream_id}" ) - profile_connections_key = f"profile_connections:{profile_id}" - - # Only decrement if the profile had a max_connections limit - current_count = int(redis_client.get(profile_connections_key) or 0) - if current_count > 0: - redis_client.decr(profile_connections_key) + release_profile_slot(profile_id, redis_client) # Clear metadata fields so duplicate release_stream() calls # (e.g. from _clean_redis_keys or ChannelService.stop_channel) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py new file mode 100644 index 00000000..b06aff10 --- /dev/null +++ b/apps/m3u/connection_pool.py @@ -0,0 +1,299 @@ +""" +Shared connection pool enforcement for M3U accounts with identical credentials. + +All Redis INCR/DECR for profile and server-group limits should go through this +module so live TV and VOD stay consistent. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + +PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" +SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}" +EXCLUDE_FROM_POOL_KEY = "exclude_from_credential_pool" + +_XC_URL_CREDENTIALS_RE = re.compile( + r"/(?:live|movie|series)/([^/]+)/([^/]+)/", + re.IGNORECASE, +) + + +def profile_connections_key(profile_id: int) -> str: + return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) + + +def server_group_connections_key(group_id: int) -> str: + return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id) + + +def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: + """Return a stable hash for grouping accounts with the same IPTV login.""" + if not username or not password: + return None + normalized = f"{username.strip().lower()}\0{password.strip()}" + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def extract_credentials_from_stream_url(url: str) -> Tuple[Optional[str], Optional[str]]: + """Parse username/password embedded in an Xtream-style stream URL.""" + if not url: + return None, None + match = _XC_URL_CREDENTIALS_RE.search(url) + if not match: + return None, None + return match.group(1), match.group(2) + + +def _fingerprint_from_profile_stream_url(profile) -> Optional[str]: + """STD/M3U: fingerprint from a sample stream URL after profile rewrite.""" + from apps.channels.models import Stream + + sample_url = ( + Stream.objects.filter(m3u_account=profile.m3u_account) + .exclude(url="") + .values_list("url", flat=True) + .first() + ) + if not sample_url: + return None + + try: + from apps.proxy.live_proxy.url_utils import transform_url + + transformed = transform_url( + sample_url, + profile.search_pattern or "", + profile.replace_pattern or "", + ) + url_user, url_pass = extract_credentials_from_stream_url( + transformed or sample_url + ) + return compute_credential_fingerprint(url_user or "", url_pass or "") + except Exception as exc: + logger.debug( + "Could not derive profile %s fingerprint from stream URL: %s", + profile.pk, + exc, + ) + return None + + +def get_profile_credential_fingerprint(profile) -> Optional[str]: + """Fingerprint for credentials this profile uses at playback time.""" + m3u_account = profile.m3u_account + + if m3u_account.account_type == "XC": + try: + from apps.m3u.tasks import get_transformed_credentials + + _url, username, password = get_transformed_credentials(m3u_account, profile) + fingerprint = compute_credential_fingerprint(username or "", password or "") + if fingerprint: + return fingerprint + except Exception as exc: + logger.debug( + "Could not resolve transformed credentials for profile %s: %s", + profile.pk, + exc, + ) + + fingerprint = _fingerprint_from_profile_stream_url(profile) + if fingerprint: + return fingerprint + + return compute_credential_fingerprint( + m3u_account.username or "", + m3u_account.password or "", + ) + + +def account_excluded_from_pool(m3u_account) -> bool: + props = m3u_account.custom_properties or {} + return bool(props.get(EXCLUDE_FROM_POOL_KEY)) + + +def _get_pool_for_fingerprint(fingerprint: str): + """Return the auto credential pool ServerGroup for a fingerprint, if configured.""" + if not fingerprint: + return None + from apps.m3u.models import ServerGroup + + group = ServerGroup.objects.filter(credential_fingerprint=fingerprint).first() + if not group or group.max_streams == 0: + return None + return group + + +def get_enforced_server_group_for_profile(profile): + """Return the shared pool for this profile's effective provider login.""" + if account_excluded_from_pool(profile.m3u_account): + return None + + group = _get_pool_for_fingerprint(get_profile_credential_fingerprint(profile)) + if group: + return group + + manual = profile.m3u_account.server_group + if manual and not manual.credential_fingerprint and manual.max_streams > 0: + return manual + return None + + +def pool_has_capacity_for_profile(profile, redis_client) -> bool: + """Non-mutating check for the profile's credential pool.""" + group = get_enforced_server_group_for_profile(profile) + if not group: + return True + key = server_group_connections_key(group.id) + current = int(redis_client.get(key) or 0) + return current < group.max_streams + + +def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: + group = get_enforced_server_group_for_profile(profile) + if not group: + return True + key = server_group_connections_key(group.id) + group_count = redis_client.incr(key) + if group_count <= group.max_streams: + return True + redis_client.decr(key) + return False + + +def _release_server_group_slot_for_profile(profile, redis_client) -> None: + group = get_enforced_server_group_for_profile(profile) + if not group: + return + key = server_group_connections_key(group.id) + new_count = redis_client.decr(key) + if new_count < 0: + redis_client.set(key, 0) + + +def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: + """ + Atomically reserve profile + shared pool slots (INCR-first). + + Returns (reserved, profile_count_after_attempt). + """ + if profile.max_streams == 0: + if _reserve_server_group_slot_for_profile(profile, redis_client): + return True, 0 + return False, 0 + + profile_key = profile_connections_key(profile.id) + new_count = redis_client.incr(profile_key) + + if new_count <= profile.max_streams: + if _reserve_server_group_slot_for_profile(profile, redis_client): + return True, new_count + redis_client.decr(profile_key) + return False, new_count - 1 + + redis_client.decr(profile_key) + return False, new_count - 1 + + +def release_profile_slot(profile_id: int, redis_client) -> None: + """Release profile and shared pool slots after a stream ends.""" + from apps.m3u.models import M3UAccountProfile + + try: + profile = M3UAccountProfile.objects.get(id=profile_id) + except M3UAccountProfile.DoesNotExist: + profile = None + + profile_key = profile_connections_key(profile_id) + if profile is None or profile.max_streams > 0: + current = int(redis_client.get(profile_key) or 0) + if current > 0: + redis_client.decr(profile_key) + + if profile: + _release_server_group_slot_for_profile(profile, redis_client) + + +def recompute_pool_max_streams(server_group) -> None: + """Set pool max_streams to the minimum positive limit across members and profiles.""" + from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup + + if not server_group.credential_fingerprint: + return + + fp = server_group.credential_fingerprint + limits = list( + M3UAccount.objects.filter( + server_group=server_group, max_streams__gt=0 + ).values_list("max_streams", flat=True) + ) + + for profile in M3UAccountProfile.objects.filter(is_active=True).select_related( + "m3u_account" + ): + if account_excluded_from_pool(profile.m3u_account): + continue + if get_profile_credential_fingerprint(profile) != fp: + continue + if profile.max_streams > 0: + limits.append(profile.max_streams) + + new_max = min(limits) if limits else 0 + if server_group.max_streams != new_max: + ServerGroup.objects.filter(pk=server_group.pk).update(max_streams=new_max) + + +def _ensure_pool_for_fingerprint(fingerprint: str): + from apps.m3u.models import ServerGroup + + group, _created = ServerGroup.objects.get_or_create( + credential_fingerprint=fingerprint, + defaults={ + "name": f"credential-pool-{fingerprint[:16]}", + "max_streams": 0, + }, + ) + recompute_pool_max_streams(group) + return group + + +def sync_account_credential_pool(m3u_account) -> None: + """ + Ensure auto credential pools exist for each distinct login on this account. + + Sets M3UAccount.server_group only when every active profile shares one login. + """ + from apps.m3u.models import M3UAccount, M3UAccountProfile + + if account_excluded_from_pool(m3u_account): + return + + if m3u_account.server_group_id: + existing = m3u_account.server_group + if existing and not existing.credential_fingerprint: + return + + profile_fps = set() + for profile in M3UAccountProfile.objects.filter( + m3u_account=m3u_account, is_active=True + ): + fp = get_profile_credential_fingerprint(profile) + if not fp: + continue + profile_fps.add(fp) + _ensure_pool_for_fingerprint(fp) + + if len(profile_fps) == 1: + group = _get_pool_for_fingerprint(next(iter(profile_fps))) + if group and m3u_account.server_group_id != group.id: + M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=group) + elif m3u_account.server_group_id: + existing = m3u_account.server_group + if existing and existing.credential_fingerprint: + M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=None) diff --git a/apps/m3u/migrations/0020_servergroup_max_streams.py b/apps/m3u/migrations/0020_servergroup_max_streams.py new file mode 100644 index 00000000..ce17d830 --- /dev/null +++ b/apps/m3u/migrations/0020_servergroup_max_streams.py @@ -0,0 +1,21 @@ +# Generated by Django 6.0.3 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("m3u", "0019_m3uaccountprofile_exp_date"), + ] + + operations = [ + migrations.AddField( + model_name="servergroup", + name="max_streams", + field=models.PositiveIntegerField( + default=0, + help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + ), + ), + ] diff --git a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py new file mode 100644 index 00000000..8d958945 --- /dev/null +++ b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py @@ -0,0 +1,102 @@ +# Generated manually for shared credential connection pools (#1137) + +from django.db import migrations, models + +from apps.m3u.connection_pool import ( + compute_credential_fingerprint, + extract_credentials_from_stream_url, +) + + +def _historical_account_fingerprint(M3UAccount, M3UAccountProfile, Stream, account): + """Resolve fingerprint for migration using historical models only.""" + fingerprint = compute_credential_fingerprint( + account.username or "", + account.password or "", + ) + if fingerprint: + return fingerprint + + sample_url = ( + Stream.objects.filter(m3u_account_id=account.pk) + .exclude(url="") + .values_list("url", flat=True) + .first() + ) + if sample_url: + url_user, url_pass = extract_credentials_from_stream_url(sample_url) + return compute_credential_fingerprint(url_user or "", url_pass or "") + + return None + + +def backfill_credential_pools(apps, schema_editor): + from django.db.models import Min + + M3UAccount = apps.get_model("m3u", "M3UAccount") + ServerGroup = apps.get_model("m3u", "ServerGroup") + M3UAccountProfile = apps.get_model("m3u", "M3UAccountProfile") + Stream = apps.get_model("dispatcharr_channels", "Stream") + + seen = {} + for account in M3UAccount.objects.all(): + props = account.custom_properties or {} + if props.get("exclude_from_credential_pool"): + continue + if account.server_group_id: + group = ServerGroup.objects.filter(pk=account.server_group_id).first() + if group and not group.credential_fingerprint: + continue + + fingerprint = _historical_account_fingerprint( + M3UAccount, M3UAccountProfile, Stream, account + ) + if not fingerprint: + continue + + if fingerprint not in seen: + short = fingerprint[:16] + group, _ = ServerGroup.objects.get_or_create( + credential_fingerprint=fingerprint, + defaults={ + "name": f"credential-pool-{short}", + "max_streams": 0, + }, + ) + seen[fingerprint] = group + else: + group = seen[fingerprint] + + M3UAccount.objects.filter(pk=account.pk).update(server_group_id=group.pk) + + for group in seen.values(): + agg = M3UAccount.objects.filter( + server_group_id=group.pk, max_streams__gt=0 + ).aggregate(min_limit=Min("max_streams")) + new_max = agg["min_limit"] or 0 + if group.max_streams != new_max: + ServerGroup.objects.filter(pk=group.pk).update(max_streams=new_max) + + +class Migration(migrations.Migration): + + dependencies = [ + ("m3u", "0020_servergroup_max_streams"), + ("dispatcharr_channels", "0037_auto_sync_overhaul"), + ] + + operations = [ + migrations.AddField( + model_name="servergroup", + name="credential_fingerprint", + field=models.CharField( + blank=True, + db_index=True, + help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", + max_length=64, + null=True, + unique=True, + ), + ), + migrations.RunPython(backfill_credential_pools, migrations.RunPython.noop), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 6fc48bec..4b1325d5 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -199,6 +199,18 @@ class ServerGroup(models.Model): name = models.CharField( max_length=100, unique=True, help_text="Unique name for this server group." ) + max_streams = models.PositiveIntegerField( + default=0, + help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + ) + credential_fingerprint = models.CharField( + max_length=64, + null=True, + blank=True, + unique=True, + db_index=True, + help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", + ) def __str__(self): return self.name @@ -334,6 +346,26 @@ class M3UAccountProfile(models.Model): return None +@receiver(models.signals.post_save, sender=M3UAccount) +def assign_credential_pool_for_m3u_account(sender, instance, **kwargs): + """Link accounts with identical credentials into a shared connection pool.""" + if kwargs.get("raw"): + return + from apps.m3u.connection_pool import sync_account_credential_pool + + sync_account_credential_pool(instance) + + +@receiver(models.signals.post_save, sender=M3UAccountProfile) +def assign_credential_pool_for_m3u_profile(sender, instance, **kwargs): + """Re-sync pools when profile URL transforms change.""" + if kwargs.get("raw"): + return + from apps.m3u.connection_pool import sync_account_credential_pool + + sync_account_credential_pool(instance.m3u_account) + + @receiver(models.signals.post_save, sender=M3UAccount) def create_profile_for_m3u_account(sender, instance, created, **kwargs): """Automatically create an M3UAccountProfile when M3UAccount is created.""" diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 485a3687..29c4157f 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -408,4 +408,4 @@ class ServerGroupSerializer(serializers.ModelSerializer): class Meta: model = ServerGroup - fields = ["id", "name"] + fields = ["id", "name", "max_streams"] diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py new file mode 100644 index 00000000..79285484 --- /dev/null +++ b/apps/m3u/tests/test_connection_pool.py @@ -0,0 +1,247 @@ +"""Tests for shared credential connection pools (#1137).""" + +from django.test import TestCase +from unittest.mock import patch + +from apps.m3u.connection_pool import ( + compute_credential_fingerprint, + extract_credentials_from_stream_url, + get_enforced_server_group_for_profile, + get_profile_credential_fingerprint, + pool_has_capacity_for_profile, + release_profile_slot, + reserve_profile_slot, + server_group_connections_key, + sync_account_credential_pool, +) +from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup + + +class FakeRedis: + """Minimal in-memory Redis stand-in for counter tests.""" + + def __init__(self): + self._data = {} + + def get(self, key): + val = self._data.get(key) + if val is None: + return None + return str(val).encode() + + def set(self, key, value, ex=None): + self._data[key] = int(value) + + def incr(self, key): + self._data[key] = self._data.get(key, 0) + 1 + return self._data[key] + + def decr(self, key): + self._data[key] = self._data.get(key, 0) - 1 + return self._data[key] + + +class CredentialFingerprintTests(TestCase): + def test_same_credentials_same_fingerprint(self): + fp1 = compute_credential_fingerprint("User", "pass") + fp2 = compute_credential_fingerprint("user", "pass") + self.assertEqual(fp1, fp2) + self.assertIsNotNone(fp1) + + def test_different_password_different_fingerprint(self): + fp1 = compute_credential_fingerprint("user", "pass1") + fp2 = compute_credential_fingerprint("user", "pass2") + self.assertNotEqual(fp1, fp2) + + def test_empty_credentials_returns_none(self): + self.assertIsNone(compute_credential_fingerprint("", "pass")) + self.assertIsNone(compute_credential_fingerprint("user", "")) + + def test_extract_credentials_from_xc_style_url(self): + url = "http://example.com/live/alice/secret123/99999.ts" + user, password = extract_credentials_from_stream_url(url) + self.assertEqual(user, "alice") + self.assertEqual(password, "secret123") + + +class AutoAssignTests(TestCase): + def test_accounts_with_same_credentials_share_server_group(self): + account1 = M3UAccount.objects.create( + name="Provider A", + account_type="XC", + username="user1", + password="secret", + server_url="http://a.example.com", + max_streams=2, + ) + account2 = M3UAccount.objects.create( + name="Provider B", + account_type="XC", + username="user1", + password="secret", + server_url="http://b.example.com", + max_streams=3, + ) + + account1.refresh_from_db() + account2.refresh_from_db() + + self.assertIsNotNone(account1.server_group_id) + self.assertEqual(account1.server_group_id, account2.server_group_id) + self.assertTrue(account1.server_group.credential_fingerprint) + self.assertEqual(account1.server_group.max_streams, 2) + + def test_exclude_from_pool_opt_out(self): + account1 = M3UAccount.objects.create( + name="Pooled", + account_type="XC", + username="shared", + password="secret", + max_streams=1, + ) + account2 = M3UAccount.objects.create( + name="Opt-out", + account_type="XC", + username="shared", + password="secret", + custom_properties={"exclude_from_credential_pool": True}, + max_streams=1, + ) + + account1.refresh_from_db() + account2.refresh_from_db() + + self.assertIsNotNone(account1.server_group_id) + self.assertIsNone(account2.server_group_id) + + +class MultiProfilePoolTests(TestCase): + def test_profiles_with_different_credentials_get_separate_pools(self): + account = M3UAccount.objects.create( + name="Multi-login XC", + account_type="XC", + username="xc_user_a", + password="xc_pass_a", + server_url="http://xc.example.com", + max_streams=1, + ) + base = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + base.search_pattern = r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" + base.replace_pattern = r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" + base.save() + + p2 = M3UAccountProfile.objects.create( + m3u_account=account, + name="login_b", + is_default=False, + is_active=True, + max_streams=1, + search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", + replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + ) + + sync_account_credential_pool(account) + + fp1 = get_profile_credential_fingerprint(base) + fp2 = get_profile_credential_fingerprint(p2) + self.assertNotEqual(fp1, fp2) + + g1 = get_enforced_server_group_for_profile(base) + g2 = get_enforced_server_group_for_profile(p2) + self.assertIsNotNone(g1) + self.assertIsNotNone(g2) + self.assertNotEqual(g1.id, g2.id) + self.assertIsNone(account.server_group_id) + + +class PoolEnforcementTests(TestCase): + def setUp(self): + self.redis = FakeRedis() + self.group = ServerGroup.objects.create( + name="test-pool", + max_streams=1, + ) + self.account = M3UAccount.objects.create( + name="Test Account", + account_type="XC", + username="user", + password="pass", + server_group=self.group, + max_streams=5, + ) + self.profile = M3UAccountProfile.objects.get( + m3u_account=self.account, is_default=True + ) + + def test_reserve_and_release_pool_counter(self): + reserved, count = reserve_profile_slot(self.profile, self.redis) + self.assertTrue(reserved) + self.assertEqual(count, 1) + + group_key = server_group_connections_key(self.group.id) + self.assertEqual(self.redis._data[group_key], 1) + + release_profile_slot(self.profile.id, self.redis) + self.assertEqual(self.redis._data[group_key], 0) + + def test_reserve_fails_when_pool_at_capacity(self): + group_key = server_group_connections_key(self.group.id) + self.redis.set(group_key, 1) + + reserved, _count = reserve_profile_slot(self.profile, self.redis) + self.assertFalse(reserved) + self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) + + def test_live_slot_blocks_second_reserve_same_profile(self): + reserved1, _ = reserve_profile_slot(self.profile, self.redis) + self.assertTrue(reserved1) + + reserved2, _ = reserve_profile_slot(self.profile, self.redis) + self.assertFalse(reserved2) + + +class VodProfileSelectionTests(TestCase): + """VOD must try alternate profiles when the default pool is full (live TV).""" + + def test_get_m3u_profile_skips_default_when_pool_full(self): + from apps.proxy.vod_proxy.views import _get_m3u_profile + + account = M3UAccount.objects.create( + name="VOD multi-login", + account_type="XC", + username="xc_user_a", + password="xc_pass_a", + server_url="http://xc.example.com", + max_streams=1, + ) + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.search_pattern = ( + r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" + ) + default.replace_pattern = ( + r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" + ) + default.save() + + alt = M3UAccountProfile.objects.create( + m3u_account=account, + name="login_b", + is_default=False, + is_active=True, + max_streams=1, + search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", + replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + ) + + sync_account_credential_pool(account) + + redis = FakeRedis() + reserved, _ = reserve_profile_slot(default, redis) + self.assertTrue(reserved) + + with patch("core.utils.RedisClient.get_client", return_value=redis): + result = _get_m3u_profile(account, None, None) + + self.assertIsNotNone(result) + selected, _connections = result + self.assertEqual(selected.id, alt.id) diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index 3f75693e..6180f317 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -390,11 +390,12 @@ class ChannelStatus: created_at = float(init_time_bytes) uptime = time.time() - created_at if created_at > 0 else 0 + stream_url = metadata.get(ChannelMetadataField.URL, "") or "" # Simplified info info = { 'channel_id': channel_id, 'state': metadata.get(ChannelMetadataField.STATE), - 'url': metadata.get(ChannelMetadataField.URL, ""), + 'url': stream_url, 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""), 'owner': metadata.get(ChannelMetadataField.OWNER), 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, @@ -402,6 +403,15 @@ class ChannelStatus: 'uptime': uptime, 'started_at': created_at if created_at > 0 else None, } + if stream_url: + try: + from apps.m3u.connection_pool import extract_credentials_from_stream_url + + provider_user, _ = extract_credentials_from_stream_url(stream_url) + if provider_user: + info['provider_username'] = provider_user + except Exception as e: + logger.debug(f"Could not parse provider username from stream URL: {e}") channel_name = metadata.get(ChannelMetadataField.CHANNEL_NAME) if channel_name: @@ -498,7 +508,15 @@ class ChannelStatus: m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE) if m3u_profile_id: try: - info['m3u_profile_id'] = int(m3u_profile_id) + profile_id_int = int(m3u_profile_id) + info['m3u_profile_id'] = profile_id_int + try: + from apps.m3u.models import M3UAccountProfile + m3u_profile = M3UAccountProfile.objects.filter(id=profile_id_int).first() + if m3u_profile: + info['m3u_profile_name'] = m3u_profile.name + except Exception as e: + logger.warning(f"Failed to get M3U profile name for ID {profile_id_int}: {e}") except ValueError: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 5227a3fa..083b0fd1 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -8,6 +8,7 @@ from typing import Optional, Tuple, List from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.m3u.connection_pool import pool_has_capacity_for_profile from core.models import UserAgent, CoreSettings, StreamProfile from .utils import get_logger from uuid import UUID @@ -15,6 +16,35 @@ import requests logger = get_logger() + +def _resolve_live_stream_url(stream, m3u_account, m3u_profile): + """ + Build the upstream URL for live playback. + + XC accounts use current transformed credentials plus provider stream_id so + playback matches the account login (not a stale stream.url from an old sync). + STD/M3U accounts keep using the URL stored on the stream row. + """ + if ( + m3u_account.account_type == M3UAccount.Types.XC + and stream.stream_id + ): + from apps.m3u.tasks import get_transformed_credentials + + server_url, username, password = get_transformed_credentials( + m3u_account, m3u_profile + ) + if server_url and username and password: + base = server_url.rstrip("/") + return f"{base}/live/{username}/{password}/{stream.stream_id}.ts" + + return transform_url( + stream.url or "", + m3u_profile.search_pattern, + m3u_profile.replace_pattern, + ) + + def get_stream_object(id: str): try: logger.info(f"Fetching channel ID {id}") @@ -63,7 +93,7 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - stream_url = transform_url(stream.url, profile.search_pattern, profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, profile) stream_profile = stream.get_stream_profile() logger.debug(f"Using stream profile: {stream_profile.name}") @@ -106,9 +136,7 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - # Generate stream URL based on the selected profile - input_url = stream.url - stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, m3u_profile) # Check if transcoding is needed stream_profile = channel.get_stream_profile() @@ -204,35 +232,41 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: - - # Check connection availability if redis_client: + if not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"Profile {profile.id} credential pool at capacity for stream switch" + ) + continue + profile_connections_key = f"profile_connections:{profile.id}" current_connections = int(redis_client.get(profile_connections_key) or 0) - # Check if this channel is already using this profile channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: - # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id - existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") + existing_profile_id = redis_client.get( + f"stream_profile:{existing_stream_id}" + ) if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True - logger.debug(f"Channel {channel.id} already using profile {profile.id}") - # Calculate effective connections (subtract 1 if channel already using this profile) - effective_connections = current_connections - (1 if channel_using_profile else 0) + effective_connections = current_connections - ( + 1 if channel_using_profile else 0 + ) - # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: selected_profile = profile - logger.debug(f"Selected profile {profile.id} with {effective_connections}/{profile.max_streams} effective connections (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Selected profile {profile.id} with " + f"{effective_connections}/{profile.max_streams} effective connections" + ) break - else: - logger.debug(f"Profile {profile.id} at max connections: {effective_connections}/{profile.max_streams} (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Profile {profile.id} at max connections: " + f"{effective_connections}/{profile.max_streams}" + ) else: - # No Redis available, assume first active profile is okay selected_profile = profile break @@ -260,8 +294,7 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] # Get the user agent from the M3U account user_agent = m3u_account.get_user_agent().user_agent - # Generate URL using the transform function directly - stream_url = transform_url(stream.url, profile.search_pattern, profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, profile) # Get transcode info from the channel's stream profile stream_profile = channel.get_stream_profile() @@ -365,6 +398,12 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: + if not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"Credential pool at capacity for profile " + f"{profile.id} (account {m3u_account.id})" + ) + continue selected_profile = profile logger.debug(f"Found available profile {profile.id} for stream {stream.id}: {effective_connections}/{profile.max_streams} effective (current: {current_connections}, already using: {channel_using_profile})") break diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index 7d367531..77db3b1b 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -735,68 +735,33 @@ class MultiWorkerVODConnectionManager: """Get Redis key for tracking connections per profile - STANDARDIZED with TS proxy""" return f"profile_connections:{profile_id}" - def _check_profile_limits(self, m3u_profile) -> bool: - """Check if profile has available connection slots""" - if m3u_profile.max_streams == 0: # Unlimited - return True - - try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - current_connections = int(self.redis_client.get(profile_connections_key) or 0) - - logger.info(f"[PROFILE-CHECK] Profile {m3u_profile.id} has {current_connections}/{m3u_profile.max_streams} connections") - return current_connections < m3u_profile.max_streams - - except Exception as e: - logger.error(f"Error checking profile limits: {e}") - return False - def _check_and_reserve_profile_slot(self, m3u_profile) -> bool: """ Atomically check and reserve a connection slot for the given profile. - Uses an INCR-first-then-check pattern to eliminate the TOCTOU race - condition where separate GET > check > INCR operations could allow - concurrent requests to both pass the capacity check. - - For profiles with max_streams=0 (unlimited), no reservation is needed. - Returns: bool: True if slot was reserved (or unlimited), False if at capacity """ - if m3u_profile.max_streams == 0: # Unlimited - return True + from apps.m3u.connection_pool import reserve_profile_slot try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - - # Atomically increment first — single Redis command eliminates race window - new_count = self.redis_client.incr(profile_connections_key) - - if new_count <= m3u_profile.max_streams: - logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: {new_count}/{m3u_profile.max_streams}") - return True - - # Over capacity — roll back the increment - self.redis_client.decr(profile_connections_key) - logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: {new_count - 1}/{m3u_profile.max_streams}") - return False + reserved, new_count = reserve_profile_slot(m3u_profile, self.redis_client) + if reserved: + logger.info( + f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: " + f"{new_count}/{m3u_profile.max_streams}" + ) + else: + logger.info( + f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: " + f"{new_count}/{m3u_profile.max_streams}" + ) + return reserved except Exception as e: logger.error(f"Error reserving profile slot: {e}") return False - def _increment_profile_connections(self, m3u_profile): - """Increment profile connection count""" - try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - new_count = self.redis_client.incr(profile_connections_key) - logger.info(f"[PROFILE-INCR] Profile {m3u_profile.id} connections: {new_count}") - return new_count - except Exception as e: - logger.error(f"Error incrementing profile connections: {e}") - return None - def _trigger_vod_stats_update(self): """Trigger a VOD stats WebSocket update in a background thread.""" threading.Thread(target=self._do_vod_stats_update, daemon=True).start() @@ -859,21 +824,14 @@ class MultiWorkerVODConnectionManager: logger.error(f"Failed to trigger VOD stats update: {e}") def _decrement_profile_connections(self, m3u_profile_id: int): - """Decrement profile connection count. + """Decrement profile and shared pool connection counters.""" + from apps.m3u.connection_pool import release_profile_slot - Uses a single atomic DECR (no GET-before-DECR) to avoid the race condition - where two concurrent decrements both pass a >0 guard and both fire, sending - the counter negative. If the counter would go below zero it is clamped to 0. - """ try: - profile_connections_key = self._get_profile_connections_key(m3u_profile_id) - new_count = self.redis_client.decr(profile_connections_key) - if new_count < 0: - self.redis_client.set(profile_connections_key, 0) - new_count = 0 - logger.warning(f"[PROFILE-DECR] Profile {m3u_profile_id} counter went negative, clamped to 0") - else: - logger.info(f"[PROFILE-DECR] Profile {m3u_profile_id} connections: {new_count}") + release_profile_slot(m3u_profile_id, self.redis_client) + profile_key = self._get_profile_connections_key(m3u_profile_id) + new_count = int(self.redis_client.get(profile_key) or 0) + logger.info(f"[PROFILE-DECR] Profile {m3u_profile_id} connections: {new_count}") return new_count except Exception as e: logger.error(f"Error decrementing profile connections: {e}") diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index f73f48b4..a7995107 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -178,6 +178,7 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): """ try: from core.utils import RedisClient + from apps.m3u.connection_pool import pool_has_capacity_for_profile redis_client = RedisClient.get_client() if not redis_client: @@ -214,7 +215,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): except Exception as e: logger.warning(f"[PROFILE-SELECTION] Error checking existing profile for session {session_id}: {e}") else: - logger.debug(f"[PROFILE-SELECTION] Session {session_id} exists but has no profile ID stored") # If specific profile requested, try to use it + logger.debug( + f"[PROFILE-SELECTION] Session {session_id} exists but has no profile ID stored" + ) + + # If specific profile requested, try to use it if profile_id: try: profile = M3UAccountProfile.objects.get( @@ -226,9 +231,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): profile_connections_key = f"profile_connections:{profile.id}" current_connections = int(redis_client.get(profile_connections_key) or 0) - if profile.max_streams == 0 or current_connections < profile.max_streams: + if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Using requested profile {profile.id}: {current_connections}/{profile.max_streams} connections") return (profile, current_connections) + elif not pool_has_capacity_for_profile(profile, redis_client): + logger.warning(f"[PROFILE-SELECTION] Shared credential pool at capacity for account {m3u_account.id}") else: logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") except M3UAccountProfile.DoesNotExist: @@ -253,9 +260,15 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): current_connections = int(redis_client.get(profile_connections_key) or 0) # Check if profile has available connection slots - if profile.max_streams == 0 or current_connections < profile.max_streams: + if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Selected profile {profile.id} ({profile.name}): {current_connections}/{profile.max_streams} connections") return (profile, current_connections) + elif not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"[PROFILE-SELECTION] Credential pool at capacity for profile " + f"{profile.id}, trying next profile" + ) + continue else: logger.debug(f"[PROFILE-SELECTION] Profile {profile.id} at capacity: {current_connections}/{profile.max_streams}") @@ -791,17 +804,30 @@ def build_vod_stats_data(redis_client): # Get M3U profile information m3u_profile_info = {} m3u_profile_id = combined_data.get('m3u_profile_id') + stream_url_for_creds = ( + combined_data.get('final_url') + or combined_data.get('stream_url') + or '' + ) if m3u_profile_id: try: from apps.m3u.models import M3UAccountProfile + from apps.m3u.connection_pool import extract_credentials_from_stream_url + profile = M3UAccountProfile.objects.select_related('m3u_account').get(id=m3u_profile_id) m3u_profile_info = { 'profile_name': profile.name, 'account_name': profile.m3u_account.name, 'account_id': profile.m3u_account.id, 'max_streams': profile.m3u_account.max_streams, - 'm3u_profile_id': int(m3u_profile_id) + 'm3u_profile_id': int(m3u_profile_id), } + if stream_url_for_creds: + provider_user, _ = extract_credentials_from_stream_url( + stream_url_for_creds + ) + if provider_user: + m3u_profile_info['provider_username'] = provider_user except Exception as e: logger.warning(f"Could not fetch M3U profile {m3u_profile_id}: {e}") diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 30509a86..0e6e221b 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -547,6 +547,7 @@ const StreamConnectionCard = ({ channel.m3u_profile?.name || channel.m3u_profile_name || 'Unknown M3U Profile'; + const providerUsername = channel.provider_username; // Create select options for available streams const streamOptions = getStreamOptions(availableStreams, m3uAccountsMap); @@ -648,12 +649,21 @@ const StreamConnectionCard = ({ {/* M3U Profile on right - absolutely positioned */} - - - - {m3uProfileName} - - + + + + + {m3uProfileName} + + + {providerUsername && ( + + + Login: {providerUsername} + + + )} + {/* Channel Name on left */} diff --git a/frontend/src/components/cards/VodConnectionCard.jsx b/frontend/src/components/cards/VodConnectionCard.jsx index 90b4c034..6539426a 100644 --- a/frontend/src/components/cards/VodConnectionCard.jsx +++ b/frontend/src/components/cards/VodConnectionCard.jsx @@ -349,6 +349,13 @@ const VodConnectionCard = ({ vodContent, stopVODClient }) => { {connection.m3u_profile.profile_name || 'Default Profile'} + {connection.m3u_profile.provider_username && ( + + + Login: {connection.m3u_profile.provider_username} + + + )} From bb9b95b4a4605674a6b8919997537bc08eb5ba0a Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Sat, 23 May 2026 20:18:45 -0400 Subject: [PATCH 02/81] Refactor connection pools to manual ServerGroups with profile rotation. Drop auto-fingerprint migration and restore per-profile selection for live/VOD. Enforce shared limits on reserve using login-scoped group counters, and add Server Groups UI for manual account assignment per maintainer feedback (#1137). Co-authored-by: Cursor --- apps/channels/models.py | 48 ++-- apps/m3u/connection_pool.py | 201 +++++--------- ...0021_servergroup_credential_fingerprint.py | 102 -------- apps/m3u/models.py | 28 -- apps/m3u/tests/test_connection_pool.py | 245 +++++++++--------- apps/proxy/live_proxy/url_utils.py | 65 ++--- apps/proxy/live_proxy/views.py | 17 +- apps/proxy/vod_proxy/views.py | 26 +- frontend/src/api.js | 54 ++++ frontend/src/components/forms/M3U.jsx | 24 ++ frontend/src/components/forms/ServerGroup.jsx | 90 +++++++ .../components/tables/ServerGroupsTable.jsx | 203 +++++++++++++++ frontend/src/pages/Settings.jsx | 16 ++ frontend/src/store/auth.jsx | 2 + frontend/src/store/serverGroups.jsx | 40 +++ frontend/src/utils/forms/M3uUtils.js | 4 + 16 files changed, 701 insertions(+), 464 deletions(-) delete mode 100644 apps/m3u/migrations/0021_servergroup_credential_fingerprint.py create mode 100644 frontend/src/components/forms/ServerGroup.jsx create mode 100644 frontend/src/components/tables/ServerGroupsTable.jsx create mode 100644 frontend/src/store/serverGroups.jsx diff --git a/apps/channels/models.py b/apps/channels/models.py index 59a2bc3d..38ebbbc7 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -210,13 +210,14 @@ class Stream(models.Model): Finds an available profile for this stream and reserves a connection slot. Returns: - Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason) + Tuple[Optional[int], Optional[int], Optional[str], bool]: + (stream_id, profile_id, error_reason, slot_reserved) """ redis_client = RedisClient.get_client() profile_id = redis_client.get(f"stream_profile:{self.id}") if profile_id: profile_id = int(profile_id) - return self.id, profile_id, None + return self.id, profile_id, None, False # Retrieve the M3U account associated with the stream. m3u_account = self.m3u_account @@ -238,9 +239,9 @@ class Stream(models.Model): if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) redis_client.set(f"stream_profile:{self.id}", profile.id) - return self.id, profile.id, None + return self.id, profile.id, None, True - return None, None, "All active M3U profiles have reached maximum connection limits" + return None, None, "All active M3U profiles have reached maximum connection limits", False def release_stream(self): """ @@ -615,7 +616,8 @@ class Channel(models.Model): Finds an available stream for the requested channel and returns the selected stream and profile. Returns: - Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason) + Tuple[Optional[int], Optional[int], Optional[str], bool]: + (stream_id, profile_id, error_reason, slot_reserved) """ redis_client = RedisClient.get_client() error_reason = None @@ -623,11 +625,11 @@ class Channel(models.Model): # Check if this channel has any streams if not self.streams.exists(): error_reason = "No streams assigned to channel" - return None, None, error_reason + return None, None, error_reason, False # Reuse assignment only when this channel is still active in the proxy. - # Stale channel_stream keys after stop/disconnect caused every tune to - # reuse the default profile without re-running rotation or INCR. + # Stale channel_stream keys after stop/disconnect skip INCR and break pool + # accounting, which lets a second stream reach the provider and fail validation. stream_id_bytes = redis_client.get(f"channel_stream:{self.id}") if stream_id_bytes: try: @@ -648,7 +650,7 @@ class Channel(models.Model): f"Channel {self.uuid}: reusing active assignment " f"stream={stream_id} profile={profile_id}" ) - return stream_id, profile_id, None + return stream_id, profile_id, None, False except (ValueError, TypeError): logger.debug( f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" @@ -707,6 +709,7 @@ class Channel(models.Model): stream.id, profile.id, None, + True, ) # Return newly assigned stream and matched profile else: # At capacity: try to preempt a lower-impact channel on this profile @@ -735,7 +738,7 @@ class Channel(models.Model): else: error_reason = "No active profiles found for any assigned stream" - return None, None, error_reason + return None, None, error_reason, False def release_stream(self): """ @@ -779,10 +782,6 @@ class Channel(models.Model): ChannelMetadataField.M3U_PROFILE, ) - stream = Stream.objects.select_related("m3u_account").filter( - id=stream_id - ).first() - m3u_account = stream.m3u_account if stream else None release_profile_slot(profile_id, redis_client) return True @@ -874,10 +873,27 @@ class Channel(models.Model): if current_profile_id == new_profile_id: return True + from apps.m3u.models import M3UAccountProfile + from apps.m3u.connection_pool import ( + get_enforced_server_group_for_profile, + profile_connections_key, + ) + + new_profile = M3UAccountProfile.objects.get(id=new_profile_id) + if get_enforced_server_group_for_profile(new_profile): + # Shared group pool: one active stream uses one group slot regardless + # of which profile supplies the URL. + redis_client.set(f"stream_profile:{stream_id}", new_profile_id) + logger.info( + f"Updated stream {stream_id} profile from {current_profile_id} to " + f"{new_profile_id} (shared group pool unchanged)" + ) + return True + # Use pipeline for atomic profile switch to prevent counter drift # if an exception occurs between DECR and INCR - old_profile_connections_key = f"profile_connections:{current_profile_id}" - new_profile_connections_key = f"profile_connections:{new_profile_id}" + old_profile_connections_key = profile_connections_key(current_profile_id) + new_profile_connections_key = profile_connections_key(new_profile_id) old_count = int(redis_client.get(old_profile_connections_key) or 0) pipe = redis_client.pipeline() diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index b06aff10..9817d86c 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -1,8 +1,10 @@ """ -Shared connection pool enforcement for M3U accounts with identical credentials. +Shared connection pool enforcement for M3U accounts in the same ServerGroup. -All Redis INCR/DECR for profile and server-group limits should go through this -module so live TV and VOD stay consistent. +Profile selection rotates across M3UAccountProfile rows using each profile's own +Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup +with max_streams > 0, the group counter is scoped by provider login fingerprint +so profiles that rewrite to different IPTV credentials keep independent limits. """ from __future__ import annotations @@ -15,8 +17,7 @@ from typing import Optional, Tuple logger = logging.getLogger(__name__) PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" -SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}" -EXCLUDE_FROM_POOL_KEY = "exclude_from_credential_pool" +SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}:{fingerprint}" _XC_URL_CREDENTIALS_RE = re.compile( r"/(?:live|movie|series)/([^/]+)/([^/]+)/", @@ -28,8 +29,10 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) -def server_group_connections_key(group_id: int) -> str: - return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id) +def server_group_connections_key(group_id: int, fingerprint: Optional[str] = None) -> str: + """Redis key for a manual ServerGroup slot, scoped by provider login.""" + fp = (fingerprint or "unknown")[:16] + return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id, fingerprint=fp) def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: @@ -113,53 +116,58 @@ def get_profile_credential_fingerprint(profile) -> Optional[str]: ) -def account_excluded_from_pool(m3u_account) -> bool: - props = m3u_account.custom_properties or {} - return bool(props.get(EXCLUDE_FROM_POOL_KEY)) - - -def _get_pool_for_fingerprint(fingerprint: str): - """Return the auto credential pool ServerGroup for a fingerprint, if configured.""" - if not fingerprint: - return None - from apps.m3u.models import ServerGroup - - group = ServerGroup.objects.filter(credential_fingerprint=fingerprint).first() - if not group or group.max_streams == 0: - return None - return group - - def get_enforced_server_group_for_profile(profile): - """Return the shared pool for this profile's effective provider login.""" - if account_excluded_from_pool(profile.m3u_account): - return None - - group = _get_pool_for_fingerprint(get_profile_credential_fingerprint(profile)) - if group: + """Return the shared ServerGroup limit for this profile's account, if configured.""" + group = profile.m3u_account.server_group + if group and group.max_streams > 0: return group - - manual = profile.m3u_account.server_group - if manual and not manual.credential_fingerprint and manual.max_streams > 0: - return manual return None -def pool_has_capacity_for_profile(profile, redis_client) -> bool: - """Non-mutating check for the profile's credential pool.""" +def _group_counter_key(profile, group) -> str: + return server_group_connections_key( + group.id, + get_profile_credential_fingerprint(profile), + ) + + +def get_profile_connection_count(profile, redis_client) -> int: + return int(redis_client.get(profile_connections_key(profile.id)) or 0) + + +def get_group_connection_count(profile, redis_client) -> int: + group = get_enforced_server_group_for_profile(profile) + if not group: + return 0 + return int(redis_client.get(_group_counter_key(profile, group)) or 0) + + +def profile_has_capacity_for_selection(profile, redis_client) -> bool: + """Per-profile capacity check used when rotating across profiles on one account.""" + if profile.max_streams == 0: + return True + return get_profile_connection_count(profile, redis_client) < profile.max_streams + + +def group_has_capacity_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) if not group: return True - key = server_group_connections_key(group.id) - current = int(redis_client.get(key) or 0) - return current < group.max_streams + return get_group_connection_count(profile, redis_client) < group.max_streams + + +def pool_has_capacity_for_profile(profile, redis_client) -> bool: + """Non-mutating check before reserve: profile slot and group slot if applicable.""" + return profile_has_capacity_for_selection(profile, redis_client) and group_has_capacity_for_profile( + profile, redis_client + ) def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) if not group: return True - key = server_group_connections_key(group.id) + key = _group_counter_key(profile, group) group_count = redis_client.incr(key) if group_count <= group.max_streams: return True @@ -171,7 +179,10 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: group = get_enforced_server_group_for_profile(profile) if not group: return - key = server_group_connections_key(group.id) + key = _group_counter_key(profile, group) + current = int(redis_client.get(key) or 0) + if current <= 0: + return new_count = redis_client.decr(key) if new_count < 0: redis_client.set(key, 0) @@ -179,30 +190,29 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: """ - Atomically reserve profile + shared pool slots (INCR-first). + Atomically reserve profile + optional group slots (INCR-first). Returns (reserved, profile_count_after_attempt). """ - if profile.max_streams == 0: - if _reserve_server_group_slot_for_profile(profile, redis_client): - return True, 0 - return False, 0 - profile_key = profile_connections_key(profile.id) - new_count = redis_client.incr(profile_key) + profile_count = 0 - if new_count <= profile.max_streams: - if _reserve_server_group_slot_for_profile(profile, redis_client): - return True, new_count - redis_client.decr(profile_key) - return False, new_count - 1 + if profile.max_streams > 0: + profile_count = redis_client.incr(profile_key) + if profile_count > profile.max_streams: + redis_client.decr(profile_key) + return False, profile_count - 1 - redis_client.decr(profile_key) - return False, new_count - 1 + if not _reserve_server_group_slot_for_profile(profile, redis_client): + if profile.max_streams > 0: + redis_client.decr(profile_key) + return False, profile_count - 1 if profile.max_streams > 0 else 0 + + return True, profile_count def release_profile_slot(profile_id: int, redis_client) -> None: - """Release profile and shared pool slots after a stream ends.""" + """Release profile and shared group slots after a stream ends.""" from apps.m3u.models import M3UAccountProfile try: @@ -218,82 +228,3 @@ def release_profile_slot(profile_id: int, redis_client) -> None: if profile: _release_server_group_slot_for_profile(profile, redis_client) - - -def recompute_pool_max_streams(server_group) -> None: - """Set pool max_streams to the minimum positive limit across members and profiles.""" - from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup - - if not server_group.credential_fingerprint: - return - - fp = server_group.credential_fingerprint - limits = list( - M3UAccount.objects.filter( - server_group=server_group, max_streams__gt=0 - ).values_list("max_streams", flat=True) - ) - - for profile in M3UAccountProfile.objects.filter(is_active=True).select_related( - "m3u_account" - ): - if account_excluded_from_pool(profile.m3u_account): - continue - if get_profile_credential_fingerprint(profile) != fp: - continue - if profile.max_streams > 0: - limits.append(profile.max_streams) - - new_max = min(limits) if limits else 0 - if server_group.max_streams != new_max: - ServerGroup.objects.filter(pk=server_group.pk).update(max_streams=new_max) - - -def _ensure_pool_for_fingerprint(fingerprint: str): - from apps.m3u.models import ServerGroup - - group, _created = ServerGroup.objects.get_or_create( - credential_fingerprint=fingerprint, - defaults={ - "name": f"credential-pool-{fingerprint[:16]}", - "max_streams": 0, - }, - ) - recompute_pool_max_streams(group) - return group - - -def sync_account_credential_pool(m3u_account) -> None: - """ - Ensure auto credential pools exist for each distinct login on this account. - - Sets M3UAccount.server_group only when every active profile shares one login. - """ - from apps.m3u.models import M3UAccount, M3UAccountProfile - - if account_excluded_from_pool(m3u_account): - return - - if m3u_account.server_group_id: - existing = m3u_account.server_group - if existing and not existing.credential_fingerprint: - return - - profile_fps = set() - for profile in M3UAccountProfile.objects.filter( - m3u_account=m3u_account, is_active=True - ): - fp = get_profile_credential_fingerprint(profile) - if not fp: - continue - profile_fps.add(fp) - _ensure_pool_for_fingerprint(fp) - - if len(profile_fps) == 1: - group = _get_pool_for_fingerprint(next(iter(profile_fps))) - if group and m3u_account.server_group_id != group.id: - M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=group) - elif m3u_account.server_group_id: - existing = m3u_account.server_group - if existing and existing.credential_fingerprint: - M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=None) diff --git a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py deleted file mode 100644 index 8d958945..00000000 --- a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py +++ /dev/null @@ -1,102 +0,0 @@ -# Generated manually for shared credential connection pools (#1137) - -from django.db import migrations, models - -from apps.m3u.connection_pool import ( - compute_credential_fingerprint, - extract_credentials_from_stream_url, -) - - -def _historical_account_fingerprint(M3UAccount, M3UAccountProfile, Stream, account): - """Resolve fingerprint for migration using historical models only.""" - fingerprint = compute_credential_fingerprint( - account.username or "", - account.password or "", - ) - if fingerprint: - return fingerprint - - sample_url = ( - Stream.objects.filter(m3u_account_id=account.pk) - .exclude(url="") - .values_list("url", flat=True) - .first() - ) - if sample_url: - url_user, url_pass = extract_credentials_from_stream_url(sample_url) - return compute_credential_fingerprint(url_user or "", url_pass or "") - - return None - - -def backfill_credential_pools(apps, schema_editor): - from django.db.models import Min - - M3UAccount = apps.get_model("m3u", "M3UAccount") - ServerGroup = apps.get_model("m3u", "ServerGroup") - M3UAccountProfile = apps.get_model("m3u", "M3UAccountProfile") - Stream = apps.get_model("dispatcharr_channels", "Stream") - - seen = {} - for account in M3UAccount.objects.all(): - props = account.custom_properties or {} - if props.get("exclude_from_credential_pool"): - continue - if account.server_group_id: - group = ServerGroup.objects.filter(pk=account.server_group_id).first() - if group and not group.credential_fingerprint: - continue - - fingerprint = _historical_account_fingerprint( - M3UAccount, M3UAccountProfile, Stream, account - ) - if not fingerprint: - continue - - if fingerprint not in seen: - short = fingerprint[:16] - group, _ = ServerGroup.objects.get_or_create( - credential_fingerprint=fingerprint, - defaults={ - "name": f"credential-pool-{short}", - "max_streams": 0, - }, - ) - seen[fingerprint] = group - else: - group = seen[fingerprint] - - M3UAccount.objects.filter(pk=account.pk).update(server_group_id=group.pk) - - for group in seen.values(): - agg = M3UAccount.objects.filter( - server_group_id=group.pk, max_streams__gt=0 - ).aggregate(min_limit=Min("max_streams")) - new_max = agg["min_limit"] or 0 - if group.max_streams != new_max: - ServerGroup.objects.filter(pk=group.pk).update(max_streams=new_max) - - -class Migration(migrations.Migration): - - dependencies = [ - ("m3u", "0020_servergroup_max_streams"), - ("dispatcharr_channels", "0037_auto_sync_overhaul"), - ] - - operations = [ - migrations.AddField( - model_name="servergroup", - name="credential_fingerprint", - field=models.CharField( - blank=True, - db_index=True, - help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", - max_length=64, - null=True, - unique=True, - ), - ), - migrations.RunPython(backfill_credential_pools, migrations.RunPython.noop), - ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 4b1325d5..e6ccaadb 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -203,14 +203,6 @@ class ServerGroup(models.Model): default=0, help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", ) - credential_fingerprint = models.CharField( - max_length=64, - null=True, - blank=True, - unique=True, - db_index=True, - help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", - ) def __str__(self): return self.name @@ -346,26 +338,6 @@ class M3UAccountProfile(models.Model): return None -@receiver(models.signals.post_save, sender=M3UAccount) -def assign_credential_pool_for_m3u_account(sender, instance, **kwargs): - """Link accounts with identical credentials into a shared connection pool.""" - if kwargs.get("raw"): - return - from apps.m3u.connection_pool import sync_account_credential_pool - - sync_account_credential_pool(instance) - - -@receiver(models.signals.post_save, sender=M3UAccountProfile) -def assign_credential_pool_for_m3u_profile(sender, instance, **kwargs): - """Re-sync pools when profile URL transforms change.""" - if kwargs.get("raw"): - return - from apps.m3u.connection_pool import sync_account_credential_pool - - sync_account_credential_pool(instance.m3u_account) - - @receiver(models.signals.post_save, sender=M3UAccount) def create_profile_for_m3u_account(sender, instance, created, **kwargs): """Automatically create an M3UAccountProfile when M3UAccount is created.""" diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 79285484..44281211 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -1,18 +1,21 @@ -"""Tests for shared credential connection pools (#1137).""" +"""Tests for shared ServerGroup connection pools (#1137).""" from django.test import TestCase from unittest.mock import patch from apps.m3u.connection_pool import ( - compute_credential_fingerprint, extract_credentials_from_stream_url, get_enforced_server_group_for_profile, + get_group_connection_count, + get_profile_connection_count, get_profile_credential_fingerprint, + group_has_capacity_for_profile, pool_has_capacity_for_profile, + profile_has_capacity_for_selection, + profile_connections_key, release_profile_slot, reserve_profile_slot, server_group_connections_key, - sync_account_credential_pool, ) from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup @@ -41,22 +44,7 @@ class FakeRedis: return self._data[key] -class CredentialFingerprintTests(TestCase): - def test_same_credentials_same_fingerprint(self): - fp1 = compute_credential_fingerprint("User", "pass") - fp2 = compute_credential_fingerprint("user", "pass") - self.assertEqual(fp1, fp2) - self.assertIsNotNone(fp1) - - def test_different_password_different_fingerprint(self): - fp1 = compute_credential_fingerprint("user", "pass1") - fp2 = compute_credential_fingerprint("user", "pass2") - self.assertNotEqual(fp1, fp2) - - def test_empty_credentials_returns_none(self): - self.assertIsNone(compute_credential_fingerprint("", "pass")) - self.assertIsNone(compute_credential_fingerprint("user", "")) - +class ExtractCredentialsTests(TestCase): def test_extract_credentials_from_xc_style_url(self): url = "http://example.com/live/alice/secret123/99999.ts" user, password = extract_credentials_from_stream_url(url) @@ -64,94 +52,76 @@ class CredentialFingerprintTests(TestCase): self.assertEqual(password, "secret123") -class AutoAssignTests(TestCase): - def test_accounts_with_same_credentials_share_server_group(self): - account1 = M3UAccount.objects.create( - name="Provider A", - account_type="XC", - username="user1", - password="secret", - server_url="http://a.example.com", - max_streams=2, - ) - account2 = M3UAccount.objects.create( - name="Provider B", - account_type="XC", - username="user1", - password="secret", - server_url="http://b.example.com", - max_streams=3, - ) - - account1.refresh_from_db() - account2.refresh_from_db() - - self.assertIsNotNone(account1.server_group_id) - self.assertEqual(account1.server_group_id, account2.server_group_id) - self.assertTrue(account1.server_group.credential_fingerprint) - self.assertEqual(account1.server_group.max_streams, 2) - - def test_exclude_from_pool_opt_out(self): - account1 = M3UAccount.objects.create( - name="Pooled", - account_type="XC", - username="shared", - password="secret", - max_streams=1, - ) - account2 = M3UAccount.objects.create( - name="Opt-out", - account_type="XC", - username="shared", - password="secret", - custom_properties={"exclude_from_credential_pool": True}, - max_streams=1, - ) - - account1.refresh_from_db() - account2.refresh_from_db() - - self.assertIsNotNone(account1.server_group_id) - self.assertIsNone(account2.server_group_id) - - -class MultiProfilePoolTests(TestCase): - def test_profiles_with_different_credentials_get_separate_pools(self): +class ManualServerGroupTests(TestCase): + def test_group_enforced_when_max_streams_set(self): + group = ServerGroup.objects.create(name="provider-a", max_streams=2) account = M3UAccount.objects.create( - name="Multi-login XC", + name="Account A", account_type="XC", - username="xc_user_a", - password="xc_pass_a", + username="user", + password="pass", + server_group=group, + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + + self.assertEqual(get_enforced_server_group_for_profile(profile), group) + + def test_accounts_in_same_group_share_counter(self): + group = ServerGroup.objects.create(name="shared", max_streams=1) + account1 = M3UAccount.objects.create( + name="XC Account", + account_type="XC", + username="user", + password="pass", server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + account2 = M3UAccount.objects.create( + name="M3U Account", + account_type="STD", + username="user", + password="pass", + server_group=group, + max_streams=5, + ) + profile1 = M3UAccountProfile.objects.get(m3u_account=account1, is_default=True) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + + redis = FakeRedis() + reserved1, _ = reserve_profile_slot(profile1, redis) + self.assertTrue(reserved1) + + reserved2, _ = reserve_profile_slot(profile2, redis) + self.assertFalse(reserved2) + self.assertFalse(group_has_capacity_for_profile(profile2, redis)) + + def test_profile_rotation_when_default_profile_full(self): + """Pre-pool behavior: try the next profile on the same account.""" + account = M3UAccount.objects.create( + name="Multi-profile", + account_type="XC", max_streams=1, ) - base = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) - base.search_pattern = r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" - base.replace_pattern = r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" - base.save() + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.max_streams = 1 + default.save() - p2 = M3UAccountProfile.objects.create( + alt = M3UAccountProfile.objects.create( m3u_account=account, - name="login_b", + name="alt_profile", is_default=False, is_active=True, max_streams=1, - search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", - replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + search_pattern="", + replace_pattern="", ) - sync_account_credential_pool(account) - - fp1 = get_profile_credential_fingerprint(base) - fp2 = get_profile_credential_fingerprint(p2) - self.assertNotEqual(fp1, fp2) - - g1 = get_enforced_server_group_for_profile(base) - g2 = get_enforced_server_group_for_profile(p2) - self.assertIsNotNone(g1) - self.assertIsNotNone(g2) - self.assertNotEqual(g1.id, g2.id) - self.assertIsNone(account.server_group_id) + redis = FakeRedis() + reserved, _ = reserve_profile_slot(default, redis) + self.assertTrue(reserved) + self.assertFalse(profile_has_capacity_for_selection(default, redis)) + self.assertTrue(profile_has_capacity_for_selection(alt, redis)) class PoolEnforcementTests(TestCase): @@ -172,42 +142,86 @@ class PoolEnforcementTests(TestCase): self.profile = M3UAccountProfile.objects.get( m3u_account=self.account, is_default=True ) + self.profile.max_streams = 5 + self.profile.save() - def test_reserve_and_release_pool_counter(self): + def test_reserve_and_release_both_counters(self): reserved, count = reserve_profile_slot(self.profile, self.redis) self.assertTrue(reserved) self.assertEqual(count, 1) - group_key = server_group_connections_key(self.group.id) + group_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) + profile_key = profile_connections_key(self.profile.id) self.assertEqual(self.redis._data[group_key], 1) + self.assertEqual(self.redis._data[profile_key], 1) release_profile_slot(self.profile.id, self.redis) self.assertEqual(self.redis._data[group_key], 0) + self.assertEqual(self.redis._data[profile_key], 0) - def test_reserve_fails_when_pool_at_capacity(self): - group_key = server_group_connections_key(self.group.id) + def test_reserve_fails_when_group_at_capacity(self): + group_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) self.redis.set(group_key, 1) reserved, _count = reserve_profile_slot(self.profile, self.redis) self.assertFalse(reserved) self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) - def test_live_slot_blocks_second_reserve_same_profile(self): - reserved1, _ = reserve_profile_slot(self.profile, self.redis) - self.assertTrue(reserved1) + def test_different_logins_in_group_do_not_block_each_other(self): + """Profiles with different provider logins keep separate group counters.""" + account = M3UAccount.objects.create( + name="Grouped multi-login", + account_type="XC", + username="login_a", + password="pass_a", + server_group=self.group, + max_streams=5, + ) + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.max_streams = 1 + default.save() - reserved2, _ = reserve_profile_slot(self.profile, self.redis) - self.assertFalse(reserved2) + alt = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt_login", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + fp_a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fp_b = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + side_effect=lambda profile: fp_a if profile.id == default.id else fp_b, + ): + reserved_default, _ = reserve_profile_slot(default, self.redis) + self.assertTrue(reserved_default) + + reserved_alt, _ = reserve_profile_slot(alt, self.redis) + self.assertTrue(reserved_alt) + + key_a = server_group_connections_key(self.group.id, fp_a) + key_b = server_group_connections_key(self.group.id, fp_b) + self.assertEqual(self.redis._data[key_a], 1) + self.assertEqual(self.redis._data[key_b], 1) class VodProfileSelectionTests(TestCase): - """VOD must try alternate profiles when the default pool is full (live TV).""" - - def test_get_m3u_profile_skips_default_when_pool_full(self): + def test_get_m3u_profile_skips_default_when_profile_full(self): from apps.proxy.vod_proxy.views import _get_m3u_profile account = M3UAccount.objects.create( - name="VOD multi-login", + name="VOD multi-profile", account_type="XC", username="xc_user_a", password="xc_pass_a", @@ -215,26 +229,19 @@ class VodProfileSelectionTests(TestCase): max_streams=1, ) default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) - default.search_pattern = ( - r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" - ) - default.replace_pattern = ( - r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" - ) + default.max_streams = 1 default.save() alt = M3UAccountProfile.objects.create( m3u_account=account, - name="login_b", + name="alt_profile", is_default=False, is_active=True, max_streams=1, - search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", - replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + search_pattern="", + replace_pattern="", ) - sync_account_credential_pool(account) - redis = FakeRedis() reserved, _ = reserve_profile_slot(default, redis) self.assertTrue(reserved) diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 083b0fd1..60d0e3d0 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -8,7 +8,7 @@ from typing import Optional, Tuple, List from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile -from apps.m3u.connection_pool import pool_has_capacity_for_profile +from apps.m3u.connection_pool import profile_connections_key from core.models import UserAgent, CoreSettings, StreamProfile from .utils import get_logger from uuid import UUID @@ -54,15 +54,12 @@ def get_stream_object(id: str): logger.info(f"Fetching stream hash {id}") return get_object_or_404(Stream, stream_hash=id) -def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]]: +def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], bool]: """ Generate the appropriate stream URL for a channel or stream based on its profile settings. - Args: - channel_id: The UUID of the channel or stream hash - Returns: - Tuple[str, str, bool, Optional[int]]: (stream_url, user_agent, transcode_flag, profile_id) + Tuple: (stream_url, user_agent, transcode_flag, profile_id, slot_reserved) """ try: channel_or_stream = get_stream_object(channel_id) @@ -74,15 +71,12 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] if not stream.m3u_account: logger.error(f"Stream {stream.id} has no M3U account") - return None, None, False, None + return None, None, False, None, False - # Use get_stream() to atomically reserve a slot and write the - # channel_stream / stream_profile Redis keys, matching the channel - # path so stream_name and stream_stats work correctly. - stream_id, profile_id, error_reason = stream.get_stream() + stream_id, profile_id, error_reason, slot_reserved = stream.get_stream() if not stream_id or not profile_id: logger.error(f"No profile available for stream {stream.id}: {error_reason}") - return None, None, False, None + return None, None, False, None, False try: profile = M3UAccountProfile.objects.get(id=profile_id) @@ -101,11 +95,12 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] transcode = not stream_profile.is_proxy() stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved except Exception as e: logger.error(f"Error generating stream URL for stream {stream.id}: {e}") - stream.release_stream() - return None, None, False, None + if slot_reserved: + stream.release_stream() + return None, None, False, None, False # Handle channel preview (existing logic) @@ -113,11 +108,11 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] # Get stream and profile for this channel # Note: get_stream now returns 3 values (stream_id, profile_id, error_reason) - stream_id, profile_id, error_reason = channel.get_stream() + stream_id, profile_id, error_reason, slot_reserved = channel.get_stream() if not stream_id or not profile_id: logger.error(f"No stream available for channel {channel_id}: {error_reason}") - return None, None, False, None + return None, None, False, None, False # get_stream() allocated a connection slot - ensure it's released on any error try: @@ -147,15 +142,16 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved except Exception as e: logger.error(f"Error generating stream URL for channel {channel_id}: {e}") - if not channel.release_stream(): - logger.warning(f"Failed to release stream for channel {channel_id} after URL generation error") - return None, None, False, None + if slot_reserved: + if not channel.release_stream(): + logger.warning(f"Failed to release stream for channel {channel_id} after URL generation error") + return None, None, False, None, False except Exception as e: logger.error(f"Error generating stream URL: {e}") - return None, None, False, None + return None, None, False, None, False def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -233,14 +229,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: if redis_client: - if not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"Profile {profile.id} credential pool at capacity for stream switch" - ) - continue - - profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = int( + redis_client.get(profile_connections_key(profile.id)) or 0 + ) channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") @@ -275,7 +266,7 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] m3u_profile_id = selected_profile.id else: - stream_id, m3u_profile_id, error_reason = channel.get_stream() + stream_id, m3u_profile_id, error_reason, _slot_reserved = channel.get_stream() if stream_id is None or m3u_profile_id is None: return {'error': error_reason or 'No stream assigned to channel'} @@ -398,14 +389,12 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: - if not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"Credential pool at capacity for profile " - f"{profile.id} (account {m3u_account.id})" - ) - continue selected_profile = profile - logger.debug(f"Found available profile {profile.id} for stream {stream.id}: {effective_connections}/{profile.max_streams} effective (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Found available profile {profile.id} for stream {stream.id}: " + f"{effective_connections}/{profile.max_streams} effective " + f"(current: {current_connections}, already using: {channel_using_profile})" + ) break else: logger.debug(f"Profile {profile.id} at max connections: {effective_connections}/{profile.max_streams} (current: {current_connections}, already using: {channel_using_profile})") diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 1369a364..e9e1b8ab 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -213,6 +213,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): stream_user_agent = None transcode = False profile_value = None + slot_reserved = False error_reason = None attempt = 0 should_retry = True @@ -220,7 +221,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # Try to get a stream with fixed interval retries while should_retry and time.time() - wait_start_time < retry_timeout: attempt += 1 - stream_url, stream_user_agent, transcode, profile_value = ( + stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( generate_stream_url(channel_id) ) @@ -232,7 +233,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # On first failure, check if the error is retryable if attempt == 1: - _, _, error_reason = channel.get_stream() + _, _, error_reason, _ = channel.get_stream() if error_reason and "maximum connection limits" not in error_reason: logger.warning( f"[{client_id}] Can't retry - error not related to connection limits: {error_reason}" @@ -265,7 +266,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.info( f"[{client_id}] Making final attempt {attempt} at timeout boundary" ) - stream_url, stream_user_agent, transcode, profile_value = ( + stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( generate_stream_url(channel_id) ) if stream_url is not None: @@ -274,9 +275,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ) if stream_url is None: - # Release any connection slot that may have been allocated - # by the error-checking get_stream() call during retries - if not channel.release_stream(): + if slot_reserved and not channel.release_stream(): logger.debug(f"[{client_id}] release_stream found no keys during failed init cleanup") # Get the specific error message if available @@ -295,7 +294,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # generate_stream_url() called get_stream() which allocated a connection # slot (INCR'd profile_connections) - track this for cleanup on error - if needs_initialization: + if needs_initialization and slot_reserved: connection_allocated = True # Read stream assignment from Redis (already set by generate_stream_url → get_stream). @@ -378,8 +377,8 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.warning( f"[{client_id}] Alternate stream #{alt['stream_id']} failed validation: {message}" ) - # Release stream lock before redirecting - if not channel.release_stream(): + # Release stream lock before redirecting only if we reserved a slot + if connection_allocated and not channel.release_stream(): logger.warning(f"[{client_id}] Failed to release stream before redirect") connection_allocated = False # Final decision based on validation results diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index a7995107..01272468 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -178,7 +178,10 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): """ try: from core.utils import RedisClient - from apps.m3u.connection_pool import pool_has_capacity_for_profile + from apps.m3u.connection_pool import ( + get_profile_connection_count, + profile_has_capacity_for_selection, + ) redis_client = RedisClient.get_client() if not redis_client: @@ -229,15 +232,12 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): ) # Check Redis-based current connections profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = get_profile_connection_count(profile, redis_client) - if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): + if profile_has_capacity_for_selection(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Using requested profile {profile.id}: {current_connections}/{profile.max_streams} connections") return (profile, current_connections) - elif not pool_has_capacity_for_profile(profile, redis_client): - logger.warning(f"[PROFILE-SELECTION] Shared credential pool at capacity for account {m3u_account.id}") - else: - logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") + logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") except M3UAccountProfile.DoesNotExist: logger.warning(f"[PROFILE-SELECTION] Requested profile {profile_id} not found") @@ -256,19 +256,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): profiles = [default_profile] + list(m3u_profiles.filter(is_default=False)) for profile in profiles: - profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = get_profile_connection_count(profile, redis_client) - # Check if profile has available connection slots - if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): + if profile_has_capacity_for_selection(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Selected profile {profile.id} ({profile.name}): {current_connections}/{profile.max_streams} connections") return (profile, current_connections) - elif not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"[PROFILE-SELECTION] Credential pool at capacity for profile " - f"{profile.id}, trying next profile" - ) - continue else: logger.debug(f"[PROFILE-SELECTION] Profile {profile.id} at capacity: {current_connections}/{profile.max_streams}") diff --git a/frontend/src/api.js b/frontend/src/api.js index 2a357a81..4f58527e 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3,6 +3,7 @@ import useAuthStore from './store/auth'; import useChannelsStore from './store/channels'; import useLogosStore from './store/logos'; import useUserAgentsStore from './store/userAgents'; +import useServerGroupsStore from './store/serverGroups'; import usePlaylistsStore from './store/playlists'; import useEPGsStore from './store/epgs'; import useStreamsStore from './store/streams'; @@ -1305,6 +1306,59 @@ export default class API { } } + static async getServerGroups() { + try { + const response = await request(`${host}/api/m3u/server-groups/`); + + return response; + } catch (e) { + errorNotification('Failed to retrieve server groups', e); + } + } + + static async addServerGroup(values) { + try { + const response = await request(`${host}/api/m3u/server-groups/`, { + method: 'POST', + body: values, + }); + + useServerGroupsStore.getState().addServerGroup(response); + + return response; + } catch (e) { + errorNotification('Failed to create server group', e); + } + } + + static async updateServerGroup(values) { + try { + const { id, ...payload } = values; + const response = await request(`${host}/api/m3u/server-groups/${id}/`, { + method: 'PUT', + body: payload, + }); + + useServerGroupsStore.getState().updateServerGroup(response); + + return response; + } catch (e) { + errorNotification('Failed to update server group', e); + } + } + + static async deleteServerGroup(id) { + try { + await request(`${host}/api/m3u/server-groups/${id}/`, { + method: 'DELETE', + }); + + useServerGroupsStore.getState().removeServerGroups([id]); + } catch (e) { + errorNotification('Failed to delete server group', e); + } + } + static async getPlaylist(id) { try { const response = await request(`${host}/api/m3u/accounts/${id}/`); diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index e27570db..3a4161cb 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -1,6 +1,7 @@ // Modal.js import React, { useEffect, useState } from 'react'; import useUserAgentsStore from '../../store/userAgents'; +import useServerGroupsStore from '../../store/serverGroups'; import M3UProfiles from './M3UProfiles'; import { Box, @@ -43,6 +44,7 @@ const M3U = ({ playlistCreated = false, }) => { const userAgents = useUserAgentsStore((s) => s.userAgents); + const serverGroups = useServerGroupsStore((s) => s.serverGroups); const fetchChannelGroups = useChannelsStore((s) => s.fetchChannelGroups); const fetchEPGs = useEPGsStore((s) => s.fetchEPGs); const fetchCategories = useVODStore((s) => s.fetchCategories); @@ -61,6 +63,7 @@ const M3U = ({ name: '', server_url: '', user_agent: '0', + server_group: '0', is_active: true, max_streams: 0, refresh_interval: 24, @@ -88,6 +91,9 @@ const M3U = ({ server_url: m3uAccount.server_url, max_streams: m3uAccount.max_streams, user_agent: m3uAccount.user_agent ? `${m3uAccount.user_agent}` : '0', + server_group: m3uAccount.server_group + ? `${m3uAccount.server_group}` + : '0', is_active: m3uAccount.is_active, refresh_interval: m3uAccount.refresh_interval, cron_expression: m3uAccount.cron_expression || '', @@ -355,6 +361,24 @@ const M3U = ({ key={form.key('max_streams')} /> + { + const defaultValues = useMemo( + () => ({ + name: serverGroup?.name || '', + max_streams: serverGroup?.max_streams ?? 0, + }), + [serverGroup] + ); + + const { + register, + handleSubmit, + formState: { errors, isSubmitting }, + reset, + setValue, + watch, + } = useForm({ + defaultValues, + resolver: yupResolver(schema), + }); + + const onSubmit = async (values) => { + const payload = { + ...values, + max_streams: Number(values.max_streams), + }; + + if (serverGroup?.id) { + await API.updateServerGroup({ id: serverGroup.id, ...payload }); + } else { + await API.addServerGroup(payload); + } + + reset(); + onClose(); + }; + + useEffect(() => { + reset(defaultValues); + }, [defaultValues, reset]); + + if (!isOpen) { + return null; + } + + const maxStreams = watch('max_streams'); + + return ( + +
+ + + setValue('max_streams', value ?? 0)} + error={errors.max_streams?.message} + /> + + + + + +
+ ); +}; + +export default ServerGroupForm; diff --git a/frontend/src/components/tables/ServerGroupsTable.jsx b/frontend/src/components/tables/ServerGroupsTable.jsx new file mode 100644 index 00000000..e4280037 --- /dev/null +++ b/frontend/src/components/tables/ServerGroupsTable.jsx @@ -0,0 +1,203 @@ +import { useEffect, useMemo, useState } from 'react'; +import API from '../../api'; +import useServerGroupsStore from '../../store/serverGroups'; +import ServerGroupForm from '../forms/ServerGroup'; +import { + ActionIcon, + Box, + Button, + Center, + Flex, + Paper, + Stack, + Text, + Tooltip, +} from '@mantine/core'; +import { SquareMinus, SquarePen, SquarePlus } from 'lucide-react'; +import { CustomTable, useTable } from './CustomTable'; +import useLocalStorage from '../../hooks/useLocalStorage'; + +const RowActions = ({ row, editServerGroup, deleteServerGroup }) => { + return ( + <> + editServerGroup(row.original)} + > + + + deleteServerGroup(row.original.id)} + > + + + + ); +}; + +const ServerGroupsTable = () => { + const [serverGroup, setServerGroup] = useState(null); + const [serverGroupModalOpen, setServerGroupModalOpen] = useState(false); + + const serverGroups = useServerGroupsStore((state) => state.serverGroups); + const fetchServerGroups = useServerGroupsStore( + (state) => state.fetchServerGroups + ); + const [tableSize] = useLocalStorage('table-size', 'default'); + + const columns = useMemo( + () => [ + { + header: 'Name', + accessorKey: 'name', + size: 175, + }, + { + header: 'Max Streams', + accessorKey: 'max_streams', + size: 100, + cell: ({ cell }) => { + const value = cell.getValue(); + return value === 0 ? 'Unlimited' : value; + }, + }, + { + id: 'actions', + header: 'Actions', + size: tableSize == 'compact' ? 50 : 75, + }, + ], + [tableSize] + ); + + const [isLoading, setIsLoading] = useState(true); + + const editServerGroup = (group = null) => { + setServerGroup(group); + setServerGroupModalOpen(true); + }; + + const deleteServerGroup = async (id) => { + await API.deleteServerGroup(id); + }; + + const closeServerGroupForm = () => { + setServerGroup(null); + setServerGroupModalOpen(false); + }; + + useEffect(() => { + fetchServerGroups().finally(() => setIsLoading(false)); + }, [fetchServerGroups]); + + const renderHeaderCell = (header) => { + return ( + + {header.column.columnDef.header} + + ); + }; + + const renderBodyCell = ({ row }) => { + return ( + + ); + }; + + const table = useTable({ + columns, + data: serverGroups, + allRowIds: serverGroups.map((group) => group.id), + bodyCellRenderFns: { + actions: renderBodyCell, + }, + headerCellRenderFns: { + name: renderHeaderCell, + max_streams: renderHeaderCell, + actions: renderHeaderCell, + }, + }); + + if (isLoading) { + return ( +
+ Loading server groups... +
+ ); + } + + return ( + + + + + + + + + + + + + +
+ +
+ + + + + + ); +}; + +export default ServerGroupsTable; diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index 29d46dd8..c3481984 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -14,6 +14,9 @@ import { const UserAgentsTable = React.lazy( () => import('../components/tables/UserAgentsTable.jsx') ); +const ServerGroupsTable = React.lazy( + () => import('../components/tables/ServerGroupsTable.jsx') +); const StreamProfilesTable = React.lazy( () => import('../components/tables/StreamProfilesTable.jsx') ); @@ -135,6 +138,19 @@ const SettingsPage = () => { + + Server Groups + + + }> + + + + + + User-Agents diff --git a/frontend/src/store/auth.jsx b/frontend/src/store/auth.jsx index cfea5d4f..8f362357 100644 --- a/frontend/src/store/auth.jsx +++ b/frontend/src/store/auth.jsx @@ -6,6 +6,7 @@ import useEPGsStore from './epgs'; import useStreamProfilesStore from './streamProfiles'; import useOutputProfilesStore from './outputProfiles'; import useUserAgentsStore from './userAgents'; +import useServerGroupsStore from './serverGroups'; import useUsersStore from './users'; import API from '../api'; import { USER_LEVELS } from '../constants'; @@ -124,6 +125,7 @@ const useAuthStore = create((set, get) => ({ useStreamProfilesStore.getState().fetchProfiles(), useOutputProfilesStore.getState().fetchProfiles(), useUserAgentsStore.getState().fetchUserAgents(), + useServerGroupsStore.getState().fetchServerGroups(), useChannelsStore.getState().fetchChannelIds(), ]); diff --git a/frontend/src/store/serverGroups.jsx b/frontend/src/store/serverGroups.jsx new file mode 100644 index 00000000..42406d70 --- /dev/null +++ b/frontend/src/store/serverGroups.jsx @@ -0,0 +1,40 @@ +import { create } from 'zustand'; +import api from '../api'; + +const useServerGroupsStore = create((set) => ({ + serverGroups: [], + isLoading: false, + error: null, + + fetchServerGroups: async () => { + set({ isLoading: true, error: null }); + try { + const serverGroups = await api.getServerGroups(); + set({ serverGroups: serverGroups || [], isLoading: false }); + } catch (error) { + console.error('Failed to fetch server groups:', error); + set({ error: 'Failed to load server groups.', isLoading: false }); + } + }, + + addServerGroup: (serverGroup) => + set((state) => ({ + serverGroups: [...state.serverGroups, serverGroup], + })), + + updateServerGroup: (serverGroup) => + set((state) => ({ + serverGroups: state.serverGroups.map((group) => + group.id === serverGroup.id ? serverGroup : group + ), + })), + + removeServerGroups: (serverGroupIds) => + set((state) => ({ + serverGroups: state.serverGroups.filter( + (group) => !serverGroupIds.includes(group.id) + ), + })), +})); + +export default useServerGroupsStore; diff --git a/frontend/src/utils/forms/M3uUtils.js b/frontend/src/utils/forms/M3uUtils.js index 7fc7e369..44692b78 100644 --- a/frontend/src/utils/forms/M3uUtils.js +++ b/frontend/src/utils/forms/M3uUtils.js @@ -50,5 +50,9 @@ export const prepareSubmitValues = (values, expDate) => { prepared.user_agent = null; } + if (prepared.server_group == '0') { + prepared.server_group = null; + } + return prepared; }; From 6e07dce3f17532fc3519c3ef2b052467c06ead05 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Fri, 29 May 2026 17:05:59 -0400 Subject: [PATCH 03/81] Fix maintainer review items without breaking multi-login rotation. Cap credential-scoped group counters at profile.max_streams (not group.max_streams), skip credential counters when fingerprint resolution fails, and always swap profile counters on update_stream_profile. Restore per-profile-only selection for VOD/live switches so a second stream can use a different login without invalid HTTP errors. Co-authored-by: Cursor --- apps/channels/models.py | 44 +++--- apps/m3u/connection_pool.py | 94 ++++++++----- apps/m3u/tests/test_connection_pool.py | 178 +++++++++++++++++++++---- 3 files changed, 233 insertions(+), 83 deletions(-) diff --git a/apps/channels/models.py b/apps/channels/models.py index 38ebbbc7..154f6fa2 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -712,6 +712,11 @@ class Channel(models.Model): True, ) # Return newly assigned stream and matched profile else: + from apps.m3u.connection_pool import ( + group_has_capacity_for_profile, + profile_has_capacity_for_selection, + ) + # At capacity: try to preempt a lower-impact channel on this profile victim_channel_id = self._pick_channel_to_preempt( profile_id=profile.id, @@ -723,12 +728,21 @@ class Channel(models.Model): logger.info(f"Preempting channel {victim_channel_id} for new stream on profile {profile.id}") # return self.id, profile.id, victim_channel_id - # This profile is at max connections has_streams_but_maxed_out = True - logger.debug( - f"Profile {profile.id} at max connections: " - f"{current_count}/{profile.max_streams}" - ) + if not profile_has_capacity_for_selection(profile, redis_client): + logger.info( + f"Profile {profile.id} at max connections: " + f"{current_count}/{profile.max_streams}, trying next profile" + ) + elif not group_has_capacity_for_profile(profile, redis_client): + logger.info( + f"Profile {profile.id} login or server group pool full, trying next profile" + ) + else: + logger.debug( + f"Profile {profile.id} reservation failed: " + f"{current_count}/{profile.max_streams}" + ) # No available streams - determine specific reason if has_streams_but_maxed_out: @@ -873,25 +887,9 @@ class Channel(models.Model): if current_profile_id == new_profile_id: return True - from apps.m3u.models import M3UAccountProfile - from apps.m3u.connection_pool import ( - get_enforced_server_group_for_profile, - profile_connections_key, - ) + from apps.m3u.connection_pool import profile_connections_key - new_profile = M3UAccountProfile.objects.get(id=new_profile_id) - if get_enforced_server_group_for_profile(new_profile): - # Shared group pool: one active stream uses one group slot regardless - # of which profile supplies the URL. - redis_client.set(f"stream_profile:{stream_id}", new_profile_id) - logger.info( - f"Updated stream {stream_id} profile from {current_profile_id} to " - f"{new_profile_id} (shared group pool unchanged)" - ) - return True - - # Use pipeline for atomic profile switch to prevent counter drift - # if an exception occurs between DECR and INCR + # Profile counters always move on switch; group totals stay unchanged (one stream). old_profile_connections_key = profile_connections_key(current_profile_id) new_profile_connections_key = profile_connections_key(new_profile_id) old_count = int(redis_client.get(old_profile_connections_key) or 0) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 9817d86c..35c35b0e 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -3,8 +3,9 @@ Shared connection pool enforcement for M3U accounts in the same ServerGroup. Profile selection rotates across M3UAccountProfile rows using each profile's own Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup -with max_streams > 0, the group counter is scoped by provider login fingerprint -so profiles that rewrite to different IPTV credentials keep independent limits. +with max_streams > 0, a credential-scoped counter is checked on reserve/release +so accounts sharing the same provider login share one limit without blocking +unrelated logins on the same group. """ from __future__ import annotations @@ -29,10 +30,12 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) -def server_group_connections_key(group_id: int, fingerprint: Optional[str] = None) -> str: - """Redis key for a manual ServerGroup slot, scoped by provider login.""" - fp = (fingerprint or "unknown")[:16] - return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id, fingerprint=fp) +def server_group_connections_key(group_id: int, fingerprint: str) -> str: + """Redis key for per-credential usage within a ServerGroup.""" + return SERVER_GROUP_CONNECTIONS_KEY.format( + group_id=group_id, + fingerprint=fingerprint[:16], + ) def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: @@ -124,22 +127,30 @@ def get_enforced_server_group_for_profile(profile): return None -def _group_counter_key(profile, group) -> str: - return server_group_connections_key( - group.id, - get_profile_credential_fingerprint(profile), - ) +def _credential_counter_key(profile, group) -> Optional[str]: + fingerprint = get_profile_credential_fingerprint(profile) + if not fingerprint: + return None + return server_group_connections_key(group.id, fingerprint) def get_profile_connection_count(profile, redis_client) -> int: return int(redis_client.get(profile_connections_key(profile.id)) or 0) -def get_group_connection_count(profile, redis_client) -> int: +def get_credential_connection_count(profile, redis_client) -> int: group = get_enforced_server_group_for_profile(profile) if not group: return 0 - return int(redis_client.get(_group_counter_key(profile, group)) or 0) + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return 0 + return int(redis_client.get(cred_key) or 0) + + +def get_group_connection_count(profile, redis_client) -> int: + """Backwards-compatible alias for credential-scoped group usage.""" + return get_credential_connection_count(profile, redis_client) def profile_has_capacity_for_selection(profile, redis_client) -> bool: @@ -151,35 +162,22 @@ def profile_has_capacity_for_selection(profile, redis_client) -> bool: def group_has_capacity_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) - if not group: + if not group or profile.max_streams == 0: return True - return get_group_connection_count(profile, redis_client) < group.max_streams + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return True + return get_credential_connection_count(profile, redis_client) < profile.max_streams def pool_has_capacity_for_profile(profile, redis_client) -> bool: - """Non-mutating check before reserve: profile slot and group slot if applicable.""" + """Non-mutating check before reserve: profile slot and credential slot if applicable.""" return profile_has_capacity_for_selection(profile, redis_client) and group_has_capacity_for_profile( profile, redis_client ) -def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: - group = get_enforced_server_group_for_profile(profile) - if not group: - return True - key = _group_counter_key(profile, group) - group_count = redis_client.incr(key) - if group_count <= group.max_streams: - return True - redis_client.decr(key) - return False - - -def _release_server_group_slot_for_profile(profile, redis_client) -> None: - group = get_enforced_server_group_for_profile(profile) - if not group: - return - key = _group_counter_key(profile, group) +def _safe_decr(redis_client, key: str) -> None: current = int(redis_client.get(key) or 0) if current <= 0: return @@ -188,9 +186,35 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: redis_client.set(key, 0) +def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: + group = get_enforced_server_group_for_profile(profile) + if not group or profile.max_streams == 0: + return True + + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return True + + cred_count = redis_client.incr(cred_key) + if cred_count <= profile.max_streams: + return True + + redis_client.decr(cred_key) + return False + + +def _release_server_group_slot_for_profile(profile, redis_client) -> None: + group = get_enforced_server_group_for_profile(profile) + if not group or profile.max_streams == 0: + return + cred_key = _credential_counter_key(profile, group) + if cred_key: + _safe_decr(redis_client, cred_key) + + def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: """ - Atomically reserve profile + optional group slots (INCR-first). + Atomically reserve profile + optional credential slots (INCR-first). Returns (reserved, profile_count_after_attempt). """ @@ -212,7 +236,7 @@ def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: def release_profile_slot(profile_id: int, redis_client) -> None: - """Release profile and shared group slots after a stream ends.""" + """Release profile and shared credential slots after a stream end.""" from apps.m3u.models import M3UAccountProfile try: diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 44281211..1dbfc2a9 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -5,6 +5,7 @@ from unittest.mock import patch from apps.m3u.connection_pool import ( extract_credentials_from_stream_url, + get_credential_connection_count, get_enforced_server_group_for_profile, get_group_connection_count, get_profile_connection_count, @@ -43,6 +44,37 @@ class FakeRedis: self._data[key] = self._data.get(key, 0) - 1 return self._data[key] + def pipeline(self): + return FakeRedisPipeline(self) + + +class FakeRedisPipeline: + def __init__(self, redis): + self.redis = redis + self._ops = [] + + def decr(self, key): + self._ops.append(("decr", key)) + return self + + def incr(self, key): + self._ops.append(("incr", key)) + return self + + def set(self, key, value): + self._ops.append(("set", key, value)) + return self + + def execute(self): + for op in self._ops: + if op[0] == "decr": + self.redis.decr(op[1]) + elif op[0] == "incr": + self.redis.incr(op[1]) + elif op[0] == "set": + self.redis.set(op[1], op[2]) + self._ops = [] + class ExtractCredentialsTests(TestCase): def test_extract_credentials_from_xc_style_url(self): @@ -66,8 +98,8 @@ class ManualServerGroupTests(TestCase): self.assertEqual(get_enforced_server_group_for_profile(profile), group) - def test_accounts_in_same_group_share_counter(self): - group = ServerGroup.objects.create(name="shared", max_streams=1) + def test_accounts_in_same_group_share_credential_counter(self): + group = ServerGroup.objects.create(name="shared", max_streams=2) account1 = M3UAccount.objects.create( name="XC Account", account_type="XC", @@ -87,6 +119,10 @@ class ManualServerGroupTests(TestCase): ) profile1 = M3UAccountProfile.objects.get(m3u_account=account1, is_default=True) profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile1.max_streams = 1 + profile1.save() + profile2.max_streams = 1 + profile2.save() redis = FakeRedis() reserved1, _ = reserve_profile_slot(profile1, redis) @@ -97,7 +133,6 @@ class ManualServerGroupTests(TestCase): self.assertFalse(group_has_capacity_for_profile(profile2, redis)) def test_profile_rotation_when_default_profile_full(self): - """Pre-pool behavior: try the next profile on the same account.""" account = M3UAccount.objects.create( name="Multi-profile", account_type="XC", @@ -129,20 +164,21 @@ class PoolEnforcementTests(TestCase): self.redis = FakeRedis() self.group = ServerGroup.objects.create( name="test-pool", - max_streams=1, + max_streams=2, ) self.account = M3UAccount.objects.create( name="Test Account", account_type="XC", username="user", password="pass", + server_url="http://xc.example.com", server_group=self.group, max_streams=5, ) self.profile = M3UAccountProfile.objects.get( m3u_account=self.account, is_default=True ) - self.profile.max_streams = 5 + self.profile.max_streams = 1 self.profile.save() def test_reserve_and_release_both_counters(self): @@ -150,31 +186,45 @@ class PoolEnforcementTests(TestCase): self.assertTrue(reserved) self.assertEqual(count, 1) - group_key = server_group_connections_key( + cred_key = server_group_connections_key( self.group.id, get_profile_credential_fingerprint(self.profile), ) profile_key = profile_connections_key(self.profile.id) - self.assertEqual(self.redis._data[group_key], 1) + self.assertEqual(self.redis._data[cred_key], 1) self.assertEqual(self.redis._data[profile_key], 1) release_profile_slot(self.profile.id, self.redis) - self.assertEqual(self.redis._data[group_key], 0) + self.assertEqual(self.redis._data[cred_key], 0) self.assertEqual(self.redis._data[profile_key], 0) - def test_reserve_fails_when_group_at_capacity(self): - group_key = server_group_connections_key( - self.group.id, - get_profile_credential_fingerprint(self.profile), + def test_same_credential_capped_at_profile_max_not_group_max(self): + """Maintainer example: group max=2 but each login only allows 1.""" + account2 = M3UAccount.objects.create( + name="Second Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=self.group, + max_streams=5, ) - self.redis.set(group_key, 1) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile2.max_streams = 1 + profile2.save() - reserved, _count = reserve_profile_slot(self.profile, self.redis) - self.assertFalse(reserved) - self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + self.assertFalse(reserve_profile_slot(profile2, self.redis)[0]) + + fp = get_profile_credential_fingerprint(self.profile) + cred_key = server_group_connections_key(self.group.id, fp) + self.assertEqual(self.redis._data[cred_key], 1) + + def test_different_logins_both_stream_when_group_max_is_one(self): + """Regression: group max=1 must not block a second login on another profile.""" + self.group.max_streams = 1 + self.group.save() - def test_different_logins_in_group_do_not_block_each_other(self): - """Profiles with different provider logins keep separate group counters.""" account = M3UAccount.objects.create( name="Grouped multi-login", account_type="XC", @@ -204,17 +254,96 @@ class PoolEnforcementTests(TestCase): "apps.m3u.connection_pool.get_profile_credential_fingerprint", side_effect=lambda profile: fp_a if profile.id == default.id else fp_b, ): - reserved_default, _ = reserve_profile_slot(default, self.redis) - self.assertTrue(reserved_default) - - reserved_alt, _ = reserve_profile_slot(alt, self.redis) - self.assertTrue(reserved_alt) + self.assertTrue(reserve_profile_slot(default, self.redis)[0]) + self.assertTrue(reserve_profile_slot(alt, self.redis)[0]) key_a = server_group_connections_key(self.group.id, fp_a) key_b = server_group_connections_key(self.group.id, fp_b) self.assertEqual(self.redis._data[key_a], 1) self.assertEqual(self.redis._data[key_b], 1) + def test_no_fingerprint_skips_credential_counter(self): + account = M3UAccount.objects.create( + name="No creds", + account_type="STD", + server_group=self.group, + max_streams=5, + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile.max_streams = 1 + profile.save() + + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + return_value=None, + ): + self.assertTrue(reserve_profile_slot(profile, self.redis)[0]) + self.assertEqual(get_credential_connection_count(profile, self.redis), 0) + self.assertEqual(get_group_connection_count(profile, self.redis), 0) + + def test_release_when_profile_row_deleted(self): + profile_id = self.profile.id + fp = get_profile_credential_fingerprint(self.profile) + cred_key = server_group_connections_key(self.group.id, fp) + + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + self.profile.delete() + + release_profile_slot(profile_id, self.redis) + + self.assertEqual(self.redis._data[profile_connections_key(profile_id)], 0) + self.assertEqual(self.redis._data[cred_key], 1) + + +class UpdateStreamProfileTests(TestCase): + def test_switch_updates_profile_counters_when_group_assigned(self): + from apps.channels.models import Channel, Stream + + redis = FakeRedis() + group = ServerGroup.objects.create(name="switch-group", max_streams=2) + account = M3UAccount.objects.create( + name="Switch Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + profile_a = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile_a.max_streams = 1 + profile_a.save() + profile_b = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + stream = Stream.objects.create(name="Test Stream", m3u_account=account) + channel = Channel.objects.create(channel_number=501, name="Switch Channel") + channel.streams.add(stream) + + reserve_profile_slot(profile_a, redis) + redis.set(f"channel_stream:{channel.id}", stream.id) + redis.set(f"stream_profile:{stream.id}", profile_a.id) + + cred_key = server_group_connections_key( + group.id, get_profile_credential_fingerprint(profile_a) + ) + cred_before = redis._data[cred_key] + + with patch("core.utils.RedisClient.get_client", return_value=redis): + self.assertTrue(channel.update_stream_profile(profile_b.id)) + + self.assertEqual(int(redis.get(f"stream_profile:{stream.id}")), profile_b.id) + self.assertEqual(redis._data[profile_connections_key(profile_a.id)], 0) + self.assertEqual(redis._data[profile_connections_key(profile_b.id)], 1) + self.assertEqual(redis._data[cred_key], cred_before) + class VodProfileSelectionTests(TestCase): def test_get_m3u_profile_skips_default_when_profile_full(self): @@ -243,8 +372,7 @@ class VodProfileSelectionTests(TestCase): ) redis = FakeRedis() - reserved, _ = reserve_profile_slot(default, redis) - self.assertTrue(reserved) + self.assertTrue(reserve_profile_slot(default, redis)[0]) with patch("core.utils.RedisClient.get_client", return_value=redis): result = _get_m3u_profile(account, None, None) From d204d6ad64ff5068850d94a6fb0684c419c91923 Mon Sep 17 00:00:00 2001 From: Five Boroughs <40718516+FiveBoroughs@users.noreply.github.com> Date: Wed, 3 Jun 2026 20:05:19 +0200 Subject: [PATCH 04/81] feat(frontend): multi-word, accent-insensitive EPG channel filter --- frontend/src/components/forms/Channel.jsx | 19 ++- .../forms/__tests__/Channel.test.jsx | 149 ++++++++++++++++++ 2 files changed, 163 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/forms/Channel.jsx b/frontend/src/components/forms/Channel.jsx index c904641c..0fad1476 100644 --- a/frontend/src/components/forms/Channel.jsx +++ b/frontend/src/components/forms/Channel.jsx @@ -485,13 +485,21 @@ const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { return <>; } + // Case- and accent-insensitive. + const foldText = (text) => + text + .toLowerCase() + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, ''); + + // Every term must match, against name and id combined. + const tvgFilterTerms = foldText(tvgFilter).split(/\s+/).filter(Boolean); const filteredTvgs = tvgs .filter((tvg) => tvg.epg_source == selectedEPG) - .filter( - (tvg) => - tvg.name.toLowerCase().includes(tvgFilter.toLowerCase()) || - tvg.tvg_id.toLowerCase().includes(tvgFilter.toLowerCase()) - ); + .filter((tvg) => { + const haystack = foldText(`${tvg.name} ${tvg.tvg_id}`); + return tvgFilterTerms.every((term) => haystack.includes(term)); + }); const filteredLogos = logoOptions.filter((logo) => logo.name.toLowerCase().includes(logoFilter.toLowerCase()) @@ -1072,6 +1080,7 @@ const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { {/* Filter Input */} setTvgFilter(event.currentTarget.value) diff --git a/frontend/src/components/forms/__tests__/Channel.test.jsx b/frontend/src/components/forms/__tests__/Channel.test.jsx index 7b00775a..3c841b61 100644 --- a/frontend/src/components/forms/__tests__/Channel.test.jsx +++ b/frontend/src/components/forms/__tests__/Channel.test.jsx @@ -1206,4 +1206,153 @@ describe('ChannelForm', () => { expect(screen.queryByText(/Clear All Overrides/)).not.toBeInTheDocument(); }); }); + + // ── EPG filter search ────────────────────────────────────────────────────── + describe('EPG filter search', () => { + const skyText = 'Sky Sports Ultra HD (SkySportUltraHD.1.uk)'; + const btText = 'BT Sport 1 (BTSport1.uk)'; + + const epgTvgs = [ + { + id: 'tvg-sky', + name: 'Sky Sports Ultra HD', + tvg_id: 'SkySportUltraHD.1.uk', + epg_source: 10, + icon_url: '', + }, + { + id: 'tvg-bt', + name: 'BT Sport 1', + tvg_id: 'BTSport1.uk', + epg_source: 10, + icon_url: '', + }, + // Accent in the name, but the id does NOT mirror it ("decale"): an ascii + // query only matches if the haystack is accent-folded. + { + id: 'tvg-canal', + name: 'Canal Décalé', + tvg_id: 'CPLUS42.fr', + epg_source: 10, + icon_url: '', + }, + // Plain ascii everywhere: an accented query only matches if the query + // itself is accent-folded. + { + id: 'tvg-decaleplus', + name: 'Decale Plus', + tvg_id: 'DECP.fr', + epg_source: 10, + icon_url: '', + }, + ]; + const canalText = 'Canal Décalé (CPLUS42.fr)'; + const decaleText = 'Decale Plus (DECP.fr)'; + + // Render the form, point the store at our two same-source tvgs, and pick the + // EPG source so the dropdown list is populated. + const renderAndSelectSource = () => { + setupMocks(); + // Stable references — recreating these per selector call would retrigger + // the epgs-dependent effect and loop forever. + const epgs = makeEpgs(); + const tvgsById = {}; + vi.mocked(useEPGsStore).mockImplementation((sel) => + sel({ epgs, tvgs: epgTvgs, tvgsById }) + ); + render(); + fireEvent.change(screen.getByTestId('select-Source'), { + target: { value: '10' }, + }); + }; + + const typeFilter = (value) => + fireEvent.change(screen.getByTestId('text-input-tvg-filter'), { + target: { value }, + }); + + it('matches when space-separated terms each appear across name and id', () => { + renderAndSelectSource(); + typeFilter('Sky uk'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('matches regardless of term order', () => { + renderAndSelectSource(); + typeFilter('uk sky'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('still matches a single substring term', () => { + renderAndSelectSource(); + typeFilter('sky'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('shows every row for the source when the filter is empty', () => { + renderAndSelectSource(); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.getByText(btText)).toBeInTheDocument(); + }); + + it('shows no rows when a term matches nothing', () => { + renderAndSelectSource(); + typeFilter('sky zzz'); + expect(screen.queryByText(skyText)).not.toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('ignores case, including all-caps', () => { + renderAndSelectSource(); + typeFilter('SKY UK'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('matches when one term hits the name and another hits the id', () => { + renderAndSelectSource(); + typeFilter('Sports uk'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('tolerates leading, trailing and repeated whitespace', () => { + renderAndSelectSource(); + typeFilter(' sky uk '); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('matches an id fragment that is not in the display name', () => { + renderAndSelectSource(); + typeFilter('skysport'); + expect(screen.getByText(skyText)).toBeInTheDocument(); + expect(screen.queryByText(btText)).not.toBeInTheDocument(); + }); + + it('matches a name word combined with a channel number', () => { + renderAndSelectSource(); + typeFilter('bt 1'); + expect(screen.getByText(btText)).toBeInTheDocument(); + expect(screen.queryByText(skyText)).not.toBeInTheDocument(); + }); + + it('folds accents in the data so a plain ascii query matches', () => { + renderAndSelectSource(); + typeFilter('canal decale'); + expect(screen.getByText(canalText)).toBeInTheDocument(); + expect(screen.queryByText(decaleText)).not.toBeInTheDocument(); + expect(screen.queryByText(skyText)).not.toBeInTheDocument(); + }); + + it('folds accents in the query so it matches plain ascii data', () => { + renderAndSelectSource(); + typeFilter('décalé'); + expect(screen.getByText(decaleText)).toBeInTheDocument(); + expect(screen.queryByText(skyText)).not.toBeInTheDocument(); + }); + }); }); From abdf2d7864d29a4e834c5c521522ad0144e129f1 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:51:20 -0400 Subject: [PATCH 05/81] Fix credential release on deleted profile and round 3 review items. Store credential Redis keys at reserve so release works when the profile row is deleted. Return reserve failure reasons to avoid fingerprint DB queries on logging paths. Document unlimited profile bypass in pool logic and Server Group UI. Co-authored-by: Cursor --- apps/channels/models.py | 19 +++-- apps/m3u/connection_pool.py | 78 +++++++++++++++---- apps/m3u/models.py | 6 +- apps/m3u/tests/test_connection_pool.py | 47 +++++++++-- .../multi_worker_connection_manager.py | 4 +- frontend/src/components/forms/M3U.jsx | 2 +- frontend/src/components/forms/ServerGroup.jsx | 2 +- 7 files changed, 123 insertions(+), 35 deletions(-) diff --git a/apps/channels/models.py b/apps/channels/models.py index 154f6fa2..9742626b 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -234,7 +234,9 @@ class Stream(models.Model): continue # Atomic slot reservation via shared connection pool helper - reserved, _count = reserve_profile_slot(profile, redis_client) + reserved, _count, _failure_reason = reserve_profile_slot( + profile, redis_client + ) if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) @@ -694,7 +696,9 @@ class Channel(models.Model): has_active_profiles = True # Atomically check and reserve a slot (INCR-first pattern) - reserved, current_count = reserve_profile_slot(profile, redis_client) + reserved, current_count, failure_reason = reserve_profile_slot( + profile, redis_client + ) if reserved: # Slot reserved — assign stream to this channel @@ -712,11 +716,6 @@ class Channel(models.Model): True, ) # Return newly assigned stream and matched profile else: - from apps.m3u.connection_pool import ( - group_has_capacity_for_profile, - profile_has_capacity_for_selection, - ) - # At capacity: try to preempt a lower-impact channel on this profile victim_channel_id = self._pick_channel_to_preempt( profile_id=profile.id, @@ -729,14 +728,14 @@ class Channel(models.Model): # return self.id, profile.id, victim_channel_id has_streams_but_maxed_out = True - if not profile_has_capacity_for_selection(profile, redis_client): + if failure_reason == "profile_full": logger.info( f"Profile {profile.id} at max connections: " f"{current_count}/{profile.max_streams}, trying next profile" ) - elif not group_has_capacity_for_profile(profile, redis_client): + elif failure_reason == "credential_full": logger.info( - f"Profile {profile.id} login or server group pool full, trying next profile" + f"Profile {profile.id} shared login pool full, trying next profile" ) else: logger.debug( diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 35c35b0e..36a74d17 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -5,7 +5,8 @@ Profile selection rotates across M3UAccountProfile rows using each profile's own Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup with max_streams > 0, a credential-scoped counter is checked on reserve/release so accounts sharing the same provider login share one limit without blocking -unrelated logins on the same group. +unrelated logins on the same group. Account profiles with max_streams=0 skip +credential enforcement for that profile. """ from __future__ import annotations @@ -13,11 +14,14 @@ from __future__ import annotations import hashlib import logging import re -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple logger = logging.getLogger(__name__) +ReserveFailureReason = Literal["profile_full", "credential_full"] + PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" +PROFILE_CREDENTIAL_RELEASE_KEY = "profile_credential_release:{profile_id}" SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}:{fingerprint}" _XC_URL_CREDENTIALS_RE = re.compile( @@ -30,6 +34,11 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) +def profile_credential_release_key(profile_id: int) -> str: + """Redis key storing the credential counter to release when the profile row is gone.""" + return PROFILE_CREDENTIAL_RELEASE_KEY.format(profile_id=profile_id) + + def server_group_connections_key(group_id: int, fingerprint: str) -> str: """Redis key for per-credential usage within a ServerGroup.""" return SERVER_GROUP_CONNECTIONS_KEY.format( @@ -161,6 +170,8 @@ def profile_has_capacity_for_selection(profile, redis_client) -> bool: def group_has_capacity_for_profile(profile, redis_client) -> bool: + # Profiles with max_streams=0 skip credential enforcement entirely. An unlimited + # profile in a pooled group can still stream while other accounts share the login. group = get_enforced_server_group_for_profile(profile) if not group or profile.max_streams == 0: return True @@ -186,21 +197,43 @@ def _safe_decr(redis_client, key: str) -> None: redis_client.set(key, 0) -def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: +def _remember_credential_release_key( + profile_id: int, cred_key: str, redis_client +) -> None: + redis_client.set(profile_credential_release_key(profile_id), cred_key) + + +def _release_credential_slot_by_profile_id(profile_id: int, redis_client) -> bool: + """Release a reserved credential counter using the key stored at reserve time.""" + release_key = profile_credential_release_key(profile_id) + cred_key = redis_client.get(release_key) + if not cred_key: + return False + + if isinstance(cred_key, bytes): + cred_key = cred_key.decode() + _safe_decr(redis_client, cred_key) + redis_client.delete(release_key) + return True + + +def _reserve_server_group_slot_for_profile( + profile, redis_client +) -> Tuple[bool, Optional[str]]: group = get_enforced_server_group_for_profile(profile) if not group or profile.max_streams == 0: - return True + return True, None cred_key = _credential_counter_key(profile, group) if not cred_key: - return True + return True, None cred_count = redis_client.incr(cred_key) if cred_count <= profile.max_streams: - return True + return True, cred_key redis_client.decr(cred_key) - return False + return False, None def _release_server_group_slot_for_profile(profile, redis_client) -> None: @@ -212,11 +245,14 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: _safe_decr(redis_client, cred_key) -def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: +def reserve_profile_slot( + profile, redis_client +) -> Tuple[bool, int, Optional[ReserveFailureReason]]: """ Atomically reserve profile + optional credential slots (INCR-first). - Returns (reserved, profile_count_after_attempt). + Returns (reserved, profile_count_after_attempt, failure_reason). + failure_reason is set when reserved is False. """ profile_key = profile_connections_key(profile.id) profile_count = 0 @@ -225,20 +261,34 @@ def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: profile_count = redis_client.incr(profile_key) if profile_count > profile.max_streams: redis_client.decr(profile_key) - return False, profile_count - 1 + return False, profile_count - 1, "profile_full" - if not _reserve_server_group_slot_for_profile(profile, redis_client): + cred_reserved, cred_key = _reserve_server_group_slot_for_profile( + profile, redis_client + ) + if not cred_reserved: if profile.max_streams > 0: redis_client.decr(profile_key) - return False, profile_count - 1 if profile.max_streams > 0 else 0 + return ( + False, + profile_count - 1 if profile.max_streams > 0 else 0, + "credential_full", + ) - return True, profile_count + if cred_key: + _remember_credential_release_key(profile.id, cred_key, redis_client) + + return True, profile_count, None def release_profile_slot(profile_id: int, redis_client) -> None: """Release profile and shared credential slots after a stream end.""" from apps.m3u.models import M3UAccountProfile + released_via_stored_key = _release_credential_slot_by_profile_id( + profile_id, redis_client + ) + try: profile = M3UAccountProfile.objects.get(id=profile_id) except M3UAccountProfile.DoesNotExist: @@ -250,5 +300,5 @@ def release_profile_slot(profile_id: int, redis_client) -> None: if current > 0: redis_client.decr(profile_key) - if profile: + if profile and not released_via_stored_key: _release_server_group_slot_for_profile(profile, redis_client) diff --git a/apps/m3u/models.py b/apps/m3u/models.py index e6ccaadb..46d1a068 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -201,7 +201,11 @@ class ServerGroup(models.Model): ) max_streams = models.PositiveIntegerField( default=0, - help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + help_text=( + "Set above 0 to enable shared credential pooling for accounts in this group. " + "Per-login limits come from each account profile's max_streams. " + "Profiles with max_streams=0 (unlimited) skip cross-account credential enforcement." + ), ) def __str__(self): diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 1dbfc2a9..e5de6c10 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -14,6 +14,7 @@ from apps.m3u.connection_pool import ( pool_has_capacity_for_profile, profile_has_capacity_for_selection, profile_connections_key, + profile_credential_release_key, release_profile_slot, reserve_profile_slot, server_group_connections_key, @@ -31,10 +32,15 @@ class FakeRedis: val = self._data.get(key) if val is None: return None + if isinstance(val, str): + return val.encode() return str(val).encode() def set(self, key, value, ex=None): - self._data[key] = int(value) + try: + self._data[key] = int(value) + except (ValueError, TypeError): + self._data[key] = value def incr(self, key): self._data[key] = self._data.get(key, 0) + 1 @@ -44,6 +50,9 @@ class FakeRedis: self._data[key] = self._data.get(key, 0) - 1 return self._data[key] + def delete(self, key): + self._data.pop(key, None) + def pipeline(self): return FakeRedisPipeline(self) @@ -125,10 +134,10 @@ class ManualServerGroupTests(TestCase): profile2.save() redis = FakeRedis() - reserved1, _ = reserve_profile_slot(profile1, redis) + reserved1, _, _ = reserve_profile_slot(profile1, redis) self.assertTrue(reserved1) - reserved2, _ = reserve_profile_slot(profile2, redis) + reserved2, _, _ = reserve_profile_slot(profile2, redis) self.assertFalse(reserved2) self.assertFalse(group_has_capacity_for_profile(profile2, redis)) @@ -153,7 +162,7 @@ class ManualServerGroupTests(TestCase): ) redis = FakeRedis() - reserved, _ = reserve_profile_slot(default, redis) + reserved, _, _ = reserve_profile_slot(default, redis) self.assertTrue(reserved) self.assertFalse(profile_has_capacity_for_selection(default, redis)) self.assertTrue(profile_has_capacity_for_selection(alt, redis)) @@ -182,7 +191,7 @@ class PoolEnforcementTests(TestCase): self.profile.save() def test_reserve_and_release_both_counters(self): - reserved, count = reserve_profile_slot(self.profile, self.redis) + reserved, count, _ = reserve_profile_slot(self.profile, self.redis) self.assertTrue(reserved) self.assertEqual(count, 1) @@ -286,13 +295,37 @@ class PoolEnforcementTests(TestCase): fp = get_profile_credential_fingerprint(self.profile) cred_key = server_group_connections_key(self.group.id, fp) - self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + reserved, _, failure_reason = reserve_profile_slot( + self.profile, self.redis + ) + self.assertTrue(reserved) + self.assertIsNone(failure_reason) self.profile.delete() release_profile_slot(profile_id, self.redis) self.assertEqual(self.redis._data[profile_connections_key(profile_id)], 0) - self.assertEqual(self.redis._data[cred_key], 1) + self.assertEqual(self.redis._data[cred_key], 0) + self.assertNotIn(profile_credential_release_key(profile_id), self.redis._data) + + def test_reserve_returns_failure_reason_without_extra_checks(self): + account2 = M3UAccount.objects.create( + name="Reason Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=self.group, + max_streams=5, + ) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile2.max_streams = 1 + profile2.save() + + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + reserved, _count, reason = reserve_profile_slot(profile2, self.redis) + self.assertFalse(reserved) + self.assertEqual(reason, "credential_full") class UpdateStreamProfileTests(TestCase): diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index 77db3b1b..8d803dcf 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -745,7 +745,9 @@ class MultiWorkerVODConnectionManager: from apps.m3u.connection_pool import reserve_profile_slot try: - reserved, new_count = reserve_profile_slot(m3u_profile, self.redis_client) + reserved, new_count, _failure_reason = reserve_profile_slot( + m3u_profile, self.redis_client + ) if reserved: logger.info( f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: " diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 3a4161cb..6e19c93a 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -365,7 +365,7 @@ const M3U = ({ id="server_group" name="server_group" label="Server Group" - description="Share a connection limit across multiple accounts (e.g. XC + M3U on the same subscription)" + description="Share login limits across accounts in a server group. Set max streams on each profile (unlimited profiles skip group enforcement)." {...form.getInputProps('server_group')} key={form.key('server_group')} data={[{ value: '0', label: '(None)' }].concat( diff --git a/frontend/src/components/forms/ServerGroup.jsx b/frontend/src/components/forms/ServerGroup.jsx index 229479ea..a01a0a76 100644 --- a/frontend/src/components/forms/ServerGroup.jsx +++ b/frontend/src/components/forms/ServerGroup.jsx @@ -70,7 +70,7 @@ const ServerGroupForm = ({ serverGroup = null, isOpen, onClose }) => { setValue('max_streams', value ?? 0)} From 57cae0e4be9023e3d7a670107e6097ba153434eb Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Sun, 7 Jun 2026 22:02:23 -0500 Subject: [PATCH 06/81] fix(m3u): correct auto-sync numbering field interactions from the auto-sync overhaul The auto-sync overhaul added a [start, end] range and a shared numbering picker, but each mode's UI exposes only a subset of the persisted fields (Provider's "Start #" writes channel_numbering_fallback; Next Available exposes no Start/End) while the backend read auto_sync_channel_start and auto_sync_channel_end in every mode. Switching modes never resets the others, so a stale or auto-computed value silently changed numbering. - Provider mode honors the provider-supplied number (stream_chno) verbatim; the group's Start (channel_numbering_fallback) and End bound only the fallback for streams the provider did not number. auto_sync_channel_start is no longer applied in provider mode. - Next Available ignores End, since its UI exposes no range. - Range enforcement (the overflow-delete) runs in fixed mode only, the one mode with a user-set [start, end] range. - Provider mode gains the Start>End guard fixed mode already had, so an inverted fallback range cannot fail every numberless stream. Includes backend and frontend regression tests. --- apps/m3u/tasks.py | 64 ++--- apps/m3u/tests/test_sync_correctness.py | 260 +++++++++++++++++- .../src/components/forms/AutoSyncBasic.jsx | 15 +- .../forms/__tests__/AutoSyncBasic.test.jsx | 19 ++ 4 files changed, 305 insertions(+), 53 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 3b504ebf..a4d863e9 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1716,42 +1716,27 @@ def _pick_target_number( fixed_cursor, fallback_start, end_number=None, - range_start=None, ): """ - Return the channel number a given stream should claim under the group's - numbering mode, or None if the configured range is exhausted. + Return the channel number a stream should claim under the group's numbering + mode, or None if the range is exhausted. Shared by the renumber and create + passes. Each mode reads only the fields its UI exposes: - Shared by the existing-channel renumber pass and the new-channel create - pass so both honor identical mode semantics: provider-supplied number - when available and free, otherwise fall back; or always next-available; - or fixed-cursor sequential. - - `range_start`, when provided, is the inclusive lower bound for the - group's configured numbering range. Provider-supplied numbers below - this bound fall back to the next-available picker so freshly-created - channels never land outside the configured range. + - provider: the provider number is authoritative and used as-is when free. + Start (`channel_numbering_fallback`) and End bound only the fallback for + streams with no provider number; `auto_sync_channel_start` does not apply. + - next_available: lowest free number from 1; End does not apply (its UI has + no range, so a stale End must not cap it). + - fixed: sequential from the cursor, bounded by End. """ if mode == "provider": chno = stream.stream_chno - if ( - chno is not None - and chno not in used_numbers - and (range_start is None or chno >= range_start) - and (end_number is None or chno <= end_number) - ): + if chno is not None and chno not in used_numbers: return chno - # No usable provider number: walk from fallback_start, bumped up - # to range_start when set so the fallback never lands below the - # configured range. - effective_start = ( - max(fallback_start, range_start) - if range_start is not None - else fallback_start - ) - return _next_available_number(used_numbers, effective_start, end=end_number) + # No usable provider number: fall back into the configured range. + return _next_available_number(used_numbers, fallback_start, end=end_number) if mode == "next_available": - return _next_available_number(used_numbers, 1, end=end_number) + return _next_available_number(used_numbers, 1) return _next_available_number(used_numbers, fixed_cursor, end=end_number) @@ -2253,7 +2238,6 @@ def sync_auto_channels(account_id, scan_start_time=None): temp_channel_number, channel_numbering_fallback, end_number=end_number, - range_start=start_number, ) # Range exhausted: leave the channel at its existing @@ -2290,17 +2274,16 @@ def sync_auto_channels(account_id, scan_start_time=None): f"Renumbered {len(channels_to_renumber)} channels to maintain sort order" ) - # When the group's configured range is narrower than its existing - # channels span, any non-hidden auto-created channel whose number - # falls outside [start, end] gets deleted. The new-channel - # creation loop below picks up the freed stream and re-creates - # the channel at a slot inside the new range, so the net user - # outcome is a renumber, not a failure. Counted in - # channels_deleted; the replacement counts in channels_created. - # Hidden channels are preserved. Runs BEFORE new-channel creation - # so slots freed by the deletions are available to incoming - # streams. - if end_number is not None: + # Range enforcement runs in fixed mode only: it is the one mode with + # a user-set [start, end]. Provider numbers are authoritative and + # next_available has no range, so their channels are never deleted + # for falling outside start/end. + # + # Channels outside the range are deleted (hidden ones preserved); + # the creation loop below re-adds the freed streams inside the range, + # so the net effect is a renumber, not a failure. Runs first so the + # freed slots are available. + if end_number is not None and channel_numbering_mode == "fixed": overflow_delete_ids = [] for stream_id, ch in list(existing_channel_map.items()): if ch.hidden_from_output: @@ -2461,7 +2444,6 @@ def sync_auto_channels(account_id, scan_start_time=None): current_channel_number, channel_numbering_fallback, end_number=end_number, - range_start=start_number, ) if target_number is None: diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 9f8b87a2..4a2986fb 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -46,13 +46,14 @@ def _attach_group_to_account(account, group, custom_properties=None): ) -def _make_stream(account, group, name="ESPN", tvg_id="espn"): +def _make_stream(account, group, name="ESPN", tvg_id="espn", stream_chno=None): return Stream.objects.create( name=name, url=f"http://example.com/{name.lower()}.m3u8", m3u_account=account, channel_group=group, tvg_id=tvg_id, + stream_chno=stream_chno, last_seen=timezone.now(), ) @@ -413,21 +414,21 @@ class RangeEnforcementTests(TestCase): "new auto-channel duplicates A's effective channel number.", ) - def test_provider_mode_fallback_respects_range_start(self): - # Provider-mode streams without a usable stream_chno fall back to - # the next-available picker. The fallback must honor the group's - # configured `auto_sync_channel_start` so freshly-created channels - # never land below the user's chosen range. Without this, a group - # configured for [100, 200] silently spawned channels at #1 when - # the provider omitted channel-number metadata. + def test_provider_mode_numberless_fallback_uses_visible_start(self): + # In provider mode the visible "Start #" is channel_numbering_fallback, + # so a numberless stream's fallback walks from there, not from the + # hidden auto_sync_channel_start (set far above the range here to prove + # it is ignored). + # Fail signature: 0 channels created, or a channel below 100 = fallback + # seeded from the wrong field. account = _make_account() group = _make_group(name="Sports") rel = _attach_group_to_account(account, group) - rel.auto_sync_channel_start = 100 + rel.auto_sync_channel_start = 5000 # hidden; must be ignored rel.auto_sync_channel_end = 200 rel.custom_properties = { "channel_numbering_mode": "provider", - "channel_numbering_fallback": 1, + "channel_numbering_fallback": 100, # the visible "Start #" } rel.save() @@ -2300,3 +2301,242 @@ class CompactNumberingIdempotencyTests(TransactionTestCase): "pass with no hide or override change. _repack_inner is following " "physical row order instead of id order.", ) + + +class ProviderNumberingHonorsProviderNumberTests(TestCase): + """ + Provider numbering uses a stream's provider number (stream_chno) verbatim. + The group start is auto-populated by the UI and is not editable in provider + mode (the UI binds "Start #" to channel_numbering_fallback), so treating it + as a lower bound silently discarded valid provider numbers: on a lineup + topping out near 5000, provider numbers 100-150 landed at ~5000. + + The start and end bound only the fallback for numberless streams. + """ + + def test_provider_number_below_high_auto_start_is_honored(self): + # Provider numbers 100-104 with an auto-set start of 5000 must land at + # their provider numbers. + # Fail signature: channels at 5000-5004 = start used as a hard floor. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 5000 + rel.auto_sync_channel_end = None + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + for i in range(5): + _make_stream( + account, group, name=f"PPV {i}", tvg_id=f"ppv{i}", + stream_chno=100 + i, + ) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + numbers = sorted( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).values_list("channel_number", flat=True) + ) + self.assertEqual(numbers, [100.0, 101.0, 102.0, 103.0, 104.0]) + + def test_provider_number_honored_when_start_unset(self): + # start blank -> defaults to 1.0; provider numbers still honored. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = None + rel.auto_sync_channel_end = None + rel.custom_properties = {"channel_numbering_mode": "provider"} + rel.save() + _make_stream(account, group, name="PPV", tvg_id="ppv", stream_chno=100) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(created.channel_number, 100.0) + + def test_numberless_stream_uses_fallback_not_hidden_start(self): + # In provider mode without a range, a stream lacking a provider + # number falls back to channel_numbering_fallback (the visible + # "Start #"), not the hidden auto_sync_channel_start. + # Fail signature: channel at 5000 = fallback bumped to hidden start. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 5000 + rel.auto_sync_channel_end = None + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 300, + } + rel.save() + _make_stream(account, group, name="NoChno", tvg_id="nc") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(created.channel_number, 300.0) + + def test_provider_number_below_range_is_honored_verbatim(self): + # A provider number below the group's Start/End is used as-is, not + # coerced into the range. + # Fail signature: channel pulled to >= 100 = range coercing a provider + # number. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_end = 200 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 100, + } + rel.save() + _make_stream(account, group, name="Low", tvg_id="low", stream_chno=50) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(created.channel_number, 50.0) + + def test_provider_number_above_end_is_honored_verbatim(self): + # Provider numbers above the configured End are also honored as-is; + # the End caps only the fallback for numberless streams. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_end = 200 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + _make_stream(account, group, name="High", tvg_id="high", stream_chno=5000) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(created.channel_number, 5000.0) + + def test_provider_number_within_range_is_honored(self): + # An in-range provider number is used as-is. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_end = 200 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + _make_stream(account, group, name="Mid", tvg_id="mid", stream_chno=150) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(created.channel_number, 150.0) + + +class CrossModeNumberingFieldTests(TestCase): + """ + Each numbering mode's UI exposes only a subset of the persisted fields, + and switching modes does not reset the others. The backend must therefore + read only the fields a mode actually owns, so a stale/hidden value left by + another mode cannot silently change numbering. These guard the two + remaining facets of that family (the provider-floor facet is covered by + ProviderNumberingHonorsProviderNumberTests). + """ + + def _restamp(self, account): + Stream.objects.filter(m3u_account=account).update( + last_seen=timezone.now() + ) + + def test_next_available_ignores_configured_end(self): + # next_available exposes no Start/End in its UI, so a stale End left + # over from a prior mode must not cap it. Every stream gets the lowest + # free number from 1 regardless of the End. + # Fail signature: streams beyond the End fail = next_available honoring + # a hidden cap. + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 1 + rel.auto_sync_channel_end = 3 # stale cap from a prior mode + rel.custom_properties = {"channel_numbering_mode": "next_available"} + rel.save() + for i in range(5): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["channels_created"], 5) + self.assertEqual(result["channels_failed"], 0) + + def test_provider_channels_outside_range_are_not_deleted(self): + # Range enforcement (the overflow-delete) is fixed-mode only. A provider + # channel whose number is outside [start, end] is authoritative and must + # survive sync, not be deleted and churned into a new row. + # Fail signature: channels_deleted > 0 on the second sync = overflow + # delete firing in provider mode. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_end = 200 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + _make_stream(account, group, name="High", tvg_id="high", stream_chno=5000) + + first = _sync(account) + self.assertEqual(first["channels_created"], 1) + original = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(original.channel_number, 5000.0) + + self._restamp(account) + second = _sync(account) + + self.assertEqual(second["channels_deleted"], 0) + survivor = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertEqual(survivor.id, original.id) + self.assertEqual(survivor.channel_number, 5000.0) + + def test_next_available_channels_outside_stale_range_not_deleted(self): + # Same gate for next_available: tightening a stale End must not delete + # already-assigned channels (range enforcement is fixed-mode only). + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 1 + rel.auto_sync_channel_end = None + rel.custom_properties = {"channel_numbering_mode": "next_available"} + rel.save() + for i in range(5): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + self.assertEqual(_sync(account)["channels_created"], 5) + + rel.auto_sync_channel_end = 3 # stale cap appears + rel.save() + self._restamp(account) + second = _sync(account) + + self.assertEqual(second["channels_deleted"], 0) + self.assertEqual( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).count(), + 5, + ) diff --git a/frontend/src/components/forms/AutoSyncBasic.jsx b/frontend/src/components/forms/AutoSyncBasic.jsx index 6cab8ea7..a3a0b5ba 100644 --- a/frontend/src/components/forms/AutoSyncBasic.jsx +++ b/frontend/src/components/forms/AutoSyncBasic.jsx @@ -37,13 +37,24 @@ const AutoSyncBasic = ({ ? 1 : clampChannelNumber(value); if (mode === 'provider') { - onApplyGroupChange({ + // Provider's Start # seeds the fallback for numberless streams. Mirror + // Fixed mode: if it exceeds End, drop End so the fallback range is never + // inverted (an inverted range fails every numberless stream). + const next = { ...group, custom_properties: { ...(group.custom_properties || {}), channel_numbering_fallback: normalized, }, - }); + }; + if ( + endValue !== null && + endValue !== undefined && + normalized > endValue + ) { + next.auto_sync_channel_end = null; + } + onApplyGroupChange(next); } else { // If End is set and the new Start exceeds it, drop End so the user // is not left holding an invalid range silently. diff --git a/frontend/src/components/forms/__tests__/AutoSyncBasic.test.jsx b/frontend/src/components/forms/__tests__/AutoSyncBasic.test.jsx index 67055a4a..8392171f 100644 --- a/frontend/src/components/forms/__tests__/AutoSyncBasic.test.jsx +++ b/frontend/src/components/forms/__tests__/AutoSyncBasic.test.jsx @@ -289,6 +289,25 @@ describe('AutoSyncBasic', () => { const call = onApplyGroupChange.mock.calls[0][0]; expect(call.custom_properties.channel_numbering_fallback).toBe(1); }); + + it('drops End when the fallback start exceeds End in provider mode', () => { + // Mirrors fixed mode: an inverted fallback range would fail every + // numberless stream, so End is dropped rather than silently kept. + const { onApplyGroupChange } = renderComponent({ + custom_properties: { + channel_numbering_mode: 'provider', + channel_numbering_fallback: 1, + }, + auto_sync_channel_end: 200, + }); + vi.mocked(clampChannelNumber).mockReturnValueOnce(500); + fireEvent.change(screen.getByTestId(/start/i), { + target: { value: '500' }, + }); + const call = onApplyGroupChange.mock.calls[0][0]; + expect(call.custom_properties.channel_numbering_fallback).toBe(500); + expect(call.auto_sync_channel_end).toBeFalsy(); + }); }); // ── updateEnd ────────────────────────────────────────────────────────────── From 7086f41d640a3f804808fd841ac3fc1fd4e59abc Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 8 Jun 2026 18:12:29 -0500 Subject: [PATCH 07/81] feat(epg): overhaul EPG auto-matching logic and improve performance - Moved matching logic to a dedicated module for better organization and testability. - Made single-channel auto-matching asynchronous, allowing for larger EPG libraries without hitting HTTP timeouts. - Enhanced memory management and throughput during EPG matching, including optimizations for fuzzy matching and bulk processing. - Fixed various reliability issues in the auto-matching process, ensuring accurate channel assignments and improved UI feedback. - Updated API views and frontend components to reflect changes in the matching process and provide real-time notifications. - Added tests for EPG matching functionality and name normalization. - Single-channel and selected-channel auto-match always run, even when the channel already has EPG assigned; match-all (no channel IDs) still only processes channels without EPG. --- CHANGELOG.md | 26 + apps/channels/api_views.py | 49 +- apps/channels/epg_matching.py | 937 ++++++++++++++++++ apps/channels/tasks.py | 901 +++-------------- apps/channels/tests/test_epg_match_apply.py | 58 ++ .../channels/tests/test_epg_name_normalize.py | 117 +++ dispatcharr/celery.py | 2 + frontend/src/WebSocket.jsx | 36 +- frontend/src/components/forms/Channel.jsx | 62 +- .../forms/__tests__/Channel.test.jsx | 19 + 10 files changed, 1393 insertions(+), 814 deletions(-) create mode 100644 apps/channels/epg_matching.py create mode 100644 apps/channels/tests/test_epg_match_apply.py create mode 100644 apps/channels/tests/test_epg_name_normalize.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d19b94..22b27745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **EPG auto-match overhaul** — matching logic moved to `apps/channels/epg_matching.py`; Celery tasks in `tasks.py` are thin wrappers. + - Single-channel auto-match is now asynchronous: the API returns `202 Accepted` and pushes the result over WebSocket (`single_channel_epg_match`), so large EPG libraries no longer hit the previous 30-second HTTP timeout. + - Progress, bulk completion, and single-channel results use `send_websocket_update` instead of `async_to_sync(channel_layer.group_send)`, so notifications work reliably under gevent-patched uWSGI and Celery workers. + - Single-channel and selected-channel auto-match always run, even when the channel already has EPG assigned; match-all (no channel IDs) still only processes channels without EPG. + - Rematching to the same EPG no longer re-saves the channel or queues program-parse tasks; only assignments that actually change are written and refreshed. + +### Performance + +- **EPG auto-match memory and throughput improvements.** + - Single-channel matching streams active EPG rows and keeps only the best match plus the top 20 candidates in memory; ML validates at most 21 names per channel instead of embedding the full catalog. + - Strong fuzzy matches (≥75% single channel, ≥80% bulk) skip ML entirely, avoiding a ~500MB PyTorch load when the fuzzy result is already reliable. + - Bulk matching uses a single fuzzy pass per channel instead of scanning the full catalog twice for best match and top candidates. + - Bulk exact `tvg_id` / Gracenote matching uses an in-memory index built alongside the EPG catalog (`build_epg_matching_catalog()`), giving O(1) lookups with no extra database queries. + - Bulk match apply uses batched queries (two fetches plus `bulk_update`) instead of one `EPGData.objects.get()` per matched channel. + - EPG normalization settings are cached once per matching run, avoiding repeated `CoreSettings` reads when normalizing thousands of names. + +### Fixed + +- **EPG auto-match reliability fixes.** + - Memory could spike to multiple GB on large EPG sources when building a full in-memory catalog before fuzzy matching; single-channel matching now streams rows and bounds ML work to a small candidate set. + - Wrong channel assignments from global ML similarity; ML validation now checks the fuzzy best match (or top fuzzy candidates as a last resort) instead of scoring the entire catalog. + - Channel form auto-match spinner could stick after errors or early task exits; all single-channel outcomes now push a WebSocket result, and the UI clears loading state after a 3-minute timeout. + - Bulk auto-match completion no longer calls `batch-set-epg` from the WebSocket handler, which had been re-applying every match and queueing redundant `parse_programs_for_tvg_id` tasks even when assignments were unchanged. + ## [0.26.0] - 2026-06-07 ### Added diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 9fd47400..965da468 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -2087,7 +2087,7 @@ class ChannelViewSet(viewsets.ModelViewSet): fields={ 'channel_ids': serializers.ListField( child=serializers.IntegerField(), - help_text='List of channel IDs to process. If empty or not provided, all channels without EPG will be processed.', + help_text='List of channel IDs to process (includes channels that already have EPG). If empty or not provided, only channels without EPG are processed.', required=False, ) } @@ -2120,23 +2120,15 @@ class ChannelViewSet(viewsets.ModelViewSet): def match_channel_epg(self, request, pk=None): channel = self.get_object() - # Import the matching logic - from apps.channels.tasks import match_single_channel_epg - - try: - # Try to match this specific channel - call synchronously for immediate response - result = match_single_channel_epg.apply_async(args=[channel.id]).get(timeout=30) - - # Refresh the channel from DB to get any updates - channel.refresh_from_db() - - return Response({ - "message": result.get("message", "Channel matching completed"), - "matched": result.get("matched", False), - "channel": self.get_serializer(channel).data - }) - except Exception as e: - return Response({"error": str(e)}, status=400) + match_single_channel_epg.delay(channel.id) + return Response( + { + "message": f"EPG matching started for channel '{channel.name}'", + "accepted": True, + "channel_id": channel.id, + }, + status=status.HTTP_202_ACCEPTED, + ) # ───────────────────────────────────────────────────────── # 7) Set EPG and Refresh @@ -2321,7 +2313,6 @@ class ChannelViewSet(viewsets.ModelViewSet): # Extract channel IDs upfront channel_updates = {} - unique_epg_ids = set() for assoc in associations: channel_id = assoc.get("channel_id") @@ -2331,24 +2322,28 @@ class ChannelViewSet(viewsets.ModelViewSet): continue channel_updates[channel_id] = epg_data_id - if epg_data_id: - unique_epg_ids.add(epg_data_id) # Batch fetch all channels (single query) channels_dict = { c.id: c for c in Channel.objects.filter(id__in=channel_updates.keys()) } - # Collect channels to update + # Collect channels whose EPG assignment actually changes channels_to_update = [] + changed_epg_ids = set() for channel_id, epg_data_id in channel_updates.items(): if channel_id not in channels_dict: logger.error(f"Channel with ID {channel_id} not found") continue channel = channels_dict[channel_id] + if channel.epg_data_id == epg_data_id: + continue + channel.epg_data_id = epg_data_id channels_to_update.append(channel) + if epg_data_id: + changed_epg_ids.add(epg_data_id) # Bulk update all channels (single query) if channels_to_update: @@ -2361,25 +2356,25 @@ class ChannelViewSet(viewsets.ModelViewSet): channels_updated = len(channels_to_update) - # Trigger program refresh for unique EPG data IDs (skip dummy EPGs) + # Trigger program refresh only for EPG ids newly assigned (skip dummy/SD) from apps.epg.tasks import parse_programs_for_tvg_id from apps.epg.models import EPGData # Batch fetch EPG data (single query) epg_data_dict = { epg.id: epg - for epg in EPGData.objects.filter(id__in=unique_epg_ids).select_related('epg_source') + for epg in EPGData.objects.filter(id__in=changed_epg_ids).select_related('epg_source') } programs_refreshed = 0 - for epg_id in unique_epg_ids: + for epg_id in changed_epg_ids: epg_data = epg_data_dict.get(epg_id) if not epg_data: logger.error(f"EPGData with ID {epg_id} not found") continue - # Only refresh non-dummy EPG sources - if epg_data.epg_source.source_type != 'dummy': + source_type = epg_data.epg_source.source_type if epg_data.epg_source else None + if source_type not in ('dummy', 'schedules_direct'): parse_programs_for_tvg_id.delay(epg_id) programs_refreshed += 1 diff --git a/apps/channels/epg_matching.py b/apps/channels/epg_matching.py new file mode 100644 index 00000000..c468568b --- /dev/null +++ b/apps/channels/epg_matching.py @@ -0,0 +1,937 @@ +""" +EPG channel matching: fuzzy scoring, optional ML validation, and UI notifications. + +Celery tasks in tasks.py call into this module; keep orchestration here and +task wiring thin so matching logic stays testable without a worker. +""" +import gc +import heapq +import logging +import os +import re + +from rapidfuzz import fuzz + +from apps.epg.models import EPGData +from core.models import CoreSettings +from core.utils import send_websocket_update + +logger = logging.getLogger(__name__) + +_ml_model_cache = {'sentence_transformer': None} +_normalize_settings_cache = None + +ML_CANDIDATE_LIMIT = 20 +SINGLE_CHANNEL_MATCH_TIMEOUT_MS = 180_000 + +COMMON_EXTRANEOUS_WORDS = [ + "tv", "channel", "network", "television", + "east", "west", "hd", "uhd", "24/7", + "1080p", "720p", "540p", "480p", + "film", "movie", "movies", +] + + +def release_ml_models(): + """Unload sentence transformer and encourage PyTorch to release memory.""" + if _ml_model_cache['sentence_transformer'] is None: + return + logger.info("Cleaning up ML models from memory") + model = _ml_model_cache['sentence_transformer'] + _ml_model_cache['sentence_transformer'] = None + del model + try: + import torch + if hasattr(torch, 'cuda') and torch.cuda.is_available(): + torch.cuda.empty_cache() + except ImportError: + pass + gc.collect() + + +def clear_normalize_settings_cache(): + """Reset cached normalization settings after a matching run.""" + global _normalize_settings_cache + _normalize_settings_cache = None + + +def cleanup_after_matching(): + """Release ML models and normalization cache after a matching run.""" + release_ml_models() + clear_normalize_settings_cache() + + +def get_sentence_transformer(): + """Lazy load the sentence transformer model only when needed.""" + if _ml_model_cache['sentence_transformer'] is None: + try: + from sentence_transformers import SentenceTransformer + from sentence_transformers import util + + model_name = "sentence-transformers/all-MiniLM-L6-v2" + cache_dir = "/data/models" + disable_downloads = os.environ.get('DISABLE_ML_DOWNLOADS', 'false').lower() == 'true' + + if disable_downloads: + hf_model_path = os.path.join(cache_dir, f"models--{model_name.replace('/', '--')}") + if not os.path.exists(hf_model_path): + logger.warning( + "ML model not found and downloads disabled (DISABLE_ML_DOWNLOADS=true). " + "Skipping ML matching." + ) + return None, None + + os.makedirs(cache_dir, exist_ok=True) + logger.info(f"Loading sentence transformer model (cache: {cache_dir})") + _ml_model_cache['sentence_transformer'] = SentenceTransformer( + model_name, + cache_folder=cache_dir, + ) + return _ml_model_cache['sentence_transformer'], util + except ImportError: + logger.warning("sentence-transformers not available - ML-enhanced matching disabled") + return None, None + except Exception as e: + logger.error(f"Failed to load sentence transformer: {e}") + return None, None + + from sentence_transformers import util + return _ml_model_cache['sentence_transformer'], util + + +def normalize_name(name: str) -> str: + """Normalize a channel/EPG name for fuzzy matching.""" + if not name: + return "" + + global _normalize_settings_cache + if _normalize_settings_cache is None: + prefixes = [] + suffixes = [] + custom_strings = [] + try: + settings = CoreSettings.get_epg_settings() + mode = settings.get("epg_match_mode", "default") + if mode == "advanced": + prefixes = settings.get("epg_match_ignore_prefixes", []) + suffixes = settings.get("epg_match_ignore_suffixes", []) + custom_strings = settings.get("epg_match_ignore_custom", []) + if not isinstance(prefixes, list): + prefixes = [] + if not isinstance(suffixes, list): + suffixes = [] + if not isinstance(custom_strings, list): + custom_strings = [] + except Exception as e: + logger.debug(f"Could not load EPG matching settings: {e}") + _normalize_settings_cache = (prefixes, suffixes, custom_strings) + + prefixes, suffixes, custom_strings = _normalize_settings_cache + result = name + + for prefix in prefixes: + if not prefix or not isinstance(prefix, str): + continue + if result.startswith(prefix): + result = result[len(prefix):] + break + + for suffix in suffixes: + if not suffix or not isinstance(suffix, str): + continue + if result.endswith(suffix): + result = result[:-len(suffix)] + break + + for custom in custom_strings: + if not custom or not isinstance(custom, str): + continue + try: + result = result.replace(custom, "") + except Exception as e: + logger.debug(f"Failed to remove custom string '{custom}': {e}") + + norm = result.lower() + norm = re.sub(r"\[.*?\]", "", norm) + + call_sign_match = re.search(r"\(([A-Z]{3,5})\)", name) + preserved_call_sign = "" + if call_sign_match: + preserved_call_sign = " " + call_sign_match.group(1).lower() + + norm = re.sub(r"\(.*?\)", "", norm) + norm = norm + preserved_call_sign + norm = re.sub(r"[^\w\s]", "", norm) + tokens = [t for t in norm.split() if t not in COMMON_EXTRANEOUS_WORDS] + return " ".join(tokens).strip() + + +def send_epg_matching_progress(total_channels, matched_channels, current_channel_name="", stage="matching"): + """Send bulk EPG matching progress via WebSocket.""" + matched_count = ( + len(matched_channels) if isinstance(matched_channels, list) else matched_channels + ) + send_websocket_update( + 'updates', + 'update', + { + 'type': 'epg_matching_progress', + 'total': total_channels, + 'matched': matched_count, + 'remaining': total_channels - matched_count, + 'current_channel': current_channel_name, + 'stage': stage, + 'progress_percent': round(matched_count / total_channels * 100, 1) if total_channels > 0 else 0, + }, + ) + + +def send_single_channel_epg_match_result(channel_id, matched, message, channel=None, epg_data=None): + """Notify the UI that a single-channel EPG match attempt has finished.""" + try: + from apps.channels.serializers import ChannelSerializer + + payload = { + "type": "single_channel_epg_match", + "channel_id": channel_id, + "matched": matched, + "message": message, + } + if channel is not None: + payload["channel"] = ChannelSerializer(channel).data + if epg_data is not None: + payload["epg_id"] = epg_data.id + payload["epg_name"] = epg_data.name + + send_websocket_update('updates', 'update', payload) + except Exception as e: + logger.warning(f"Failed to send single channel EPG match result: {e}") + + +def _compute_fuzzy_score(chan_norm, row, region_code=None): + """Compute fuzzy match score with optional region bonus/penalty.""" + if not row.get("norm_name"): + return 0 + base_score = fuzz.ratio(chan_norm, row["norm_name"]) + bonus = 0 + if region_code and row.get("tvg_id"): + combined_text = row["tvg_id"].lower() + " " + row["name"].lower() + dot_regions = re.findall(r'\.([a-z]{2})', combined_text) + if dot_regions: + bonus = 15 if region_code in dot_regions else -15 + elif region_code in combined_text: + bonus = 10 + return base_score + bonus + + +def _ml_cosine_similarities(st_model, util, query_text, candidate_texts): + """Encode only the query plus candidate texts (not the full EPG database).""" + if not candidate_texts: + return [] + texts = [query_text] + list(candidate_texts) + embeddings = st_model.encode(texts, convert_to_tensor=True, show_progress_bar=False) + sim_scores = util.cos_sim(embeddings[0:1], embeddings[1:])[0] + return [float(s) for s in sim_scores] + + +def _active_epg_lookup_queryset(): + """Lightweight queryset for exact EPG lookups (includes nameless entries).""" + return ( + EPGData.objects + .filter(epg_source__is_active=True) + .values('id', 'tvg_id', 'name', 'epg_source_id', 'epg_source__priority') + ) + + +def _active_epg_fuzzy_queryset(): + """Lightweight queryset for fuzzy EPG matching (requires a display name).""" + return ( + _active_epg_lookup_queryset() + .filter(name__isnull=False) + .exclude(name='') + ) + + +def _row_from_epg_values(values_row): + tvg_id = values_row.get('tvg_id') or '' + normalized_tvg_id = tvg_id.strip().lower() if tvg_id else '' + return { + 'id': values_row['id'], + 'tvg_id': normalized_tvg_id, + 'original_tvg_id': tvg_id, + 'name': values_row['name'], + 'epg_source_id': values_row['epg_source_id'], + 'epg_source_priority': values_row.get('epg_source__priority') or 0, + } + + +def lookup_epg_by_tvg_id(tvg_id): + """Exact tvg_id lookup without loading the full EPG catalog into memory.""" + if not tvg_id: + return None + values_row = _active_epg_lookup_queryset().filter(tvg_id__iexact=tvg_id.strip()).first() + return _row_from_epg_values(values_row) if values_row else None + + +def build_epg_matching_catalog(): + """ + Build the in-memory EPG catalog for bulk matching using a streaming DB cursor. + + Returns (epg_data, tvg_id_index): the full catalog plus an O(1) in-memory + tvg_id lookup table (no extra DB queries). The index prefers the first entry + per tvg_id after priority sorting. + """ + epg_data = [] + for values_row in _active_epg_fuzzy_queryset().iterator(chunk_size=500): + row = _row_from_epg_values(values_row) + row['norm_name'] = normalize_name(row['name']) + epg_data.append(row) + epg_data.sort(key=lambda x: x['epg_source_priority'], reverse=True) + return epg_data, build_epg_tvg_id_index(epg_data) + + +def build_epg_tvg_id_index(epg_data): + """ + Build an in-memory tvg_id -> row index from an EPG catalog (no DB queries). + epg_data must be sorted by source priority (highest first) so the first + entry wins when multiple sources share the same tvg_id. + """ + index = {} + for row in epg_data: + tvg_id = row.get("tvg_id") + if tvg_id and tvg_id not in index: + index[tvg_id] = row + return index + + +def _dispatch_program_parse_for_epg_assignments(changed_associations): + """ + Queue parse_programs once per unique EPG id newly assigned to a channel. + + bulk_update bypasses post_save, so callers must invoke this when epg_data + actually changes (mirrors the M3U sync path). + """ + if not changed_associations: + return 0 + + from apps.epg.tasks import parse_programs_for_tvg_id + + epg_ids = { + assoc["epg_data_id"] + for assoc in changed_associations + if assoc.get("epg_data_id") + } + if not epg_ids: + return 0 + + dispatched = 0 + for epg in EPGData.objects.filter(id__in=epg_ids).select_related("epg_source"): + source_type = epg.epg_source.source_type if epg.epg_source else None + if source_type in ("dummy", "schedules_direct"): + continue + parse_programs_for_tvg_id.delay(epg.id) + dispatched += 1 + return dispatched + + +def _log_unchanged_epg_assignment(chan, epg_id, epg_name, epg_tvg_id, match_method): + chan_name = chan.get("name") or f"id={chan['id']}" + chan_tvg = chan.get("original_tvg_id") or chan.get("tvg_id") or "" + logger.debug( + f"Channel '{chan_name}' (id={chan['id']}, tvg_id={chan_tvg!r}) " + f"unchanged - already on EPG '{epg_name or '?'}' " + f"(id={epg_id}, tvg_id={(epg_tvg_id or '?')!r}, via {match_method})" + ) + + +def _record_epg_match( + chan, + epg_id, + *, + epg_name, + epg_tvg_id, + match_method, + channels_to_update, + matched_channels, + unchanged_channels, +): + """Record a match result; skip channels_to_update when assignment is already correct.""" + if chan.get("current_epg_data_id") == epg_id: + unchanged_channels.append((chan["id"], chan.get("name") or "", epg_tvg_id or "")) + _log_unchanged_epg_assignment(chan, epg_id, epg_name, epg_tvg_id, match_method) + return + + chan_name = chan.get("name") or f"id={chan['id']}" + chan_tvg = chan.get("original_tvg_id") or chan.get("tvg_id") or "" + fallback_name = chan.get("fallback_name") or chan_name + chan["epg_data_id"] = epg_id + channels_to_update.append(chan) + matched_channels.append((chan["id"], fallback_name, epg_tvg_id or "")) + logger.info( + f"Channel '{chan_name}' (id={chan['id']}, tvg_id={chan_tvg!r}) " + f"=> EPG '{epg_name or '?'}' (id={epg_id}, tvg_id={(epg_tvg_id or '?')!r}, via {match_method})" + ) + + +def apply_matched_epg_to_channels(channels_to_update_dicts): + """ + Assign matched EPG rows to channels using two DB queries (channels + EPG). + + Skips channels that already have the matched EPG. Returns association dicts + for channels whose epg_data assignment actually changed, and dispatches + program-parse tasks only for those new assignments. + """ + from apps.channels.models import Channel + + if not channels_to_update_dicts: + return [] + + channel_ids = [d["id"] for d in channels_to_update_dicts] + epg_mapping = {d["id"]: d["epg_data_id"] for d in channels_to_update_dicts} + epg_ids = {epg_id for epg_id in epg_mapping.values() if epg_id} + + epg_by_id = {epg.id: epg for epg in EPGData.objects.filter(id__in=epg_ids)} + channels_list = list(Channel.objects.filter(id__in=channel_ids)) + + changed_associations = [] + channels_to_bulk = [] + for channel_obj in channels_list: + epg_data_id = epg_mapping.get(channel_obj.id) + if not epg_data_id: + continue + if channel_obj.epg_data_id == epg_data_id: + epg_row = epg_by_id.get(epg_data_id) + _log_unchanged_epg_assignment( + { + "id": channel_obj.id, + "name": channel_obj.name, + "original_tvg_id": channel_obj.tvg_id, + }, + epg_data_id, + epg_row.name if epg_row else None, + epg_row.tvg_id if epg_row else None, + "apply", + ) + continue + epg_data_obj = epg_by_id.get(epg_data_id) + if epg_data_obj: + channel_obj.epg_data = epg_data_obj + channels_to_bulk.append(channel_obj) + changed_associations.append( + {"channel_id": channel_obj.id, "epg_data_id": epg_data_id} + ) + else: + logger.error(f"EPG data {epg_data_id} not found for channel {channel_obj.id}") + + if channels_to_bulk: + Channel.objects.bulk_update(channels_to_bulk, ["epg_data"]) + + parse_dispatched = _dispatch_program_parse_for_epg_assignments(changed_associations) + if parse_dispatched: + logger.info( + f"Dispatched {parse_dispatched} EPG program parse task(s) for changed assignments" + ) + + return changed_associations + + +def get_preferred_region_code(): + try: + region_obj = CoreSettings.objects.get(key="preferred-region") + return region_obj.value.strip().lower() + except CoreSettings.DoesNotExist: + return None + + +def _fuzzy_scan_core(chan_norm, rows, region_code=None, candidate_limit=ML_CANDIDATE_LIMIT): + """ + Single-pass fuzzy scan: track best match and top-K candidates. + Rows must already include norm_name when scanning an in-memory catalog. + """ + best_score = 0 + best_epg = None + top_heap = [] + seq = 0 + scanned = 0 + + for row in rows: + if not row.get("norm_name"): + continue + + scanned += 1 + score = _compute_fuzzy_score(chan_norm, row, region_code) + if score <= 0: + continue + + if score > 50: + logger.debug(f" EPG '{row['name']}' (norm: '{row['norm_name']}') => score: {score}") + + priority = row['epg_source_priority'] + if score > best_score or ( + score == best_score + and priority > (best_epg.get('epg_source_priority', 0) if best_epg else -1) + ): + best_score = score + best_epg = row + + seq += 1 + if len(top_heap) < candidate_limit: + heapq.heappush(top_heap, (score, priority, seq, row)) + else: + smallest_score, smallest_priority, _, _ = top_heap[0] + if score > smallest_score or (score == smallest_score and priority > smallest_priority): + heapq.heapreplace(top_heap, (score, priority, seq, row)) + + top_candidates = sorted(top_heap, key=lambda item: (item[0], item[1]), reverse=True) + return best_score, best_epg, [(score, row) for score, _, _, row in top_candidates], scanned + + +def fuzzy_scan_epg_list(chan_norm, epg_data, region_code=None, candidate_limit=ML_CANDIDATE_LIMIT): + """Fuzzy scan over a pre-built in-memory EPG catalog (bulk matching).""" + logger.debug(f"Fuzzy matching '{chan_norm}' against EPG entries...") + return _fuzzy_scan_core(chan_norm, epg_data, region_code, candidate_limit) + + +def stream_fuzzy_epg_scan(chan_norm, region_code=None, candidate_limit=ML_CANDIDATE_LIMIT): + """Stream fuzzy scan over active EPG entries (single-channel matching).""" + + def row_iterator(): + for values_row in _active_epg_fuzzy_queryset().iterator(chunk_size=500): + row = _row_from_epg_values(values_row) + row['norm_name'] = normalize_name(row['name']) + yield row + + logger.debug(f"Fuzzy matching '{chan_norm}' against EPG entries...") + return _fuzzy_scan_core(chan_norm, row_iterator(), region_code, candidate_limit) + + +def _get_epg_match_thresholds(is_bulk_matching): + if is_bulk_matching: + return { + 'FUZZY_HIGH_CONFIDENCE': 90, + 'FUZZY_SKIP_ML': 80, + 'FUZZY_MEDIUM_CONFIDENCE': 70, + 'ML_HIGH_CONFIDENCE': 0.75, + 'ML_LAST_RESORT': 0.65, + 'FUZZY_LAST_RESORT_MIN': 50, + } + return { + 'FUZZY_HIGH_CONFIDENCE': 85, + 'FUZZY_SKIP_ML': 75, + 'FUZZY_MEDIUM_CONFIDENCE': 40, + 'ML_HIGH_CONFIDENCE': 0.65, + 'ML_LAST_RESORT': 0.50, + 'FUZZY_LAST_RESORT_MIN': 20, + } + + +def try_epg_name_match(chan, best_score, best_epg, top_candidates, is_bulk_matching, + use_ml=True, ml_state=None): + """ + Apply fuzzy/ML thresholds to a channel's best fuzzy result. + Returns the matched EPG row dict, or None. + """ + if not best_epg: + return None + + thresholds = _get_epg_match_thresholds(is_bulk_matching) + fuzzy_high = thresholds['FUZZY_HIGH_CONFIDENCE'] + fuzzy_skip_ml = thresholds['FUZZY_SKIP_ML'] + fuzzy_medium = thresholds['FUZZY_MEDIUM_CONFIDENCE'] + ml_high = thresholds['ML_HIGH_CONFIDENCE'] + ml_last_resort = thresholds['ML_LAST_RESORT'] + fuzzy_last_resort_min = thresholds['FUZZY_LAST_RESORT_MIN'] + + if best_score >= fuzzy_high: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => matched tvg_id={best_epg['tvg_id']} " + f"(score={best_score})" + ) + return best_epg + + if best_score >= fuzzy_skip_ml: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => matched tvg_id={best_epg['tvg_id']} " + f"(fuzzy={best_score}, ML skipped)" + ) + return best_epg + + if ml_state is None: + ml_state = {} + + st_model = ml_state.get('st_model') + util = ml_state.get('util') + + if best_score >= fuzzy_medium and use_ml: + if st_model is None: + st_model, util = get_sentence_transformer() + ml_state['st_model'] = st_model + ml_state['util'] = util + + if st_model: + try: + logger.info("Validating fuzzy best match with ML model (single candidate)") + sims = _ml_cosine_similarities(st_model, util, chan["norm_chan"], [best_epg["norm_name"]]) + top_value = sims[0] if sims else 0.0 + + if top_value >= ml_high - 1e-9: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => matched EPG tvg_id={best_epg['tvg_id']} " + f"(fuzzy={best_score}, ML-sim={top_value:.2f})" + ) + return best_epg + if top_value >= ml_last_resort - 1e-9: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => LAST RESORT match EPG " + f"tvg_id={best_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})" + ) + return best_epg + logger.info( + f"Channel {chan['id']} '{chan['name']}' => fuzzy={best_score}, " + f"ML-sim={top_value:.2f} < {ml_last_resort}, skipping" + ) + except Exception as e: + logger.warning(f"ML matching failed for channel {chan['id']}: {e}") + logger.info( + f"Channel {chan['id']} '{chan['name']}' => fuzzy score {best_score} below threshold, skipping" + ) + return None + + if best_score >= fuzzy_last_resort_min and use_ml: + if st_model is None: + st_model, util = get_sentence_transformer() + ml_state['st_model'] = st_model + ml_state['util'] = util + + if st_model and top_candidates: + try: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => trying ML last resort against " + f"top {len(top_candidates)} fuzzy candidates (fuzzy={best_score})" + ) + candidate_rows = [row for _, row in top_candidates] + sims = _ml_cosine_similarities( + st_model, + util, + chan["norm_chan"], + [row["norm_name"] for row in candidate_rows], + ) + top_index = max(range(len(sims)), key=lambda i: sims[i]) + top_value = sims[top_index] + matched_epg = candidate_rows[top_index] + + if top_value >= ml_last_resort - 1e-9: + logger.info( + f"Channel {chan['id']} '{chan['name']}' => DESPERATE LAST RESORT match " + f"EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})" + ) + return matched_epg + logger.info( + f"Channel {chan['id']} '{chan['name']}' => desperate last resort " + f"ML-sim {top_value:.2f} < {ml_last_resort}, giving up" + ) + except Exception as e: + logger.warning(f"Last resort ML matching failed for channel {chan['id']}: {e}") + logger.info( + f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} " + f"< {fuzzy_medium}, giving up" + ) + return None + + logger.info( + f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} " + f"< {fuzzy_medium}, no ML fallback available" + ) + return None + + +def prepare_channel_match_data(channel): + """Build the channel dict used by matching logic.""" + normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" + normalized_gracenote_id = ( + channel.tvc_guide_stationid.strip().lower() if channel.tvc_guide_stationid else "" + ) + return { + "id": channel.id, + "name": channel.name, + "tvg_id": normalized_tvg_id, + "original_tvg_id": channel.tvg_id, + "gracenote_id": normalized_gracenote_id, + "original_gracenote_id": channel.tvc_guide_stationid, + "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, + "norm_chan": normalize_name(channel.name), + "current_epg_data_id": channel.epg_data_id, + } + + +def match_channels_to_epg( + channels_data, + epg_data, + region_code=None, + use_ml=True, + send_progress=True, + epg_tvg_id_index=None, +): + """ + Match channels to EPG rows using exact ID, fuzzy, and optional ML strategies. + + epg_tvg_id_index: optional pre-built tvg_id -> row map from build_epg_matching_catalog(). + """ + channels_to_update = [] + matched_channels = [] + unchanged_channels = [] + total_channels = len(channels_data) + + if send_progress: + send_epg_matching_progress(total_channels, 0, stage="starting") + + is_bulk_matching = len(channels_data) > 1 + ml_state = {} + epg_by_tvg_id = epg_tvg_id_index if epg_tvg_id_index is not None else build_epg_tvg_id_index(epg_data) + + if is_bulk_matching: + logger.info(f"Using conservative thresholds for bulk matching ({total_channels} channels)") + else: + logger.info("Using aggressive thresholds for single channel matching") + + for index, chan in enumerate(channels_data): + normalized_tvg_id = chan.get("tvg_id", "") + fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] + + resolved_count = len(matched_channels) + len(unchanged_channels) + if send_progress and (index < 5 or index % 5 == 0 or index == total_channels - 1): + send_epg_matching_progress( + total_channels, + resolved_count, + current_channel_name=chan["name"][:50], + stage="matching", + ) + + if normalized_tvg_id: + epg_row = epg_by_tvg_id.get(normalized_tvg_id) + if epg_row: + _record_epg_match( + chan, + epg_row["id"], + epg_name=epg_row.get("name"), + epg_tvg_id=epg_row.get("original_tvg_id") or epg_row.get("tvg_id"), + match_method="exact tvg_id", + channels_to_update=channels_to_update, + matched_channels=matched_channels, + unchanged_channels=unchanged_channels, + ) + continue + + normalized_gracenote_id = chan.get("gracenote_id", "") + if normalized_gracenote_id: + epg_by_gracenote_id = epg_by_tvg_id.get(normalized_gracenote_id) + if epg_by_gracenote_id: + _record_epg_match( + chan, + epg_by_gracenote_id["id"], + epg_name=epg_by_gracenote_id.get("name"), + epg_tvg_id=epg_by_gracenote_id.get("original_tvg_id") + or epg_by_gracenote_id.get("tvg_id"), + match_method="exact gracenote_id", + channels_to_update=channels_to_update, + matched_channels=matched_channels, + unchanged_channels=unchanged_channels, + ) + continue + + if not chan["norm_chan"]: + logger.debug(f"Channel {chan['id']} '{chan['name']}' => empty after normalization, skipping") + continue + + best_score, best_epg, top_candidates, _scanned = fuzzy_scan_epg_list( + chan["norm_chan"], epg_data, region_code + ) + if not best_epg: + logger.debug(f"Channel {chan['id']} '{chan['name']}' => no EPG entries with valid norm_name found") + continue + + matched_epg = try_epg_name_match( + chan, + best_score, + best_epg, + top_candidates, + is_bulk_matching, + use_ml=use_ml, + ml_state=ml_state, + ) + if matched_epg: + _record_epg_match( + chan, + matched_epg["id"], + epg_name=matched_epg.get("name"), + epg_tvg_id=matched_epg.get("original_tvg_id") or matched_epg.get("tvg_id"), + match_method=f"fuzzy (score={best_score})", + channels_to_update=channels_to_update, + matched_channels=matched_channels, + unchanged_channels=unchanged_channels, + ) + + if send_progress: + send_epg_matching_progress( + total_channels, + len(matched_channels) + len(unchanged_channels), + stage="completed", + ) + + return { + "channels_to_update": channels_to_update, + "matched_channels": matched_channels, + "unchanged_channels": unchanged_channels, + } + + +def run_single_channel_epg_match(channel_id): + """ + Match one channel to EPG data. Always notifies the UI via WebSocket before returning. + """ + from apps.channels.models import Channel + + channel = None + try: + logger.info(f"Starting integrated single channel EPG matching for channel ID {channel_id}") + + try: + channel = Channel.objects.get(id=channel_id) + except Channel.DoesNotExist: + message = "Channel not found" + send_single_channel_epg_match_result(channel_id, False, message) + return {"matched": False, "message": message} + + channel_data = prepare_channel_match_data(channel) + logger.info( + f"Channel data prepared: name='{channel.name}', tvg_id='{channel_data['tvg_id']}', " + f"gracenote_id='{channel_data['gracenote_id']}', norm_chan='{channel_data['norm_chan']}'" + ) + + send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="matching") + region_code = get_preferred_region_code() + + fallback_name = channel_data["tvg_id"] if channel_data["tvg_id"] else channel.name + matched_epg_row = None + match_via = None + + if channel_data["tvg_id"]: + matched_epg_row = lookup_epg_by_tvg_id(channel_data["tvg_id"]) + if matched_epg_row: + match_via = matched_epg_row["tvg_id"] + logger.info( + f"Channel {channel.id} '{fallback_name}' => EPG found by exact tvg_id={match_via}" + ) + + if not matched_epg_row and channel_data["gracenote_id"]: + matched_epg_row = lookup_epg_by_tvg_id(channel_data["gracenote_id"]) + if matched_epg_row: + match_via = f"gracenote:{matched_epg_row['tvg_id']}" + logger.info( + f"Channel {channel.id} '{fallback_name}' => EPG found by exact " + f"gracenote_id={channel_data['gracenote_id']}" + ) + + if not matched_epg_row and channel_data["norm_chan"]: + best_score, best_epg, top_candidates, scanned = stream_fuzzy_epg_scan( + channel_data["norm_chan"], region_code + ) + logger.info( + f"Matching single channel '{channel.name}' against {scanned} EPG entries" + ) + if best_epg: + logger.info( + f"Channel {channel.id} '{channel.name}' => best match: '{best_epg['name']}' " + f"(score: {best_score})" + ) + matched_epg_row = try_epg_name_match( + channel_data, + best_score, + best_epg, + top_candidates, + is_bulk_matching=False, + use_ml=True, + ) + if matched_epg_row: + match_via = matched_epg_row["tvg_id"] + elif not channel_data["norm_chan"]: + logger.debug(f"Channel {channel.id} '{channel.name}' => empty after normalization, skipping") + + if not matched_epg_row: + has_fuzzy_epg = _active_epg_fuzzy_queryset().exists() + if not has_fuzzy_epg and not channel_data["tvg_id"] and not channel_data["gracenote_id"]: + message = "No EPG data available for matching (from active sources)" + send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="completed") + send_single_channel_epg_match_result(channel.id, False, message, channel=channel) + return {"matched": False, "message": message} + + if matched_epg_row: + try: + matched_epg_id = matched_epg_row["id"] + epg_data = ( + channel.epg_data + if channel.epg_data_id == matched_epg_id + else EPGData.objects.get(id=matched_epg_id) + ) + + if channel.epg_data_id == matched_epg_id: + success_msg = ( + f"Channel '{channel.name}' already matched with EPG '{epg_data.name}'" + ) + if match_via: + success_msg += f" (matched via: {match_via})" + logger.info(success_msg) + send_epg_matching_progress(1, 1, current_channel_name=channel.name, stage="completed") + send_single_channel_epg_match_result( + channel.id, True, success_msg, channel=channel, epg_data=epg_data + ) + return { + "matched": True, + "unchanged": True, + "message": success_msg, + "epg_name": epg_data.name, + "epg_id": epg_data.id, + } + + channel.epg_data = epg_data + channel.save(update_fields=["epg_data"]) + + success_msg = f"Channel '{channel.name}' matched with EPG '{epg_data.name}'" + if match_via: + success_msg += f" (matched via: {match_via})" + + logger.info(success_msg) + send_epg_matching_progress(1, 1, current_channel_name=channel.name, stage="completed") + channel.refresh_from_db() + send_single_channel_epg_match_result( + channel.id, True, success_msg, channel=channel, epg_data=epg_data + ) + return { + "matched": True, + "message": success_msg, + "epg_name": epg_data.name, + "epg_id": epg_data.id, + } + except EPGData.DoesNotExist: + message = "Matched EPG data not found" + send_single_channel_epg_match_result(channel.id, False, message, channel=channel) + return {"matched": False, "message": message} + + send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="completed") + message = f"No suitable EPG match found for channel '{channel.name}'" + send_single_channel_epg_match_result(channel.id, False, message, channel=channel) + return {"matched": False, "message": message} + + except Exception as e: + logger.error(f"Error in integrated single channel EPG matching: {e}", exc_info=True) + message = f"Error during matching: {str(e)}" + send_single_channel_epg_match_result( + channel_id, + False, + message, + channel=channel, + ) + return {"matched": False, "message": message} + + finally: + cleanup_after_matching() diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index aae62ef4..79058c3c 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -18,8 +18,15 @@ import gc from celery import shared_task from celery.signals import worker_shutting_down from django.utils.text import slugify -from rapidfuzz import fuzz +from apps.channels.epg_matching import ( + apply_matched_epg_to_channels, + build_epg_matching_catalog, + cleanup_after_matching, + match_channels_to_epg, + normalize_name, + run_single_channel_epg_match, +) from apps.channels.models import Channel from apps.epg.models import EPGData from core.models import CoreSettings @@ -200,469 +207,6 @@ def validate_logo_url(logo_url, max_length=2000): return None return logo_url -def send_epg_matching_progress(total_channels, matched_channels, current_channel_name="", stage="matching"): - """ - Send EPG matching progress via WebSocket - """ - try: - channel_layer = get_channel_layer() - if channel_layer: - progress_data = { - 'type': 'epg_matching_progress', - 'total': total_channels, - 'matched': len(matched_channels) if isinstance(matched_channels, list) else matched_channels, - 'remaining': total_channels - (len(matched_channels) if isinstance(matched_channels, list) else matched_channels), - 'current_channel': current_channel_name, - 'stage': stage, - 'progress_percent': round((len(matched_channels) if isinstance(matched_channels, list) else matched_channels) / total_channels * 100, 1) if total_channels > 0 else 0 - } - - async_to_sync(channel_layer.group_send)( - "updates", - { - "type": "update", - "data": { - "type": "epg_matching_progress", - **progress_data - } - } - ) - except Exception as e: - logger.warning(f"Failed to send EPG matching progress: {e}") - -# Lazy loading for ML models - only imported/loaded when needed -_ml_model_cache = { - 'sentence_transformer': None -} - -def get_sentence_transformer(): - """Lazy load the sentence transformer model only when needed""" - if _ml_model_cache['sentence_transformer'] is None: - try: - from sentence_transformers import SentenceTransformer - from sentence_transformers import util - - model_name = "sentence-transformers/all-MiniLM-L6-v2" - cache_dir = "/data/models" - - # Check environment variable to disable downloads - disable_downloads = os.environ.get('DISABLE_ML_DOWNLOADS', 'false').lower() == 'true' - - if disable_downloads: - # Check if model exists before attempting to load - hf_model_path = os.path.join(cache_dir, f"models--{model_name.replace('/', '--')}") - if not os.path.exists(hf_model_path): - logger.warning("ML model not found and downloads disabled (DISABLE_ML_DOWNLOADS=true). Skipping ML matching.") - return None, None - - # Ensure cache directory exists - os.makedirs(cache_dir, exist_ok=True) - - # Let sentence-transformers handle all cache detection and management - logger.info(f"Loading sentence transformer model (cache: {cache_dir})") - _ml_model_cache['sentence_transformer'] = SentenceTransformer( - model_name, - cache_folder=cache_dir - ) - - return _ml_model_cache['sentence_transformer'], util - except ImportError: - logger.warning("sentence-transformers not available - ML-enhanced matching disabled") - return None, None - except Exception as e: - logger.error(f"Failed to load sentence transformer: {e}") - return None, None - else: - from sentence_transformers import util - return _ml_model_cache['sentence_transformer'], util - -# ML matching thresholds (same as original script) -BEST_FUZZY_THRESHOLD = 85 -LOWER_FUZZY_THRESHOLD = 40 -EMBED_SIM_THRESHOLD = 0.65 - -# Words we remove to help with fuzzy + embedding matching -COMMON_EXTRANEOUS_WORDS = [ - "tv", "channel", "network", "television", - "east", "west", "hd", "uhd", "24/7", - "1080p", "720p", "540p", "480p", - "film", "movie", "movies" -] - -def normalize_name(name: str) -> str: - """ - A more aggressive normalization that: - - Removes user-configured prefixes/suffixes/custom strings (only if mode is 'advanced') - - Lowercases - - Removes bracketed/parenthesized text - - Removes punctuation - - Strips extraneous words - - Collapses extra spaces - """ - if not name: - return "" - - # Load user-configured EPG matching rules (fail gracefully) - prefixes = [] - suffixes = [] - custom_strings = [] - - try: - from core.models import CoreSettings - settings = CoreSettings.get_epg_settings() - - # Check if user has enabled advanced mode - mode = settings.get("epg_match_mode", "default") - - # Only use custom settings if mode is 'advanced' - if mode == "advanced": - prefixes = settings.get("epg_match_ignore_prefixes", []) - suffixes = settings.get("epg_match_ignore_suffixes", []) - custom_strings = settings.get("epg_match_ignore_custom", []) - - # Ensure we have lists - if not isinstance(prefixes, list): - prefixes = [] - if not isinstance(suffixes, list): - suffixes = [] - if not isinstance(custom_strings, list): - custom_strings = [] - - except Exception as e: - # Settings unavailable or error - continue with empty lists (graceful degradation) - logger.debug(f"Could not load EPG matching settings: {e}") - prefixes = [] - suffixes = [] - custom_strings = [] - - result = name - - # Step 1: Remove prefixes (from START only - exact string match) - for prefix in prefixes: - # Skip empty or non-string entries - if not prefix or not isinstance(prefix, str): - continue - # Exact match at start - if result.startswith(prefix): - result = result[len(prefix):] - break # Only remove first matching prefix - - # Step 2: Remove suffixes (from END only - exact string match) - for suffix in suffixes: - # Skip empty or non-string entries - if not suffix or not isinstance(suffix, str): - continue - # Exact match at end - if result.endswith(suffix): - result = result[:-len(suffix)] - break # Only remove first matching suffix - - # Step 3: Remove custom strings (from ANYWHERE - exact string match) - for custom in custom_strings: - # Skip empty or non-string entries - if not custom or not isinstance(custom, str): - continue - try: - # Exact string removal (replace with empty string) - result = result.replace(custom, "") - except Exception as e: - # If removal fails for any reason, skip this entry - logger.debug(f"Failed to remove custom string '{custom}': {e}") - continue - - # Step 4: Existing normalization logic (unchanged) - norm = result.lower() - norm = re.sub(r"\[.*?\]", "", norm) - - # Extract and preserve important call signs from parentheses before removing them - # This captures call signs like (KVLY), (KING), (KARE), etc. - call_sign_match = re.search(r"\(([A-Z]{3,5})\)", name) - preserved_call_sign = "" - if call_sign_match: - preserved_call_sign = " " + call_sign_match.group(1).lower() - - # Now remove all parentheses content - norm = re.sub(r"\(.*?\)", "", norm) - - # Add back the preserved call sign - norm = norm + preserved_call_sign - - norm = re.sub(r"[^\w\s]", "", norm) - tokens = norm.split() - tokens = [t for t in tokens if t not in COMMON_EXTRANEOUS_WORDS] - norm = " ".join(tokens).strip() - return norm - -def match_channels_to_epg(channels_data, epg_data, region_code=None, use_ml=True, send_progress=True): - """ - EPG matching logic that finds the best EPG matches for channels using - multiple matching strategies including fuzzy matching and ML models. - - Automatically uses conservative thresholds for bulk matching (multiple channels) - to avoid bad matches that create user cleanup work, and aggressive thresholds - for single channel matching where users specifically requested a match attempt. - """ - channels_to_update = [] - matched_channels = [] - total_channels = len(channels_data) - - # Send initial progress - if send_progress: - send_epg_matching_progress(total_channels, 0, stage="starting") - - # Try to get ML models if requested (but don't load yet - lazy loading) - st_model, util = None, None - epg_embeddings = None - ml_available = use_ml - - # Automatically determine matching strategy based on number of channels - is_bulk_matching = len(channels_data) > 1 - - # Adjust matching thresholds based on operation type - if is_bulk_matching: - # Conservative thresholds for bulk matching to avoid creating cleanup work - FUZZY_HIGH_CONFIDENCE = 90 # Only very high fuzzy scores - FUZZY_MEDIUM_CONFIDENCE = 70 # Higher threshold for ML enhancement - ML_HIGH_CONFIDENCE = 0.75 # Higher ML confidence required - ML_LAST_RESORT = 0.65 # More conservative last resort - FUZZY_LAST_RESORT_MIN = 50 # Higher fuzzy minimum for last resort - logger.info(f"Using conservative thresholds for bulk matching ({total_channels} channels)") - else: - # More aggressive thresholds for single channel matching (user requested specific match) - FUZZY_HIGH_CONFIDENCE = 85 # Original threshold - FUZZY_MEDIUM_CONFIDENCE = 40 # Original threshold - ML_HIGH_CONFIDENCE = 0.65 # Original threshold - ML_LAST_RESORT = 0.50 # Original desperate threshold - FUZZY_LAST_RESORT_MIN = 20 # Original minimum - logger.info("Using aggressive thresholds for single channel matching") # Process each channel - for index, chan in enumerate(channels_data): - normalized_tvg_id = chan.get("tvg_id", "") - fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] - - # Send progress update every 5 channels or for the first few - if send_progress and (index < 5 or index % 5 == 0 or index == total_channels - 1): - send_epg_matching_progress( - total_channels, - len(matched_channels), - current_channel_name=chan["name"][:50], # Truncate long names - stage="matching" - ) - normalized_tvg_id = chan.get("tvg_id", "") - fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] - - # Step 1: Exact TVG ID match - epg_by_tvg_id = next((epg for epg in epg_data if epg["tvg_id"] == normalized_tvg_id), None) - if normalized_tvg_id and epg_by_tvg_id: - chan["epg_data_id"] = epg_by_tvg_id["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], fallback_name, epg_by_tvg_id["tvg_id"])) - logger.info(f"Channel {chan['id']} '{fallback_name}' => EPG found by exact tvg_id={epg_by_tvg_id['tvg_id']}") - continue - - # Step 2: Secondary TVG ID check (legacy compatibility) - if chan["tvg_id"]: - epg_match = [epg["id"] for epg in epg_data if epg["tvg_id"] == chan["tvg_id"]] - if epg_match: - chan["epg_data_id"] = epg_match[0] - channels_to_update.append(chan) - matched_channels.append((chan['id'], fallback_name, chan["tvg_id"])) - logger.info(f"Channel {chan['id']} '{chan['name']}' => EPG found by secondary tvg_id={chan['tvg_id']}") - continue - - # Step 2.5: Exact Gracenote ID match - normalized_gracenote_id = chan.get("gracenote_id", "") - if normalized_gracenote_id: - epg_by_gracenote_id = next((epg for epg in epg_data if epg["tvg_id"] == normalized_gracenote_id), None) - if epg_by_gracenote_id: - chan["epg_data_id"] = epg_by_gracenote_id["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], fallback_name, f"gracenote:{epg_by_gracenote_id['tvg_id']}")) - logger.info(f"Channel {chan['id']} '{fallback_name}' => EPG found by exact gracenote_id={normalized_gracenote_id}") - continue - - # Step 3: Name-based fuzzy matching - if not chan["norm_chan"]: - logger.debug(f"Channel {chan['id']} '{chan['name']}' => empty after normalization, skipping") - continue - - best_score = 0 - best_epg = None - - # Debug: show what we're matching against - logger.debug(f"Fuzzy matching '{chan['norm_chan']}' against EPG entries...") - - # Find best fuzzy match - for row in epg_data: - if not row.get("norm_name"): - continue - - base_score = fuzz.ratio(chan["norm_chan"], row["norm_name"]) - bonus = 0 - - # Apply region-based bonus/penalty - if region_code and row.get("tvg_id"): - combined_text = row["tvg_id"].lower() + " " + row["name"].lower() - dot_regions = re.findall(r'\.([a-z]{2})', combined_text) - - if dot_regions: - if region_code in dot_regions: - bonus = 15 # Bigger bonus for matching region - else: - bonus = -15 # Penalty for different region - elif region_code in combined_text: - bonus = 10 - - score = base_score + bonus - - # Debug the best few matches - if score > 50: # Only show decent matches - logger.debug(f" EPG '{row['name']}' (norm: '{row['norm_name']}') => score: {score} (base: {base_score}, bonus: {bonus})") - - # When scores are equal, prefer higher priority EPG source - row_priority = row.get('epg_source_priority', 0) - best_priority = best_epg.get('epg_source_priority', 0) if best_epg else -1 - - if score > best_score or (score == best_score and row_priority > best_priority): - best_score = score - best_epg = row - - # Log the best score we found - if best_epg: - logger.info(f"Channel {chan['id']} '{chan['name']}' => best match: '{best_epg['name']}' (score: {best_score})") - else: - logger.debug(f"Channel {chan['id']} '{chan['name']}' => no EPG entries with valid norm_name found") - continue - - # High confidence match - accept immediately - if best_score >= FUZZY_HIGH_CONFIDENCE: - chan["epg_data_id"] = best_epg["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], chan['name'], best_epg["tvg_id"])) - logger.info(f"Channel {chan['id']} '{chan['name']}' => matched tvg_id={best_epg['tvg_id']} (score={best_score})") - - # Medium confidence - use ML if available (lazy load models here) - elif best_score >= FUZZY_MEDIUM_CONFIDENCE and ml_available: - # Lazy load ML models only when we actually need them - if st_model is None: - st_model, util = get_sentence_transformer() - - # Lazy generate embeddings only when we actually need them - if epg_embeddings is None and st_model and any(row.get("norm_name") for row in epg_data): - try: - logger.info("Generating embeddings for EPG data using ML model (lazy loading)") - epg_embeddings = st_model.encode( - [row["norm_name"] for row in epg_data if row.get("norm_name")], - convert_to_tensor=True - ) - except Exception as e: - logger.warning(f"Failed to generate embeddings: {e}") - epg_embeddings = None - - if epg_embeddings is not None and st_model: - try: - # Generate embedding for this channel - chan_embedding = st_model.encode(chan["norm_chan"], convert_to_tensor=True) - - # Calculate similarity with all EPG embeddings - sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] - top_index = int(sim_scores.argmax()) - top_value = float(sim_scores[top_index]) - - if top_value >= ML_HIGH_CONFIDENCE: - # Find the EPG entry that corresponds to this embedding index - epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] - matched_epg = epg_with_names[top_index] - - chan["epg_data_id"] = matched_epg["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) - logger.info(f"Channel {chan['id']} '{chan['name']}' => matched EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") - else: - logger.info(f"Channel {chan['id']} '{chan['name']}' => fuzzy={best_score}, ML-sim={top_value:.2f} < {ML_HIGH_CONFIDENCE}, trying last resort...") - - # Last resort: try ML with very low fuzzy threshold - if top_value >= ML_LAST_RESORT: # Dynamic last resort threshold - epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] - matched_epg = epg_with_names[top_index] - - chan["epg_data_id"] = matched_epg["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) - logger.info(f"Channel {chan['id']} '{chan['name']}' => LAST RESORT match EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") - else: - logger.info(f"Channel {chan['id']} '{chan['name']}' => even last resort ML-sim {top_value:.2f} < {ML_LAST_RESORT}, skipping") - - except Exception as e: - logger.warning(f"ML matching failed for channel {chan['id']}: {e}") - # Fall back to non-ML decision - logger.info(f"Channel {chan['id']} '{chan['name']}' => fuzzy score {best_score} below threshold, skipping") - - # Last resort: Try ML matching even with very low fuzzy scores - elif best_score >= FUZZY_LAST_RESORT_MIN and ml_available: - # Lazy load ML models for last resort attempts - if st_model is None: - st_model, util = get_sentence_transformer() - - # Lazy generate embeddings for last resort attempts - if epg_embeddings is None and st_model and any(row.get("norm_name") for row in epg_data): - try: - logger.info("Generating embeddings for EPG data using ML model (last resort lazy loading)") - epg_embeddings = st_model.encode( - [row["norm_name"] for row in epg_data if row.get("norm_name")], - convert_to_tensor=True - ) - except Exception as e: - logger.warning(f"Failed to generate embeddings for last resort: {e}") - epg_embeddings = None - - if epg_embeddings is not None and st_model: - try: - logger.info(f"Channel {chan['id']} '{chan['name']}' => trying ML as last resort (fuzzy={best_score})") - # Generate embedding for this channel - chan_embedding = st_model.encode(chan["norm_chan"], convert_to_tensor=True) - - # Calculate similarity with all EPG embeddings - sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] - top_index = int(sim_scores.argmax()) - top_value = float(sim_scores[top_index]) - - if top_value >= ML_LAST_RESORT: # Dynamic threshold for desperate attempts - # Find the EPG entry that corresponds to this embedding index - epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] - matched_epg = epg_with_names[top_index] - - chan["epg_data_id"] = matched_epg["id"] - channels_to_update.append(chan) - matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) - logger.info(f"Channel {chan['id']} '{chan['name']}' => DESPERATE LAST RESORT match EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") - else: - logger.info(f"Channel {chan['id']} '{chan['name']}' => desperate last resort ML-sim {top_value:.2f} < {ML_LAST_RESORT}, giving up") - except Exception as e: - logger.warning(f"Last resort ML matching failed for channel {chan['id']}: {e}") - logger.info(f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} < {FUZZY_MEDIUM_CONFIDENCE}, giving up") - else: - # No ML available or very low fuzzy score - logger.info(f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} < {FUZZY_MEDIUM_CONFIDENCE}, no ML fallback available") - - # Clean up ML models from memory after matching (infrequent operation) - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory") - _ml_model_cache['sentence_transformer'] = None - gc.collect() - - # Send final progress update - if send_progress: - send_epg_matching_progress( - total_channels, - len(matched_channels), - stage="completed" - ) - - return { - "channels_to_update": channels_to_update, - "matched_channels": matched_channels - } - @shared_task def match_epg_channels(): """ @@ -695,97 +239,74 @@ def match_epg_channels(): "gracenote_id": normalized_gracenote_id, "original_gracenote_id": channel.tvc_guide_stationid, "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, - "norm_chan": normalize_name(channel.name) # Always use channel name for fuzzy matching! + "norm_chan": normalize_name(channel.name), # Always use channel name for fuzzy matching! + "current_epg_data_id": channel.epg_data_id, }) - # Get all EPG data from active sources, ordered by source priority (highest first) so we prefer higher priority matches - epg_data = [] - for epg in EPGData.objects.select_related('epg_source').filter(epg_source__is_active=True): - normalized_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" - epg_data.append({ - 'id': epg.id, - 'tvg_id': normalized_tvg_id, - 'original_tvg_id': epg.tvg_id, - 'name': epg.name, - 'norm_name': normalize_name(epg.name), - 'epg_source_id': epg.epg_source.id if epg.epg_source else None, - 'epg_source_priority': epg.epg_source.priority if epg.epg_source else 0, - }) - - # Sort EPG data by source priority (highest first) so we prefer higher priority matches - epg_data.sort(key=lambda x: x['epg_source_priority'], reverse=True) - + epg_data, epg_tvg_id_index = build_epg_matching_catalog() logger.info(f"Processing {len(channels_data)} channels against {len(epg_data)} EPG entries (from active sources only)") # Run EPG matching with progress updates - automatically uses conservative thresholds for bulk operations - result = match_channels_to_epg(channels_data, epg_data, region_code, use_ml=True, send_progress=True) + result = match_channels_to_epg( + channels_data, + epg_data, + region_code, + use_ml=True, + send_progress=True, + epg_tvg_id_index=epg_tvg_id_index, + ) channels_to_update_dicts = result["channels_to_update"] matched_channels = result["matched_channels"] + unchanged_channels = result.get("unchanged_channels", []) - # Update channels in database - if channels_to_update_dicts: - channel_ids = [d["id"] for d in channels_to_update_dicts] - channels_qs = Channel.objects.filter(id__in=channel_ids) - channels_list = list(channels_qs) + changed_associations = apply_matched_epg_to_channels(channels_to_update_dicts) - # Create mapping from channel_id to epg_data_id - epg_mapping = {d["id"]: d["epg_data_id"] for d in channels_to_update_dicts} - - # Update each channel with matched EPG data - for channel_obj in channels_list: - epg_data_id = epg_mapping.get(channel_obj.id) - if epg_data_id: - try: - epg_data_obj = EPGData.objects.get(id=epg_data_id) - channel_obj.epg_data = epg_data_obj - except EPGData.DoesNotExist: - logger.error(f"EPG data {epg_data_id} not found for channel {channel_obj.id}") - - # Bulk update all channels - Channel.objects.bulk_update(channels_list, ["epg_data"]) - - total_matched = len(matched_channels) - if total_matched: - logger.info(f"Match Summary: {total_matched} channel(s) matched.") + channels_updated = len(changed_associations) + if channels_updated: + logger.info(f"Match Summary: {channels_updated} channel(s) updated.") for (cid, cname, tvg) in matched_channels: - logger.info(f" - Channel ID={cid}, Name='{cname}' => tvg_id='{tvg}'") - else: - logger.info("No new channels were matched.") + logger.info(f" - Channel '{cname}' (id={cid}) => tvg_id={tvg!r}") + if unchanged_channels: + logger.debug( + f"{len(unchanged_channels)} channel(s) already correctly matched (unchanged)" + ) + if not channels_updated and not unchanged_channels: + logger.info("No channels were matched.") logger.info("Finished integrated EPG matching.") - # Send WebSocket update - channel_layer = get_channel_layer() - associations = [ - {"channel_id": chan["id"], "epg_data_id": chan["epg_data_id"]} - for chan in channels_to_update_dicts - ] + from core.utils import send_websocket_update - async_to_sync(channel_layer.group_send)( + if channels_updated: + match_message = f"EPG matching complete: {channels_updated} channel(s) updated" + elif unchanged_channels: + match_message = ( + f"EPG matching complete: {len(unchanged_channels)} channel(s) " + f"already correctly matched" + ) + else: + match_message = "EPG matching complete: no matches found" + + send_websocket_update( 'updates', + 'update', { - 'type': 'update', - "data": { - "success": True, - "type": "epg_match", - "refresh_channels": True, - "matches_count": total_matched, - "message": f"EPG matching complete: {total_matched} channel(s) matched", - "associations": associations - } - } + "success": True, + "type": "epg_match", + "refresh_channels": True, + "matches_count": channels_updated, + "message": match_message, + "associations": changed_associations, + }, ) - return f"Done. Matched {total_matched} channel(s)." + return ( + f"Done. {channels_updated} channel(s) updated " + f"({len(unchanged_channels)} unchanged)." + ) finally: - # Clean up ML models from memory after bulk matching - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory") - _ml_model_cache['sentence_transformer'] = None - - # Memory cleanup - gc.collect() + cleanup_after_matching() from core.utils import cleanup_memory cleanup_memory(log_usage=True, force_collection=True) @@ -806,36 +327,31 @@ def match_selected_channels_epg(channel_ids): except CoreSettings.DoesNotExist: region_code = None - # Get only the specified channels that don't have EPG data assigned - channels_without_epg = Channel.objects.filter( - id__in=channel_ids, - epg_data__isnull=True - ) - logger.info(f"Found {channels_without_epg.count()} selected channels without EPG data") + # Selected-channel matching always runs, including channels that already have EPG. + selected_channels = Channel.objects.filter(id__in=channel_ids) + logger.info(f"Processing {selected_channels.count()} selected channel(s) for EPG matching") - if not channels_without_epg.exists(): - logger.info("No selected channels need EPG matching.") + if not selected_channels.exists(): + logger.info("No selected channels found for EPG matching.") - # Send WebSocket update - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( + from core.utils import send_websocket_update + + send_websocket_update( 'updates', + 'update', { - 'type': 'update', - "data": { - "success": True, - "type": "epg_match", - "refresh_channels": True, - "matches_count": 0, - "message": "No selected channels need EPG matching", - "associations": [] - } - } + "success": True, + "type": "epg_match", + "refresh_channels": True, + "matches_count": 0, + "message": "No selected channels found for EPG matching", + "associations": [], + }, ) - return "No selected channels needed EPG matching." + return "No selected channels found for EPG matching." channels_data = [] - for channel in channels_without_epg: + for channel in selected_channels: normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" normalized_gracenote_id = channel.tvc_guide_stationid.strip().lower() if channel.tvc_guide_stationid else "" channels_data.append({ @@ -846,246 +362,91 @@ def match_selected_channels_epg(channel_ids): "gracenote_id": normalized_gracenote_id, "original_gracenote_id": channel.tvc_guide_stationid, "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, - "norm_chan": normalize_name(channel.name) + "norm_chan": normalize_name(channel.name), + "current_epg_data_id": channel.epg_data_id, }) - # Get all EPG data from active sources, ordered by source priority (highest first) so we prefer higher priority matches - epg_data = [] - for epg in EPGData.objects.select_related('epg_source').filter(epg_source__is_active=True): - normalized_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" - epg_data.append({ - 'id': epg.id, - 'tvg_id': normalized_tvg_id, - 'original_tvg_id': epg.tvg_id, - 'name': epg.name, - 'norm_name': normalize_name(epg.name), - 'epg_source_id': epg.epg_source.id if epg.epg_source else None, - 'epg_source_priority': epg.epg_source.priority if epg.epg_source else 0, - }) - - # Sort EPG data by source priority (highest first) so we prefer higher priority matches - epg_data.sort(key=lambda x: x['epg_source_priority'], reverse=True) - + epg_data, epg_tvg_id_index = build_epg_matching_catalog() logger.info(f"Processing {len(channels_data)} selected channels against {len(epg_data)} EPG entries (from active sources only)") # Run EPG matching with progress updates - automatically uses appropriate thresholds - result = match_channels_to_epg(channels_data, epg_data, region_code, use_ml=True, send_progress=True) + result = match_channels_to_epg( + channels_data, + epg_data, + region_code, + use_ml=True, + send_progress=True, + epg_tvg_id_index=epg_tvg_id_index, + ) channels_to_update_dicts = result["channels_to_update"] matched_channels = result["matched_channels"] + unchanged_channels = result.get("unchanged_channels", []) - # Update channels in database - if channels_to_update_dicts: - channel_ids_to_update = [d["id"] for d in channels_to_update_dicts] - channels_qs = Channel.objects.filter(id__in=channel_ids_to_update) - channels_list = list(channels_qs) + changed_associations = apply_matched_epg_to_channels(channels_to_update_dicts) - # Create mapping from channel_id to epg_data_id - epg_mapping = {d["id"]: d["epg_data_id"] for d in channels_to_update_dicts} - - # Update each channel with matched EPG data - for channel_obj in channels_list: - epg_data_id = epg_mapping.get(channel_obj.id) - if epg_data_id: - try: - epg_data_obj = EPGData.objects.get(id=epg_data_id) - channel_obj.epg_data = epg_data_obj - except EPGData.DoesNotExist: - logger.error(f"EPG data {epg_data_id} not found for channel {channel_obj.id}") - - # Bulk update all channels - Channel.objects.bulk_update(channels_list, ["epg_data"]) - - total_matched = len(matched_channels) - if total_matched: - logger.info(f"Selected Channel Match Summary: {total_matched} channel(s) matched.") + channels_updated = len(changed_associations) + if channels_updated: + logger.info( + f"Selected Channel Match Summary: {channels_updated} channel(s) updated." + ) for (cid, cname, tvg) in matched_channels: - logger.info(f" - Channel ID={cid}, Name='{cname}' => tvg_id='{tvg}'") - else: + logger.info(f" - Channel '{cname}' (id={cid}) => tvg_id={tvg!r}") + if unchanged_channels: + logger.debug( + f"{len(unchanged_channels)} selected channel(s) already correctly matched " + f"(unchanged)" + ) + if not channels_updated and not unchanged_channels: logger.info("No selected channels were matched.") logger.info("Finished integrated EPG matching for selected channels.") - # Send WebSocket update - channel_layer = get_channel_layer() - associations = [ - {"channel_id": chan["id"], "epg_data_id": chan["epg_data_id"]} - for chan in channels_to_update_dicts - ] + from core.utils import send_websocket_update - async_to_sync(channel_layer.group_send)( + if channels_updated: + match_message = ( + f"EPG matching complete: {channels_updated} selected channel(s) updated" + ) + elif unchanged_channels: + match_message = ( + f"EPG matching complete: {len(unchanged_channels)} selected channel(s) " + f"already correctly matched" + ) + else: + match_message = "EPG matching complete: no matches found" + + send_websocket_update( 'updates', + 'update', { - 'type': 'update', - "data": { - "success": True, - "type": "epg_match", - "refresh_channels": True, - "matches_count": total_matched, - "message": f"EPG matching complete: {total_matched} selected channel(s) matched", - "associations": associations - } - } + "success": True, + "type": "epg_match", + "refresh_channels": True, + "matches_count": channels_updated, + "message": match_message, + "associations": changed_associations, + }, ) - return f"Done. Matched {total_matched} selected channel(s)." + return ( + f"Done. {channels_updated} selected channel(s) updated " + f"({len(unchanged_channels)} unchanged)." + ) finally: - # Clean up ML models from memory after bulk matching - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory") - _ml_model_cache['sentence_transformer'] = None - - # Memory cleanup - gc.collect() + cleanup_after_matching() from core.utils import cleanup_memory cleanup_memory(log_usage=True, force_collection=True) @shared_task def match_single_channel_epg(channel_id): - """ - Try to match a single channel with EPG data using the integrated matching logic - that includes both fuzzy and ML-enhanced matching. Returns a dict with match status and message. - """ + """Match one channel to EPG data (async; results pushed via WebSocket).""" try: - from apps.channels.models import Channel - from apps.epg.models import EPGData - - logger.info(f"Starting integrated single channel EPG matching for channel ID {channel_id}") - - # Get the channel - try: - channel = Channel.objects.get(id=channel_id) - except Channel.DoesNotExist: - return {"matched": False, "message": "Channel not found"} - - # If channel already has EPG data, skip - if channel.epg_data: - return {"matched": False, "message": f"Channel '{channel.name}' already has EPG data assigned"} - - # Prepare single channel data for matching (same format as bulk matching) - normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" - normalized_gracenote_id = channel.tvc_guide_stationid.strip().lower() if channel.tvc_guide_stationid else "" - channel_data = { - "id": channel.id, - "name": channel.name, - "tvg_id": normalized_tvg_id, - "original_tvg_id": channel.tvg_id, - "gracenote_id": normalized_gracenote_id, - "original_gracenote_id": channel.tvc_guide_stationid, - "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, - "norm_chan": normalize_name(channel.name) # Always use channel name for fuzzy matching! - } - - logger.info(f"Channel data prepared: name='{channel.name}', tvg_id='{normalized_tvg_id}', gracenote_id='{normalized_gracenote_id}', norm_chan='{channel_data['norm_chan']}'") - - # Debug: Test what the normalization does to preserve call signs - test_name = "NBC 11 (KVLY) - Fargo" # Example for testing - test_normalized = normalize_name(test_name) - logger.debug(f"DEBUG normalization example: '{test_name}' → '{test_normalized}' (call sign preserved)") - - # Get all EPG data for matching from active sources - must include norm_name field - # Ordered by source priority (highest first) so we prefer higher priority matches - epg_data_list = [] - for epg in EPGData.objects.select_related('epg_source').filter(epg_source__is_active=True, name__isnull=False).exclude(name=''): - normalized_epg_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" - epg_data_list.append({ - 'id': epg.id, - 'tvg_id': normalized_epg_tvg_id, - 'original_tvg_id': epg.tvg_id, - 'name': epg.name, - 'norm_name': normalize_name(epg.name), - 'epg_source_id': epg.epg_source.id if epg.epg_source else None, - 'epg_source_priority': epg.epg_source.priority if epg.epg_source else 0, - }) - - # Sort EPG data by source priority (highest first) so we prefer higher priority matches - epg_data_list.sort(key=lambda x: x['epg_source_priority'], reverse=True) - - if not epg_data_list: - return {"matched": False, "message": "No EPG data available for matching (from active sources)"} - - logger.info(f"Matching single channel '{channel.name}' against {len(epg_data_list)} EPG entries") - - # Send progress for single channel matching - send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="matching") - - # Use the EPG matching function - automatically uses aggressive thresholds for single channel - result = match_channels_to_epg([channel_data], epg_data_list, send_progress=False) - channels_to_update = result.get("channels_to_update", []) - matched_channels = result.get("matched_channels", []) - - if channels_to_update: - # Find our channel in the results - channel_match = None - for update in channels_to_update: - if update["id"] == channel.id: - channel_match = update - break - - if channel_match: - # Apply the match to the channel - try: - epg_data = EPGData.objects.get(id=channel_match['epg_data_id']) - channel.epg_data = epg_data - channel.save(update_fields=["epg_data"]) - - # Find match details from matched_channels for better reporting - match_details = None - for match_info in matched_channels: - if match_info[0] == channel.id: # matched_channels format: (channel_id, channel_name, epg_info) - match_details = match_info - break - - success_msg = f"Channel '{channel.name}' matched with EPG '{epg_data.name}'" - if match_details: - success_msg += f" (matched via: {match_details[2]})" - - logger.info(success_msg) - - # Send completion progress for single channel - send_epg_matching_progress(1, 1, current_channel_name=channel.name, stage="completed") - - # Clean up ML models from memory after single channel matching - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory") - _ml_model_cache['sentence_transformer'] = None - gc.collect() - - return { - "matched": True, - "message": success_msg, - "epg_name": epg_data.name, - "epg_id": epg_data.id - } - except EPGData.DoesNotExist: - return {"matched": False, "message": "Matched EPG data not found"} - - # No match found - # Send completion progress for single channel (failed) - send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="completed") - - # Clean up ML models from memory after single channel matching - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory") - _ml_model_cache['sentence_transformer'] = None - gc.collect() - - return { - "matched": False, - "message": f"No suitable EPG match found for channel '{channel.name}'" - } - - except Exception as e: - logger.error(f"Error in integrated single channel EPG matching: {e}", exc_info=True) - - # Clean up ML models from memory even on error - if _ml_model_cache['sentence_transformer'] is not None: - logger.info("Cleaning up ML models from memory after error") - _ml_model_cache['sentence_transformer'] = None - gc.collect() - - return {"matched": False, "message": f"Error during matching: {str(e)}"} + return run_single_channel_epg_match(channel_id) + finally: + from core.utils import cleanup_memory + cleanup_memory(log_usage=True, force_collection=True) def evaluate_series_rules_impl(tvg_id: str | None = None): diff --git a/apps/channels/tests/test_epg_match_apply.py b/apps/channels/tests/test_epg_match_apply.py new file mode 100644 index 00000000..4d0cc2e5 --- /dev/null +++ b/apps/channels/tests/test_epg_match_apply.py @@ -0,0 +1,58 @@ +"""Tests for applying EPG auto-match results to channels.""" +from unittest.mock import patch + +from django.test import TestCase + +from apps.channels.epg_matching import apply_matched_epg_to_channels +from apps.channels.models import Channel +from apps.epg.models import EPGData, EPGSource + + +class ApplyMatchedEpgToChannelsTests(TestCase): + def setUp(self): + self.source = EPGSource.objects.create( + name="XML EPG", + source_type="xmltv", + url="http://example.com/epg.xml", + ) + self.epg_one = EPGData.objects.create( + tvg_id="ch.one", + name="Channel One", + epg_source=self.source, + ) + self.epg_two = EPGData.objects.create( + tvg_id="ch.two", + name="Channel Two", + epg_source=self.source, + ) + self.channel = Channel.objects.create( + channel_number=1, + name="Channel One", + tvg_id="ch.one", + epg_data=self.epg_one, + ) + + @patch("apps.epg.tasks.parse_programs_for_tvg_id.delay") + def test_skips_unchanged_assignment(self, mock_delay): + changed = apply_matched_epg_to_channels( + [{"id": self.channel.id, "epg_data_id": self.epg_one.id}] + ) + + self.assertEqual(changed, []) + mock_delay.assert_not_called() + self.channel.refresh_from_db() + self.assertEqual(self.channel.epg_data_id, self.epg_one.id) + + @patch("apps.epg.tasks.parse_programs_for_tvg_id.delay") + def test_updates_changed_assignment_and_dispatches_parse(self, mock_delay): + changed = apply_matched_epg_to_channels( + [{"id": self.channel.id, "epg_data_id": self.epg_two.id}] + ) + + self.assertEqual( + changed, + [{"channel_id": self.channel.id, "epg_data_id": self.epg_two.id}], + ) + mock_delay.assert_called_once_with(self.epg_two.id) + self.channel.refresh_from_db() + self.assertEqual(self.channel.epg_data_id, self.epg_two.id) diff --git a/apps/channels/tests/test_epg_name_normalize.py b/apps/channels/tests/test_epg_name_normalize.py new file mode 100644 index 00000000..0fa6df5f --- /dev/null +++ b/apps/channels/tests/test_epg_name_normalize.py @@ -0,0 +1,117 @@ +"""Tests for EPG channel name normalization (prefix/suffix/custom ignore rules).""" +from django.test import TestCase + +from apps.channels.epg_matching import ( + build_epg_tvg_id_index, + clear_normalize_settings_cache, + normalize_name, +) +from core.models import CoreSettings, EPG_SETTINGS_KEY + + +class NormalizeNameSettingsTest(TestCase): + def _set_epg_settings(self, **kwargs): + obj, _ = CoreSettings.objects.get_or_create( + key=EPG_SETTINGS_KEY, + defaults={"name": "EPG Settings", "value": {}}, + ) + current = obj.value if isinstance(obj.value, dict) else {} + current.update(kwargs) + obj.value = current + obj.save() + clear_normalize_settings_cache() + + def test_default_mode_does_not_apply_ignore_lists(self): + self._set_epg_settings( + epg_match_mode="default", + epg_match_ignore_prefixes=["HD:"], + epg_match_ignore_suffixes=[" 4K"], + epg_match_ignore_custom=["Plus"], + ) + result_default = normalize_name("HD:HBO Plus East 4K") + + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_prefixes=["HD:"], + epg_match_ignore_suffixes=[" 4K"], + epg_match_ignore_custom=["Plus"], + ) + result_advanced = normalize_name("HD:HBO Plus East 4K") + + self.assertNotEqual(result_default, result_advanced) + self.assertEqual(result_advanced, "hbo east") + + def test_advanced_mode_strips_prefix(self): + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_prefixes=["HD:"], + ) + self.assertEqual( + normalize_name("HD:ABC 7 (WXYZ) - Springfield"), + "abc 7 springfield wxyz", + ) + + def test_advanced_mode_strips_suffix(self): + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_suffixes=[" 4K"], + ) + self.assertEqual( + normalize_name("NBC 5 (KABC) - Metro 4K"), + "nbc 5 metro kabc", + ) + + def test_advanced_mode_removes_custom_strings(self): + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_custom=["Plus"], + ) + self.assertEqual( + normalize_name("HBO Plus East"), + "hbo east", + ) + + def test_advanced_mode_applies_prefix_suffix_and_custom_in_order(self): + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_prefixes=["Sling:"], + epg_match_ignore_suffixes=[" HD"], + epg_match_ignore_custom=["Plus"], + ) + self.assertEqual( + normalize_name("Sling:HBO Plus East HD"), + "hbo east", + ) + + def test_only_first_matching_prefix_is_removed(self): + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_prefixes=["HD:", "SD:"], + ) + self.assertEqual(normalize_name("HD:SD:Channel 5"), "sd channel 5") + + def test_call_sign_preserved_from_original_name(self): + self._set_epg_settings(epg_match_mode="default") + self.assertEqual( + normalize_name("NBC 5 (KABC) - Metro"), + "nbc 5 metro kabc", + ) + + def test_tvg_id_index_prefers_first_entry_when_catalog_sorted_by_priority(self): + # Catalog from build_epg_matching_catalog() is highest-priority first. + epg_data = [ + {"id": 2, "tvg_id": "abc.us", "epg_source_priority": 50, "name": "High"}, + {"id": 1, "tvg_id": "abc.us", "epg_source_priority": 10, "name": "Low"}, + ] + index = build_epg_tvg_id_index(epg_data) + self.assertEqual(index["abc.us"]["id"], 2) + + def test_settings_cache_refresh_picks_up_new_rules(self): + self._set_epg_settings(epg_match_mode="default") + self.assertEqual(normalize_name("HD:ABC"), "hdabc") + + self._set_epg_settings( + epg_match_mode="advanced", + epg_match_ignore_prefixes=["HD:"], + ) + self.assertEqual(normalize_name("HD:ABC"), "abc") diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 81f26fe2..7ce0ab29 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -95,6 +95,8 @@ def cleanup_task_memory(**kwargs): 'apps.epg.tasks.parse_programs_for_source', 'apps.epg.tasks.parse_programs_for_tvg_id', 'apps.channels.tasks.match_epg_channels', + 'apps.channels.tasks.match_selected_channels_epg', + 'apps.channels.tasks.match_single_channel_epg', 'core.tasks.rehash_streams' ] diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index 0e70a46c..4cbb3d89 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -366,22 +366,26 @@ export const WebsocketProvider = ({ children }) => { fetchEPGData(); break; + case 'single_channel_epg_match': { + const matchResult = parsedEvent.data; + if (matchResult.channel) { + useChannelsStore.getState().updateChannel(matchResult.channel); + } + window.dispatchEvent( + new CustomEvent('single-channel-epg-match', { + detail: matchResult, + }) + ); + break; + } + case 'epg_match': notifications.show({ message: parsedEvent.data.message || 'EPG match is complete!', color: 'green.5', }); - // Check if we have associations data and use the more efficient batch API - if ( - parsedEvent.data.associations && - parsedEvent.data.associations.length > 0 - ) { - API.batchSetEPG(parsedEvent.data.associations); - } - - // Refresh EPG store first, then requery channels so the table - // cross-references updated epg_data_id assignments immediately + // Celery already applied assignments server-side; refresh local state. fetchEPGData(); API.requeryChannels(); break; @@ -647,8 +651,13 @@ export const WebsocketProvider = ({ children }) => { // Read from the store directly. connectWebSocket closes over a stale // epgs snapshot, so a newly created source is missed and the old early- // return path never reached fetchEPGData on parsing_channels completion. - let { epgs: epgsState, updateEPG, updateEPGProgress, fetchEPGs, fetchEPGData } = - useEPGsStore.getState(); + let { + epgs: epgsState, + updateEPG, + updateEPGProgress, + fetchEPGs, + fetchEPGData, + } = useEPGsStore.getState(); if (!epgsState[sourceId]) { try { @@ -694,8 +703,7 @@ export const WebsocketProvider = ({ children }) => { updateEPG({ ...epg, status: parsedEvent.data.status || 'success', - last_message: - parsedEvent.data.message || epg.last_message, + last_message: parsedEvent.data.message || epg.last_message, ...(parsedEvent.data.updated_at && { updated_at: parsedEvent.data.updated_at, }), diff --git a/frontend/src/components/forms/Channel.jsx b/frontend/src/components/forms/Channel.jsx index c904641c..8f39e8f3 100644 --- a/frontend/src/components/forms/Channel.jsx +++ b/frontend/src/components/forms/Channel.jsx @@ -163,12 +163,24 @@ const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { } setAutoMatchLoading(true); + let accepted = false; try { const response = await matchChannelEpg(channel); + if (response?.accepted) { + accepted = true; + showNotification({ + title: 'Matching in Progress', + message: + response.message || + 'EPG auto-match is running. Results will appear when complete.', + color: 'blue', + }); + return; + } + if (response.matched) { - // Update the form with the new EPG data - if (response.channel && response.channel.epg_data_id) { + if (response.channel?.epg_data_id) { setValue('epg_data_id', response.channel.epg_data_id); } @@ -192,7 +204,9 @@ const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { }); console.error('Auto-match error:', error); } finally { - setAutoMatchLoading(false); + if (!accepted) { + setAutoMatchLoading(false); + } } }; @@ -354,6 +368,48 @@ const ChannelForm = ({ channel: channelProp = null, isOpen, onClose }) => { resolver: yupResolver(validationSchema), }); + useEffect(() => { + const onMatchResult = (event) => { + const data = event.detail; + if (!channel?.id || String(data.channel_id) !== String(channel.id)) { + return; + } + + if (data.matched && data.channel?.epg_data_id) { + setValue('epg_data_id', data.channel.epg_data_id); + } + + showNotification({ + title: data.matched ? 'Success' : 'No Match Found', + message: data.message, + color: data.matched ? 'green' : 'orange', + }); + setAutoMatchLoading(false); + }; + + window.addEventListener('single-channel-epg-match', onMatchResult); + return () => + window.removeEventListener('single-channel-epg-match', onMatchResult); + }, [channel?.id, setValue]); + + useEffect(() => { + if (!autoMatchLoading) { + return undefined; + } + + const timeoutId = window.setTimeout(() => { + setAutoMatchLoading(false); + showNotification({ + title: 'Matching Timed Out', + message: + 'EPG auto-match is taking longer than expected. Check back shortly or try again.', + color: 'orange', + }); + }, 180_000); + + return () => window.clearTimeout(timeoutId); + }, [autoMatchLoading]); + const clearOverrides = async () => { if (!channel) return; try { diff --git a/frontend/src/components/forms/__tests__/Channel.test.jsx b/frontend/src/components/forms/__tests__/Channel.test.jsx index 7b00775a..efdd01e2 100644 --- a/frontend/src/components/forms/__tests__/Channel.test.jsx +++ b/frontend/src/components/forms/__tests__/Channel.test.jsx @@ -703,6 +703,25 @@ describe('ChannelForm', () => { expect(autoMatch).not.toBeDisabled(); }); + it('shows in-progress notification when matchChannelEpg returns accepted', async () => { + const channel = makeChannel(); + vi.mocked(ChannelUtils.matchChannelEpg).mockResolvedValue({ + accepted: true, + message: 'EPG matching started', + }); + setupMocks({ channel }); + render(); + fireEvent.click(screen.getByText('Auto Match')); + await waitFor(() => { + expect(showNotification).toHaveBeenCalledWith( + expect.objectContaining({ + title: 'Matching in Progress', + color: 'blue', + }) + ); + }); + }); + it('calls matchChannelEpg with the channel on click', async () => { const channel = makeChannel(); vi.mocked(ChannelUtils.matchChannelEpg).mockResolvedValue({ From abf16fd104a64cb6ec8b5daa8ac9c188ab1423e9 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:22:16 -0500 Subject: [PATCH 08/81] test(m3u): cover provider-number collisions in auto-sync numbering - A duplicate provider number keeps one channel and falls the collider back to a free number. - A provider number matching an existing channel falls back instead of overwriting it. --- apps/m3u/tests/test_sync_correctness.py | 53 ++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 4a2986fb..beeeef0d 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -2445,6 +2445,55 @@ class ProviderNumberingHonorsProviderNumberTests(TestCase): created = Channel.objects.get(auto_created=True, auto_created_by=account) self.assertEqual(created.channel_number, 150.0) + def test_duplicate_provider_numbers_keep_one_and_fall_back_the_other(self): + # Two streams claim the same provider number (common in messy event + # feeds). One keeps it; the colliding one falls back to a different free + # number rather than being dropped or overwriting the first. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + _make_stream(account, group, name="A", tvg_id="a", stream_chno=77250) + _make_stream(account, group, name="B", tvg_id="b", stream_chno=77250) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + numbers = sorted( + Channel.objects.filter( + auto_created=True, auto_created_by=account + ).values_list("channel_number", flat=True) + ) + self.assertEqual(len(numbers), 2) + self.assertIn(77250.0, numbers) + self.assertNotEqual(numbers[0], numbers[1]) + + def test_provider_number_colliding_with_manual_channel_falls_back(self): + # A provider number matching an existing channel must not overwrite it; + # the auto-created channel falls back to a different free number. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 1, + } + rel.save() + manual = Channel.objects.create(name="Manual", channel_number=88250) + _make_stream(account, group, name="P", tvg_id="p", stream_chno=88250) + + result = _sync(account) + + self.assertEqual(result["status"], "ok") + created = Channel.objects.get(auto_created=True, auto_created_by=account) + self.assertNotEqual(created.channel_number, 88250.0) + manual.refresh_from_db() + self.assertEqual(manual.channel_number, 88250.0) + class CrossModeNumberingFieldTests(TestCase): """ @@ -2468,7 +2517,7 @@ class CrossModeNumberingFieldTests(TestCase): # Fail signature: streams beyond the End fail = next_available honoring # a hidden cap. account = _make_account() - group = _make_group(name="Sports") + group = _make_group(name="PPV") rel = _attach_group_to_account(account, group) rel.auto_sync_channel_start = 1 rel.auto_sync_channel_end = 3 # stale cap from a prior mode @@ -2517,7 +2566,7 @@ class CrossModeNumberingFieldTests(TestCase): # Same gate for next_available: tightening a stale End must not delete # already-assigned channels (range enforcement is fixed-mode only). account = _make_account() - group = _make_group(name="Sports") + group = _make_group(name="PPV") rel = _attach_group_to_account(account, group) rel.auto_sync_channel_start = 1 rel.auto_sync_channel_end = None From bc046b3c92a1bec47c144c80082eb28126042439 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 13:40:16 -0500 Subject: [PATCH 09/81] refactor(m3u, channels): streamline stream assignment and server group management - Renamed `_clear_stream_assignment_keys` to `_release_stale_stream_assignment` for clarity and updated its logic to release pool counters. - Introduced new functions for managing credential slots during profile switches, enhancing the handling of shared connection limits across server groups. - Removed the `max_streams` field from the `ServerGroup` model and updated related components to reflect this change, simplifying the server group management. - Updated frontend components to integrate server group management, allowing for dynamic creation and editing of server groups. - Enhanced error handling in stream URL generation to provide more informative feedback on connection issues. - Added tests for stale assignment release and credential management during profile switches. --- apps/channels/models.py | 47 ++++- apps/m3u/connection_pool.py | 61 +++++- .../0020_servergroup_max_streams.py | 21 -- apps/m3u/models.py | 16 +- apps/m3u/serializers.py | 2 +- apps/m3u/tests/test_connection_pool.py | 185 ++++++++++++++++-- apps/proxy/live_proxy/url_utils.py | 88 +++++---- apps/proxy/live_proxy/views.py | 23 ++- apps/proxy/vod_proxy/views.py | 11 +- .../components/ServerGroupsManagerModal.jsx | 28 +++ .../ServerGroupsManagerModal.test.jsx | 59 ++++++ frontend/src/components/forms/M3U.jsx | 53 ++++- frontend/src/components/forms/ServerGroup.jsx | 50 +++-- frontend/src/components/tables/M3UsTable.jsx | 47 +++-- .../components/tables/ServerGroupsTable.jsx | 170 ++++++++++++---- .../__tests__/ServerGroupsTable.test.jsx | 62 ++++++ frontend/src/pages/Settings.jsx | 16 -- 17 files changed, 710 insertions(+), 229 deletions(-) delete mode 100644 apps/m3u/migrations/0020_servergroup_max_streams.py create mode 100644 frontend/src/components/ServerGroupsManagerModal.jsx create mode 100644 frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx create mode 100644 frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx diff --git a/apps/channels/models.py b/apps/channels/models.py index d09a50f5..e84f9768 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -607,11 +607,22 @@ class Channel(models.Model): ChannelState.CONNECTING, ) - def _clear_stream_assignment_keys(self, redis_client, stream_id=None) -> None: - """Remove channel/stream profile assignment keys from Redis.""" + def _release_stale_stream_assignment(self, redis_client, stream_id: int) -> None: + """Release pool counters and remove stale channel/stream assignment keys.""" + profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") + if profile_id_bytes: + try: + profile_id = int(profile_id_bytes) + release_profile_slot(profile_id, redis_client) + except (ValueError, TypeError): + logger.debug( + "Invalid profile ID for stale assignment on stream %s: %s", + stream_id, + profile_id_bytes, + ) + redis_client.delete(f"channel_stream:{self.id}") - if stream_id is not None: - redis_client.delete(f"stream_profile:{stream_id}") + redis_client.delete(f"stream_profile:{stream_id}") def get_stream(self, requester=None): """ @@ -659,10 +670,10 @@ class Channel(models.Model): ) else: logger.info( - f"Channel {self.uuid}: clearing stale stream assignment " + f"Channel {self.uuid}: releasing stale stream assignment " f"(stream={stream_id}, proxy not active)" ) - self._clear_stream_assignment_keys(redis_client, stream_id) + self._release_stale_stream_assignment(redis_client, stream_id) # No existing active stream, attempt to assign a new one has_streams_but_maxed_out = False @@ -886,9 +897,29 @@ class Channel(models.Model): if current_profile_id == new_profile_id: return True - from apps.m3u.connection_pool import profile_connections_key + from apps.m3u.connection_pool import ( + move_credential_slot_on_profile_switch, + profile_connections_key, + ) + from apps.m3u.models import M3UAccountProfile - # Profile counters always move on switch; group totals stay unchanged (one stream). + old_profile = M3UAccountProfile.objects.select_related( + "m3u_account__server_group" + ).get(id=current_profile_id) + new_profile = M3UAccountProfile.objects.select_related( + "m3u_account__server_group" + ).get(id=new_profile_id) + + if not move_credential_slot_on_profile_switch( + old_profile, new_profile, redis_client + ): + logger.warning( + "Shared login pool full for profile %s during stream profile switch", + new_profile_id, + ) + return False + + # Profile counters always move on switch; credential totals move only when login changes. old_profile_connections_key = profile_connections_key(current_profile_id) new_profile_connections_key = profile_connections_key(new_profile_id) old_count = int(redis_client.get(old_profile_connections_key) or 0) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 36a74d17..2d4c5830 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -2,8 +2,7 @@ Shared connection pool enforcement for M3U accounts in the same ServerGroup. Profile selection rotates across M3UAccountProfile rows using each profile's own -Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup -with max_streams > 0, a credential-scoped counter is checked on reserve/release +Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup, a credential-scoped counter is checked on reserve/release so accounts sharing the same provider login share one limit without blocking unrelated logins on the same group. Account profiles with max_streams=0 skip credential enforcement for that profile. @@ -129,9 +128,9 @@ def get_profile_credential_fingerprint(profile) -> Optional[str]: def get_enforced_server_group_for_profile(profile): - """Return the shared ServerGroup limit for this profile's account, if configured.""" + """Return the ServerGroup for credential pooling when the account is assigned to one.""" group = profile.m3u_account.server_group - if group and group.max_streams > 0: + if group: return group return None @@ -157,11 +156,6 @@ def get_credential_connection_count(profile, redis_client) -> int: return int(redis_client.get(cred_key) or 0) -def get_group_connection_count(profile, redis_client) -> int: - """Backwards-compatible alias for credential-scoped group usage.""" - return get_credential_connection_count(profile, redis_client) - - def profile_has_capacity_for_selection(profile, redis_client) -> bool: """Per-profile capacity check used when rotating across profiles on one account.""" if profile.max_streams == 0: @@ -188,6 +182,55 @@ def pool_has_capacity_for_profile(profile, redis_client) -> bool: ) +def profile_available_for_channel_switch( + profile, redis_client, *, channel_already_on_profile: bool +) -> bool: + """ + Non-mutating capacity check when selecting a profile for an in-flight channel. + + If the channel already holds this profile's slots, skip re-checking capacity. + """ + if channel_already_on_profile: + return True + return pool_has_capacity_for_profile(profile, redis_client) + + +def move_credential_slot_on_profile_switch( + old_profile, new_profile, redis_client +) -> bool: + """ + Move the shared credential counter when switching to a different provider login. + + Profile counters are managed separately by Channel.update_stream_profile(). + Returns False when the new profile's credential pool is full. + """ + old_fp = get_profile_credential_fingerprint(old_profile) + new_fp = get_profile_credential_fingerprint(new_profile) + if old_fp == new_fp: + return True + + released = _release_credential_slot_by_profile_id(old_profile.id, redis_client) + if not released: + _release_server_group_slot_for_profile(old_profile, redis_client) + + cred_reserved, cred_key = _reserve_server_group_slot_for_profile( + new_profile, redis_client + ) + if not cred_reserved: + restore_reserved, restore_key = _reserve_server_group_slot_for_profile( + old_profile, redis_client + ) + if restore_reserved and restore_key: + _remember_credential_release_key( + old_profile.id, restore_key, redis_client + ) + return False + + if cred_key: + _remember_credential_release_key(new_profile.id, cred_key, redis_client) + return True + + def _safe_decr(redis_client, key: str) -> None: current = int(redis_client.get(key) or 0) if current <= 0: diff --git a/apps/m3u/migrations/0020_servergroup_max_streams.py b/apps/m3u/migrations/0020_servergroup_max_streams.py deleted file mode 100644 index ce17d830..00000000 --- a/apps/m3u/migrations/0020_servergroup_max_streams.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 6.0.3 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("m3u", "0019_m3uaccountprofile_exp_date"), - ] - - operations = [ - migrations.AddField( - model_name="servergroup", - name="max_streams", - field=models.PositiveIntegerField( - default=0, - help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", - ), - ), - ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 46d1a068..53c18d38 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -194,19 +194,17 @@ class M3UFilter(models.Model): class ServerGroup(models.Model): - """Represents a logical grouping of servers or channels.""" + """ + Groups M3U accounts that share provider credentials. + + Accounts assigned to the same server group share credential-scoped connection + counters when their logins match. Limits come from each account profile's + max_streams, not from the group itself. + """ name = models.CharField( max_length=100, unique=True, help_text="Unique name for this server group." ) - max_streams = models.PositiveIntegerField( - default=0, - help_text=( - "Set above 0 to enable shared credential pooling for accounts in this group. " - "Per-login limits come from each account profile's max_streams. " - "Profiles with max_streams=0 (unlimited) skip cross-account credential enforcement." - ), - ) def __str__(self): return self.name diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 29c4157f..485a3687 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -408,4 +408,4 @@ class ServerGroupSerializer(serializers.ModelSerializer): class Meta: model = ServerGroup - fields = ["id", "name", "max_streams"] + fields = ["id", "name"] diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index e5de6c10..5b05535a 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -7,7 +7,6 @@ from apps.m3u.connection_pool import ( extract_credentials_from_stream_url, get_credential_connection_count, get_enforced_server_group_for_profile, - get_group_connection_count, get_profile_connection_count, get_profile_credential_fingerprint, group_has_capacity_for_profile, @@ -94,8 +93,8 @@ class ExtractCredentialsTests(TestCase): class ManualServerGroupTests(TestCase): - def test_group_enforced_when_max_streams_set(self): - group = ServerGroup.objects.create(name="provider-a", max_streams=2) + def test_group_enforced_when_account_assigned(self): + group = ServerGroup.objects.create(name="provider-a") account = M3UAccount.objects.create( name="Account A", account_type="XC", @@ -108,7 +107,7 @@ class ManualServerGroupTests(TestCase): self.assertEqual(get_enforced_server_group_for_profile(profile), group) def test_accounts_in_same_group_share_credential_counter(self): - group = ServerGroup.objects.create(name="shared", max_streams=2) + group = ServerGroup.objects.create(name="shared") account1 = M3UAccount.objects.create( name="XC Account", account_type="XC", @@ -171,10 +170,7 @@ class ManualServerGroupTests(TestCase): class PoolEnforcementTests(TestCase): def setUp(self): self.redis = FakeRedis() - self.group = ServerGroup.objects.create( - name="test-pool", - max_streams=2, - ) + self.group = ServerGroup.objects.create(name="test-pool") self.account = M3UAccount.objects.create( name="Test Account", account_type="XC", @@ -207,8 +203,8 @@ class PoolEnforcementTests(TestCase): self.assertEqual(self.redis._data[cred_key], 0) self.assertEqual(self.redis._data[profile_key], 0) - def test_same_credential_capped_at_profile_max_not_group_max(self): - """Maintainer example: group max=2 but each login only allows 1.""" + def test_same_credential_capped_at_profile_max(self): + """Shared credential counter is capped by each profile's max_streams.""" account2 = M3UAccount.objects.create( name="Second Account", account_type="XC", @@ -229,11 +225,8 @@ class PoolEnforcementTests(TestCase): cred_key = server_group_connections_key(self.group.id, fp) self.assertEqual(self.redis._data[cred_key], 1) - def test_different_logins_both_stream_when_group_max_is_one(self): - """Regression: group max=1 must not block a second login on another profile.""" - self.group.max_streams = 1 - self.group.save() - + def test_different_logins_both_stream_in_same_group(self): + """Different provider logins keep separate credential counters in one group.""" account = M3UAccount.objects.create( name="Grouped multi-login", account_type="XC", @@ -288,7 +281,7 @@ class PoolEnforcementTests(TestCase): ): self.assertTrue(reserve_profile_slot(profile, self.redis)[0]) self.assertEqual(get_credential_connection_count(profile, self.redis), 0) - self.assertEqual(get_group_connection_count(profile, self.redis), 0) + self.assertEqual(get_credential_connection_count(profile, self.redis), 0) def test_release_when_profile_row_deleted(self): profile_id = self.profile.id @@ -328,12 +321,55 @@ class PoolEnforcementTests(TestCase): self.assertEqual(reason, "credential_full") +class StaleAssignmentTests(TestCase): + def test_stale_assignment_releases_counters(self): + from apps.channels.models import Channel, Stream + + redis = FakeRedis() + group = ServerGroup.objects.create(name="stale-group") + account = M3UAccount.objects.create( + name="Stale Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile.max_streams = 1 + profile.save() + + stream = Stream.objects.create(name="Stale Stream", m3u_account=account) + channel = Channel.objects.create(channel_number=502, name="Stale Channel") + channel.streams.add(stream) + + reserve_profile_slot(profile, redis) + redis.set(f"channel_stream:{channel.id}", stream.id) + redis.set(f"stream_profile:{stream.id}", profile.id) + + profile_key = profile_connections_key(profile.id) + cred_key = server_group_connections_key( + group.id, get_profile_credential_fingerprint(profile) + ) + self.assertEqual(redis._data[profile_key], 1) + self.assertEqual(redis._data[cred_key], 1) + + with patch("core.utils.RedisClient.get_client", return_value=redis): + channel._release_stale_stream_assignment(redis, stream.id) + + self.assertNotIn(f"channel_stream:{channel.id}", redis._data) + self.assertNotIn(f"stream_profile:{stream.id}", redis._data) + self.assertEqual(redis._data[profile_key], 0) + self.assertEqual(redis._data[cred_key], 0) + + class UpdateStreamProfileTests(TestCase): def test_switch_updates_profile_counters_when_group_assigned(self): from apps.channels.models import Channel, Stream redis = FakeRedis() - group = ServerGroup.objects.create(name="switch-group", max_streams=2) + group = ServerGroup.objects.create(name="switch-group") account = M3UAccount.objects.create( name="Switch Account", account_type="XC", @@ -377,6 +413,58 @@ class UpdateStreamProfileTests(TestCase): self.assertEqual(redis._data[profile_connections_key(profile_b.id)], 1) self.assertEqual(redis._data[cred_key], cred_before) + def test_switch_moves_credential_counter_when_login_changes(self): + from apps.channels.models import Channel, Stream + + redis = FakeRedis() + group = ServerGroup.objects.create(name="cred-switch-group") + account = M3UAccount.objects.create( + name="Cred Switch Account", + account_type="XC", + username="login_a", + password="pass_a", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + profile_a = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile_a.max_streams = 1 + profile_a.save() + profile_b = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt_login", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + fp_a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fp_b = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + key_a = server_group_connections_key(group.id, fp_a) + key_b = server_group_connections_key(group.id, fp_b) + + stream = Stream.objects.create(name="Cred Switch Stream", m3u_account=account) + channel = Channel.objects.create(channel_number=503, name="Cred Switch Channel") + channel.streams.add(stream) + + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + side_effect=lambda profile: fp_a if profile.id == profile_a.id else fp_b, + ): + reserve_profile_slot(profile_a, redis) + redis.set(f"channel_stream:{channel.id}", stream.id) + redis.set(f"stream_profile:{stream.id}", profile_a.id) + self.assertEqual(redis._data[key_a], 1) + self.assertNotIn(key_b, redis._data) + + with patch("core.utils.RedisClient.get_client", return_value=redis): + self.assertTrue(channel.update_stream_profile(profile_b.id)) + + self.assertEqual(redis._data[key_a], 0) + self.assertEqual(redis._data[key_b], 1) + class VodProfileSelectionTests(TestCase): def test_get_m3u_profile_skips_default_when_profile_full(self): @@ -413,3 +501,66 @@ class VodProfileSelectionTests(TestCase): self.assertIsNotNone(result) selected, _connections = result self.assertEqual(selected.id, alt.id) + + def test_get_m3u_profile_skips_default_when_credential_pool_full(self): + from apps.proxy.vod_proxy.views import _get_m3u_profile + + group = ServerGroup.objects.create(name="vod-cred-pool") + account = M3UAccount.objects.create( + name="VOD pooled", + account_type="XC", + username="shared_user", + password="shared_pass", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.max_streams = 1 + default.save() + + alt = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt_login", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + other_account = M3UAccount.objects.create( + name="Other pooled account", + account_type="XC", + username="shared_user", + password="shared_pass", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + other_profile = M3UAccountProfile.objects.get( + m3u_account=other_account, is_default=True + ) + other_profile.max_streams = 1 + other_profile.save() + + fp_shared = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + fp_alt = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + + redis = FakeRedis() + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + side_effect=lambda profile: ( + fp_shared + if profile.id in (default.id, other_profile.id) + else fp_alt + ), + ): + self.assertTrue(reserve_profile_slot(other_profile, redis)[0]) + + with patch("core.utils.RedisClient.get_client", return_value=redis): + result = _get_m3u_profile(account, None, None) + + self.assertIsNotNone(result) + selected, _connections = result + self.assertEqual(selected.id, alt.id) diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 60d0e3d0..899ffc9a 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -8,7 +8,10 @@ from typing import Optional, Tuple, List from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile -from apps.m3u.connection_pool import profile_connections_key +from apps.m3u.connection_pool import ( + get_profile_connection_count, + profile_available_for_channel_switch, +) from core.models import UserAgent, CoreSettings, StreamProfile from .utils import get_logger from uuid import UUID @@ -54,12 +57,14 @@ def get_stream_object(id: str): logger.info(f"Fetching stream hash {id}") return get_object_or_404(Stream, stream_hash=id) -def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], bool]: +def generate_stream_url( + channel_id: str, +) -> Tuple[str, str, bool, Optional[int], bool, Optional[str]]: """ Generate the appropriate stream URL for a channel or stream based on its profile settings. Returns: - Tuple: (stream_url, user_agent, transcode_flag, profile_id, slot_reserved) + Tuple: (stream_url, user_agent, transcode_flag, profile_id, slot_reserved, error_reason) """ try: channel_or_stream = get_stream_object(channel_id) @@ -71,12 +76,12 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], if not stream.m3u_account: logger.error(f"Stream {stream.id} has no M3U account") - return None, None, False, None, False + return None, None, False, None, False, "Stream has no M3U account" stream_id, profile_id, error_reason, slot_reserved = stream.get_stream() if not stream_id or not profile_id: logger.error(f"No profile available for stream {stream.id}: {error_reason}") - return None, None, False, None, False + return None, None, False, None, False, error_reason try: profile = M3UAccountProfile.objects.get(id=profile_id) @@ -95,24 +100,23 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], transcode = not stream_profile.is_proxy() stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved, None except Exception as e: logger.error(f"Error generating stream URL for stream {stream.id}: {e}") if slot_reserved: stream.release_stream() - return None, None, False, None, False + return None, None, False, None, False, str(e) # Handle channel preview (existing logic) channel = channel_or_stream # Get stream and profile for this channel - # Note: get_stream now returns 3 values (stream_id, profile_id, error_reason) stream_id, profile_id, error_reason, slot_reserved = channel.get_stream() if not stream_id or not profile_id: logger.error(f"No stream available for channel {channel_id}: {error_reason}") - return None, None, False, None, False + return None, None, False, None, False, error_reason # get_stream() allocated a connection slot - ensure it's released on any error try: @@ -142,16 +146,16 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved, None except Exception as e: logger.error(f"Error generating stream URL for channel {channel_id}: {e}") if slot_reserved: if not channel.release_stream(): logger.warning(f"Failed to release stream for channel {channel_id} after URL generation error") - return None, None, False, None, False + return None, None, False, None, False, str(e) except Exception as e: logger.error(f"Error generating stream URL: {e}") - return None, None, False, None, False + return None, None, False, None, False, str(e) def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -229,10 +233,6 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: if redis_client: - current_connections = int( - redis_client.get(profile_connections_key(profile.id)) or 0 - ) - channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: @@ -242,20 +242,22 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True - effective_connections = current_connections - ( - 1 if channel_using_profile else 0 - ) - - if profile.max_streams == 0 or effective_connections < profile.max_streams: + if profile_available_for_channel_switch( + profile, + redis_client, + channel_already_on_profile=channel_using_profile, + ): + current_connections = get_profile_connection_count( + profile, redis_client + ) selected_profile = profile logger.debug( f"Selected profile {profile.id} with " - f"{effective_connections}/{profile.max_streams} effective connections" + f"{current_connections}/{profile.max_streams} connections" ) break logger.debug( - f"Profile {profile.id} at max connections: " - f"{effective_connections}/{profile.max_streams}" + f"Profile {profile.id} unavailable for channel switch" ) else: selected_profile = profile @@ -368,38 +370,38 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No selected_profile = None for profile in profiles: - # Check connection availability if redis_client: - profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) - - # Check if this channel is already using this profile channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: - # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id - existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") + existing_profile_id = redis_client.get( + f"stream_profile:{existing_stream_id}" + ) if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True - logger.debug(f"Channel {channel.id} already using profile {profile.id}") + logger.debug( + f"Channel {channel.id} already using profile {profile.id}" + ) - # Calculate effective connections (subtract 1 if channel already using this profile) - effective_connections = current_connections - (1 if channel_using_profile else 0) - - # Check if profile has available slots - if profile.max_streams == 0 or effective_connections < profile.max_streams: + if profile_available_for_channel_switch( + profile, + redis_client, + channel_already_on_profile=channel_using_profile, + ): + current_connections = get_profile_connection_count( + profile, redis_client + ) selected_profile = profile logger.debug( f"Found available profile {profile.id} for stream {stream.id}: " - f"{effective_connections}/{profile.max_streams} effective " - f"(current: {current_connections}, already using: {channel_using_profile})" + f"{current_connections}/{profile.max_streams} " + f"(already using: {channel_using_profile})" ) break - else: - logger.debug(f"Profile {profile.id} at max connections: {effective_connections}/{profile.max_streams} (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Profile {profile.id} unavailable for alternate stream {stream.id}" + ) else: - # No Redis available, assume first active profile is okay selected_profile = profile break diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index eb845743..d60d9703 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -221,9 +221,14 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # Try to get a stream with fixed interval retries while should_retry and time.time() - wait_start_time < retry_timeout: attempt += 1 - stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( - generate_stream_url(channel_id) - ) + ( + stream_url, + stream_user_agent, + transcode, + profile_value, + slot_reserved, + error_reason, + ) = generate_stream_url(channel_id) if stream_url is not None: logger.info( @@ -233,7 +238,6 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # On first failure, check if the error is retryable if attempt == 1: - _, _, error_reason, _ = channel.get_stream() if error_reason and "maximum connection limits" not in error_reason: logger.warning( f"[{client_id}] Can't retry - error not related to connection limits: {error_reason}" @@ -266,9 +270,14 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.info( f"[{client_id}] Making final attempt {attempt} at timeout boundary" ) - stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( - generate_stream_url(channel_id) - ) + ( + stream_url, + stream_user_agent, + transcode, + profile_value, + slot_reserved, + error_reason, + ) = generate_stream_url(channel_id) if stream_url is not None: logger.info( f"[{client_id}] Successfully obtained stream on final attempt for channel {channel_id}" diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index c4e84986..fc825cf4 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -262,7 +262,7 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): from core.utils import RedisClient from apps.m3u.connection_pool import ( get_profile_connection_count, - profile_has_capacity_for_selection, + pool_has_capacity_for_profile, ) redis_client = RedisClient.get_client() @@ -316,7 +316,7 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): profile_connections_key = f"profile_connections:{profile.id}" current_connections = get_profile_connection_count(profile, redis_client) - if profile_has_capacity_for_selection(profile, redis_client): + if pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Using requested profile {profile.id}: {current_connections}/{profile.max_streams} connections") return (profile, current_connections) logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") @@ -340,11 +340,14 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): for profile in profiles: current_connections = get_profile_connection_count(profile, redis_client) - if profile_has_capacity_for_selection(profile, redis_client): + if pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Selected profile {profile.id} ({profile.name}): {current_connections}/{profile.max_streams} connections") return (profile, current_connections) else: - logger.debug(f"[PROFILE-SELECTION] Profile {profile.id} at capacity: {current_connections}/{profile.max_streams}") + logger.debug( + f"[PROFILE-SELECTION] Profile {profile.id} unavailable " + f"(profile={current_connections}/{profile.max_streams})" + ) # All profiles are at capacity - return None to trigger error response logger.error(f"[PROFILE-SELECTION] All profiles at capacity for M3U account {m3u_account.id}, rejecting request") diff --git a/frontend/src/components/ServerGroupsManagerModal.jsx b/frontend/src/components/ServerGroupsManagerModal.jsx new file mode 100644 index 00000000..5e3005ec --- /dev/null +++ b/frontend/src/components/ServerGroupsManagerModal.jsx @@ -0,0 +1,28 @@ +import { Modal } from '@mantine/core'; +import ServerGroupsTable from './tables/ServerGroupsTable'; + +const ServerGroupsManagerModal = ({ + isOpen, + onClose, + onGroupCreated, + openCreateOnMount = false, +}) => { + return ( + + {isOpen ? ( + + ) : null} + + ); +}; + +export default ServerGroupsManagerModal; diff --git a/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx b/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx new file mode 100644 index 00000000..a60ae4db --- /dev/null +++ b/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import { MantineProvider } from '@mantine/core'; +import ServerGroupsManagerModal from '../ServerGroupsManagerModal'; + +vi.mock('../../api', () => ({ + default: { + deleteServerGroup: vi.fn(), + }, +})); + +vi.mock('../../store/serverGroups', () => ({ + default: (selector) => + selector({ + serverGroups: [ + { id: 1, name: 'Pool A' }, + { id: 2, name: 'Pool B' }, + ], + fetchServerGroups: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock('../../store/playlists', () => ({ + default: (selector) => + selector({ + playlists: [{ id: 10, server_group: 2 }], + }), +})); + +vi.mock('../forms/ServerGroup', () => ({ + default: () => null, +})); + +vi.mock('../../hooks/useLocalStorage', () => ({ + default: () => ['default', vi.fn()], +})); + +const renderModal = (props = {}) => + render( + + + + ); + +describe('ServerGroupsManagerModal', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('opens without Mantine Tooltip errors', async () => { + expect(() => renderModal()).not.toThrow(); + + await waitFor(() => { + expect(screen.getByText('Pool A')).toBeInTheDocument(); + expect(screen.getByText('Pool B')).toBeInTheDocument(); + expect(screen.getByText('Accounts')).toBeInTheDocument(); + }); + }); +}); diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 6e19c93a..bfc015de 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -36,6 +36,7 @@ import { prepareSubmitValues, updatePlaylist, } from '../../utils/forms/M3uUtils.js'; +import ServerGroupsManagerModal from '../ServerGroupsManagerModal'; const M3U = ({ m3uAccount = null, @@ -56,6 +57,8 @@ const M3U = ({ const [groupFilterModalOpen, setGroupFilterModalOpen] = useState(false); const [filterModalOpen, setFilterModalOpen] = useState(false); const [scheduleType, setScheduleType] = useState('interval'); + const [serverGroupsManagerOpen, setServerGroupsManagerOpen] = useState(false); + const [serverGroupsCreateOnOpen, setServerGroupsCreateOnOpen] = useState(false); const form = useForm({ mode: 'uncontrolled', @@ -366,18 +369,36 @@ const M3U = ({ name="server_group" label="Server Group" description="Share login limits across accounts in a server group. Set max streams on each profile (unlimited profiles skip group enforcement)." - {...form.getInputProps('server_group')} key={form.key('server_group')} - data={[{ value: '0', label: '(None)' }].concat( - serverGroups.map((group) => ({ - label: - group.max_streams > 0 - ? `${group.name} (max: ${group.max_streams})` - : `${group.name} (unlimited)`, + value={form.getValues().server_group} + onChange={(value) => { + if (value === '__new__') { + setServerGroupsCreateOnOpen(true); + setServerGroupsManagerOpen(true); + return; + } + form.setFieldValue('server_group', value); + }} + data={[ + { value: '0', label: '(None)' }, + ...serverGroups.map((group) => ({ + label: group.name, value: `${group.id}`, - })) - )} + })), + { value: '__new__', label: '+ Add server group...' }, + ]} /> + } data={[ - { - value: 'STD', - label: 'Standard', - }, - { - value: 'XC', - label: 'Xtream Codes', - }, + { value: 'STD', label: 'Standard' }, + { value: 'XC', label: 'Xtream Codes' }, ]} key={form.key('account_type')} {...form.getInputProps('account_type')} /> {form.getValues().account_type == 'XC' && ( - - {!m3uAccount && ( - - Create EPG - - - )} - - - Enable VOD Scanning - - - + <> - - + )} {form.getValues().account_type != 'XC' && ( @@ -335,7 +297,6 @@ const M3U = ({ }, }} /> - - + - + + + + } /> - - + {form.getValues().account_type == 'XC' && ( + + + + + Enable VOD Scanning + + + + {!m3uAccount && ( + + Create EPG + + + )} + + )} - - + - - - {playlist && ( - <> - - - - - )} + + {playlist && ( + <> + + + + + )} - + + From 4fc12bca4a449826d9c3b5fb54c85c26ca651411 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 17:59:12 -0500 Subject: [PATCH 13/81] refactor(m3u, channels, proxy): enhance stream assignment logic and error handling - Improved logging in the Stream model for better debugging of profile evaluations. - Introduced a new method `_stream_assignment_is_reusable` in the Channel model to determine if existing stream assignments can be reused, enhancing efficiency. - Updated the release logic in `release_profile_slot` to utilize stored credential keys, reducing unnecessary database lookups. - Simplified error handling in the `get_stream_info_for_switch` function to ensure proper stream release on exceptions. - Enhanced tests for connection pool management and error handling in the ServerGroupsTable component to improve reliability and user feedback. --- apps/channels/models.py | 23 ++- .../tests/test_get_stream_assignment.py | 173 ++++++++++++++++++ apps/m3u/connection_pool.py | 32 ++-- apps/m3u/tests/test_connection_pool.py | 28 +++ apps/proxy/live_proxy/channel_status.py | 10 +- apps/proxy/live_proxy/url_utils.py | 16 +- .../ServerGroupsManagerModal.test.jsx | 2 + .../components/tables/ServerGroupsTable.jsx | 17 +- .../__tests__/ServerGroupsTable.test.jsx | 41 ++++- 9 files changed, 293 insertions(+), 49 deletions(-) create mode 100644 apps/channels/tests/test_get_stream_assignment.py diff --git a/apps/channels/models.py b/apps/channels/models.py index e84f9768..f900a817 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -228,7 +228,7 @@ class Stream(models.Model): ] for profile in profiles: - logger.info(profile) + logger.debug("Evaluating profile %s for stream %s", profile.id, self.id) # Skip inactive profiles if profile.is_active == False: continue @@ -607,6 +607,23 @@ class Channel(models.Model): ChannelState.CONNECTING, ) + def _stream_assignment_is_reusable(self, redis_client, stream_id: int) -> bool: + """ + Return True when an existing channel_stream assignment should be reused. + + Reuse when the proxy is active, or when metadata is not written yet + (between get_stream() reserving slots and initialize_channel() starting). + When metadata exists but the proxy is inactive, the assignment is stale. + """ + if self._channel_proxy_is_active(redis_client): + return True + + metadata_key = RedisKeys.channel_metadata(str(self.uuid)) + if not redis_client.exists(metadata_key): + return redis_client.get(f"stream_profile:{stream_id}") is not None + + return False + def _release_stale_stream_assignment(self, redis_client, stream_id: int) -> None: """Release pool counters and remove stale channel/stream assignment keys.""" profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") @@ -654,13 +671,13 @@ class Channel(models.Model): stream_id = None if stream_id is not None: - if self._channel_proxy_is_active(redis_client): + if self._stream_assignment_is_reusable(redis_client, stream_id): profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") if profile_id_bytes: try: profile_id = int(profile_id_bytes) logger.debug( - f"Channel {self.uuid}: reusing active assignment " + f"Channel {self.uuid}: reusing stream assignment " f"stream={stream_id} profile={profile_id}" ) return stream_id, profile_id, None, False diff --git a/apps/channels/tests/test_get_stream_assignment.py b/apps/channels/tests/test_get_stream_assignment.py new file mode 100644 index 00000000..bd5fea2e --- /dev/null +++ b/apps/channels/tests/test_get_stream_assignment.py @@ -0,0 +1,173 @@ +"""Tests for Channel.get_stream() assignment reuse and stale cleanup.""" + +from unittest.mock import patch + +from django.test import TestCase + +from apps.channels.models import Channel, ChannelStream, Stream +from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState +from apps.proxy.live_proxy.redis_keys import RedisKeys + + +class FakeAssignmentRedis: + """In-memory Redis for channel_stream assignment tests.""" + + def __init__(self): + self._strings = {} + self._hashes = {} + + def _decode(self, value): + if isinstance(value, bytes): + return value.decode() + return value + + def get(self, key): + value = self._strings.get(key) + if value is None: + return None + if isinstance(value, int): + return str(value).encode() + return str(value).encode() + + def set(self, key, value): + self._strings[key] = value + + def delete(self, key): + self._strings.pop(key, None) + self._hashes.pop(key, None) + + def exists(self, key): + return key in self._strings or key in self._hashes + + def hget(self, key, field): + return self._hashes.get(key, {}).get(field) + + def hset(self, key, mapping=None, **kwargs): + bucket = self._hashes.setdefault(key, {}) + if mapping: + bucket.update(mapping) + bucket.update(kwargs) + + def incr(self, key): + current = int(self._decode(self.get(key)) or 0) + current += 1 + self._strings[key] = current + return current + + def decr(self, key): + current = int(self._decode(self.get(key)) or 0) + current -= 1 + self._strings[key] = current + return current + + +class ChannelGetStreamAssignmentTests(TestCase): + def setUp(self): + self.redis = FakeAssignmentRedis() + self.account = M3UAccount.objects.create( + name="assignment-test", + account_type="XC", + username="user", + password="pass", + max_streams=5, + ) + self.profile = M3UAccountProfile.objects.get( + m3u_account=self.account, is_default=True + ) + self.profile.max_streams = 2 + self.profile.save() + + self.stream = Stream.objects.create( + name="Test Stream", + url="http://example.com/live/user/pass/1.ts", + m3u_account=self.account, + ) + self.channel = Channel.objects.create(channel_number=501, name="Assignment Ch") + ChannelStream.objects.create(channel=self.channel, stream=self.stream, order=0) + + self.metadata_key = RedisKeys.channel_metadata(str(self.channel.uuid)) + + def _seed_assignment(self): + self.redis.set(f"channel_stream:{self.channel.id}", self.stream.id) + self.redis.set(f"stream_profile:{self.stream.id}", self.profile.id) + + @patch("apps.channels.models.RedisClient.get_client") + @patch("apps.channels.models.reserve_profile_slot") + def test_reuses_assignment_when_proxy_active( + self, mock_reserve, mock_get_client + ): + mock_get_client.return_value = self.redis + self._seed_assignment() + self.redis.hset( + self.metadata_key, + {ChannelMetadataField.STATE: ChannelState.ACTIVE}, + ) + + stream_id, profile_id, error, slot_reserved = self.channel.get_stream() + + self.assertEqual(stream_id, self.stream.id) + self.assertEqual(profile_id, self.profile.id) + self.assertIsNone(error) + self.assertFalse(slot_reserved) + mock_reserve.assert_not_called() + + @patch("apps.channels.models.RedisClient.get_client") + @patch("apps.channels.models.reserve_profile_slot") + def test_reuses_assignment_during_init_before_metadata( + self, mock_reserve, mock_get_client + ): + mock_get_client.return_value = self.redis + self._seed_assignment() + + stream_id, profile_id, error, slot_reserved = self.channel.get_stream() + + self.assertEqual(stream_id, self.stream.id) + self.assertEqual(profile_id, self.profile.id) + self.assertIsNone(error) + self.assertFalse(slot_reserved) + mock_reserve.assert_not_called() + + @patch("apps.channels.models.RedisClient.get_client") + @patch("apps.channels.models.release_profile_slot") + @patch("apps.channels.models.reserve_profile_slot") + def test_releases_stale_assignment_when_proxy_stopped( + self, mock_reserve, mock_release, mock_get_client + ): + mock_get_client.return_value = self.redis + mock_reserve.return_value = (True, 1, None) + self._seed_assignment() + self.redis.hset( + self.metadata_key, + {ChannelMetadataField.STATE: ChannelState.STOPPED}, + ) + + stream_id, profile_id, error, slot_reserved = self.channel.get_stream() + + mock_release.assert_called_once_with(self.profile.id, self.redis) + mock_reserve.assert_called_once() + self.assertEqual(stream_id, self.stream.id) + self.assertEqual(profile_id, self.profile.id) + self.assertTrue(slot_reserved) + + @patch("apps.channels.models.RedisClient.get_client") + def test_stream_assignment_is_reusable_during_init_pending(self, mock_get_client): + mock_get_client.return_value = self.redis + self._seed_assignment() + + self.assertTrue( + self.channel._stream_assignment_is_reusable(self.redis, self.stream.id) + ) + + @patch("apps.channels.models.RedisClient.get_client") + def test_stream_assignment_not_reusable_when_stopped(self, mock_get_client): + mock_get_client.return_value = self.redis + self._seed_assignment() + self.redis.hset( + self.metadata_key, + {ChannelMetadataField.STATE: ChannelState.STOPPED}, + ) + + self.assertFalse( + self.channel._stream_assignment_is_reusable(self.redis, self.stream.id) + ) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 2d4c5830..0fa0864f 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -172,7 +172,7 @@ def group_has_capacity_for_profile(profile, redis_client) -> bool: cred_key = _credential_counter_key(profile, group) if not cred_key: return True - return get_credential_connection_count(profile, redis_client) < profile.max_streams + return int(redis_client.get(cred_key) or 0) < profile.max_streams def pool_has_capacity_for_profile(profile, redis_client) -> bool: @@ -326,22 +326,26 @@ def reserve_profile_slot( def release_profile_slot(profile_id: int, redis_client) -> None: """Release profile and shared credential slots after a stream end.""" - from apps.m3u.models import M3UAccountProfile - released_via_stored_key = _release_credential_slot_by_profile_id( profile_id, redis_client ) - try: - profile = M3UAccountProfile.objects.get(id=profile_id) - except M3UAccountProfile.DoesNotExist: - profile = None - profile_key = profile_connections_key(profile_id) - if profile is None or profile.max_streams > 0: - current = int(redis_client.get(profile_key) or 0) - if current > 0: - redis_client.decr(profile_key) + current = int(redis_client.get(profile_key) or 0) + if current > 0: + redis_client.decr(profile_key) - if profile and not released_via_stored_key: - _release_server_group_slot_for_profile(profile, redis_client) + if released_via_stored_key: + return + + # Legacy fallback for reservations made before credential release keys were stored. + from apps.m3u.models import M3UAccountProfile + + try: + profile = M3UAccountProfile.objects.select_related( + "m3u_account__server_group" + ).get(id=profile_id) + except M3UAccountProfile.DoesNotExist: + return + + _release_server_group_slot_for_profile(profile, redis_client) diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 5b05535a..443c5bd8 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -186,6 +186,18 @@ class PoolEnforcementTests(TestCase): self.profile.max_streams = 1 self.profile.save() + def test_group_has_capacity_reads_credential_counter_directly(self): + cred_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) + self.redis.set(cred_key, 1) + + self.assertFalse(group_has_capacity_for_profile(self.profile, self.redis)) + + self.redis.set(cred_key, 0) + self.assertTrue(group_has_capacity_for_profile(self.profile, self.redis)) + def test_reserve_and_release_both_counters(self): reserved, count, _ = reserve_profile_slot(self.profile, self.redis) self.assertTrue(reserved) @@ -301,6 +313,22 @@ class PoolEnforcementTests(TestCase): self.assertEqual(self.redis._data[cred_key], 0) self.assertNotIn(profile_credential_release_key(profile_id), self.redis._data) + def test_release_uses_stored_credential_key_without_db_lookup(self): + profile_id = self.profile.id + cred_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) + + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + + with patch("apps.m3u.models.M3UAccountProfile.objects.get") as mock_get: + release_profile_slot(profile_id, self.redis) + mock_get.assert_not_called() + + self.assertEqual(self.redis._data[profile_connections_key(profile_id)], 0) + self.assertEqual(self.redis._data[cred_key], 0) + def test_reserve_returns_failure_reason_without_extra_checks(self): account2 = M3UAccount.objects.create( name="Reason Account", diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index f52a5fbb..9bd1d284 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -501,15 +501,7 @@ class ChannelStatus: m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE) if m3u_profile_id: try: - profile_id_int = int(m3u_profile_id) - info['m3u_profile_id'] = profile_id_int - try: - from apps.m3u.models import M3UAccountProfile - m3u_profile = M3UAccountProfile.objects.filter(id=profile_id_int).first() - if m3u_profile: - info['m3u_profile_name'] = m3u_profile.name - except Exception as e: - logger.warning(f"Failed to get M3U profile name for ID {profile_id_int}: {e}") + info['m3u_profile_id'] = int(m3u_profile_id) except ValueError: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 899ffc9a..7cdc92ef 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -203,6 +203,8 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] Returns: dict: Stream information including URL, user agent and transcode flag """ + slot_reserved = False + channel = None try: from core.utils import RedisClient @@ -268,28 +270,18 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] m3u_profile_id = selected_profile.id else: - stream_id, m3u_profile_id, error_reason, _slot_reserved = channel.get_stream() + stream_id, m3u_profile_id, error_reason, slot_reserved = channel.get_stream() if stream_id is None or m3u_profile_id is None: return {'error': error_reason or 'No stream assigned to channel'} - # Get the stream and profile objects directly stream = get_object_or_404(Stream, pk=stream_id) profile = get_object_or_404(M3UAccountProfile, pk=m3u_profile_id) - # Check connections left m3u_account = M3UAccount.objects.get(id=profile.m3u_account.id) - #connections_left = get_connections_left(m3u_profile_id) - - #if connections_left <= 0: - #logger.warning(f"No connections left for M3U account {m3u_account.id}") - #return {'error': 'No connections left'} - - # Get the user agent from the M3U account user_agent = m3u_account.get_user_agent().user_agent stream_url = _resolve_live_stream_url(stream, m3u_account, profile) - # Get transcode info from the channel's stream profile stream_profile = channel.get_stream_profile() transcode = not (stream_profile.is_proxy() or stream_profile is None) profile_value = stream_profile.id @@ -304,6 +296,8 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] 'stream_name': stream.name, } except Exception as e: + if slot_reserved and channel is not None: + channel.release_stream() logger.error(f"Error getting stream info for switch: {e}", exc_info=True) return {'error': f'Error: {str(e)}'} diff --git a/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx b/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx index a60ae4db..c35b1be1 100644 --- a/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx +++ b/frontend/src/components/__tests__/ServerGroupsManagerModal.test.jsx @@ -16,6 +16,8 @@ vi.mock('../../store/serverGroups', () => ({ { id: 1, name: 'Pool A' }, { id: 2, name: 'Pool B' }, ], + isLoading: false, + error: null, fetchServerGroups: vi.fn().mockResolvedValue(undefined), }), })); diff --git a/frontend/src/components/tables/ServerGroupsTable.jsx b/frontend/src/components/tables/ServerGroupsTable.jsx index 54500294..ee3d07e4 100644 --- a/frontend/src/components/tables/ServerGroupsTable.jsx +++ b/frontend/src/components/tables/ServerGroupsTable.jsx @@ -56,6 +56,8 @@ const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { const suppressWarning = useWarningsStore((s) => s.suppressWarning); const serverGroups = useServerGroupsStore((state) => state.serverGroups); + const isLoading = useServerGroupsStore((state) => state.isLoading); + const error = useServerGroupsStore((state) => state.error); const fetchServerGroups = useServerGroupsStore( (state) => state.fetchServerGroups ); @@ -106,8 +108,6 @@ const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { [] ); - const [isLoading, setIsLoading] = useState(true); - const editServerGroup = (group = null) => { setServerGroup(group); setServerGroupModalOpen(true); @@ -147,7 +147,7 @@ const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { }; useEffect(() => { - fetchServerGroups().finally(() => setIsLoading(false)); + fetchServerGroups(); }, [fetchServerGroups]); useEffect(() => { @@ -179,7 +179,6 @@ const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { columns, data: tableData, allRowIds: tableData.map((group) => group.id), - enableColumnResizing: false, bodyCellRenderFns: { actions: renderBodyCell, }, @@ -198,6 +197,16 @@ const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { ); } + if (error) { + return ( +
+ + {error} + +
+ ); + } + return ( diff --git a/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx b/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx index 73719512..3896d133 100644 --- a/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx +++ b/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx @@ -3,6 +3,16 @@ import { render, screen, waitFor } from '@testing-library/react'; import { MantineProvider } from '@mantine/core'; import ServerGroupsTable from '../ServerGroupsTable'; +const mockServerGroupsState = vi.hoisted(() => ({ + serverGroups: [ + { id: 1, name: 'Pool A' }, + { id: 2, name: 'Pool B' }, + ], + isLoading: false, + error: null, + fetchServerGroups: vi.fn().mockResolvedValue(undefined), +})); + const renderTable = () => render( @@ -17,14 +27,7 @@ vi.mock('../../../api', () => ({ })); vi.mock('../../../store/serverGroups', () => ({ - default: (selector) => - selector({ - serverGroups: [ - { id: 1, name: 'Pool A' }, - { id: 2, name: 'Pool B' }, - ], - fetchServerGroups: vi.fn().mockResolvedValue(undefined), - }), + default: (selector) => selector(mockServerGroupsState), })); vi.mock('../../../store/playlists', () => ({ @@ -44,6 +47,15 @@ vi.mock('../../forms/ServerGroup', () => ({ describe('ServerGroupsTable', () => { beforeEach(() => { vi.clearAllMocks(); + mockServerGroupsState.serverGroups = [ + { id: 1, name: 'Pool A' }, + { id: 2, name: 'Pool B' }, + ]; + mockServerGroupsState.isLoading = false; + mockServerGroupsState.error = null; + mockServerGroupsState.fetchServerGroups = vi + .fn() + .mockResolvedValue(undefined); }); it('renders server groups without tooltip errors', async () => { @@ -55,4 +67,17 @@ describe('ServerGroupsTable', () => { expect(screen.getByText('Accounts')).toBeInTheDocument(); }); }); + + it('shows store error message when fetch fails', async () => { + mockServerGroupsState.serverGroups = []; + mockServerGroupsState.error = 'Failed to load server groups.'; + + renderTable(); + + await waitFor(() => { + expect( + screen.getByText('Failed to load server groups.') + ).toBeInTheDocument(); + }); + }); }); From c6b23690f4e4aedbe15b748cdda841e7e0dcbeec Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 18:09:55 -0500 Subject: [PATCH 14/81] changelog: enhance M3U connection management and server group features --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22b27745..19ff2f53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Manual Server Groups for shared M3U connection limits.** The existing `ServerGroup` model and account FK are now wired into live and VOD playback. Accounts assigned to the same group share a credential-scoped Redis counter when their provider logins match (hashed fingerprint); unrelated logins in the same group keep separate counters. Enforcement uses each profile's `max_streams` - not a group-wide cap - and profiles with `max_streams=0` skip credential pooling for that profile while still rotating on their own per-profile counter. New `apps/m3u/connection_pool.py` centralizes reserve/release, profile rotation, and credential moves on profile switch. (Closes #1137) — Thanks [@Goldenfreddy0703](https://github.com/Goldenfreddy0703) + - **Server Groups manager UI** on the M3U Accounts page: create, rename, and delete groups with account counts and delete confirmation. Groups load on login via a new Zustand store and REST helpers (`/api/m3u/server-groups/`). + - **M3U account form** adds a Server Group picker (including inline “add group”), reorganized into three columns (source/auth, connection limits, sync/content), and a Manage server groups shortcut. + - **VOD profile selection** uses `pool_has_capacity_for_profile()` so grouped accounts respect shared credential limits before a profile is chosen (non-grouped accounts behave as before). + - **Live profile switches** move the shared credential counter when the new profile uses a different provider login; same-login switches leave the credential counter unchanged. + - **Credential release keys** stored at reserve time allow counters to be released even if the M3U profile row is deleted afterward. + ### Changed +- **Live and VOD connection slot logic now routes through `connection_pool`.** `Channel.get_stream()`, direct `Stream.get_stream()`, VOD profile selection, and the multi-worker VOD connection manager all call `reserve_profile_slot()` / `release_profile_slot()` instead of inline Redis INCR/DECR. VOD profile selection also checks `pool_has_capacity_for_profile()` before choosing a profile. +- **Live XC upstream URLs use current credentials.** `_resolve_live_stream_url()` builds `/live/{user}/{pass}/{stream_id}.ts` from transformed account credentials and the stream's provider `stream_id`, so playback stays aligned with the active login after credential or profile changes instead of reusing a stale `stream.url` from sync. +- **Channel stream switches check pooled capacity.** `get_stream_info_for_switch()` uses `profile_available_for_channel_switch()` when targeting a specific stream, and releases a reserved slot if URL assembly fails after `get_stream()` allocated one. +- **Live proxy init reads Redis assignment once.** After `generate_stream_url()` reserves a slot, the stream handler reads `channel_stream` / `stream_profile` from Redis instead of calling `get_stream()` again, avoiding double INCR under concurrent release. - **EPG auto-match overhaul** — matching logic moved to `apps/channels/epg_matching.py`; Celery tasks in `tasks.py` are thin wrappers. - Single-channel auto-match is now asynchronous: the API returns `202 Accepted` and pushes the result over WebSocket (`single_channel_epg_match`), so large EPG libraries no longer hit the previous 30-second HTTP timeout. - Progress, bulk completion, and single-channel results use `send_websocket_update` instead of `async_to_sync(channel_layer.group_send)`, so notifications work reliably under gevent-patched uWSGI and Celery workers. @@ -27,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** - Memory could spike to multiple GB on large EPG sources when building a full in-memory catalog before fuzzy matching; single-channel matching now streams rows and bounds ML work to a small candidate set. - Wrong channel assignments from global ML similarity; ML validation now checks the fuzzy best match (or top fuzzy candidates as a last resort) instead of scoring the entire catalog. From 95975d5756858da2597a3830cd6a2c92d20beb4e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 18:38:52 -0500 Subject: [PATCH 15/81] refactor(connection_pool): streamline credential slot management - Removed redundant logic for releasing server group slots in `move_credential_slot_on_profile_switch` and `release_profile_slot`. - Simplified the `_release_server_group_slot_for_profile` function by eliminating it, as its functionality was integrated into other methods. --- apps/m3u/connection_pool.py | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 0fa0864f..375714ee 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -209,9 +209,7 @@ def move_credential_slot_on_profile_switch( if old_fp == new_fp: return True - released = _release_credential_slot_by_profile_id(old_profile.id, redis_client) - if not released: - _release_server_group_slot_for_profile(old_profile, redis_client) + _release_credential_slot_by_profile_id(old_profile.id, redis_client) cred_reserved, cred_key = _reserve_server_group_slot_for_profile( new_profile, redis_client @@ -279,15 +277,6 @@ def _reserve_server_group_slot_for_profile( return False, None -def _release_server_group_slot_for_profile(profile, redis_client) -> None: - group = get_enforced_server_group_for_profile(profile) - if not group or profile.max_streams == 0: - return - cred_key = _credential_counter_key(profile, group) - if cred_key: - _safe_decr(redis_client, cred_key) - - def reserve_profile_slot( profile, redis_client ) -> Tuple[bool, int, Optional[ReserveFailureReason]]: @@ -326,26 +315,9 @@ def reserve_profile_slot( def release_profile_slot(profile_id: int, redis_client) -> None: """Release profile and shared credential slots after a stream end.""" - released_via_stored_key = _release_credential_slot_by_profile_id( - profile_id, redis_client - ) + _release_credential_slot_by_profile_id(profile_id, redis_client) profile_key = profile_connections_key(profile_id) current = int(redis_client.get(profile_key) or 0) if current > 0: redis_client.decr(profile_key) - - if released_via_stored_key: - return - - # Legacy fallback for reservations made before credential release keys were stored. - from apps.m3u.models import M3UAccountProfile - - try: - profile = M3UAccountProfile.objects.select_related( - "m3u_account__server_group" - ).get(id=profile_id) - except M3UAccountProfile.DoesNotExist: - return - - _release_server_group_slot_for_profile(profile, redis_client) From 887a177581edf6280d24c7df7fadbef1b0dec063 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 10 Jun 2026 08:28:59 -0400 Subject: [PATCH 16/81] proxy SD country list fetch through backend to fix CORS failure --- apps/epg/api_views.py | 27 ++++++++++++++++++- frontend/src/api.js | 8 ++++++ frontend/src/components/forms/EPG.jsx | 22 +++++++-------- .../components/forms/__tests__/EPG.test.jsx | 1 + 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 547be152..58ab0334 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -53,7 +53,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): try: return [perm() for perm in permission_classes_by_action[self.action]] except KeyError: - if self.action in ('sd_lineups', 'sd_lineups_search'): + if self.action in ('sd_lineups', 'sd_lineups_search', 'sd_countries'): if self.request.method == 'GET': return [IsStandardUser()] return [IsAdmin()] @@ -390,6 +390,31 @@ class EPGSourceViewSet(viewsets.ModelViewSet): status=status.HTTP_502_BAD_GATEWAY ) + @action(detail=True, methods=["get"], url_path="sd-countries") + def sd_countries(self, request, pk=None): + """Proxy /available/countries from the SD API to avoid browser CORS restrictions.""" + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + + source = self.get_object() + if source.source_type != 'schedules_direct': + return Response( + {"error": "This action is only available for Schedules Direct sources."}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + resp = http_requests.get( + f"{SD_BASE_URL}/available/countries", + timeout=15, + ) + resp.raise_for_status() + return Response(resp.json()) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to fetch countries: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY, + ) + @action(detail=True, methods=["post"], url_path="sd-lineups/search") def sd_lineups_search(self, request, pk=None): """ diff --git a/frontend/src/api.js b/frontend/src/api.js index 1c826465..445e7e82 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3889,6 +3889,14 @@ export default class API { } } + static async getSDCountries(sourceId) { + try { + return await request(`${host}/api/epg/sources/${sourceId}/sd-countries/`); + } catch (e) { + return null; + } + } + static async getSDLineups(sourceId) { try { const response = await request( diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index f7e6ec92..ebfea46b 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -325,18 +325,16 @@ const SDLineupManager = ({ sourceId }) => { useEffect(() => { if (sourceId) { fetchActiveLineups(); - // Fetch country list from SD API per their recommendation to not hardcode - fetch('https://json.schedulesdirect.org/20141201/available/countries') - .then((r) => r.json()) - .then((data) => { - const all = Object.values(data).flat(); - const mapped = all - .filter((c) => c.shortName && c.fullName) - .map((c) => ({ value: c.shortName, label: c.fullName })) - .sort((a, b) => a.label.localeCompare(b.label)); - if (mapped.length > 0) setCountries(mapped); - }) - .catch(() => {}); // fallback list remains if fetch fails + // Fetch country list via backend proxy to avoid browser CORS restrictions + API.getSDCountries(sourceId).then((data) => { + if (!data) return; + const all = Object.values(data).flat(); + const mapped = all + .filter((c) => c.shortName && c.fullName) + .map((c) => ({ value: c.shortName, label: c.fullName })) + .sort((a, b) => a.label.localeCompare(b.label)); + if (mapped.length > 0) setCountries(mapped); + }); } }, [sourceId, fetchActiveLineups]); diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index 1e64a563..66ead589 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -8,6 +8,7 @@ global.fetch = vi.fn().mockResolvedValue({ vi.mock('../../../api.js', () => ({ default: { + getSDCountries: vi.fn().mockResolvedValue({}), getSDLineups: vi.fn().mockResolvedValue([]), addSDLineup: vi.fn().mockResolvedValue({ success: true }), deleteSDLineup: vi.fn().mockResolvedValue({ success: true }), From bfa70193241cbfdafb916b46b7cbab22093fd3c7 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 10 Jun 2026 10:01:47 -0400 Subject: [PATCH 17/81] force-fetch SD schedules for newly mapped stations with no existing ProgramData --- apps/epg/tasks.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index d868dfeb..6da11218 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2665,6 +2665,49 @@ def fetch_schedules_direct(source, stations_only=False, force=False): send_epg_update(source.id, "parsing_programs", 38, message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") + # Force-fetch for mapped stations with no existing ProgramData. + # The MD5 cache stores hashes for all lineup stations, including unmapped ones. + # If a station was in the lineup (MD5 cached) but not yet mapped to a channel + # (no ProgramData created), a later mapping causes those cached dates to appear + # "unchanged", skipping the fetch and leaving the channel with no guide data. + mapped_epg_ids_early = set( + Channel.objects.filter( + epg_data__epg_source=source, epg_data__isnull=False + ).values_list('epg_data_id', flat=True) + ) + mapped_tvg_ids_early = set( + EPGData.objects.filter( + id__in=mapped_epg_ids_early, epg_source=source + ).values_list('tvg_id', flat=True) + ) + newly_mapped = set() + if mapped_tvg_ids_early: + stations_with_data = set( + ProgramData.objects.filter( + epg__epg_source=source, + tvg_id__in=mapped_tvg_ids_early, + ).values_list('tvg_id', flat=True).distinct() + ) + newly_mapped = mapped_tvg_ids_early - stations_with_data + if newly_mapped: + server_dates_by_sid = {} + for (sid, ds) in server_md5s: + if sid in newly_mapped: + server_dates_by_sid.setdefault(sid, set()).add(ds) + forced_count = 0 + for sid in newly_mapped: + available = server_dates_by_sid.get(sid, set()) + already_changing = set(changed_by_station.get(sid, [])) + force_dates = [ds for ds in date_list if ds in available and ds not in already_changing] + if force_dates: + changed_by_station.setdefault(sid, []).extend(force_dates) + forced_count += len(force_dates) + if forced_count: + logger.info( + f"Force-fetching {forced_count} station/date combinations for " + f"{len(newly_mapped)} stations with no existing ProgramData (newly mapped channels)." + ) + # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} schedules_by_station = {sid: [] for sid in station_ids} program_ids_needed = set() @@ -2814,6 +2857,16 @@ def fetch_schedules_direct(source, stations_only=False, force=False): ).only('program_id', 'md5') } + # For newly mapped stations with no ProgramData, invalidate cached program MD5s so + # metadata gets re-fetched. Without this, programs whose MD5s were cached when the + # station was unmapped would be skipped, producing "No Title" records. + if newly_mapped: + for sid in newly_mapped: + for airing in schedules_by_station.get(sid, []): + pid = airing.get('programID') + if pid: + cached_prog_md5s.pop(pid, None) + # Only fetch programs where MD5 differs from our cached value programs_to_fetch = { pid for pid in program_ids_needed From ed364a26c9884fbabb7885f7eebd22ef4c92c66c Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 09:00:35 -0500 Subject: [PATCH 18/81] refactor(EPGSourceViewSet, frontend): streamline SD countries fetching and remove proxy endpoint - Introduced a new method `_fetch_sd_countries` in `EPGSourceViewSet` to fetch SD countries directly from the backend, eliminating the need for a separate proxy endpoint. - Updated the `sd_lineups` action to include the fetched countries in the response. - Removed the `getSDCountries` method from the API and adjusted the frontend to fetch countries alongside lineups, improving efficiency - Updated related components to handle the new data structure for countries. --- apps/epg/api_views.py | 48 +++++++++---------- frontend/src/api.js | 8 ---- frontend/src/components/forms/EPG.jsx | 24 +++++----- .../components/forms/__tests__/EPG.test.jsx | 22 +++++++-- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 58ab0334..f9810a04 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -53,7 +53,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): try: return [perm() for perm in permission_classes_by_action[self.action]] except KeyError: - if self.action in ('sd_lineups', 'sd_lineups_search', 'sd_countries'): + if self.action in ('sd_lineups', 'sd_lineups_search'): if self.request.method == 'GET': return [IsStandardUser()] return [IsAdmin()] @@ -227,6 +227,24 @@ class EPGSourceViewSet(viewsets.ModelViewSet): f"Lockout set until {reset_at.isoformat()}." ) + def _fetch_sd_countries(self): + """Fetch the SD country list (token not required; User-Agent is).""" + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + try: + resp = http_requests.get( + f"{SD_BASE_URL}/available/countries", + headers={'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, + timeout=15, + ) + resp.raise_for_status() + return resp.json() + except http_requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch SD countries: {e}") + return None + @action(detail=True, methods=["get", "post", "delete"], url_path="sd-lineups") def sd_lineups(self, request, pk=None): """ @@ -256,6 +274,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): } if request.method == "GET": + countries = self._fetch_sd_countries() try: resp = http_requests.get( f"{SD_BASE_URL}/lineups", @@ -272,6 +291,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): "changes_remaining": self._get_sd_changes_remaining(source), "changes_reset_at": self._get_sd_reset_at(source), "notice": "No lineups are currently configured on this Schedules Direct account. Use the search below to add one.", + "countries": countries, }) resp.raise_for_status() data = resp.json() @@ -281,6 +301,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): "max_lineups": 4, "changes_remaining": self._get_sd_changes_remaining(source), "changes_reset_at": self._get_sd_reset_at(source), + "countries": countries, }) except http_requests.exceptions.RequestException as e: return Response( @@ -390,31 +411,6 @@ class EPGSourceViewSet(viewsets.ModelViewSet): status=status.HTTP_502_BAD_GATEWAY ) - @action(detail=True, methods=["get"], url_path="sd-countries") - def sd_countries(self, request, pk=None): - """Proxy /available/countries from the SD API to avoid browser CORS restrictions.""" - import requests as http_requests - from apps.epg.tasks import SD_BASE_URL - - source = self.get_object() - if source.source_type != 'schedules_direct': - return Response( - {"error": "This action is only available for Schedules Direct sources."}, - status=status.HTTP_400_BAD_REQUEST, - ) - try: - resp = http_requests.get( - f"{SD_BASE_URL}/available/countries", - timeout=15, - ) - resp.raise_for_status() - return Response(resp.json()) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Failed to fetch countries: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY, - ) - @action(detail=True, methods=["post"], url_path="sd-lineups/search") def sd_lineups_search(self, request, pk=None): """ diff --git a/frontend/src/api.js b/frontend/src/api.js index 445e7e82..1c826465 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3889,14 +3889,6 @@ export default class API { } } - static async getSDCountries(sourceId) { - try { - return await request(`${host}/api/epg/sources/${sourceId}/sd-countries/`); - } catch (e) { - return null; - } - } - static async getSDLineups(sourceId) { try { const response = await request( diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index ebfea46b..7b8118de 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -29,7 +29,7 @@ import { showNotification } from '../../utils/notificationUtils.js'; import API from '../../api.js'; import useEPGsStore from '../../store/epgs'; -// Countries are fetched dynamically from the SD API on component mount. +// Countries are fetched with lineups from the backend on component mount. // Fallback list used if the API call fails. const SD_COUNTRIES_FALLBACK = [ { value: 'USA', label: 'United States' }, @@ -48,6 +48,16 @@ const SD_COUNTRIES_FALLBACK = [ { value: 'NZL', label: 'New Zealand' }, ]; +const mapSDCountries = (data) => { + if (!data) return null; + const all = Object.values(data).flat(); + const mapped = all + .filter((c) => c.shortName && c.fullName) + .map((c) => ({ value: c.shortName, label: c.fullName })) + .sort((a, b) => a.label.localeCompare(b.label)); + return mapped.length > 0 ? mapped : null; +}; + // ESPN HD logo previews — packaged as static URLs so no API call is needed. // These are publicly accessible S3 URLs that don't require authentication. const SD_LOGO_PREVIEW_BASE = @@ -309,6 +319,8 @@ const SDLineupManager = ({ sourceId }) => { if (data) { setActiveLineups(data.lineups || []); setLineupNotice(data.notice || null); + const mappedCountries = mapSDCountries(data.countries); + if (mappedCountries) setCountries(mappedCountries); // Always update changesRemaining from server — includes null (unknown) and 0 (locked) if (data.changes_remaining !== undefined) { setChangesRemaining(data.changes_remaining); @@ -325,16 +337,6 @@ const SDLineupManager = ({ sourceId }) => { useEffect(() => { if (sourceId) { fetchActiveLineups(); - // Fetch country list via backend proxy to avoid browser CORS restrictions - API.getSDCountries(sourceId).then((data) => { - if (!data) return; - const all = Object.values(data).flat(); - const mapped = all - .filter((c) => c.shortName && c.fullName) - .map((c) => ({ value: c.shortName, label: c.fullName })) - .sort((a, b) => a.label.localeCompare(b.label)); - if (mapped.length > 0) setCountries(mapped); - }); } }, [sourceId, fetchActiveLineups]); diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index 66ead589..0597f0a0 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -8,7 +8,6 @@ global.fetch = vi.fn().mockResolvedValue({ vi.mock('../../../api.js', () => ({ default: { - getSDCountries: vi.fn().mockResolvedValue({}), getSDLineups: vi.fn().mockResolvedValue([]), addSDLineup: vi.fn().mockResolvedValue({ success: true }), deleteSDLineup: vi.fn().mockResolvedValue({ success: true }), @@ -216,12 +215,20 @@ vi.mock('@mantine/core', async () => ({ {(data ?? []).map((opt) => { const val = typeof opt === 'string' ? opt : opt.value; const lbl = typeof opt === 'string' ? opt : opt.label; - return ; + return ( + + ); })} ), Loader: ({ size }) =>
, - Badge: ({ children, color }) => {children}, + Badge: ({ children, color }) => ( + + {children} + + ), ScrollArea: ({ children }) =>
{children}
, Table: ({ children }) => {children}
, Tooltip: ({ children }) =>
{children}
, @@ -238,10 +245,15 @@ vi.mock('@mantine/core', async () => ({ ), UnstyledButton: ({ children, onClick, ...props }) => ( - + ), Alert: ({ children, title, color, icon }) => ( -
{title}{children}
+
+ {title} + {children} +
), Stack: ({ children, gap }) =>
{children}
, Text: ({ children, ...props }) => {children}, From 843940f5525cac416005ae826048435176f182ad Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 09:45:35 -0500 Subject: [PATCH 19/81] refactor(http headers): centralize User-Agent construction and streamline API requests - Introduced `dispatcharr_user_agent`, `dispatcharr_dvr_user_agent`, and `dispatcharr_http_headers` functions in `core.utils` to standardize User-Agent strings and HTTP headers for outbound requests. - Updated various components, including `LogoViewSet`, `EPGSourceViewSet`, and VOD proxy views, to utilize the new header functions, ensuring consistent User-Agent usage across the application. - Enhanced the handling of Schedules Direct API requests by including proper User-Agent headers, addressing previous API compliance issues. --- CHANGELOG.md | 3 +++ apps/channels/api_views.py | 5 +++-- apps/channels/tasks.py | 3 ++- apps/epg/api_views.py | 33 +++++++++++---------------------- apps/epg/tasks.py | 11 ++--------- apps/proxy/vod_proxy/views.py | 3 ++- core/tasks.py | 4 ++-- core/tests.py | 29 +++++++++++++++++++++++++++++ core/utils.py | 27 +++++++++++++++++++++++++++ 9 files changed, 81 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19ff2f53..49637717 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Live XC upstream URLs use current credentials.** `_resolve_live_stream_url()` builds `/live/{user}/{pass}/{stream_id}.ts` from transformed account credentials and the stream's provider `stream_id`, so playback stays aligned with the active login after credential or profile changes instead of reusing a stale `stream.url` from sync. - **Channel stream switches check pooled capacity.** `get_stream_info_for_switch()` uses `profile_available_for_channel_switch()` when targeting a specific stream, and releases a reserved slot if URL assembly fails after `get_stream()` allocated one. - **Live proxy init reads Redis assignment once.** After `generate_stream_url()` reserves a slot, the stream handler reads `channel_stream` / `stream_profile` from Redis instead of calling `get_stream()` again, avoiding double INCR under concurrent release. +- **Centralized Dispatcharr User-Agent construction in `core.utils`.** Outbound HTTP calls (Schedules Direct API, update checks, logo/VOD fallbacks, DVR recording clients) now use `dispatcharr_user_agent()`, `dispatcharr_dvr_user_agent()`, and `dispatcharr_http_headers()` instead of ad-hoc `Dispatcharr/{version}` strings and stale `Dispatcharr/1.0` fallbacks. - **EPG auto-match overhaul** — matching logic moved to `apps/channels/epg_matching.py`; Celery tasks in `tasks.py` are thin wrappers. - Single-channel auto-match is now asynchronous: the API returns `202 Accepted` and pushes the result over WebSocket (`single_channel_epg_match`), so large EPG libraries no longer hit the previous 30-second HTTP timeout. - Progress, bulk completion, and single-channel results use `send_websocket_update` instead of `async_to_sync(channel_layer.group_send)`, so notifications work reliably under gevent-patched uWSGI and Celery workers. @@ -46,6 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Wrong channel assignments from global ML similarity; ML validation now checks the fuzzy best match (or top fuzzy candidates as a last resort) instead of scoring the entire catalog. - Channel form auto-match spinner could stick after errors or early task exits; all single-channel outcomes now push a WebSocket result, and the UI clears loading state after a 3-minute timeout. - Bulk auto-match completion no longer calls `batch-set-epg` from the WebSocket handler, which had been re-applying every match and queueing redundant `parse_programs_for_tvg_id` tasks even when assignments were unchanged. +- **Schedules Direct lineup search country dropdown.** The country list was fetched directly from the SD API in the browser, which failed due to CORS and silently fell back to a 14-country hardcoded list. Countries are now included in the `GET sd-lineups` response (server-side fetch with proper User-Agent) so the full SD country list populates correctly. — Thanks [@sethwv](https://github.com/sethwv) +- **Schedules Direct program poster proxy omitted User-Agent on image requests.** The poster endpoint authenticated with SD correctly but fetched images with only the `token` header, which violates SD's API requirements (error 1003). Image requests now include the standard `Dispatcharr/{version}` User-Agent via `dispatcharr_http_headers()`. ## [0.26.0] - 2026-06-07 diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 965da468..e9711026 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -2800,8 +2800,9 @@ class LogoViewSet(viewsets.ModelViewSet): user_agent_obj = UserAgent.objects.get(id=int(default_user_agent_id)) user_agent = user_agent_obj.user_agent except (CoreSettings.DoesNotExist, UserAgent.DoesNotExist, ValueError): - # Fallback to hardcoded if default not found - user_agent = 'Dispatcharr/1.0' + # Fallback if default not found + from core.utils import dispatcharr_user_agent + user_agent = dispatcharr_user_agent() # Hard total timeout (connect + full download) prevents a slow # server dripping bytes from holding a greenlet indefinitely. diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 79058c3c..1250607f 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1181,12 +1181,13 @@ def _dvr_ffmpeg_retry_backoff_seconds(retry_index): def _dvr_build_ffmpeg_cmd(stream_url, recording_id, hls_m3u8, hls_seg_pattern, hls_start_number): """Build the FFmpeg command for DVR HLS segment recording.""" + from core.utils import dispatcharr_dvr_user_agent return [ "ffmpeg", "-y", "-reconnect", "1", "-reconnect_streamed", "1", "-reconnect_delay_max", "5", - "-user_agent", f"Dispatcharr-DVR/recording-{recording_id}", + "-user_agent", dispatcharr_dvr_user_agent(recording_id), # Regenerate monotonic PTS to handle erratic/discontinuous timestamps # from IPTV sources. "-fflags", "+genpts", diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index f9810a04..b268ef65 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -128,7 +128,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): import hashlib import requests as http_requests from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version + from core.utils import dispatcharr_http_headers username = (source.username or '').strip() password = (source.password or '').strip() @@ -143,7 +143,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): auth_response = http_requests.post( f"{SD_BASE_URL}/token", json={'username': username, 'password': sha1_password}, - headers={'Content-Type': 'application/json', 'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, + headers=dispatcharr_http_headers(), timeout=15, ) auth_response.raise_for_status() @@ -231,12 +231,12 @@ class EPGSourceViewSet(viewsets.ModelViewSet): """Fetch the SD country list (token not required; User-Agent is).""" import requests as http_requests from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version + from core.utils import dispatcharr_http_headers try: resp = http_requests.get( f"{SD_BASE_URL}/available/countries", - headers={'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, + headers=dispatcharr_http_headers(content_type=None), timeout=15, ) resp.raise_for_status() @@ -254,7 +254,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): """ import requests as http_requests from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version + from core.utils import dispatcharr_http_headers source = self.get_object() if source.source_type != 'schedules_direct': @@ -267,11 +267,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): if error: return error - headers = { - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - 'token': token, - } + headers = dispatcharr_http_headers(token=token) if request.method == "GET": countries = self._fetch_sd_countries() @@ -420,7 +416,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): """ import requests as http_requests from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version + from core.utils import dispatcharr_http_headers source = self.get_object() if source.source_type != 'schedules_direct': @@ -441,11 +437,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): if error: return error - headers = { - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - 'token': token, - } + headers = dispatcharr_http_headers(token=token) try: resp = http_requests.get( @@ -514,6 +506,7 @@ class ProgramViewSet(viewsets.ModelViewSet): """Proxy endpoint for SD program poster images. Nginx caches the response.""" import requests as http_requests from apps.epg.tasks import SD_BASE_URL + from core.utils import dispatcharr_http_headers program = self.get_object() poster_sd_url = (program.custom_properties or {}).get('sd_icon') @@ -537,14 +530,10 @@ class ProgramViewSet(viewsets.ModelViewSet): if not token: sha1_password = hashlib.sha1(source.password.encode('utf-8')).hexdigest() try: - from version import __version__ as dispatcharr_version auth_resp = http_requests.post( f"{SD_BASE_URL}/token", json={'username': source.username, 'password': sha1_password}, - headers={ - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - }, + headers=dispatcharr_http_headers(), timeout=10, ) auth_data = auth_resp.json() @@ -570,7 +559,7 @@ class ProgramViewSet(viewsets.ModelViewSet): try: img_resp = http_requests.get( poster_sd_url, - headers={'token': token}, + headers=dispatcharr_http_headers(token=token, content_type=None), timeout=15, allow_redirects=True, ) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index d868dfeb..b7c98e24 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2173,17 +2173,10 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # SD API spec requires the User-Agent to identify the application and version. # SergeantPanda confirmed Dispatcharr should identify itself properly. # ------------------------------------------------------------------------- - from version import __version__ as dispatcharr_version - sd_user_agent = f"Dispatcharr/{dispatcharr_version}" + from core.utils import dispatcharr_http_headers def _sd_headers(token=None): - h = { - 'Content-Type': 'application/json', - 'User-Agent': sd_user_agent, - } - if token: - h['token'] = token - return h + return dispatcharr_http_headers(token=token) def _sd_post_refresh_tasks(mapped_epg_ids, program_metadata, today): """Poster fetch, logo auto-apply, and pruning — runs even when schedules are unchanged.""" diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 27208ec9..f9740aa1 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -23,6 +23,7 @@ from rest_framework_simplejwt.authentication import JWTAuthentication from apps.accounts.authentication import ApiKeyAuthentication, QueryParamJWTAuthentication from apps.proxy.utils import check_user_stream_limits from dispatcharr.utils import network_access_allowed +from core.utils import dispatcharr_user_agent logger = logging.getLogger(__name__) @@ -704,7 +705,7 @@ def head_vod(request, content_type, content_id, session_id=None, profile_id=None # Use M3U account's user agent as primary, client user agent as fallback m3u_user_agent = m3u_account.get_user_agent().user_agent if m3u_account.get_user_agent() else None headers = { - 'User-Agent': m3u_user_agent or client_user_agent or 'Dispatcharr/1.0', + 'User-Agent': m3u_user_agent or client_user_agent or dispatcharr_user_agent(), 'Accept': '*/*', 'Range': 'bytes=0-1' # Request only first 2 bytes } diff --git a/core/tasks.py b/core/tasks.py index e9d79d37..b562f335 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -796,11 +796,11 @@ def check_for_version_update(): from packaging import version as pkg_version from version import __version__, __timestamp__ from core.models import SystemNotification - from core.utils import send_websocket_notification + from core.utils import dispatcharr_http_headers, send_websocket_notification try: is_dev_build = __timestamp__ is not None - DISPATCHARR_HEADERS = {'User-Agent': f'Dispatcharr/{__version__}'} + DISPATCHARR_HEADERS = dispatcharr_http_headers(content_type=None) if is_dev_build: # Check Docker Hub for newer dev builds diff --git a/core/tests.py b/core/tests.py index e7f45c5a..7b2f1986 100644 --- a/core/tests.py +++ b/core/tests.py @@ -6,6 +6,35 @@ from apps.epg.models import EPGSource from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY +class DispatcharrUserAgentTests(TestCase): + @patch('version.__version__', '1.2.3') + def test_dispatcharr_user_agent(self): + from core.utils import dispatcharr_user_agent + self.assertEqual(dispatcharr_user_agent(), 'Dispatcharr/1.2.3') + + def test_dispatcharr_dvr_user_agent(self): + from core.utils import dispatcharr_dvr_user_agent + self.assertEqual(dispatcharr_dvr_user_agent(42), 'Dispatcharr-DVR/recording-42') + + @patch('version.__version__', '1.2.3') + def test_dispatcharr_http_headers_with_token(self): + from core.utils import dispatcharr_http_headers + headers = dispatcharr_http_headers(token='tok123') + self.assertEqual(headers, { + 'User-Agent': 'Dispatcharr/1.2.3', + 'Content-Type': 'application/json', + 'token': 'tok123', + }) + + @patch('version.__version__', '1.2.3') + def test_dispatcharr_http_headers_without_content_type(self): + from core.utils import dispatcharr_http_headers + self.assertEqual( + dispatcharr_http_headers(content_type=None), + {'User-Agent': 'Dispatcharr/1.2.3'}, + ) + + class ProgrammeIndexRebuildTests(TestCase): def test_startup_rebuild_does_not_lock_out_queued_build_task(self): EPGSource.objects.update( diff --git a/core/utils.py b/core/utils.py index 6a13c472..c0a70987 100644 --- a/core/utils.py +++ b/core/utils.py @@ -21,6 +21,33 @@ logger = logging.getLogger(__name__) # Import the command detector from .command_utils import is_management_command + +def dispatcharr_user_agent(): + """Return the standard Dispatcharr User-Agent string (Dispatcharr/{version}).""" + from version import __version__ + return f'Dispatcharr/{__version__}' + + +def dispatcharr_dvr_user_agent(recording_id): + """Return the User-Agent string used by DVR FFmpeg clients for a recording.""" + return f'Dispatcharr-DVR/recording-{recording_id}' + + +def dispatcharr_http_headers(*, token=None, content_type='application/json'): + """ + Build HTTP headers for outbound Dispatcharr requests. + + content_type=None omits Content-Type (e.g. simple GET proxies). + token is included when authenticating with Schedules Direct. + """ + headers = {'User-Agent': dispatcharr_user_agent()} + if content_type: + headers['Content-Type'] = content_type + if token: + headers['token'] = token + return headers + + def natural_sort_key(text): """ Convert a string into a list of string and number chunks for natural sorting. From 205deb150beb63c33849863002fe48937ca6cd1d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 12:31:30 -0500 Subject: [PATCH 20/81] Implement schedule MD5 delta detection and backfill logic for improved cache management - Added functions to compute schedule changes based on MD5 hashes and backfill missing dates for mapped stations. - Enhanced handling of orphaned ProgramData records for unmapped EPG entries during schedule fetch. - Introduced unit tests to validate MD5 comparison, backfill logic, and metadata fetching requirements. --- apps/epg/tasks.py | 309 +++++++++------ apps/epg/tests/test_schedules_direct.py | 474 +++++++++++++++++++++++- 2 files changed, 675 insertions(+), 108 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 6da11218..9d244e79 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -35,6 +35,73 @@ SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' SD_DAYS_TO_FETCH = 20 SD_PROGRAM_BATCH_SIZE = 5000 + +def _sd_compute_schedule_changes_from_md5(server_md5s, cached_md5s, date_list): + """Return station_id -> [date_str] for dates whose schedule MD5 differs from cache.""" + changed_by_station = {} + for (sid, date_str), server_info in server_md5s.items(): + if date_str not in date_list: + continue + cached = cached_md5s.get((sid, date_str)) + if cached != server_info['md5']: + changed_by_station.setdefault(sid, []).append(date_str) + return changed_by_station + + +def _sd_backfill_schedule_dates_without_data( + changed_by_station, + server_md5s, + date_list, + mapped_station_ids, + epg_id_map, + dates_with_data, + cached_md5s, + stations_without_any_data, +): + """ + Add fetch-window dates that lack ProgramData to changed_by_station. + + Dates with a cached schedule MD5 are treated as already fetched (e.g. legitimately + empty airings). Stations with zero ProgramData still backfill all missing dates + so stale cache from unmapped lineup refreshes cannot block guide population. + """ + from datetime import date as date_type + + stations_without_any_data = set(stations_without_any_data) + backfilled_count = 0 + for sid in mapped_station_ids: + epg_db_id = epg_id_map.get(sid) + if not epg_db_id: + continue + force_despite_cache = sid in stations_without_any_data + already_changing = set(changed_by_station.get(sid, [])) + for ds in date_list: + if ds in already_changing or (sid, ds) not in server_md5s: + continue + if (epg_db_id, date_type.fromisoformat(ds)) in dates_with_data: + continue + if (sid, ds) in cached_md5s and not force_despite_cache: + continue + changed_by_station.setdefault(sid, []).append(ds) + backfilled_count += 1 + return backfilled_count + + +def _sd_programs_needing_metadata( + program_ids_needed, + schedule_program_md5s, + cached_prog_md5s, + programs_with_data, +): + """Return programIDs that need metadata download from Schedules Direct.""" + programs_with_data = set(programs_with_data) + return { + pid for pid in program_ids_needed + if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) + or pid not in programs_with_data + } + + SD_POSTER_CATEGORIES = ( 'Iconic', 'Banner-L1', 'Banner-L2', 'Banner-L3', 'Banner', 'Staple', 'Poster Art', 'Box Art', @@ -2070,6 +2137,8 @@ def parse_programs_for_source(epg_source, tvg_id=None): logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") # Explicitly clear the process object to prevent potential memory leaks process = None + + @shared_task(bind=True) def fetch_schedules_direct_stations(self, source_id): """ @@ -2299,6 +2368,24 @@ def fetch_schedules_direct(source, stations_only=False, force=False): from apps.channels.utils import maybe_auto_apply_epg_logos maybe_auto_apply_epg_logos(source) + try: + unmapped_epg_ids = list( + EPGData.objects.filter(epg_source=source).exclude( + id__in=mapped_epg_ids, + ).values_list('id', flat=True) + ) + if unmapped_epg_ids: + orphaned_count = ProgramData.objects.filter( + epg_id__in=unmapped_epg_ids, + ).delete()[0] + if orphaned_count: + logger.info( + f"Cleaned up {orphaned_count} orphaned ProgramData records " + f"for {len(unmapped_epg_ids)} unmapped EPG entries." + ) + except Exception as prune_err: + logger.warning(f"Failed to clean up orphaned SD ProgramData: {prune_err}") + today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) try: expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] @@ -2598,32 +2685,77 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # ------------------------------------------------------------------------- # Step 5: MD5-delta schedule fetch - # First fetch MD5 hashes for all stations/dates. Compare against our - # locally cached hashes to determine which schedules have changed. - # Only download schedules that have actually changed; this minimises - # API calls against SD's rate-limited endpoints. + # Only mapped channels need guide data. Fetch MD5 hashes and schedules for + # mapped stations only; never cache schedule MD5s for unmapped lineup entries. # ------------------------------------------------------------------------- - from apps.epg.models import SDScheduleMD5 from django.utils.dateparse import parse_datetime - send_epg_update(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") station_ids = list(station_map.keys()) today = date.today() date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] - # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. - # These accumulate one row per station per day and are never useful once past. - pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] + mapped_epg_ids = set( + Channel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) + ) + mapped_tvg_ids = set( + EPGData.objects.filter( + id__in=mapped_epg_ids, + epg_source=source, + ).values_list('tvg_id', flat=True) + ) + mapped_station_ids = [sid for sid in station_ids if sid in mapped_tvg_ids] + + # Prune expired schedule MD5s and drop cache for unmapped stations. + pruned_sched_md5_count = SDScheduleMD5.objects.filter( + epg_source=source, date__lt=today, + ).delete()[0] if pruned_sched_md5_count: logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") - # Fetch MD5 hashes for all stations in batches of 5000 + if mapped_tvg_ids: + unmapped_cache_pruned = SDScheduleMD5.objects.filter( + epg_source=source, + ).exclude(station_id__in=mapped_tvg_ids).delete()[0] + else: + unmapped_cache_pruned = SDScheduleMD5.objects.filter(epg_source=source).delete()[0] + if unmapped_cache_pruned: + logger.info(f"Pruned {unmapped_cache_pruned} SDScheduleMD5 records for unmapped lineup stations.") + + if not mapped_station_ids: + logger.info("No channels mapped to this SD source; skipping schedule MD5 check and downloads.") + _sd_post_refresh_tasks(mapped_epg_ids, {}, today) + success_msg = ( + f"{len(station_map)} lineup stations synced. " + "Map channels to EPG entries, then refresh to populate guide data." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + send_epg_update(source.id, "parsing_programs", 100, status="success", message=success_msg) + return + + send_epg_update( + source.id, "parsing_programs", 33, + message=f"Checking schedule MD5s for {len(mapped_station_ids)} mapped stations over {SD_DAYS_TO_FETCH} days...", + ) + + # Fetch MD5 hashes for mapped stations in batches of 5000 STATION_BATCH_SIZE = 5000 server_md5s = {} # (station_id, date) -> {md5, last_modified} - logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") + logger.info( + f"Fetching schedule MD5s for {len(mapped_station_ids)} mapped stations " + f"(of {len(station_ids)} lineup stations) over {SD_DAYS_TO_FETCH} days." + ) - station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] + station_batches = [ + mapped_station_ids[i:i + STATION_BATCH_SIZE] + for i in range(0, len(mapped_station_ids), STATION_BATCH_SIZE) + ] for batch in station_batches: try: md5_response = requests.post( @@ -2644,84 +2776,65 @@ def fetch_schedules_direct(source, stations_only=False, force=False): except requests.exceptions.RequestException as e: logger.warning(f"Failed to fetch schedule MD5s: {e}") - # Load our cached MD5s from DB + # Load our cached MD5s from DB (mapped stations only) cached_md5s = { (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 - for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) + for r in SDScheduleMD5.objects.filter( + epg_source=source, station_id__in=mapped_station_ids, + ) } - # Determine which station/date combinations need downloading - changed_by_station = {} # station_id -> [date_str, ...] - for (sid, date_str), server_info in server_md5s.items(): - if date_str not in date_list: - continue - cached = cached_md5s.get((sid, date_str)) - if cached != server_info['md5']: - changed_by_station.setdefault(sid, []).append(date_str) + changed_by_station = _sd_compute_schedule_changes_from_md5( + server_md5s, cached_md5s, date_list, + ) + + window_start = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) + window_end = window_start + timedelta(days=SD_DAYS_TO_FETCH) + dates_with_data = set() + if mapped_epg_ids: + for epg_id, start_time in ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + start_time__gte=window_start, + start_time__lt=window_end, + ).values_list('epg_id', 'start_time'): + dates_with_data.add((epg_id, start_time.date())) + + stations_without_any_data = mapped_tvg_ids - set( + ProgramData.objects.filter(epg_id__in=mapped_epg_ids) + .values_list('tvg_id', flat=True).distinct() + ) + backfilled_count = _sd_backfill_schedule_dates_without_data( + changed_by_station, + server_md5s, + date_list, + mapped_station_ids, + epg_id_map, + dates_with_data, + cached_md5s, + stations_without_any_data, + ) + if backfilled_count: + logger.info( + f"Backfilling {backfilled_count} station/date combinations with no ProgramData " + f"in the {SD_DAYS_TO_FETCH}-day fetch window." + ) total_changed = sum(len(v) for v in changed_by_station.values()) - total_possible = len(station_ids) * len(date_list) - logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") + total_possible = len(mapped_station_ids) * len(date_list) + logger.info( + f"Schedule MD5 check: {len(server_md5s)} hashes checked, " + f"{total_changed} station/date combinations to fetch (of {total_possible} possible)." + ) send_epg_update(source.id, "parsing_programs", 38, message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") - # Force-fetch for mapped stations with no existing ProgramData. - # The MD5 cache stores hashes for all lineup stations, including unmapped ones. - # If a station was in the lineup (MD5 cached) but not yet mapped to a channel - # (no ProgramData created), a later mapping causes those cached dates to appear - # "unchanged", skipping the fetch and leaving the channel with no guide data. - mapped_epg_ids_early = set( - Channel.objects.filter( - epg_data__epg_source=source, epg_data__isnull=False - ).values_list('epg_data_id', flat=True) - ) - mapped_tvg_ids_early = set( - EPGData.objects.filter( - id__in=mapped_epg_ids_early, epg_source=source - ).values_list('tvg_id', flat=True) - ) - newly_mapped = set() - if mapped_tvg_ids_early: - stations_with_data = set( - ProgramData.objects.filter( - epg__epg_source=source, - tvg_id__in=mapped_tvg_ids_early, - ).values_list('tvg_id', flat=True).distinct() - ) - newly_mapped = mapped_tvg_ids_early - stations_with_data - if newly_mapped: - server_dates_by_sid = {} - for (sid, ds) in server_md5s: - if sid in newly_mapped: - server_dates_by_sid.setdefault(sid, set()).add(ds) - forced_count = 0 - for sid in newly_mapped: - available = server_dates_by_sid.get(sid, set()) - already_changing = set(changed_by_station.get(sid, [])) - force_dates = [ds for ds in date_list if ds in available and ds not in already_changing] - if force_dates: - changed_by_station.setdefault(sid, []).extend(force_dates) - forced_count += len(force_dates) - if forced_count: - logger.info( - f"Force-fetching {forced_count} station/date combinations for " - f"{len(newly_mapped)} stations with no existing ProgramData (newly mapped channels)." - ) - # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} - schedules_by_station = {sid: [] for sid in station_ids} + schedules_by_station = {sid: [] for sid in mapped_station_ids} program_ids_needed = set() if not changed_by_station: logger.info("No schedule changes detected, skipping schedule and program downloads.") - from apps.channels.models import Channel as ChannelModel - mapped_epg_ids_no_change = set( - ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).values_list('epg_data_id', flat=True) - ) - _sd_post_refresh_tasks(mapped_epg_ids_no_change, {}, today) + _sd_post_refresh_tasks(mapped_epg_ids, {}, today) send_epg_update(source.id, "parsing_programs", 100, status="success", message="No schedule changes detected since last refresh. Guide data is up to date.") source.status = EPGSource.STATUS_SUCCESS @@ -2857,21 +2970,21 @@ def fetch_schedules_direct(source, stations_only=False, force=False): ).only('program_id', 'md5') } - # For newly mapped stations with no ProgramData, invalidate cached program MD5s so - # metadata gets re-fetched. Without this, programs whose MD5s were cached when the - # station was unmapped would be skipped, producing "No Title" records. - if newly_mapped: - for sid in newly_mapped: - for airing in schedules_by_station.get(sid, []): - pid = airing.get('programID') - if pid: - cached_prog_md5s.pop(pid, None) + programs_with_data = set() + if program_ids_needed: + programs_with_data = set( + ProgramData.objects.filter( + epg__epg_source=source, + program_id__in=program_ids_needed, + ).values_list('program_id', flat=True).distinct() + ) - # Only fetch programs where MD5 differs from our cached value - programs_to_fetch = { - pid for pid in program_ids_needed - if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) - } + programs_to_fetch = _sd_programs_needing_metadata( + program_ids_needed, + schedule_program_md5s, + cached_prog_md5s, + programs_with_data, + ) logger.info( f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " @@ -2928,22 +3041,6 @@ def fetch_schedules_direct(source, stations_only=False, force=False): logger.info("Building program records...") send_epg_update(source.id, "parsing_programs", 80) - # Only process stations that are mapped to channels to match the existing - # XMLTV flow (parse_programs_for_source only processes mapped channels). - from apps.channels.models import Channel as ChannelModel - mapped_epg_ids = set( - ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).values_list('epg_data_id', flat=True) - ) - mapped_tvg_ids = set( - EPGData.objects.filter( - id__in=mapped_epg_ids, - epg_source=source, - ).values_list('tvg_id', flat=True) - ) - # Cache existing program data for unchanged programs BEFORE surgical delete. # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only # programs with changed program MD5s get metadata re-downloaded. The surgical delete diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index 77334112..47860f92 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -7,19 +7,26 @@ Covers: - fetch_schedules_direct: credential validation - fetch_schedules_direct: SHA1 password hashing and token exchange - fetch_schedules_direct: graceful error handling on auth failure +- fetch_schedules_direct: schedule MD5 delta, backfill, and cache invalidation - parse_schedules_direct_time: correct UTC parsing - EPG signals: SD sources skip the XMLTV program parser """ import hashlib -from datetime import datetime +from datetime import date, datetime, timedelta, timezone as dt_timezone from unittest.mock import MagicMock, patch from django.test import TestCase from django.utils import timezone -from apps.epg.models import EPGSource, EPGData +from apps.channels.models import Channel +from apps.epg.models import EPGSource, EPGData, ProgramData, SDScheduleMD5 from apps.epg.serializers import EPGSourceSerializer +from apps.epg.tasks import ( + _sd_backfill_schedule_dates_without_data, + _sd_compute_schedule_changes_from_md5, + _sd_programs_needing_metadata, +) # --------------------------------------------------------------------------- @@ -270,6 +277,469 @@ class FetchSchedulesDirectStationsOnlyTests(TestCase): self.assertEqual(complete_call[1]['channels_count'], 1) +# --------------------------------------------------------------------------- +# Schedule MD5 delta, backfill, and cache tests +# --------------------------------------------------------------------------- + +class SDScheduleMd5DeltaTests(TestCase): + """Pure-function tests for schedule MD5 comparison.""" + + DATE_LIST = ['2026-06-11', '2026-06-12', '2026-06-13'] + + def test_detects_changed_md5(self): + server_md5s = { + ('10001', '2026-06-11'): {'md5': 'abc', 'last_modified': ''}, + ('10001', '2026-06-12'): {'md5': 'def', 'last_modified': ''}, + } + cached_md5s = { + ('10001', '2026-06-11'): 'abc', + ('10001', '2026-06-12'): 'old', + } + changed = _sd_compute_schedule_changes_from_md5( + server_md5s, cached_md5s, self.DATE_LIST, + ) + self.assertEqual(changed['10001'], ['2026-06-12']) + + def test_missing_cache_treated_as_changed(self): + server_md5s = { + ('10001', '2026-06-11'): {'md5': 'abc', 'last_modified': ''}, + } + changed = _sd_compute_schedule_changes_from_md5(server_md5s, {}, self.DATE_LIST) + self.assertEqual(changed['10001'], ['2026-06-11']) + + def test_ignores_dates_outside_fetch_window(self): + server_md5s = { + ('10001', '2026-06-01'): {'md5': 'abc', 'last_modified': ''}, + } + changed = _sd_compute_schedule_changes_from_md5(server_md5s, {}, self.DATE_LIST) + self.assertEqual(changed, {}) + + +class SDScheduleBackfillTests(TestCase): + """Backfill must fix stale-cache gaps without re-fetching empty cached days.""" + + DATE_LIST = ['2026-06-11', '2026-06-12', '2026-06-13'] + EPG_ID = 42 + EPG_ID_MAP = {'10001': 42} + + def _server_md5s(self): + return { + (sid, ds): {'md5': 'hash', 'last_modified': ''} + for sid in ('10001', '10002') + for ds in self.DATE_LIST + } + + def test_backfills_when_no_cache_and_no_program_data(self): + changed = {} + count = _sd_backfill_schedule_dates_without_data( + changed, + self._server_md5s(), + self.DATE_LIST, + ['10001'], + self.EPG_ID_MAP, + set(), + {}, + {'10001'}, + ) + self.assertEqual(count, 3) + self.assertEqual(len(changed['10001']), 3) + + def test_stale_cache_ignored_when_station_has_zero_program_data(self): + """Newly mapped channel: stale MD5 cache must not block backfill.""" + changed = {} + cached_md5s = { + (sid, ds): 'hash' + for sid in ('10001',) + for ds in self.DATE_LIST + } + count = _sd_backfill_schedule_dates_without_data( + changed, + self._server_md5s(), + self.DATE_LIST, + ['10001'], + self.EPG_ID_MAP, + set(), + cached_md5s, + {'10001'}, + ) + self.assertEqual(count, 3) + self.assertEqual(len(changed['10001']), 3) + + def test_skips_cached_empty_day_when_station_has_other_program_data(self): + """Legitimately empty schedule day must not be re-fetched every refresh.""" + changed = {} + cached_md5s = {('10001', '2026-06-12'): 'hash'} + dates_with_data = {(self.EPG_ID, date(2026, 6, 11))} + count = _sd_backfill_schedule_dates_without_data( + changed, + self._server_md5s(), + self.DATE_LIST, + ['10001'], + self.EPG_ID_MAP, + dates_with_data, + cached_md5s, + set(), + ) + self.assertEqual(count, 1) + self.assertEqual(changed['10001'], ['2026-06-13']) + + def test_does_not_duplicate_dates_already_marked_changed(self): + changed = {'10001': ['2026-06-11']} + count = _sd_backfill_schedule_dates_without_data( + changed, + self._server_md5s(), + self.DATE_LIST, + ['10001'], + self.EPG_ID_MAP, + set(), + {}, + {'10001'}, + ) + self.assertEqual(count, 2) + self.assertEqual(sorted(changed['10001']), self.DATE_LIST) + + +class SDProgramMetadataDeltaTests(TestCase): + def test_fetches_when_md5_changed(self): + needed = _sd_programs_needing_metadata( + {'EP0001'}, + {'EP0001': 'new'}, + {'EP0001': 'old'}, + {'EP0001'}, + ) + self.assertEqual(needed, {'EP0001'}) + + def test_fetches_when_no_local_program_data(self): + needed = _sd_programs_needing_metadata( + {'EP0001', 'EP0002'}, + {'EP0001': 'same', 'EP0002': 'same'}, + {'EP0001': 'same', 'EP0002': 'same'}, + {'EP0001'}, + ) + self.assertEqual(needed, {'EP0002'}) + + def test_skips_when_md5_matches_and_program_data_exists(self): + needed = _sd_programs_needing_metadata( + {'EP0001'}, + {'EP0001': 'same'}, + {'EP0001': 'same'}, + {'EP0001'}, + ) + self.assertEqual(needed, set()) + + +class SDScheduleDeltaIntegrationTests(TestCase): + """DB-backed tests for cache pruning and mapped-only MD5 API calls.""" + + MAPPED_STATION = '10001' + UNMAPPED_STATION = '10002' + + def _make_sd_source(self): + return EPGSource.objects.create( + name='SD Integration', + source_type='schedules_direct', + username='sduser', + password='sdpass', + ) + + def _lineup_get_side_effect(self, url, **kwargs): + if url.endswith('/status'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'systemStatus': [{'status': 'Online'}]}), + ) + if url.endswith('/lineups'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'lineups': [{'lineupID': 'USA-TEST-X'}]}), + ) + if '/lineups/USA-TEST-X' in url: + return MagicMock( + status_code=200, + json=MagicMock(return_value={ + 'stations': [ + { + 'stationID': self.MAPPED_STATION, + 'name': 'Mapped Station', + 'callsign': 'MAP', + }, + { + 'stationID': self.UNMAPPED_STATION, + 'name': 'Unmapped Station', + 'callsign': 'UNM', + }, + ], + }), + ) + raise AssertionError(f'Unexpected GET URL: {url}') + + def _build_date_list(self, days=3): + today = date.today() + return [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(days)] + + def _seed_full_window_program_data(self, epg, days=3): + today = date.today() + for i in range(days): + day = today + timedelta(days=i) + start = datetime(day.year, day.month, day.day, 12, 0, tzinfo=dt_timezone.utc) + ProgramData.objects.create( + epg=epg, + start_time=start, + end_time=start + timedelta(hours=1), + title='Show', + tvg_id=epg.tvg_id, + ) + + @patch('apps.epg.tasks.SD_DAYS_TO_FETCH', 3) + @patch('apps.epg.tasks.send_epg_update') + @patch('apps.epg.tasks.requests.get') + @patch('apps.epg.tasks.requests.post') + def test_md5_api_only_requests_mapped_stations( + self, mock_post, mock_get, mock_send_epg_update, + ): + from apps.epg.tasks import fetch_schedules_direct + + source = self._make_sd_source() + mapped_epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + Channel.objects.create(name='Mapped Ch', epg_data=mapped_epg) + + date_list = self._build_date_list(3) + self._seed_full_window_program_data(mapped_epg, days=3) + + today = date.today() + for i, ds in enumerate(date_list): + SDScheduleMD5.objects.create( + epg_source=source, + station_id=self.MAPPED_STATION, + date=today + timedelta(days=i), + md5=f'md5-{ds}', + last_modified=timezone.now(), + ) + SDScheduleMD5.objects.create( + epg_source=source, + station_id=self.UNMAPPED_STATION, + date=today, + md5='unmapped-stale', + last_modified=timezone.now(), + ) + + mock_get.side_effect = self._lineup_get_side_effect + + md5_response_payload = { + self.MAPPED_STATION: { + ds: {'code': 0, 'md5': f'md5-{ds}', 'lastModified': '2026-06-11T00:00:00Z'} + for ds in date_list + }, + } + + def post_side_effect(url, **kwargs): + if url.endswith('/token'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'code': 0, 'token': 'tok'}), + ) + if url.endswith('/schedules/md5'): + return MagicMock( + status_code=200, + json=MagicMock(return_value=md5_response_payload), + ) + raise AssertionError(f'Unexpected POST URL: {url}') + + mock_post.side_effect = post_side_effect + + fetch_schedules_direct(source, force=True) + + md5_calls = [c for c in mock_post.call_args_list if c[0][0].endswith('/schedules/md5')] + self.assertEqual(len(md5_calls), 1) + request_body = md5_calls[0][1]['json'] + station_ids_in_request = {entry['stationID'] for entry in request_body} + self.assertEqual(station_ids_in_request, {self.MAPPED_STATION}) + + self.assertFalse( + SDScheduleMD5.objects.filter( + epg_source=source, + station_id=self.UNMAPPED_STATION, + ).exists() + ) + + schedule_calls = [c for c in mock_post.call_args_list if c[0][0].endswith('/schedules')] + self.assertEqual(len(schedule_calls), 0) + + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_SUCCESS) + + @patch('apps.epg.tasks.SD_DAYS_TO_FETCH', 3) + @patch('apps.epg.tasks.send_epg_update') + @patch('apps.epg.tasks.requests.get') + @patch('apps.epg.tasks.requests.post') + def test_newly_mapped_station_fetches_despite_stale_cache( + self, mock_post, mock_get, mock_send_epg_update, + ): + from apps.epg.tasks import fetch_schedules_direct + + source = self._make_sd_source() + mapped_epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + Channel.objects.create(name='Mapped Ch', epg_data=mapped_epg) + + date_list = self._build_date_list(3) + today = date.today() + for i, ds in enumerate(date_list): + SDScheduleMD5.objects.create( + epg_source=source, + station_id=self.MAPPED_STATION, + date=today + timedelta(days=i), + md5=f'md5-{ds}', + last_modified=timezone.now(), + ) + + mock_get.side_effect = self._lineup_get_side_effect + + schedule_payload = [{ + 'stationID': self.MAPPED_STATION, + 'metadata': {'startDate': date_list[0], 'md5': 'md5-' + date_list[0], 'modified': '2026-06-11T00:00:00Z'}, + 'programs': [{ + 'programID': 'EP000000000001', + 'airDateTime': f'{date_list[0]}T12:00:00Z', + 'duration': 3600, + 'md5': 'prog-md5-1', + }], + }] + program_payload = [{ + 'programID': 'EP000000000001', + 'titles': [{'title120': 'Test Show'}], + }] + + def post_side_effect(url, **kwargs): + if url.endswith('/token'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'code': 0, 'token': 'tok'}), + ) + if url.endswith('/schedules/md5'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={ + self.MAPPED_STATION: { + ds: {'code': 0, 'md5': f'md5-{ds}', 'lastModified': '2026-06-11T00:00:00Z'} + for ds in date_list + }, + }), + ) + if url.endswith('/schedules'): + return MagicMock(status_code=200, json=MagicMock(return_value=schedule_payload)) + if url.endswith('/programs'): + return MagicMock(status_code=200, json=MagicMock(return_value=program_payload)) + raise AssertionError(f'Unexpected POST URL: {url}') + + mock_post.side_effect = post_side_effect + + fetch_schedules_direct(source, force=True) + + schedule_calls = [c for c in mock_post.call_args_list if c[0][0].endswith('/schedules')] + self.assertGreaterEqual(len(schedule_calls), 1) + self.assertEqual( + ProgramData.objects.filter(epg=mapped_epg).count(), + 1, + ) + + @patch('apps.epg.tasks.SD_DAYS_TO_FETCH', 3) + @patch('apps.epg.tasks.send_epg_update') + @patch('apps.epg.tasks.requests.get') + @patch('apps.epg.tasks.requests.post') + def test_orphan_program_data_removed_on_post_refresh( + self, mock_post, mock_get, mock_send_epg_update, + ): + from apps.epg.tasks import fetch_schedules_direct + + source = self._make_sd_source() + mapped_epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + orphan_epg = EPGData.objects.create( + tvg_id=self.UNMAPPED_STATION, + name='Orphan', + epg_source=source, + ) + Channel.objects.create(name='Mapped Ch', epg_data=mapped_epg) + + date_list = self._build_date_list(3) + self._seed_full_window_program_data(mapped_epg, days=3) + + start = timezone.now() + ProgramData.objects.create( + epg=orphan_epg, + start_time=start, + end_time=start + timedelta(hours=1), + title='Orphan Show', + tvg_id=orphan_epg.tvg_id, + ) + + today = date.today() + for i, ds in enumerate(date_list): + SDScheduleMD5.objects.create( + epg_source=source, + station_id=self.MAPPED_STATION, + date=today + timedelta(days=i), + md5=f'md5-{ds}', + last_modified=timezone.now(), + ) + + mock_get.side_effect = self._lineup_get_side_effect + + def post_side_effect(url, **kwargs): + if url.endswith('/token'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'code': 0, 'token': 'tok'}), + ) + if url.endswith('/schedules/md5'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={ + self.MAPPED_STATION: { + ds: {'code': 0, 'md5': f'md5-{ds}', 'lastModified': '2026-06-11T00:00:00Z'} + for ds in date_list + }, + }), + ) + raise AssertionError(f'Unexpected POST URL: {url}') + + mock_post.side_effect = post_side_effect + + fetch_schedules_direct(source, force=True) + + self.assertFalse(ProgramData.objects.filter(epg=orphan_epg).exists()) + + def test_stale_program_md5_fetched_when_no_program_data(self): + programs_with_data = set() + needed = _sd_programs_needing_metadata( + {'EP0001'}, + {'EP0001': 'cached-md5'}, + {'EP0001': 'cached-md5'}, + programs_with_data, + ) + self.assertEqual(needed, {'EP0001'}) + + def test_shared_program_skips_metadata_when_cached(self): + needed = _sd_programs_needing_metadata( + {'EP0001'}, + {'EP0001': 'cached-md5'}, + {'EP0001': 'cached-md5'}, + {'EP0001'}, + ) + self.assertEqual(needed, set()) + + # --------------------------------------------------------------------------- # parse_schedules_direct_time tests # --------------------------------------------------------------------------- From 8d59b0b27b830b3e31bb56169272326be00b27a4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 12:43:16 -0500 Subject: [PATCH 21/81] changelog: Update for SD PR. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49637717..f349d186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Schedules Direct refresh only fetches guide data for mapped channels.** Schedule MD5 checks and schedule downloads now target mapped lineup stations instead of the entire lineup, and schedule MD5 cache for unmapped stations is pruned each refresh. Unmapped lineup entries no longer trigger wasted schedule API calls when their MD5 changes. - **EPG auto-match memory and throughput improvements.** - Single-channel matching streams active EPG rows and keeps only the best match plus the top 20 candidates in memory; ML validates at most 21 names per channel instead of embedding the full catalog. - Strong fuzzy matches (≥75% single channel, ≥80% bulk) skip ML entirely, avoiding a ~500MB PyTorch load when the fuzzy result is already reliable. @@ -49,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bulk auto-match completion no longer calls `batch-set-epg` from the WebSocket handler, which had been re-applying every match and queueing redundant `parse_programs_for_tvg_id` tasks even when assignments were unchanged. - **Schedules Direct lineup search country dropdown.** The country list was fetched directly from the SD API in the browser, which failed due to CORS and silently fell back to a 14-country hardcoded list. Countries are now included in the `GET sd-lineups` response (server-side fetch with proper User-Agent) so the full SD country list populates correctly. — Thanks [@sethwv](https://github.com/sethwv) - **Schedules Direct program poster proxy omitted User-Agent on image requests.** The poster endpoint authenticated with SD correctly but fetched images with only the `token` header, which violates SD's API requirements (error 1003). Image requests now include the standard `Dispatcharr/{version}` User-Agent via `dispatcharr_http_headers()`. +- **Schedules Direct guide data missing after mapping a channel.** Lineup refreshes cached schedule MD5s for all stations, but `ProgramData` was only written for mapped channels. Mapping a channel later could leave MD5s looking unchanged so schedules were skipped and the channel had no guide until dates rolled outside the cached window. Refreshes now backfill fetch-window dates that lack `ProgramData` (including newly mapped stations with stale cache), fetch program metadata when no local `ProgramData` exists for a `programID`, and clean up orphaned `ProgramData` on unmapped `EPGData` entries. — Thanks [@sethwv](https://github.com/sethwv) ## [0.26.0] - 2026-06-07 From c274bd7fbbf894d9ffe7407c969e9f8a29d63660 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 13:54:19 -0500 Subject: [PATCH 22/81] Enhance Schedules Direct integration with per-channel guide fetch and improved EPG handling - Implemented a targeted guide fetch for Schedules Direct when mapping a channel, ensuring immediate data availability without redundant API calls. - Updated EPG signals to queue guide fetches for newly mapped channels, streamlining the process and improving user experience. - Refactored existing logic to skip unnecessary program refreshes for dummy sources and ensure proper handling of Schedules Direct data. - Added unit tests to validate the new fetching behavior and ensure robustness of the integration. --- CHANGELOG.md | 1 + apps/channels/api_views.py | 14 +- apps/channels/epg_matching.py | 2 +- apps/channels/signals.py | 9 +- apps/epg/tasks.py | 483 +++++++++++++++--------- apps/epg/tests/test_schedules_direct.py | 167 +++++++- 6 files changed, 466 insertions(+), 210 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f349d186..ebf11f26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Schedules Direct lineup search country dropdown.** The country list was fetched directly from the SD API in the browser, which failed due to CORS and silently fell back to a 14-country hardcoded list. Countries are now included in the `GET sd-lineups` response (server-side fetch with proper User-Agent) so the full SD country list populates correctly. — Thanks [@sethwv](https://github.com/sethwv) - **Schedules Direct program poster proxy omitted User-Agent on image requests.** The poster endpoint authenticated with SD correctly but fetched images with only the `token` header, which violates SD's API requirements (error 1003). Image requests now include the standard `Dispatcharr/{version}` User-Agent via `dispatcharr_http_headers()`. - **Schedules Direct guide data missing after mapping a channel.** Lineup refreshes cached schedule MD5s for all stations, but `ProgramData` was only written for mapped channels. Mapping a channel later could leave MD5s looking unchanged so schedules were skipped and the channel had no guide until dates rolled outside the cached window. Refreshes now backfill fetch-window dates that lack `ProgramData` (including newly mapped stations with stale cache), fetch program metadata when no local `ProgramData` exists for a `programID`, and clean up orphaned `ProgramData` on unmapped `EPGData` entries. — Thanks [@sethwv](https://github.com/sethwv) +- **Schedules Direct guide fetch on channel map.** Mapping a channel (or bulk-assigning EPG) now triggers a targeted guide fetch for that `EPGData` entry—mirroring XMLTV's `parse_programs_for_tvg_id` flow—without waiting for the next full source refresh. Skips the fetch when `ProgramData` already exists so additional channels sharing the same `tvg-id` do not trigger redundant API calls. ## [0.26.0] - 2026-06-07 diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index e9711026..cfd6b68f 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -2163,22 +2163,14 @@ class ChannelViewSet(viewsets.ModelViewSet): epg_data = EPGData.objects.get(pk=epg_data_id) - # Set the EPG data and save + # Set the EPG data and save. refresh_epg_programs (post_save) queues + # parse_programs_for_tvg_id for non-dummy sources — no second dispatch here. channel.epg_data = epg_data channel.save(update_fields=["epg_data"]) - # Only trigger program refresh for non-dummy EPG sources status_message = None if epg_data.epg_source.source_type != 'dummy': - # Explicitly trigger program refresh for this EPG - from apps.epg.tasks import parse_programs_for_tvg_id - - task_result = parse_programs_for_tvg_id.delay(epg_data.id) - - # Prepare response with task status info status_message = "EPG refresh queued" - if task_result.result == "Task already running": - status_message = "EPG refresh already in progress" # Build response message message = f"EPG data set to {epg_data.tvg_id} for channel {channel.name}" @@ -2374,7 +2366,7 @@ class ChannelViewSet(viewsets.ModelViewSet): continue source_type = epg_data.epg_source.source_type if epg_data.epg_source else None - if source_type not in ('dummy', 'schedules_direct'): + if source_type != 'dummy': parse_programs_for_tvg_id.delay(epg_id) programs_refreshed += 1 diff --git a/apps/channels/epg_matching.py b/apps/channels/epg_matching.py index c468568b..c13e108c 100644 --- a/apps/channels/epg_matching.py +++ b/apps/channels/epg_matching.py @@ -327,7 +327,7 @@ def _dispatch_program_parse_for_epg_assignments(changed_associations): dispatched = 0 for epg in EPGData.objects.filter(id__in=epg_ids).select_related("epg_source"): source_type = epg.epg_source.source_type if epg.epg_source else None - if source_type in ("dummy", "schedules_direct"): + if source_type == "dummy": continue parse_programs_for_tvg_id.delay(epg.id) dispatched += 1 diff --git a/apps/channels/signals.py b/apps/channels/signals.py index 54dd30fc..53f1ea33 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -201,18 +201,13 @@ def refresh_epg_programs(sender, instance, created, **kwargs): if not created and kwargs.get('update_fields') and 'epg_data' in kwargs['update_fields']: logger.info(f"Channel {instance.id} ({instance.name}) EPG data updated, refreshing program data") if instance.epg_data: - # Skip XMLTV program parser for SD sources — program data is handled - # by fetch_schedules_direct() directly. - if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'schedules_direct': - logger.info(f"Skipping XMLTV parse for SD source {instance.epg_data.tvg_id}") + if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'dummy': return logger.info(f"Triggering EPG program refresh for {instance.epg_data.tvg_id}") parse_programs_for_tvg_id.delay(instance.epg_data.id) # For new channels with EPG data, also refresh elif created and instance.epg_data: - # Skip XMLTV program parser for SD sources - if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'schedules_direct': - logger.info(f"Skipping XMLTV parse for SD source {instance.epg_data.tvg_id}") + if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'dummy': return logger.info(f"New channel {instance.id} ({instance.name}) created with EPG data, refreshing program data") parse_programs_for_tvg_id.delay(instance.epg_data.id) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index ad2afb4c..0401394a 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -1496,14 +1496,11 @@ def parse_channels_only(source): @shared_task(time_limit=3600, soft_time_limit=3500) def parse_programs_for_tvg_id(epg_id, force=False): - # Skip XMLTV file parsing for Schedules Direct sources. Program data is - # fetched and persisted directly by fetch_schedules_direct(). try: from apps.epg.models import EPGData epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': - logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") - return "Skipped (Schedules Direct source)" + return fetch_sd_guide_for_epg(epg_id, force=force) except Exception as e: logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") @@ -2139,6 +2136,73 @@ def parse_programs_for_source(epg_source, tvg_id=None): process = None +def _sd_setup_single_epg_fetch(source, epg_id, token, sd_headers_fn): + """Build station_map / epg_id_map for a single mapped EPG entry.""" + epg = EPGData.objects.filter(id=epg_id, epg_source=source).first() + if not epg or not epg.tvg_id: + msg = f"Schedules Direct EPG entry {epg_id} not found or missing station ID." + logger.error(msg) + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="error", error=msg) + return None + + sd_lineup_country = None + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=sd_headers_fn(token), + timeout=30, + ) + if lineups_response.ok: + for lineup in lineups_response.json().get('lineups', []): + lid = lineup.get('lineupID') or lineup.get('lineup') or '' + if '-' in lid: + sd_lineup_country = lid.split('-')[0] + break + except requests.exceptions.RequestException as e: + logger.warning(f"Could not fetch lineups for country code: {e}") + + send_epg_update( + source.id, "parsing_programs", 15, + message=f"Fetching guide data for {epg.name or epg.tvg_id}...", + ) + station_map = {epg.tvg_id: {'name': epg.name or epg.tvg_id}} + epg_id_map = {epg.tvg_id: epg.id} + return station_map, epg_id_map, sd_lineup_country, epg + + +@shared_task(time_limit=3600, soft_time_limit=3500) +def fetch_sd_guide_for_epg(epg_id, force=False): + """ + Fetch Schedules Direct guide data for one mapped EPG entry (channel map flow). + + Skips when ProgramData already exists so additional channels sharing the + same EPGData / tvg_id do not trigger redundant API calls. + """ + epg = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() + if not epg or not epg.epg_source or epg.epg_source.source_type != 'schedules_direct': + return "Not a Schedules Direct EPG entry" + + if not force and ProgramData.objects.filter(epg_id=epg_id).exists(): + logger.info(f"SD guide fetch skipped for EPG {epg_id}: ProgramData already present") + return "Guide data already present" + + if not acquire_task_lock('parse_epg_programs', epg_id): + logger.info(f"SD guide fetch for EPG {epg_id} already in progress, skipping duplicate task") + return "Task already running" + + lock_renewer = TaskLockRenewer('parse_epg_programs', epg_id) + lock_renewer.start() + try: + logger.info(f"Fetching Schedules Direct guide for EPG {epg_id} ({epg.tvg_id})") + fetch_schedules_direct(epg.epg_source, epg_id_only=epg_id, force=force) + return "SD guide fetch complete" + finally: + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + + @shared_task(bind=True) def fetch_schedules_direct_stations(self, source_id): """ @@ -2154,7 +2218,7 @@ def fetch_schedules_direct_stations(self, source_id): fetch_schedules_direct(source, stations_only=True) -def fetch_schedules_direct(source, stations_only=False, force=False): +def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only=None): """ Fetch EPG data from the Schedules Direct JSON API and persist it to the EPGData / ProgramData models. @@ -2185,8 +2249,15 @@ def fetch_schedules_direct(source, stations_only=False, force=False): import hashlib from datetime import date + single_epg_fetch = epg_id_only is not None - logger.info(f"Fetching Schedules Direct data for source: {source.name}") + if single_epg_fetch: + logger.info( + f"Fetching Schedules Direct guide for EPG {epg_id_only} " + f"(source: {source.name})" + ) + else: + logger.info(f"Fetching Schedules Direct data for source: {source.name}") # ------------------------------------------------------------------------- # Validate credentials @@ -2213,7 +2284,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # updated_at, which would otherwise incorrectly trigger this guard). Always # allow the first full refresh through so guide data is immediately available. # ------------------------------------------------------------------------- - if not stations_only and not force and source.updated_at: + if not stations_only and not force and not single_epg_fetch and source.updated_at: from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() if has_prior_full_refresh: @@ -2234,7 +2305,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): return else: logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") - elif force and not stations_only: + elif force and not stations_only and not single_epg_fetch: logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") # ------------------------------------------------------------------------- @@ -2406,9 +2477,10 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # This is a requirement of the Schedules Direct API specification, not an # architectural choice. # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_FETCHING - source.last_message = "Authenticating with Schedules Direct..." - source.save(update_fields=['status', 'last_message']) + if not single_epg_fetch: + source.status = EPGSource.STATUS_FETCHING + source.last_message = "Authenticating with Schedules Direct..." + source.save(update_fields=['status', 'last_message']) send_epg_update(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") try: @@ -2491,191 +2563,201 @@ def fetch_schedules_direct(source, stations_only=False, force=False): except requests.exceptions.RequestException as e: logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") - # ------------------------------------------------------------------------- - # Step 3: Fetch subscribed lineups and build station map - # ------------------------------------------------------------------------- - send_epg_update(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") - try: - lineups_response = requests.get( - f"{SD_BASE_URL}/lineups", - headers=_sd_headers(token), - timeout=30, - ) - # SD returns 400 with code 4102 when no lineups are configured. - # This is a valid account state. The user needs to add lineups via - # the Manage Lineups UI. Treat as idle rather than error. - if lineups_response.status_code == 400: - sd_data = lineups_response.json() - if sd_data.get('code') == 4102: + station_map = None + epg_id_map = None + sd_lineup_country = None + + if epg_id_only is not None: + setup = _sd_setup_single_epg_fetch(source, epg_id_only, token, _sd_headers) + if setup is None: + return + station_map, epg_id_map, sd_lineup_country, _single_epg = setup + else: + # ------------------------------------------------------------------------- + # Step 3: Fetch subscribed lineups and build station map + # ------------------------------------------------------------------------- + send_epg_update(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=_sd_headers(token), + timeout=30, + ) + # SD returns 400 with code 4102 when no lineups are configured. + # This is a valid account state. The user needs to add lineups via + # the Manage Lineups UI. Treat as idle rather than error. + if lineups_response.status_code == 400: + sd_data = lineups_response.json() + if sd_data.get('code') == 4102: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "refresh", 100, status="idle", message=msg) + return + lineups_response.raise_for_status() + lineups_data = lineups_response.json() + lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] + if not lineups: msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") + logger.warning(f"SD source {source.id}: no active lineups found.") source.status = EPGSource.STATUS_IDLE source.last_message = msg source.save(update_fields=['status', 'last_message']) send_epg_update(source.id, "refresh", 100, status="idle", message=msg) return - lineups_response.raise_for_status() - lineups_data = lineups_response.json() - lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] - if not lineups: - msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no active lineups found.") - source.status = EPGSource.STATUS_IDLE + logger.info(f"Found {len(lineups)} lineup(s) in SD account.") + + # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) + sd_lineup_country = None + for l in lineups: + lid = l.get('lineupID') or l.get('lineup') or '' + if '-' in lid: + sd_lineup_country = lid.split('-')[0] + break + logger.debug(f"SD lineup country: {sd_lineup_country}") + except requests.exceptions.RequestException as e: + msg = f"Failed to fetch Schedules Direct lineups: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR source.last_message = msg source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "refresh", 100, status="idle", message=msg) + send_epg_update(source.id, "refresh", 100, status="error", error=msg) return - logger.info(f"Found {len(lineups)} lineup(s) in SD account.") - - # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) - sd_lineup_country = None - for l in lineups: - lid = l.get('lineupID') or l.get('lineup') or '' - if '-' in lid: - sd_lineup_country = lid.split('-')[0] - break - logger.debug(f"SD lineup country: {sd_lineup_country}") - except requests.exceptions.RequestException as e: - msg = f"Failed to fetch Schedules Direct lineups: {e}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg + + # Build station metadata map: stationID -> {name, callsign, logo_url} + station_map = {} + send_epg_update(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") + for lineup in lineups: + lineup_id = lineup.get('lineupID') or lineup.get('lineup') + if not lineup_id: + continue + try: + detail_response = requests.get( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=_sd_headers(token), + timeout=30, + ) + detail_response.raise_for_status() + detail_data = detail_response.json() + for station in detail_data.get('stations', []): + sid = station.get('stationID') + if not sid: + continue + logo_url = None + logos = station.get('stationLogo') or station.get('logo') or [] + if isinstance(logos, list) and logos: + # Read preferred logo style from source settings; default to 'dark' + logo_style = (source.custom_properties or {}).get('logo_style', 'dark') + preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) + logo_url = preferred.get('URL') or preferred.get('url') + elif isinstance(logos, dict): + logo_url = logos.get('URL') or logos.get('url') + station_map[sid] = { + 'name': station.get('name', sid), + 'callsign': station.get('callsign', ''), + 'logo_url': logo_url, + } + logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") + + if not station_map: + msg = "No stations found across all Schedules Direct lineups." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info(f"Built station map with {len(station_map)} stations.") + + # ------------------------------------------------------------------------- + # Step 4: Persist station metadata to EPGData + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_PARSING + source.last_message = f"Syncing {len(station_map)} stations..." source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "refresh", 100, status="error", error=msg) - return - - # Build station metadata map: stationID -> {name, callsign, logo_url} - station_map = {} - send_epg_update(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") - for lineup in lineups: - lineup_id = lineup.get('lineupID') or lineup.get('lineup') - if not lineup_id: - continue - try: - detail_response = requests.get( - f"{SD_BASE_URL}/lineups/{lineup_id}", - headers=_sd_headers(token), - timeout=30, + send_epg_update(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") + + existing_epg_map = { + epg.tvg_id: epg + for epg in EPGData.objects.filter(epg_source=source) + } + + epgs_to_create = [] + epgs_to_update = [] + icon_max_length = EPGData._meta.get_field('icon_url').max_length + name_max_length = EPGData._meta.get_field('name').max_length + + for sid, info in station_map.items(): + display_name = (info['name'] or sid)[:name_max_length] + logo = info['logo_url'] + if logo and len(logo) > icon_max_length: + logo = None + + if sid in existing_epg_map: + epg_obj = existing_epg_map[sid] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != logo: + epg_obj.icon_url = logo + needs_update = True + if needs_update: + epgs_to_update.append(epg_obj) + else: + epgs_to_create.append(EPGData( + tvg_id=sid, + name=display_name, + icon_url=logo, + epg_source=source, + )) + + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) + logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") + + gc.collect() + + # Rebuild map with fresh DB ids for all stations + epg_id_map = { + epg.tvg_id: epg.id + for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) + } + + # Station sync complete. Send progress update before continuing into programs phase. + # We deliberately do NOT send parsing_channels at 100 with status=success here + # because that would cause the frontend to mark the source as complete and + # stop rendering progress updates for the subsequent program fetch phases. + send_epg_update(source.id, "parsing_programs", 30, + message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") + + # ------------------------------------------------------------------------- + # Stations-only mode. Used on initial source creation. + # Stop here so the user can run Auto-match EPG before the full program fetch. + # ------------------------------------------------------------------------- + if stations_only: + success_msg = ( + f"{len(station_map)} stations loaded from Schedules Direct. " + f"Run Auto-match EPG to map your channels, then use the Refresh " + f"button to populate guide data." ) - detail_response.raise_for_status() - detail_data = detail_response.json() - for station in detail_data.get('stations', []): - sid = station.get('stationID') - if not sid: - continue - logo_url = None - logos = station.get('stationLogo') or station.get('logo') or [] - if isinstance(logos, list) and logos: - # Read preferred logo style from source settings; default to 'dark' - logo_style = (source.custom_properties or {}).get('logo_style', 'dark') - preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) - logo_url = preferred.get('URL') or preferred.get('url') - elif isinstance(logos, dict): - logo_url = logos.get('URL') or logos.get('url') - station_map[sid] = { - 'name': station.get('name', sid), - 'callsign': station.get('callsign', ''), - 'logo_url': logo_url, - } - logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") - - if not station_map: - msg = "No stations found across all Schedules Direct lineups." - logger.warning(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "refresh", 100, status="error", error=msg) - return - - logger.info(f"Built station map with {len(station_map)} stations.") - - # ------------------------------------------------------------------------- - # Step 4: Persist station metadata to EPGData - # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_PARSING - source.last_message = f"Syncing {len(station_map)} stations..." - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") - - existing_epg_map = { - epg.tvg_id: epg - for epg in EPGData.objects.filter(epg_source=source) - } - - epgs_to_create = [] - epgs_to_update = [] - icon_max_length = EPGData._meta.get_field('icon_url').max_length - name_max_length = EPGData._meta.get_field('name').max_length - - for sid, info in station_map.items(): - display_name = (info['name'] or sid)[:name_max_length] - logo = info['logo_url'] - if logo and len(logo) > icon_max_length: - logo = None - - if sid in existing_epg_map: - epg_obj = existing_epg_map[sid] - needs_update = False - if epg_obj.name != display_name: - epg_obj.name = display_name - needs_update = True - if epg_obj.icon_url != logo: - epg_obj.icon_url = logo - needs_update = True - if needs_update: - epgs_to_update.append(epg_obj) - else: - epgs_to_create.append(EPGData( - tvg_id=sid, - name=display_name, - icon_url=logo, - epg_source=source, - )) - - if epgs_to_create: - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") - if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) - logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") - - gc.collect() - - # Rebuild map with fresh DB ids for all stations - epg_id_map = { - epg.tvg_id: epg.id - for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) - } - - # Station sync complete. Send progress update before continuing into programs phase. - # We deliberately do NOT send parsing_channels at 100 with status=success here - # because that would cause the frontend to mark the source as complete and - # stop rendering progress updates for the subsequent program fetch phases. - send_epg_update(source.id, "parsing_programs", 30, - message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") - - # ------------------------------------------------------------------------- - # Stations-only mode. Used on initial source creation. - # Stop here so the user can run Auto-match EPG before the full program fetch. - # ------------------------------------------------------------------------- - if stations_only: - success_msg = ( - f"{len(station_map)} stations loaded from Schedules Direct. " - f"Run Auto-match EPG to map your channels, then use the Refresh " - f"button to populate guide data." - ) - source.status = EPGSource.STATUS_SUCCESS - source.last_message = success_msg - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - send_epg_update(source.id, "parsing_channels", 100, status="success", - message=success_msg, channels_count=len(station_map)) - logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") - return - + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + send_epg_update(source.id, "parsing_channels", 100, status="success", + message=success_msg, channels_count=len(station_map)) + logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") + return + # ------------------------------------------------------------------------- # Step 5: MD5-delta schedule fetch # Only mapped channels need guide data. Fetch MD5 hashes and schedules for @@ -2720,6 +2802,12 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if not mapped_station_ids: logger.info("No channels mapped to this SD source; skipping schedule MD5 check and downloads.") _sd_post_refresh_tasks(mapped_epg_ids, {}, today) + if single_epg_fetch: + msg = "No mapped channel found for this EPG entry; guide fetch skipped." + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="idle", message=msg) + return success_msg = ( f"{len(station_map)} lineup stations synced. " "Map channels to EPG entries, then refresh to populate guide data." @@ -2828,6 +2916,12 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if not changed_by_station: logger.info("No schedule changes detected, skipping schedule and program downloads.") _sd_post_refresh_tasks(mapped_epg_ids, {}, today) + if single_epg_fetch: + msg = "No schedule updates needed; guide data is up to date." + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="success", message=msg) + return send_epg_update(source.id, "parsing_programs", 100, status="success", message="No schedule changes detected since last refresh. Guide data is up to date.") source.status = EPGSource.STATUS_SUCCESS @@ -3383,6 +3477,25 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # ------------------------------------------------------------------------- # Done # ------------------------------------------------------------------------- + if single_epg_fetch: + epg_label = EPGData.objects.filter(id=epg_id_only).values_list('name', flat=True).first() + success_msg = ( + f"Fetched {total_programs:,} programs for " + f"{epg_label or epg_id_only} from Schedules Direct." + ) + source.last_message = success_msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="success", message=success_msg) + log_system_event( + event_type='epg_refresh', + source_name=source.name, + programs=total_programs, + channels=1, + skipped_programs=skipped_unmapped, + ) + logger.info(f"Schedules Direct single-EPG fetch complete for source: {source.name}") + return + success_msg = ( f"Successfully fetched {total_programs:,} programs for " f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index 47860f92..b2ec41fd 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -9,7 +9,8 @@ Covers: - fetch_schedules_direct: graceful error handling on auth failure - fetch_schedules_direct: schedule MD5 delta, backfill, and cache invalidation - parse_schedules_direct_time: correct UTC parsing -- EPG signals: SD sources skip the XMLTV program parser +- fetch_sd_guide_for_epg: per-channel guide fetch on map +- EPG signals: SD sources queue guide fetch when a channel is mapped """ import hashlib @@ -772,13 +773,167 @@ class ParseSchedulesDirectTimeTests(TestCase): # Signal tests # --------------------------------------------------------------------------- +class SDSingleEpgFetchTests(TestCase): + """Per-channel SD guide fetch on map (epg_id_only path).""" + + MAPPED_STATION = '10001' + + def _make_sd_source(self, updated_at=None): + source = EPGSource.objects.create( + name='SD Single EPG', + source_type='schedules_direct', + username='sduser', + password='sdpass', + ) + if updated_at is not None: + EPGSource.objects.filter(id=source.id).update(updated_at=updated_at) + source.refresh_from_db() + return source + + def _lineup_get_side_effect(self, url, **kwargs): + if url.endswith('/status'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'systemStatus': [{'status': 'Online'}]}), + ) + if url.endswith('/lineups'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'lineups': [{'lineupID': 'USA-TEST-X'}]}), + ) + raise AssertionError(f'Unexpected GET URL: {url}') + + @patch('apps.epg.tasks.acquire_task_lock', return_value=True) + @patch('apps.epg.tasks.release_task_lock') + @patch('apps.epg.tasks.TaskLockRenewer') + def test_fetch_sd_guide_skips_when_program_data_exists( + self, mock_renewer, mock_release, mock_acquire, + ): + from apps.epg.tasks import fetch_sd_guide_for_epg + + source = self._make_sd_source() + epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + start = timezone.now() + ProgramData.objects.create( + epg=epg, + start_time=start, + end_time=start + timedelta(hours=1), + title='Existing', + tvg_id=epg.tvg_id, + ) + + with patch('apps.epg.tasks.fetch_schedules_direct') as mock_fetch: + result = fetch_sd_guide_for_epg(epg.id) + + self.assertEqual(result, 'Guide data already present') + mock_fetch.assert_not_called() + + @patch('apps.epg.tasks.acquire_task_lock', return_value=True) + @patch('apps.epg.tasks.release_task_lock') + @patch('apps.epg.tasks.TaskLockRenewer') + def test_parse_programs_for_tvg_id_delegates_to_sd_fetch( + self, mock_renewer, mock_release, mock_acquire, + ): + from apps.epg.tasks import parse_programs_for_tvg_id + + source = self._make_sd_source() + epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + + with patch('apps.epg.tasks.fetch_sd_guide_for_epg', return_value='SD guide fetch complete') as mock_sd: + result = parse_programs_for_tvg_id(epg.id) + + mock_sd.assert_called_once_with(epg.id, force=False) + self.assertEqual(result, 'SD guide fetch complete') + + @patch('apps.epg.tasks.SD_DAYS_TO_FETCH', 3) + @patch('apps.epg.tasks.send_epg_update') + @patch('apps.epg.tasks.requests.get') + @patch('apps.epg.tasks.requests.post') + def test_single_epg_fetch_skips_lineup_sync_and_updated_at( + self, mock_post, mock_get, mock_send_epg_update, + ): + from apps.epg.tasks import fetch_schedules_direct + + prior_updated = timezone.now() - timedelta(hours=1) + source = self._make_sd_source(updated_at=prior_updated) + epg = EPGData.objects.create( + tvg_id=self.MAPPED_STATION, + name='Mapped', + epg_source=source, + ) + Channel.objects.create(name='Mapped Ch', epg_data=epg) + + date_list = [ + (date.today() + timedelta(days=i)).strftime('%Y-%m-%d') + for i in range(3) + ] + mock_get.side_effect = self._lineup_get_side_effect + + schedule_payload = [{ + 'stationID': self.MAPPED_STATION, + 'metadata': {'startDate': date_list[0], 'md5': 'md5-new', 'modified': '2026-06-11T00:00:00Z'}, + 'programs': [{ + 'programID': 'EP000000000001', + 'airDateTime': f'{date_list[0]}T12:00:00Z', + 'duration': 3600, + 'md5': 'prog-md5-1', + }], + }] + program_payload = [{ + 'programID': 'EP000000000001', + 'titles': [{'title120': 'Test Show'}], + }] + + def post_side_effect(url, **kwargs): + if url.endswith('/token'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={'code': 0, 'token': 'tok'}), + ) + if url.endswith('/schedules/md5'): + return MagicMock( + status_code=200, + json=MagicMock(return_value={ + self.MAPPED_STATION: { + ds: {'code': 0, 'md5': f'md5-{ds}', 'lastModified': '2026-06-11T00:00:00Z'} + for ds in date_list + }, + }), + ) + if url.endswith('/schedules'): + return MagicMock(status_code=200, json=MagicMock(return_value=schedule_payload)) + if url.endswith('/programs'): + return MagicMock(status_code=200, json=MagicMock(return_value=program_payload)) + raise AssertionError(f'Unexpected POST URL: {url}') + + mock_post.side_effect = post_side_effect + + fetch_schedules_direct(source, epg_id_only=epg.id) + + lineup_detail_calls = [ + c for c in mock_get.call_args_list + if '/lineups/' in c[0][0] and not c[0][0].endswith('/lineups') + ] + self.assertEqual(lineup_detail_calls, []) + + source.refresh_from_db() + self.assertEqual(source.updated_at, prior_updated) + self.assertEqual(ProgramData.objects.filter(epg=epg).count(), 1) + + class SDSourceSignalTests(TestCase): - """SD EPG sources must skip the XMLTV program parser signal.""" + """SD EPG sources queue per-EPG guide fetch when a channel is mapped.""" @patch('apps.channels.signals.parse_programs_for_tvg_id') - def test_sd_source_skips_xmltv_parse_on_channel_create(self, mock_parse): - """Creating a channel linked to an SD EPG source must not trigger - the XMLTV program parser — SD data is handled by fetch_schedules_direct.""" + def test_sd_source_queues_guide_fetch_on_channel_create(self, mock_parse): from apps.epg.models import EPGData from apps.channels.models import Channel @@ -799,7 +954,7 @@ class SDSourceSignalTests(TestCase): epg_data=epg_data, ) - mock_parse.delay.assert_not_called() + mock_parse.delay.assert_called_once_with(epg_data.id) # --------------------------------------------------------------------------- From 7ed9b11a89a60a43911860518296bbd5f9efcfd9 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 16:41:30 -0500 Subject: [PATCH 23/81] Refactor Schedules Direct EPG handling and enhance guide fetch logic - Replaced individual EPG program parse tasks with a centralized dispatch function to streamline guide refresh for newly assigned EPG IDs. - Implemented batching for guide fetches when multiple EPGs are mapped, reducing redundant API calls and improving efficiency. - Updated related utility functions to support the new fetching strategy and added tests to ensure correct behavior under various scenarios. --- CHANGELOG.md | 2 +- apps/channels/api_views.py | 22 +- apps/channels/epg_matching.py | 16 +- apps/epg/tasks.py | 322 ++++++++++++++++++++---- apps/epg/tests/test_schedules_direct.py | 229 +++++++++++++++++ apps/m3u/tasks.py | 17 +- core/utils.py | 9 + 7 files changed, 529 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf11f26..12a9da75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Schedules Direct lineup search country dropdown.** The country list was fetched directly from the SD API in the browser, which failed due to CORS and silently fell back to a 14-country hardcoded list. Countries are now included in the `GET sd-lineups` response (server-side fetch with proper User-Agent) so the full SD country list populates correctly. — Thanks [@sethwv](https://github.com/sethwv) - **Schedules Direct program poster proxy omitted User-Agent on image requests.** The poster endpoint authenticated with SD correctly but fetched images with only the `token` header, which violates SD's API requirements (error 1003). Image requests now include the standard `Dispatcharr/{version}` User-Agent via `dispatcharr_http_headers()`. - **Schedules Direct guide data missing after mapping a channel.** Lineup refreshes cached schedule MD5s for all stations, but `ProgramData` was only written for mapped channels. Mapping a channel later could leave MD5s looking unchanged so schedules were skipped and the channel had no guide until dates rolled outside the cached window. Refreshes now backfill fetch-window dates that lack `ProgramData` (including newly mapped stations with stale cache), fetch program metadata when no local `ProgramData` exists for a `programID`, and clean up orphaned `ProgramData` on unmapped `EPGData` entries. — Thanks [@sethwv](https://github.com/sethwv) -- **Schedules Direct guide fetch on channel map.** Mapping a channel (or bulk-assigning EPG) now triggers a targeted guide fetch for that `EPGData` entry—mirroring XMLTV's `parse_programs_for_tvg_id` flow—without waiting for the next full source refresh. Skips the fetch when `ProgramData` already exists so additional channels sharing the same `tvg-id` do not trigger redundant API calls. +- **Schedules Direct guide fetch on channel map.** Mapping a channel (or bulk-assigning EPG) now triggers a targeted guide fetch for that `EPGData` entry—mirroring XMLTV's `parse_programs_for_tvg_id` flow—without waiting for the next full source refresh. Skips the fetch when `ProgramData` already exists so additional channels sharing the same `tvg-id` do not trigger redundant API calls. Bulk assignment of three or more SD stations without guide data on the same source queues one batched mapped-station fetch instead of separate per-station API sessions. Concurrent batch and single-EPG fetches coordinate via source-level locks with deferred retries so mappings are not dropped and overlapping SD API sessions are avoided when possible. ## [0.26.0] - 2026-06-07 diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index cfd6b68f..3d94cb10 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -2348,27 +2348,9 @@ class ChannelViewSet(viewsets.ModelViewSet): channels_updated = len(channels_to_update) - # Trigger program refresh only for EPG ids newly assigned (skip dummy/SD) - from apps.epg.tasks import parse_programs_for_tvg_id - from apps.epg.models import EPGData + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids - # Batch fetch EPG data (single query) - epg_data_dict = { - epg.id: epg - for epg in EPGData.objects.filter(id__in=changed_epg_ids).select_related('epg_source') - } - - programs_refreshed = 0 - for epg_id in changed_epg_ids: - epg_data = epg_data_dict.get(epg_id) - if not epg_data: - logger.error(f"EPGData with ID {epg_id} not found") - continue - - source_type = epg_data.epg_source.source_type if epg_data.epg_source else None - if source_type != 'dummy': - parse_programs_for_tvg_id.delay(epg_id) - programs_refreshed += 1 + programs_refreshed = dispatch_program_refresh_for_epg_ids(changed_epg_ids) return Response( { diff --git a/apps/channels/epg_matching.py b/apps/channels/epg_matching.py index c13e108c..76e17c06 100644 --- a/apps/channels/epg_matching.py +++ b/apps/channels/epg_matching.py @@ -306,7 +306,7 @@ def build_epg_tvg_id_index(epg_data): def _dispatch_program_parse_for_epg_assignments(changed_associations): """ - Queue parse_programs once per unique EPG id newly assigned to a channel. + Queue guide/program refresh for newly assigned EPG ids. bulk_update bypasses post_save, so callers must invoke this when epg_data actually changes (mirrors the M3U sync path). @@ -314,24 +314,14 @@ def _dispatch_program_parse_for_epg_assignments(changed_associations): if not changed_associations: return 0 - from apps.epg.tasks import parse_programs_for_tvg_id + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids epg_ids = { assoc["epg_data_id"] for assoc in changed_associations if assoc.get("epg_data_id") } - if not epg_ids: - return 0 - - dispatched = 0 - for epg in EPGData.objects.filter(id__in=epg_ids).select_related("epg_source"): - source_type = epg.epg_source.source_type if epg.epg_source else None - if source_type == "dummy": - continue - parse_programs_for_tvg_id.delay(epg.id) - dispatched += 1 - return dispatched + return dispatch_program_refresh_for_epg_ids(epg_ids) def _log_unchanged_epg_assignment(chan, epg_id, epg_name, epg_tvg_id, match_method): diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0401394a..248e9a5f 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -27,14 +27,24 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 -from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event +from core.utils import ( + acquire_task_lock, + is_task_lock_held, + release_task_lock, + TaskLockRenewer, + send_websocket_update, + cleanup_memory, + log_system_event, +) logger = logging.getLogger(__name__) SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' SD_DAYS_TO_FETCH = 20 SD_PROGRAM_BATCH_SIZE = 5000 - +SD_BULK_GUIDE_FETCH_THRESHOLD = 3 +SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS = 90 +SD_MAPPED_GUIDE_FETCH_DEFER_MAX_RETRIES = 2 def _sd_compute_schedule_changes_from_md5(server_md5s, cached_md5s, date_list): """Return station_id -> [date_str] for dates whose schedule MD5 differs from cache.""" @@ -2136,6 +2146,24 @@ def parse_programs_for_source(epg_source, tvg_id=None): process = None +def _sd_fetch_lineup_country(token, sd_headers_fn): + """Return country code prefix from the first subscribed lineup (poster metadata).""" + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=sd_headers_fn(token), + timeout=30, + ) + if lineups_response.ok: + for lineup in lineups_response.json().get('lineups', []): + lid = lineup.get('lineupID') or lineup.get('lineup') or '' + if '-' in lid: + return lid.split('-')[0] + except requests.exceptions.RequestException as e: + logger.warning(f"Could not fetch lineups for country code: {e}") + return None + + def _sd_setup_single_epg_fetch(source, epg_id, token, sd_headers_fn): """Build station_map / epg_id_map for a single mapped EPG entry.""" epg = EPGData.objects.filter(id=epg_id, epg_source=source).first() @@ -2147,33 +2175,179 @@ def _sd_setup_single_epg_fetch(source, epg_id, token, sd_headers_fn): send_epg_update(source.id, "parsing_programs", 100, status="error", error=msg) return None - sd_lineup_country = None - try: - lineups_response = requests.get( - f"{SD_BASE_URL}/lineups", - headers=sd_headers_fn(token), - timeout=30, - ) - if lineups_response.ok: - for lineup in lineups_response.json().get('lineups', []): - lid = lineup.get('lineupID') or lineup.get('lineup') or '' - if '-' in lid: - sd_lineup_country = lid.split('-')[0] - break - except requests.exceptions.RequestException as e: - logger.warning(f"Could not fetch lineups for country code: {e}") + sd_lineup_country = _sd_fetch_lineup_country(token, sd_headers_fn) send_epg_update( source.id, "parsing_programs", 15, message=f"Fetching guide data for {epg.name or epg.tvg_id}...", ) - station_map = {epg.tvg_id: {'name': epg.name or epg.tvg_id}} + station_map = {epg.tvg_id: {'name': epg.name or epg.tvg_id, 'logo_url': epg.icon_url}} epg_id_map = {epg.tvg_id: epg.id} return station_map, epg_id_map, sd_lineup_country, epg +def _sd_setup_mapped_guide_fetch(source, token, sd_headers_fn): + """Build station_map / epg_id_map for all channels mapped to this SD source.""" + from apps.channels.models import Channel + + mapped_epg_ids = set( + Channel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) + ) + if not mapped_epg_ids: + msg = "No channels mapped to this Schedules Direct source." + logger.info(msg) + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="idle", message=msg) + return None + + station_map = {} + epg_id_map = {} + for epg in EPGData.objects.filter(id__in=mapped_epg_ids, epg_source=source): + if not epg.tvg_id: + continue + station_map[epg.tvg_id] = { + 'name': epg.name or epg.tvg_id, + 'logo_url': epg.icon_url, + } + epg_id_map[epg.tvg_id] = epg.id + + if not station_map: + msg = "Mapped channels have no valid Schedules Direct station IDs." + logger.warning(msg) + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="error", error=msg) + return None + + sd_lineup_country = _sd_fetch_lineup_country(token, sd_headers_fn) + send_epg_update( + source.id, "parsing_programs", 15, + message=f"Fetching guide data for {len(station_map)} mapped stations...", + ) + return station_map, epg_id_map, sd_lineup_country + + +def dispatch_program_refresh_for_epg_ids(epg_ids): + """ + Queue guide/program refresh for newly assigned EPGData rows. + + XMLTV and other non-SD sources use parse_programs_for_tvg_id per id. + Schedules Direct uses per-EPG fetches for small batches and one batched + mapped-station fetch per source when the threshold is exceeded. + """ + if not epg_ids: + return 0 + + epg_ids = {eid for eid in epg_ids if eid} + if not epg_ids: + return 0 + + epgs = list( + EPGData.objects.filter(id__in=epg_ids).select_related('epg_source') + ) + epgs_with_program_data = set( + ProgramData.objects.filter(epg_id__in=epg_ids) + .values_list('epg_id', flat=True) + .distinct() + ) + + non_sd_epg_ids = [] + sd_by_source = {} + for epg in epgs: + if not epg.epg_source: + non_sd_epg_ids.append(epg.id) + continue + source_type = epg.epg_source.source_type + if source_type == 'dummy': + continue + if source_type == 'schedules_direct': + sd_by_source.setdefault(epg.epg_source_id, []).append(epg) + else: + non_sd_epg_ids.append(epg.id) + + dispatched = 0 + for epg_id in non_sd_epg_ids: + parse_programs_for_tvg_id.delay(epg_id) + dispatched += 1 + + for source_id, source_epgs in sd_by_source.items(): + needs_fetch = [ + epg for epg in source_epgs + if epg.id not in epgs_with_program_data + ] + if not needs_fetch: + continue + if len(needs_fetch) >= SD_BULK_GUIDE_FETCH_THRESHOLD: + logger.info( + f"SD source {source_id}: {len(needs_fetch)} new mapping(s) exceed " + f"threshold ({SD_BULK_GUIDE_FETCH_THRESHOLD}); " + "queueing batched mapped guide fetch" + ) + fetch_sd_mapped_guide_batch.delay(source_id) + dispatched += 1 + else: + for epg in needs_fetch: + parse_programs_for_tvg_id.delay(epg.id) + dispatched += 1 + + return dispatched + + @shared_task(time_limit=3600, soft_time_limit=3500) -def fetch_sd_guide_for_epg(epg_id, force=False): +def fetch_sd_mapped_guide_batch(source_id, force=False, _defer_retry=0): + """ + Fetch Schedules Direct guide data for all mapped stations on one source. + + Used when bulk EPG assignment would otherwise queue many per-EPG tasks. + """ + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + logger.error(f"EPGSource {source_id} not found for SD mapped guide batch") + return + + if source.source_type != 'schedules_direct': + return "Not a Schedules Direct source" + + if not acquire_task_lock('sd_mapped_guide_fetch', source_id): + if _defer_retry < SD_MAPPED_GUIDE_FETCH_DEFER_MAX_RETRIES: + logger.info( + f"SD mapped guide batch for source {source_id} already in progress, " + f"deferring retry {_defer_retry + 1}/" + f"{SD_MAPPED_GUIDE_FETCH_DEFER_MAX_RETRIES}" + ) + fetch_sd_mapped_guide_batch.apply_async( + args=[source_id], + kwargs={ + 'force': force, + '_defer_retry': _defer_retry + 1, + }, + countdown=SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + return "Deferred - batch already in progress" + logger.warning( + f"SD mapped guide batch for source {source_id} still locked after " + f"{_defer_retry} deferrals; giving up" + ) + return "Task already running" + + lock_renewer = TaskLockRenewer('sd_mapped_guide_fetch', source_id) + lock_renewer.start() + try: + logger.info(f"Fetching Schedules Direct guide for mapped stations (source: {source.name})") + fetch_schedules_direct(source, mapped_guide_batch=True, force=force) + return "SD mapped guide batch complete" + finally: + lock_renewer.stop() + release_task_lock('sd_mapped_guide_fetch', source_id) + + +@shared_task(time_limit=3600, soft_time_limit=3500) +def fetch_sd_guide_for_epg(epg_id, force=False, _defer_retry=0): """ Fetch Schedules Direct guide data for one mapped EPG entry (channel map flow). @@ -2188,6 +2362,28 @@ def fetch_sd_guide_for_epg(epg_id, force=False): logger.info(f"SD guide fetch skipped for EPG {epg_id}: ProgramData already present") return "Guide data already present" + source_id = epg.epg_source_id + if is_task_lock_held('sd_mapped_guide_fetch', source_id): + if _defer_retry < SD_MAPPED_GUIDE_FETCH_DEFER_MAX_RETRIES: + logger.info( + f"SD mapped batch in progress for source {source_id}; " + f"deferring single-EPG fetch for {epg_id} " + f"(retry {_defer_retry + 1}/{SD_MAPPED_GUIDE_FETCH_DEFER_MAX_RETRIES})" + ) + fetch_sd_guide_for_epg.apply_async( + args=[epg_id], + kwargs={ + 'force': force, + '_defer_retry': _defer_retry + 1, + }, + countdown=SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + return "Deferred - mapped batch in progress" + logger.warning( + f"SD mapped batch still running for source {source_id} after " + f"{_defer_retry} deferrals; proceeding with single-EPG fetch for {epg_id}" + ) + if not acquire_task_lock('parse_epg_programs', epg_id): logger.info(f"SD guide fetch for EPG {epg_id} already in progress, skipping duplicate task") return "Task already running" @@ -2218,7 +2414,13 @@ def fetch_schedules_direct_stations(self, source_id): fetch_schedules_direct(source, stations_only=True) -def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only=None): +def fetch_schedules_direct( + source, + stations_only=False, + force=False, + epg_id_only=None, + mapped_guide_batch=False, +): """ Fetch EPG data from the Schedules Direct JSON API and persist it to the EPGData / ProgramData models. @@ -2250,12 +2452,18 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only from datetime import date single_epg_fetch = epg_id_only is not None + lightweight_sd_fetch = single_epg_fetch or mapped_guide_batch if single_epg_fetch: logger.info( f"Fetching Schedules Direct guide for EPG {epg_id_only} " f"(source: {source.name})" ) + elif mapped_guide_batch: + logger.info( + f"Fetching Schedules Direct guide for mapped stations " + f"(source: {source.name})" + ) else: logger.info(f"Fetching Schedules Direct data for source: {source.name}") @@ -2284,7 +2492,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only # updated_at, which would otherwise incorrectly trigger this guard). Always # allow the first full refresh through so guide data is immediately available. # ------------------------------------------------------------------------- - if not stations_only and not force and not single_epg_fetch and source.updated_at: + if not stations_only and not force and not lightweight_sd_fetch and source.updated_at: from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() if has_prior_full_refresh: @@ -2305,7 +2513,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only return else: logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") - elif force and not stations_only and not single_epg_fetch: + elif force and not stations_only and not lightweight_sd_fetch: logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") # ------------------------------------------------------------------------- @@ -2344,7 +2552,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only ) if fetch_posters and poster_program_ids: - logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") + logger.info("Poster fetch enabled, retrieving program artwork from Schedules Direct.") send_epg_update(source.id, "parsing_programs", 98, message="Fetching program artwork...") try: @@ -2477,7 +2685,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only # This is a requirement of the Schedules Direct API specification, not an # architectural choice. # ------------------------------------------------------------------------- - if not single_epg_fetch: + if not lightweight_sd_fetch: source.status = EPGSource.STATUS_FETCHING source.last_message = "Authenticating with Schedules Direct..." source.save(update_fields=['status', 'last_message']) @@ -2572,6 +2780,11 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only if setup is None: return station_map, epg_id_map, sd_lineup_country, _single_epg = setup + elif mapped_guide_batch: + setup = _sd_setup_mapped_guide_fetch(source, token, _sd_headers) + if setup is None: + return + station_map, epg_id_map, sd_lineup_country = setup else: # ------------------------------------------------------------------------- # Step 3: Fetch subscribed lineups and build station map @@ -2608,7 +2821,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only send_epg_update(source.id, "refresh", 100, status="idle", message=msg) return logger.info(f"Found {len(lineups)} lineup(s) in SD account.") - + # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) sd_lineup_country = None for l in lineups: @@ -2625,7 +2838,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only source.save(update_fields=['status', 'last_message']) send_epg_update(source.id, "refresh", 100, status="error", error=msg) return - + # Build station metadata map: stationID -> {name, callsign, logo_url} station_map = {} send_epg_update(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") @@ -2662,7 +2875,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") except requests.exceptions.RequestException as e: logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") - + if not station_map: msg = "No stations found across all Schedules Direct lineups." logger.warning(msg) @@ -2671,9 +2884,9 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only source.save(update_fields=['status', 'last_message']) send_epg_update(source.id, "refresh", 100, status="error", error=msg) return - + logger.info(f"Built station map with {len(station_map)} stations.") - + # ------------------------------------------------------------------------- # Step 4: Persist station metadata to EPGData # ------------------------------------------------------------------------- @@ -2681,23 +2894,23 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only source.last_message = f"Syncing {len(station_map)} stations..." source.save(update_fields=['status', 'last_message']) send_epg_update(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") - + existing_epg_map = { epg.tvg_id: epg for epg in EPGData.objects.filter(epg_source=source) } - + epgs_to_create = [] epgs_to_update = [] icon_max_length = EPGData._meta.get_field('icon_url').max_length name_max_length = EPGData._meta.get_field('name').max_length - + for sid, info in station_map.items(): display_name = (info['name'] or sid)[:name_max_length] logo = info['logo_url'] if logo and len(logo) > icon_max_length: logo = None - + if sid in existing_epg_map: epg_obj = existing_epg_map[sid] needs_update = False @@ -2716,29 +2929,29 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only icon_url=logo, epg_source=source, )) - + if epgs_to_create: EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") if epgs_to_update: EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") - + gc.collect() - + # Rebuild map with fresh DB ids for all stations epg_id_map = { epg.tvg_id: epg.id for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) } - + # Station sync complete. Send progress update before continuing into programs phase. # We deliberately do NOT send parsing_channels at 100 with status=success here # because that would cause the frontend to mark the source as complete and # stop rendering progress updates for the subsequent program fetch phases. send_epg_update(source.id, "parsing_programs", 30, message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") - + # ------------------------------------------------------------------------- # Stations-only mode. Used on initial source creation. # Stop here so the user can run Auto-match EPG before the full program fetch. @@ -2757,7 +2970,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only message=success_msg, channels_count=len(station_map)) logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") return - + # ------------------------------------------------------------------------- # Step 5: MD5-delta schedule fetch # Only mapped channels need guide data. Fetch MD5 hashes and schedules for @@ -2808,6 +3021,12 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only source.save(update_fields=['last_message']) send_epg_update(source.id, "parsing_programs", 100, status="idle", message=msg) return + if mapped_guide_batch: + msg = "No mapped channels with guide data to fetch." + source.last_message = msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="idle", message=msg) + return success_msg = ( f"{len(station_map)} lineup stations synced. " "Map channels to EPG entries, then refresh to populate guide data." @@ -2916,7 +3135,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only if not changed_by_station: logger.info("No schedule changes detected, skipping schedule and program downloads.") _sd_post_refresh_tasks(mapped_epg_ids, {}, today) - if single_epg_fetch: + if lightweight_sd_fetch: msg = "No schedule updates needed; guide data is up to date." source.last_message = msg source.save(update_fields=['last_message']) @@ -3496,6 +3715,25 @@ def fetch_schedules_direct(source, stations_only=False, force=False, epg_id_only logger.info(f"Schedules Direct single-EPG fetch complete for source: {source.name}") return + if mapped_guide_batch: + success_msg = ( + f"Fetched {total_programs:,} programs for " + f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " + f"({skipped_unmapped:,} programs skipped for unmapped stations)." + ) + source.last_message = success_msg + source.save(update_fields=['last_message']) + send_epg_update(source.id, "parsing_programs", 100, status="success", message=success_msg) + log_system_event( + event_type='epg_refresh', + source_name=source.name, + programs=total_programs, + channels=len(mapped_tvg_ids), + skipped_programs=skipped_unmapped, + ) + logger.info(f"Schedules Direct mapped guide batch complete for source: {source.name}") + return + success_msg = ( f"Successfully fetched {total_programs:,} programs for " f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " @@ -3565,9 +3803,7 @@ def parse_xmltv_time(time_str): def parse_schedules_direct_time(time_str): try: dt_obj = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%SZ') - aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - logger.debug(f"Parsed Schedules Direct time '{time_str}' to {aware_dt}") - return aware_dt + return timezone.make_aware(dt_obj, timezone=dt_timezone.utc) except Exception as e: logger.error(f"Error parsing Schedules Direct time '{time_str}': {e}", exc_info=True) raise diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index b2ec41fd..52a59330 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -769,6 +769,235 @@ class ParseSchedulesDirectTimeTests(TestCase): parse_schedules_direct_time('not-a-timestamp') +# --------------------------------------------------------------------------- +# dispatch_program_refresh_for_epg_ids tests +# --------------------------------------------------------------------------- + +class SDDispatchProgramRefreshTests(TestCase): + """Bulk SD assignment should batch guide fetches above the threshold.""" + + STATION = '10001' + + def _make_sd_source(self): + return EPGSource.objects.create( + name='SD Dispatch Test', + source_type='schedules_direct', + username='sduser', + password='sdpass', + ) + + def _make_xml_source(self): + return EPGSource.objects.create( + name='XML Dispatch Test', + source_type='xmltv', + url='http://example.com/epg.xml', + ) + + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.delay') + @patch('apps.epg.tasks.parse_programs_for_tvg_id.delay') + def test_xmltv_still_uses_parse_programs_per_id( + self, mock_parse_delay, mock_batch_delay, + ): + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids + + xml_source = self._make_xml_source() + epg = EPGData.objects.create( + tvg_id='xml-1', + name='XML Channel', + epg_source=xml_source, + ) + + count = dispatch_program_refresh_for_epg_ids({epg.id}) + + self.assertEqual(count, 1) + mock_parse_delay.assert_called_once_with(epg.id) + mock_batch_delay.assert_not_called() + + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.delay') + @patch('apps.epg.tasks.parse_programs_for_tvg_id.delay') + def test_sd_below_threshold_uses_per_epg_tasks( + self, mock_parse_delay, mock_batch_delay, + ): + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids + + source = self._make_sd_source() + epgs = [ + EPGData.objects.create( + tvg_id=f'{self.STATION}{i}', + name=f'Station {i}', + epg_source=source, + ) + for i in range(2) + ] + + count = dispatch_program_refresh_for_epg_ids({e.id for e in epgs}) + + self.assertEqual(count, 2) + self.assertEqual(mock_parse_delay.call_count, 2) + mock_batch_delay.assert_not_called() + + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.delay') + @patch('apps.epg.tasks.parse_programs_for_tvg_id.delay') + def test_sd_at_threshold_uses_batched_fetch( + self, mock_parse_delay, mock_batch_delay, + ): + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids + + source = self._make_sd_source() + epgs = [ + EPGData.objects.create( + tvg_id=f'{self.STATION}{i}', + name=f'Station {i}', + epg_source=source, + ) + for i in range(3) + ] + + count = dispatch_program_refresh_for_epg_ids({e.id for e in epgs}) + + self.assertEqual(count, 1) + mock_batch_delay.assert_called_once_with(source.id) + mock_parse_delay.assert_not_called() + + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.delay') + @patch('apps.epg.tasks.parse_programs_for_tvg_id.delay') + def test_sd_skips_when_program_data_exists( + self, mock_parse_delay, mock_batch_delay, + ): + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids + + source = self._make_sd_source() + epg = EPGData.objects.create( + tvg_id=self.STATION, + name='Has Data', + epg_source=source, + ) + start = timezone.now() + ProgramData.objects.create( + epg=epg, + start_time=start, + end_time=start + timedelta(hours=1), + title='Show', + tvg_id=epg.tvg_id, + ) + + count = dispatch_program_refresh_for_epg_ids({epg.id}) + + self.assertEqual(count, 0) + mock_parse_delay.assert_not_called() + mock_batch_delay.assert_not_called() + + +class SDGuideFetchCoordinationTests(TestCase): + """Batch and single-EPG SD fetches coordinate via locks and deferred retries.""" + + STATION = '10001' + + def _make_sd_source(self): + return EPGSource.objects.create( + name='SD Coordination', + source_type='schedules_direct', + username='sduser', + password='sdpass', + ) + + @patch('apps.epg.tasks.fetch_schedules_direct') + @patch('apps.epg.tasks.acquire_task_lock', return_value=False) + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.apply_async') + def test_batch_fetch_defers_when_lock_held( + self, mock_apply_async, mock_acquire, mock_fetch, + ): + from apps.epg.tasks import ( + fetch_sd_mapped_guide_batch, + SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + + source = self._make_sd_source() + result = fetch_sd_mapped_guide_batch(source.id) + + self.assertEqual(result, 'Deferred - batch already in progress') + mock_apply_async.assert_called_once_with( + args=[source.id], + kwargs={'force': False, '_defer_retry': 1}, + countdown=SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + mock_fetch.assert_not_called() + + @patch('apps.epg.tasks.fetch_schedules_direct') + @patch('apps.epg.tasks.acquire_task_lock', return_value=False) + @patch('apps.epg.tasks.fetch_sd_mapped_guide_batch.apply_async') + def test_batch_fetch_stops_after_max_defer_retries( + self, mock_apply_async, mock_acquire, mock_fetch, + ): + from apps.epg.tasks import fetch_sd_mapped_guide_batch + + source = self._make_sd_source() + result = fetch_sd_mapped_guide_batch(source.id, _defer_retry=2) + + self.assertEqual(result, 'Task already running') + mock_apply_async.assert_not_called() + mock_fetch.assert_not_called() + + @patch('apps.epg.tasks.fetch_schedules_direct') + @patch('apps.epg.tasks.acquire_task_lock', return_value=True) + @patch('apps.epg.tasks.release_task_lock') + @patch('apps.epg.tasks.TaskLockRenewer') + @patch('apps.epg.tasks.is_task_lock_held', return_value=True) + @patch('apps.epg.tasks.fetch_sd_guide_for_epg.apply_async') + def test_single_epg_defers_while_batch_running( + self, mock_apply_async, mock_batch_held, mock_renewer, + mock_release, mock_acquire, mock_fetch, + ): + from apps.epg.tasks import ( + fetch_sd_guide_for_epg, + SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + + source = self._make_sd_source() + epg = EPGData.objects.create( + tvg_id=self.STATION, + name='Deferred Station', + epg_source=source, + ) + + result = fetch_sd_guide_for_epg(epg.id) + + self.assertEqual(result, 'Deferred - mapped batch in progress') + mock_apply_async.assert_called_once_with( + args=[epg.id], + kwargs={'force': False, '_defer_retry': 1}, + countdown=SD_MAPPED_GUIDE_BATCH_DEFER_SECONDS, + ) + mock_fetch.assert_not_called() + mock_acquire.assert_not_called() + + @patch('apps.epg.tasks.fetch_schedules_direct') + @patch('apps.epg.tasks.acquire_task_lock', return_value=True) + @patch('apps.epg.tasks.release_task_lock') + @patch('apps.epg.tasks.TaskLockRenewer') + @patch('apps.epg.tasks.is_task_lock_held', return_value=True) + @patch('apps.epg.tasks.fetch_sd_guide_for_epg.apply_async') + def test_single_epg_proceeds_after_max_batch_deferrals( + self, mock_apply_async, mock_batch_held, mock_renewer, + mock_release, mock_acquire, mock_fetch, + ): + from apps.epg.tasks import fetch_sd_guide_for_epg + + source = self._make_sd_source() + epg = EPGData.objects.create( + tvg_id=self.STATION, + name='Fallback Station', + epg_source=source, + ) + + result = fetch_sd_guide_for_epg(epg.id, _defer_retry=2) + + self.assertEqual(result, 'SD guide fetch complete') + mock_apply_async.assert_not_called() + mock_fetch.assert_called_once() + mock_acquire.assert_called_once_with('parse_epg_programs', epg.id) + + # --------------------------------------------------------------------------- # Signal tests # --------------------------------------------------------------------------- diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 3b504ebf..dc623db6 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -2570,20 +2570,18 @@ def sync_auto_channels(account_id, scan_start_time=None): channels_created += len(channel_objs) - # One EPG parse task per unique EPGData replaces the - # per-channel post_save dispatch bypassed by bulk_create. - from apps.epg.tasks import parse_programs_for_tvg_id + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids unique_epg_ids = { ch.epg_data_id for ch in channel_objs if ch.epg_data_id } - for epg_id in unique_epg_ids: - parse_programs_for_tvg_id.delay(epg_id) + parse_dispatched = dispatch_program_refresh_for_epg_ids(unique_epg_ids) logger.debug( f"Bulk created {len(channel_objs)} channels in group " f"'{channel_group.name}'; dispatched " - f"{len(unique_epg_ids)} unique EPG parse task(s)" + f"{parse_dispatched} EPG refresh task(s) for " + f"{len(unique_epg_ids)} unique EPG id(s)" ) # bulk_update writes only the columns named in `fields` and @@ -2597,17 +2595,14 @@ def sync_auto_channels(account_id, scan_start_time=None): batch_size=500, ) if epg_dirty_channel_ids: - from apps.epg.tasks import parse_programs_for_tvg_id + from apps.epg.tasks import dispatch_program_refresh_for_epg_ids - # Dispatch only for channels whose epg_data_id changed. - # Other dirty channels would queue redundant parses. unique_epg_ids = { ch.epg_data_id for ch in existing_dirty_channels if ch.id in epg_dirty_channel_ids and ch.epg_data_id } - for epg_id in unique_epg_ids: - parse_programs_for_tvg_id.delay(epg_id) + dispatch_program_refresh_for_epg_ids(unique_epg_ids) logger.debug( f"Bulk updated {len(existing_dirty_channels)} existing " f"channels (fields: {sorted(existing_dirty_field_set)})" diff --git a/core/utils.py b/core/utils.py index c0a70987..08dda3dc 100644 --- a/core/utils.py +++ b/core/utils.py @@ -271,6 +271,15 @@ def release_task_lock(task_name, id): redis_client.delete(lock_id) +def is_task_lock_held(task_name, id): + """Return True when another worker holds the task lock (read-only check).""" + redis_client = RedisClient.get_client() + if redis_client is None: + return False + lock_id = f"task_lock_{task_name}_{id}" + return bool(redis_client.exists(lock_id)) + + class TaskLockRenewer: """Periodically renews a Redis task lock to prevent expiry during long-running tasks. From 703926bca5ee16ac5779e220d2c278392b122bbd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 16:58:43 -0500 Subject: [PATCH 24/81] fix(m3u): improve RANGE_EXHAUSTED error reporting for provider mode - Introduced a new helper function to generate user-facing error messages for RANGE_EXHAUSTED failures, ensuring the fallback range is correctly displayed. - Updated tests to verify that the error message reflects the fallback range instead of the hidden auto_sync_channel_start, enhancing clarity for users. --- apps/m3u/tasks.py | 16 +++++++++++++--- apps/m3u/tests/test_sync_correctness.py | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index a4d863e9..90037c0f 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1740,6 +1740,14 @@ def _pick_target_number( return _next_available_number(used_numbers, fixed_cursor, end=end_number) +def _range_exhausted_error(mode, start_number, end_number, fallback_start): + """User-facing range text for RANGE_EXHAUSTED failures.""" + range_start = ( + int(fallback_start) if mode == "provider" else int(start_number) + ) + return f"Channel number range {range_start}-{int(end_number)} is full" + + def _custom_properties_as_dict(value): """ Normalize a JSONField-backed custom_properties value into a dict. @@ -2454,9 +2462,11 @@ def sync_auto_channels(account_id, scan_start_time=None): "stream_id": stream.id, "group": channel_group.name, "reason": "RANGE_EXHAUSTED", - "error": ( - f"Channel number range " - f"{int(start_number)}-{int(end_number)} is full" + "error": _range_exhausted_error( + channel_numbering_mode, + start_number, + end_number, + channel_numbering_fallback, ), }) processed_stream_ids.add(stream.id) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index beeeef0d..3cc24942 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -2494,6 +2494,30 @@ class ProviderNumberingHonorsProviderNumberTests(TestCase): manual.refresh_from_db() self.assertEqual(manual.channel_number, 88250.0) + def test_provider_fallback_exhaustion_reports_visible_start(self): + # RANGE_EXHAUSTED must cite the fallback range (channel_numbering_fallback + # to End), not the hidden auto_sync_channel_start left from another mode. + account = _make_account() + group = _make_group(name="PPV") + rel = _attach_group_to_account(account, group) + rel.auto_sync_channel_start = 5000 + rel.auto_sync_channel_end = 102 + rel.custom_properties = { + "channel_numbering_mode": "provider", + "channel_numbering_fallback": 100, + } + rel.save() + for i in range(4): + _make_stream(account, group, name=f"S{i}", tvg_id=f"s{i}") + + result = _sync(account) + + self.assertEqual(result["channels_created"], 3) + self.assertEqual(result["channels_failed"], 1) + error = result["failed_stream_details"][0]["error"] + self.assertIn("100-102", error) + self.assertNotIn("5000", error) + class CrossModeNumberingFieldTests(TestCase): """ From 546a05ff0b7b8c816b255475e7c959576039c963 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 17:04:46 -0500 Subject: [PATCH 25/81] changelog: Update for auto sync pr. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12a9da75..a9cd96f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Schedules Direct program poster proxy omitted User-Agent on image requests.** The poster endpoint authenticated with SD correctly but fetched images with only the `token` header, which violates SD's API requirements (error 1003). Image requests now include the standard `Dispatcharr/{version}` User-Agent via `dispatcharr_http_headers()`. - **Schedules Direct guide data missing after mapping a channel.** Lineup refreshes cached schedule MD5s for all stations, but `ProgramData` was only written for mapped channels. Mapping a channel later could leave MD5s looking unchanged so schedules were skipped and the channel had no guide until dates rolled outside the cached window. Refreshes now backfill fetch-window dates that lack `ProgramData` (including newly mapped stations with stale cache), fetch program metadata when no local `ProgramData` exists for a `programID`, and clean up orphaned `ProgramData` on unmapped `EPGData` entries. — Thanks [@sethwv](https://github.com/sethwv) - **Schedules Direct guide fetch on channel map.** Mapping a channel (or bulk-assigning EPG) now triggers a targeted guide fetch for that `EPGData` entry—mirroring XMLTV's `parse_programs_for_tvg_id` flow—without waiting for the next full source refresh. Skips the fetch when `ProgramData` already exists so additional channels sharing the same `tvg-id` do not trigger redundant API calls. Bulk assignment of three or more SD stations without guide data on the same source queues one batched mapped-station fetch instead of separate per-station API sessions. Concurrent batch and single-EPG fetches coordinate via source-level locks with deferred retries so mappings are not dropped and overlapping SD API sessions are avoided when possible. +- **Auto-sync numbering modes now read only the fields each mode's UI exposes.** After the auto-sync overhaul, switching between Provider, Next Available, and Fixed modes left stale `auto_sync_channel_start` / `auto_sync_channel_end` values in the database while each mode's UI only edits a subset of those fields. Sync treated the hidden values as authoritative in every mode, which discarded valid provider numbers (floored at an auto-computed start), capped Next Available at a stale End, and ran range-enforcement deletes against provider- and next-available-numbered channels. Provider mode now honors `stream_chno` verbatim when free (Start/End bound only the fallback for numberless streams); Next Available ignores End; overflow-delete runs in Fixed mode only. Duplicate or already-taken provider numbers fall back to a free slot instead of being dropped or overwriting an existing channel. Provider mode's Start # field now drops End when it would invert the fallback range (matching Fixed mode). (Closes #1273) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.26.0] - 2026-06-07 From f9336a81153c1ccda807f61e2f4ed63564ae99f0 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 11 Jun 2026 18:26:14 -0500 Subject: [PATCH 26/81] changelog: Update changelog for pr. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9cd96f4..d41c6150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Progress, bulk completion, and single-channel results use `send_websocket_update` instead of `async_to_sync(channel_layer.group_send)`, so notifications work reliably under gevent-patched uWSGI and Celery workers. - Single-channel and selected-channel auto-match always run, even when the channel already has EPG assigned; match-all (no channel IDs) still only processes channels without EPG. - Rematching to the same EPG no longer re-saves the channel or queues program-parse tasks; only assignments that actually change are written and refreshed. +- **Easier EPG search when editing a channel.** The filter in the EPG picker now works more like a normal search box. You can type several words at once (e.g. `sky uk`) and it finds channels where every word appears somewhere in the name or TVG-ID, the order doesn't matter, and a word in the name can pair with one in the ID. Accents are ignored too, so `decale` matches `Décalé` and the other way around. — Thanks [@FiveBoroughs](https://github.com/FiveBoroughs) ### Performance From 38389a81e64328689c26cbc3c37c1223ef516ba9 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 12 Jun 2026 11:31:09 -0500 Subject: [PATCH 27/81] feat(settings): add DATABASE_POOL_CONN_MAX_LIFETIME for connection management and update database engine path (Fixes #1343) - Introduced DATABASE_POOL_CONN_MAX_LIFETIME to manage pooled connection lifecycle, improving performance and resource management. - Updated database engine path to use 'dispatcharr.db.backends.postgresql_psycopg3' for consistency. - Enhanced logging configuration for geventpool connection lifecycle tracking. --- dispatcharr/db/__init__.py | 0 dispatcharr/db/backends/__init__.py | 0 .../backends/postgresql_psycopg3/__init__.py | 0 .../db/backends/postgresql_psycopg3/base.py | 44 +++++++++ .../db/backends/postgresql_psycopg3/pool.py | 92 +++++++++++++++++++ dispatcharr/settings.py | 15 ++- tests/test_geventpool_conn_lifetime.py | 65 +++++++++++++ 7 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 dispatcharr/db/__init__.py create mode 100644 dispatcharr/db/backends/__init__.py create mode 100644 dispatcharr/db/backends/postgresql_psycopg3/__init__.py create mode 100644 dispatcharr/db/backends/postgresql_psycopg3/base.py create mode 100644 dispatcharr/db/backends/postgresql_psycopg3/pool.py create mode 100644 tests/test_geventpool_conn_lifetime.py diff --git a/dispatcharr/db/__init__.py b/dispatcharr/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dispatcharr/db/backends/__init__.py b/dispatcharr/db/backends/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dispatcharr/db/backends/postgresql_psycopg3/__init__.py b/dispatcharr/db/backends/postgresql_psycopg3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dispatcharr/db/backends/postgresql_psycopg3/base.py b/dispatcharr/db/backends/postgresql_psycopg3/base.py new file mode 100644 index 00000000..f5a49ffd --- /dev/null +++ b/dispatcharr/db/backends/postgresql_psycopg3/base.py @@ -0,0 +1,44 @@ +try: + import psycopg +except ImportError as e: + from django.core.exceptions import ImproperlyConfigured + + raise ImproperlyConfigured("Error loading psycopg3 module: %s" % e) from e + +from django.db.backends.postgresql.base import DatabaseWrapper as OriginalDatabaseWrapper +from django_db_geventpool.backends.base import DatabaseWrapperMixin + +from .pool import DatabaseConnectionPool + + +class PostgresConnectionPool(DatabaseConnectionPool): + DBERROR = psycopg.DatabaseError + + def __init__(self, *args, **kwargs): + self.connect = kwargs.pop("connect", psycopg.connect) + self.connection = None + maxsize = kwargs.pop("MAX_CONNS", 4) + reuse = kwargs.pop("REUSE_CONNS", maxsize) + max_lifetime = kwargs.pop("CONN_MAX_LIFETIME", None) + self.args = args + self.kwargs = kwargs + self.kwargs["client_encoding"] = "UTF8" + super().__init__(maxsize, reuse, max_lifetime=max_lifetime) + + def create_connection(self): + return self.connect(*self.args, **self.kwargs) + + def check_usable(self, connection): + connection.cursor().execute("SELECT 1") + + +class DatabaseWrapper(DatabaseWrapperMixin, OriginalDatabaseWrapper): + pool_class = PostgresConnectionPool + INTRANS = psycopg.pq.TransactionStatus.INTRANS + + def get_connection_params(self) -> dict: + conn_params = super().get_connection_params() + for attr in ("MAX_CONNS", "REUSE_CONNS", "CONN_MAX_LIFETIME"): + if attr in self.settings_dict["OPTIONS"]: + conn_params[attr] = self.settings_dict["OPTIONS"][attr] + return conn_params diff --git a/dispatcharr/db/backends/postgresql_psycopg3/pool.py b/dispatcharr/db/backends/postgresql_psycopg3/pool.py new file mode 100644 index 00000000..8d50a2ae --- /dev/null +++ b/dispatcharr/db/backends/postgresql_psycopg3/pool.py @@ -0,0 +1,92 @@ +"""geventpool with bounded connection lifetime. + +django-db-geventpool keeps warm connections open indefinitely. psycopg3 accumulates +cache on long-lived handles. We close and replace after +CONN_MAX_LIFETIME rather than recycling uWSGI workers, which would interrupt +live stream backends. +""" +import logging +import time + +try: + from gevent import queue +except ImportError: + from eventlet import queue + +from django_db_geventpool.backends.pool import DatabaseConnectionPool as BasePool + +logger = logging.getLogger("django.geventpool") + + +class DatabaseConnectionPool(BasePool): + def __init__(self, maxsize: int = 100, reuse: int = 100, max_lifetime: float | None = None): + super().__init__(maxsize, reuse) + self.max_lifetime = max_lifetime + + def _stamp_connection(self, conn) -> None: + conn._dispatcharr_pool_created_at = time.monotonic() + + def _connection_expired(self, conn) -> bool: + if not self.max_lifetime: + return False + created_at = getattr(conn, "_dispatcharr_pool_created_at", None) + if created_at is None: + return False + return (time.monotonic() - created_at) >= self.max_lifetime + + def _close_connection(self, conn) -> None: + try: + conn.close() + except Exception: + logger.debug("Error closing pool connection", exc_info=True) + finally: + self._conns.discard(conn) + + def get(self): + conn = None + try: + if self.size >= self.maxsize or self.pool.qsize(): + conn = self.pool.get() + else: + conn = self.pool.get_nowait() + + if conn is not None and self._connection_expired(conn): + logger.debug( + "DB connection expired after %ss, replacing", + int(self.max_lifetime), + ) + self._close_connection(conn) + conn = None + elif conn is not None: + try: + self.check_usable(conn) + logger.trace("DB connection reused") + except self.DBERROR: + logger.debug("DB connection was closed, creating a new one") + self._close_connection(conn) + conn = None + except queue.Empty: + conn = None + logger.trace("DB connection queue empty, creating a new one") + + if conn is None: + conn = self.create_connection() + self._stamp_connection(conn) + self._conns.add(conn) + + return conn + + def put(self, item): + if self._connection_expired(item): + logger.debug( + "DB connection expired after %ss on return, closing", + int(self.max_lifetime), + ) + self._close_connection(item) + return + + try: + self.pool.put_nowait(item) + logger.trace("DB connection returned to the pool") + except queue.Full: + self._close_connection(item) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 51d5bc49..a22e5f9c 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -114,6 +114,12 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries DATABASE_CONN_MAX_AGE = 0 # geventpool intercepts close(); pool handles reuse +# Pooled connections are closed and replaced after this many seconds (per worker). +# psycopg3 cache grows on long-lived handles; rotating bounds RAM without recycling +# uWSGI workers (which would interrupt live streams). Reuse within the window keeps +# the warm-pool performance win over opening a new TCP session every request. +# Override via DATABASE_POOL_CONN_MAX_LIFETIME; set 0 to disable. Default 600 (10 min). +DATABASE_POOL_CONN_MAX_LIFETIME = int(os.environ.get("DATABASE_POOL_CONN_MAX_LIFETIME", "600")) # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -223,7 +229,7 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django_db_geventpool.backends.postgresql_psycopg3", + "ENGINE": "dispatcharr.db.backends.postgresql_psycopg3", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), @@ -234,6 +240,7 @@ else: "MAX_CONNS": 8, # Per-worker pool size; 4 workers × 8 = 32 total < pg max_connections=100 "REUSE_CONNS": 3, # Connections to keep warm between requests "pool": False, # Disable Django's native psycopg3 pool; geventpool manages connections + "CONN_MAX_LIFETIME": DATABASE_POOL_CONN_MAX_LIFETIME or None, }, } } @@ -557,6 +564,12 @@ LOGGING = { "level": LOG_LEVEL, # Use configured log level for scheduler logs "propagate": False, }, + # geventpool connection lifecycle + "django.geventpool": { + "handlers": ["console"], + "level": LOG_LEVEL, + "propagate": False, + }, # Add any other loggers you need to capture TRACE logs from }, "root": { diff --git a/tests/test_geventpool_conn_lifetime.py b/tests/test_geventpool_conn_lifetime.py new file mode 100644 index 00000000..3f939034 --- /dev/null +++ b/tests/test_geventpool_conn_lifetime.py @@ -0,0 +1,65 @@ +"""Tests for dispatcharr geventpool connection max lifetime.""" +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from dispatcharr.db.backends.postgresql_psycopg3.pool import DatabaseConnectionPool + + +class _TestPool(DatabaseConnectionPool): + DBERROR = Exception + + def create_connection(self): + return MagicMock(name="connection", closed=False) + + def check_usable(self, connection): + return None + + +class GeventPoolConnLifetimeTests(TestCase): + def test_fresh_connection_is_stamped_on_create(self): + pool = _TestPool(maxsize=2, reuse=2, max_lifetime=3600) + conn = pool.get() + self.assertIsNotNone(getattr(conn, "_dispatcharr_pool_created_at", None)) + + def test_expired_connection_on_get_is_replaced(self): + pool = _TestPool(maxsize=2, reuse=2, max_lifetime=60) + conn = pool.create_connection() + pool._stamp_connection(conn) + conn._dispatcharr_pool_created_at = 0 + conn.close = MagicMock() + pool._conns.add(conn) + pool.pool.put_nowait(conn) + + with patch("dispatcharr.db.backends.postgresql_psycopg3.pool.time.monotonic", return_value=120): + new_conn = pool.get() + + conn.close.assert_called_once() + self.assertIsNot(new_conn, conn) + + def test_expired_connection_on_put_is_closed_not_pooled(self): + pool = _TestPool(maxsize=2, reuse=2, max_lifetime=60) + conn = pool.get() + conn.close = MagicMock() + conn._dispatcharr_pool_created_at = 0 + + with patch("dispatcharr.db.backends.postgresql_psycopg3.pool.time.monotonic", return_value=120): + pool.put(conn) + + conn.close.assert_called_once() + self.assertEqual(pool.pool.qsize(), 0) + + def test_non_expired_connection_is_returned_to_pool(self): + pool = _TestPool(maxsize=2, reuse=2, max_lifetime=3600) + conn = pool.get() + pool.put(conn) + self.assertEqual(pool.pool.qsize(), 1) + + def test_max_lifetime_disabled_when_none(self): + pool = _TestPool(maxsize=2, reuse=2, max_lifetime=None) + conn = pool.get() + conn._dispatcharr_pool_created_at = 0 + + with patch("dispatcharr.db.backends.postgresql_psycopg3.pool.time.monotonic", return_value=999999): + pool.put(conn) + + self.assertEqual(pool.pool.qsize(), 1) From 092ac2c7356bef3164e5fe203b3c78a4536f6510 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sun, 14 Jun 2026 09:34:33 -0400 Subject: [PATCH 28/81] Split manifest base URL --- Plugin_repo.md | 70 +++++++++++++++++++++++++++++++-------- apps/plugins/api_views.py | 48 +++++++++++++++++++-------- 2 files changed, 91 insertions(+), 27 deletions(-) diff --git a/Plugin_repo.md b/Plugin_repo.md index 1e02efb7..608233a3 100644 --- a/Plugin_repo.md +++ b/Plugin_repo.md @@ -120,12 +120,14 @@ If the name contains any of these, the repo will be rejected on add and skipped ### Top-Level Metadata -| Field | Required | Description | -| --------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `registry_name` | **Yes** | Display name for the repo. Must not contain words like "official" or "dispatcharr" that could be mistaken for an official repo (see [Name Restrictions](#name-restrictions)). | -| `registry_url` | No | URL to the repo's home page (e.g. GitHub). Used as a fallback for generating icon URLs. | -| `root_url` | No | Base URL for resolving relative URLs in plugin entries. Trailing slashes are stripped. | -| `plugins` | **Yes** | Array of plugin entry objects. | +| Field | Required | Description | +| -------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `registry_name` | **Yes** | Display name for the repo. Must not contain words like "official" or "dispatcharr" that could be mistaken for an official repo (see [Name Restrictions](#name-restrictions)). | +| `registry_url` | No | URL to the repo's home page (e.g. GitHub). Used as a fallback for generating icon URLs. | +| `root_url` | No | Generic base URL for resolving all relative URLs in plugin entries. Trailing slashes are stripped. Used as the fallback when neither `download_base_url` nor `metadata_base_url` is set. | +| `download_base_url` | No | Base URL for resolving relative download URLs (`latest_url` in plugin entries; `url` and `latest_url` inside per-plugin manifest `versions`/`latest`). Overrides `root_url` for download assets when set. | +| `metadata_base_url` | No | Base URL for resolving relative metadata URLs (`manifest_url` and `icon_url` in plugin entries). Overrides `root_url` for metadata assets when set. | +| `plugins` | **Yes** | Array of plugin entry objects. | ### Plugin Entry Fields @@ -155,13 +157,18 @@ Extra fields in a plugin entry are passed through to the frontend as-is, so you ### URL Resolution -If `root_url` is set and a URL field (`manifest_url`, `latest_url`, `icon_url`) does not start with `http://` or `https://`, it is treated as relative and resolved as: +Relative URL fields are resolved against a base URL. Dispatcharr uses two separate base URLs (one for metadata assets and one for download assets) so you can serve them from different origins (e.g., manifests and icons on GitHub Pages, release zips on a CDN). -``` -{root_url}/{field_value} -``` +**Resolution priority:** -This lets you keep plugin entries compact: +| Field(s) | Priority | +| --------- | -------- | +| `manifest_url`, `icon_url` | `metadata_base_url` → `root_url` | +| `latest_url` (plugin entries); `url`, `latest_url` (per-plugin manifest versions/latest) | `download_base_url` → `root_url` | + +A field value is treated as relative if it does not start with `http://` or `https://`. Relative values are resolved as `{base_url}/{field_value}`. All base URL fields are optional; if none are set, URL fields must be absolute. + +**Single base URL (simplest):** use `root_url` for everything: ```json { @@ -177,12 +184,45 @@ This lets you keep plugin entries compact: } ``` -**Icon fallback:** If `icon_url` is missing and `registry_url` is set, Dispatcharr generates a fallback URL by converting the GitHub URL to a raw content URL: +**Split base URLs:** use `metadata_base_url` and `download_base_url` when assets are served from different origins: +```json +{ + "metadata_base_url": "https://raw.githubusercontent.com/myorg/my-plugins/main", + "download_base_url": "https://cdn.example.com/releases", + "plugins": [ + { + "slug": "my_plugin", + "manifest_url": "plugins/my_plugin/manifest.json", + "icon_url": "plugins/my_plugin/logo.png", + "latest_url": "my_plugin/my_plugin-1.0.0.zip" + } + ] +} ``` -{registry_url => raw.githubusercontent.com}/refs/heads/main/plugins/{slug}/logo.png + +You can also combine `root_url` with one specific field. The specific field overrides for its consumers, and `root_url` covers the rest: + +```json +{ + "root_url": "https://raw.githubusercontent.com/myorg/my-plugins/main", + "download_base_url": "https://cdn.example.com/releases" +} ``` +**Icon fallback:** If `icon_url` is missing, Dispatcharr tries two fallbacks in order: + +1. **Manifest-directory fallback**: if a base URL is set (`root_url`, `metadata_base_url`, etc.) and `manifest_url` is present, the logo is assumed to live in the same directory as the per-plugin manifest: + ``` + {directory of resolved manifest_url}/logo.png + ``` + For example, if `manifest_url` resolves to `https://example.com/plugins/my_plugin/manifest.json`, the fallback icon URL is `https://example.com/plugins/my_plugin/logo.png`. + +2. **GitHub fallback**: if `registry_url` is a GitHub URL, Dispatcharr converts it to a raw content URL: + ``` + {registry_url => raw.githubusercontent.com}/refs/heads/main/plugins/{slug}/logo.png + ``` + --- ## Per-Plugin Manifest (Optional) @@ -563,7 +603,9 @@ You can host release zips as GitHub Release assets and reference them with absol "manifest": { "registry_name": "string (required)", "registry_url": "string (optional)", - "root_url": "string (optional)", + "root_url": "string (optional, generic base URL fallback)", + "download_base_url": "string (optional, overrides root_url for zip download URLs)", + "metadata_base_url": "string (optional, overrides root_url for manifest_url and icon_url)", "plugins": [ { "slug": "string (required)", diff --git a/apps/plugins/api_views.py b/apps/plugins/api_views.py index 80ab591a..08abe00e 100644 --- a/apps/plugins/api_views.py +++ b/apps/plugins/api_views.py @@ -335,16 +335,28 @@ def _save_fetched_manifest_to_repo(repo, data, verified): return None +def _resolve_manifest_base_urls(manifest: dict) -> tuple[str, str]: + """Return (download_base_url, metadata_base_url) from a manifest dict. + + Both fields fall back to root_url when not set. All base URL fields are + optional; callers must guard against empty strings before building URLs. + """ + root_url = manifest.get("root_url", "").rstrip("/") + download_base_url = manifest.get("download_base_url", "").rstrip("/") or root_url + metadata_base_url = manifest.get("metadata_base_url", "").rstrip("/") or root_url + return download_base_url, metadata_base_url + + def _invalidate_plugin_detail_cache(repo_id, manifest_data): manifest = manifest_data.get("manifest", manifest_data) - root_url = manifest.get("root_url", "").rstrip("/") + _, metadata_base_url = _resolve_manifest_base_urls(manifest) keys = [] for p in manifest.get("plugins", []): url = p.get("manifest_url", "") if not url: continue - if root_url and not url.startswith(("http://", "https://")): - url = f"{root_url}/{url}" + if metadata_base_url and not url.startswith(("http://", "https://")): + url = f"{metadata_base_url}/{url}" keys.append(f"plugin_detail:{repo_id}:{hashlib.md5(url.encode()).hexdigest()}") if keys: cache.delete_many(keys) @@ -1045,18 +1057,28 @@ class AvailablePluginsAPIView(PluginAuthMixin, APIView): for repo in repos: manifest_data = repo.cached_manifest or {} manifest = manifest_data.get("manifest", manifest_data) - root_url = manifest.get("root_url", "").rstrip("/") + download_base_url, metadata_base_url = _resolve_manifest_base_urls(manifest) registry_url = manifest.get("registry_url", "").rstrip("/") repo_plugins = manifest.get("plugins", []) for p in repo_plugins: slug = p.get("slug", "") plugin_data = {**p} - # Resolve relative URLs against root_url; absolute URLs pass through - if root_url: - for url_field in ("manifest_url", "latest_url", "icon_url"): + # Resolve relative URLs; metadata and download assets use separate bases + if metadata_base_url: + for url_field in ("manifest_url", "icon_url"): val = plugin_data.get(url_field, "") if val and not val.startswith(("http://", "https://")): - plugin_data[url_field] = f"{root_url}/{val}" + plugin_data[url_field] = f"{metadata_base_url}/{val}" + if download_base_url: + val = plugin_data.get("latest_url", "") + if val and not val.startswith(("http://", "https://")): + plugin_data["latest_url"] = f"{download_base_url}/{val}" + # Fallback icon_url: if metadata_base_url is explicitly set and manifest_url + # is known, assume logo.png lives in the same directory as the per-plugin + # manifest. Guard against root_url-only manifests to preserve the GitHub fallback. + if not plugin_data.get("icon_url") and manifest.get("metadata_base_url") and plugin_data.get("manifest_url"): + manifest_dir = plugin_data["manifest_url"].rsplit("/", 1)[0] + plugin_data["icon_url"] = f"{manifest_dir}/logo.png" # Fallback icon_url from main branch when not provided if not plugin_data.get("icon_url") and registry_url: # registry_url is e.g. https://github.com/Dispatcharr/Plugins @@ -1161,18 +1183,18 @@ class PluginDetailManifestAPIView(PluginAuthMixin, APIView): # Resolve relative URLs in versions repo_manifest = repo.cached_manifest or {} inner = repo_manifest.get("manifest", repo_manifest) - root_url = inner.get("root_url", "").rstrip("/") + download_base_url, _ = _resolve_manifest_base_urls(inner) - if root_url and isinstance(manifest_obj.get("versions"), list): + if download_base_url and isinstance(manifest_obj.get("versions"), list): for v in manifest_obj["versions"]: url_val = v.get("url", "") if url_val and not url_val.startswith(("http://", "https://")): - v["url"] = f"{root_url}/{url_val}" - if root_url and isinstance(manifest_obj.get("latest"), dict): + v["url"] = f"{download_base_url}/{url_val}" + if download_base_url and isinstance(manifest_obj.get("latest"), dict): for url_field in ("url", "latest_url"): url_val = manifest_obj["latest"].get(url_field, "") if url_val and not url_val.startswith(("http://", "https://")): - manifest_obj["latest"][url_field] = f"{root_url}/{url_val}" + manifest_obj["latest"][url_field] = f"{download_base_url}/{url_val}" result = { "manifest": manifest_obj, From 99c2b3b4d7370d046194909aad0393fcc85bf98a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 14 Jun 2026 11:50:50 -0500 Subject: [PATCH 29/81] fix(proxy): improve channel stop handling and logging - Enhanced the `stop_channel` method to prevent blocking during teardown by logging stop events asynchronously. - Updated stream buffer behavior to ignore late writes during shutdown, improving resource management. --- CHANGELOG.md | 1 + apps/proxy/live_proxy/input/buffer.py | 2 +- apps/proxy/live_proxy/input/manager.py | 2 + apps/proxy/live_proxy/server.py | 208 +++++++++++++++---------- 4 files changed, 130 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d41c6150..c5aa324a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Live proxy channels could remain running with no clients after disconnect.** `stop_channel` called `log_system_event('channel_stop')` synchronously before local cleanup and Redis key deletion; Connect integrations or DB event writes on that path could block teardown indefinitely while the cleanup thread kept refreshing metadata TTL for a stale `stream_buffers` entry. Teardown now stops ffmpeg/output managers, releases ownership, removes local buffers and managers, deletes Redis keys, and only then logs the stop event asynchronously. Metadata TTL refresh skips channels mid-shutdown and non-owned channels; a stuck-stop watchdog forces cleanup if `stop_channel` does not return within ~30s. Stream buffers also ignore late `add_chunk()` writes once shutdown begins (`buffer.stopping` set at the start of `stream_manager.stop()`). - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** - Memory could spike to multiple GB on large EPG sources when building a full in-memory catalog before fuzzy matching; single-channel matching now streams rows and bounds ML work to a small candidate set. diff --git a/apps/proxy/live_proxy/input/buffer.py b/apps/proxy/live_proxy/input/buffer.py index e26ed3ff..b345ead3 100644 --- a/apps/proxy/live_proxy/input/buffer.py +++ b/apps/proxy/live_proxy/input/buffer.py @@ -64,7 +64,7 @@ class StreamBuffer: def add_chunk(self, chunk): """Add data with optimized Redis storage and TS packet alignment""" - if not chunk: + if not chunk or self.stopping: return False try: diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index ccc52318..b560923b 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -1094,6 +1094,8 @@ class StreamManager: # Add at the beginning of your stop method self.stopping = True + if self.buffer is not None: + self.buffer.stopping = True # Release stream resources if we're the owner if self.current_stream_id and hasattr(self, 'worker_id') and self.worker_id: diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 879de2ec..7145fcd4 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -71,6 +71,7 @@ class ProxyServer: self.profile_buffers = {} # {channel_id: {profile_id: StreamBuffer}} self._channel_names = {} self._stopping_channels = set() # channels with an active stop_channel call in progress + self._stopping_since = {} # channel_id -> time.time() when stop_channel began # Generate a unique worker ID pid = os.getpid() @@ -1266,12 +1267,119 @@ class ProxyServer: for pid in list(self.profile_managers.get(channel_id, {}).keys()): self.stop_output_profile(channel_id, pid) + def _collect_channel_stop_event_data(self, channel_id): + """Snapshot metadata for channel_stop logging before Redis keys are deleted.""" + channel_name = self._channel_names.pop(channel_id, None) or str(channel_id) + runtime = None + total_bytes = None + if self.redis_client: + metadata_key = RedisKeys.channel_metadata(channel_id) + metadata = self.redis_client.hgetall(metadata_key) + if metadata: + if 'init_time' in metadata: + try: + init_time = float(metadata['init_time']) + runtime = round(time.time() - init_time, 2) + except Exception: + pass + if 'total_bytes' in metadata: + try: + total_bytes = int(metadata['total_bytes']) + except Exception: + pass + return { + 'channel_id': channel_id, + 'channel_name': channel_name, + 'runtime': runtime, + 'total_bytes': total_bytes, + } + + def _spawn_channel_stop_event(self, stop_event_data): + """Log channel_stop without blocking teardown (DB/connect dispatch can hang).""" + if not stop_event_data: + return + + def _log_stop(): + try: + close_old_connections() + log_system_event('channel_stop', **stop_event_data) + except Exception as e: + logger.error(f"Could not log channel stop event: {e}") + + gevent.spawn(_log_stop) + + def _cleanup_stopped_channel_local(self, channel_id): + """Remove local managers/buffers for a channel that is shutting down.""" + if channel_id in self.stream_managers: + del self.stream_managers[channel_id] + logger.info(f"Removed stream manager for channel {channel_id}") + + if channel_id in self.stream_buffers: + buffer = self.stream_buffers[channel_id] + if hasattr(buffer, 'stop'): + try: + buffer.stop() + logger.debug(f"Buffer for channel {channel_id} properly stopped") + except Exception as e: + logger.error(f"Error stopping buffer: {e}") + + try: + if channel_id in self.stream_buffers: + del self.stream_buffers[channel_id] + logger.info(f"Removed stream buffer for channel {channel_id}") + except KeyError: + logger.debug(f"Buffer for channel {channel_id} already removed") + + if channel_id in self.client_managers: + try: + client_manager = self.client_managers[channel_id] + if hasattr(client_manager, 'stop'): + client_manager.stop() + del self.client_managers[channel_id] + logger.info(f"Removed client manager for channel {channel_id}") + except KeyError: + logger.debug(f"Client manager for channel {channel_id} already removed") + + self.profile_managers.pop(channel_id, None) + self.profile_buffers.pop(channel_id, None) + + def _recover_stuck_channel_stops(self): + """Force cleanup when stop_channel never finishes (e.g. blocked on logging).""" + if not self._stopping_channels: + return + + now = time.time() + stuck_threshold = max(30, ConfigHelper.channel_shutdown_delay() * 2) + + for channel_id in list(self._stopping_channels): + started = self._stopping_since.get(channel_id, now) + if now - started < stuck_threshold: + continue + + logger.error( + f"Channel {channel_id} stop_channel stuck for {now - started:.0f}s " + f"- forcing local and Redis cleanup" + ) + self._stopping_channels.discard(channel_id) + self._stopping_since.pop(channel_id, None) + + if (channel_id in self.stream_buffers + or channel_id in self.client_managers + or channel_id in self.stream_managers): + try: + self._cleanup_stopped_channel_local(channel_id) + self._clean_redis_keys(channel_id) + except Exception as e: + logger.error(f"Error during forced cleanup for channel {channel_id}: {e}") + def stop_channel(self, channel_id): """Stop a channel with proper ownership handling""" if channel_id in self._stopping_channels: logger.debug(f"stop_channel already in progress for {channel_id}, ignoring duplicate call") return self._stopping_channels.add(channel_id) + self._stopping_since[channel_id] = time.time() + stop_event_data = None try: logger.info(f"Stopping channel {channel_id}") @@ -1319,90 +1427,16 @@ class ProxyServer: # Stop all output profile transcodes before releasing ownership self.stop_all_output_profiles(channel_id) + stop_event_data = self._collect_channel_stop_event_data(channel_id) + # Release ownership self.release_ownership(channel_id) - # Log channel stop event (after cleanup, before releasing ownership section ends) - try: - channel_name = self._channel_names.pop(channel_id, None) or str(channel_id) - - # Calculate runtime and get total bytes from metadata - runtime = None - total_bytes = None - if self.redis_client: - metadata_key = RedisKeys.channel_metadata(channel_id) - metadata = self.redis_client.hgetall(metadata_key) - if metadata: - # Calculate runtime from init_time - if 'init_time' in metadata: - try: - init_time = float(metadata['init_time']) - runtime = round(time.time() - init_time, 2) - except Exception: - pass - # Get total bytes transferred - if 'total_bytes' in metadata: - try: - total_bytes = int(metadata['total_bytes']) - except Exception: - pass - - log_system_event( - 'channel_stop', - channel_id=channel_id, - channel_name=channel_name, - runtime=runtime, - total_bytes=total_bytes - ) - except Exception as e: - logger.error(f"Could not log channel stop event: {e}") - - # Always clean up local resources - WITH SAFE CHECKS - if channel_id in self.stream_managers: - del self.stream_managers[channel_id] - logger.info(f"Removed stream manager for channel {channel_id}") - - # Stop buffer and ensure all its timers are cancelled - SAFE CHECK HERE - if channel_id in self.stream_buffers: - buffer = self.stream_buffers[channel_id] - # Call stop on buffer to properly shut it down - if hasattr(buffer, 'stop'): - try: - buffer.stop() - logger.debug(f"Buffer for channel {channel_id} properly stopped") - except Exception as e: - logger.error(f"Error stopping buffer: {e}") - - # Save reference and check again before deleting - try: - if channel_id in self.stream_buffers: # Check again to prevent race conditions - del self.stream_buffers[channel_id] - logger.info(f"Removed stream buffer for channel {channel_id}") - except KeyError: - logger.debug(f"Buffer for channel {channel_id} already removed") - - # Clean up client manager - SAFE CHECK HERE TOO - if channel_id in self.client_managers: - try: - client_manager = self.client_managers[channel_id] - # Stop the heartbeat thread before deleting - if hasattr(client_manager, 'stop'): - client_manager.stop() - del self.client_managers[channel_id] - logger.info(f"Removed client manager for channel {channel_id}") - except KeyError: - logger.debug(f"Client manager for channel {channel_id} already removed") - - # Clean up profile managers and buffers. Owner workers clean profile_managers - # (and their profile_buffers) via stop_all_output_profiles above. Non-owner - # workers only populate profile_buffers (not profile_managers), and that block - # is skipped for them, so we always clean both here to avoid stale entries on - # the next connect. - self.profile_managers.pop(channel_id, None) - self.profile_buffers.pop(channel_id, None) - - # Clean up Redis keys + # Always clean up local resources before Redis/logging so teardown + # cannot be blocked by connect integrations or DB event writes. + self._cleanup_stopped_channel_local(channel_id) self._clean_redis_keys(channel_id) + self._spawn_channel_stop_event(stop_event_data) return True except Exception as e: @@ -1410,6 +1444,7 @@ class ProxyServer: return False finally: self._stopping_channels.discard(channel_id) + self._stopping_since.pop(channel_id, None) def check_inactive_channels(self): """Check for inactive channels (no clients) and stop them""" @@ -1450,6 +1485,9 @@ class ProxyServer: # Refresh channel registry self.refresh_channel_registry() + # Recover channels whose stop_channel call never returned + self._recover_stuck_channel_stops() + # Create a unified list of all channels we have locally all_local_channels = set(self.stream_managers.keys()) | set(self.client_managers.keys()) @@ -1927,9 +1965,15 @@ class ProxyServer: if not self.redis_client: return - # Refresh registry entries for channels we own + # Refresh registry entries for channels we own and are actively serving. + # Skip channels mid-shutdown. Refreshing their TTL keeps zombie metadata alive. for channel_id in list(self.stream_buffers.keys()): - # Use standard key pattern + if channel_id in self._stopping_channels: + continue + + if not self.am_i_owner(channel_id): + continue + metadata_key = RedisKeys.channel_metadata(channel_id) # Update activity timestamp in metadata only From 9393f72cc120200537709ec6996483cd846db170 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 14 Jun 2026 18:59:59 -0500 Subject: [PATCH 30/81] fix(proxy): enhance channel teardown and client management - Improved handling of channel teardown to prevent client reconnect issues across uWSGI workers, ensuring proper resource cleanup and ownership management. - Added functionality to clear all client entries from Redis for a channel, enhancing client management during shutdown. - Updated logging and response mechanisms to inform clients of channel availability during teardown,. - Enhanced stream manager behavior to ensure upstream connections are properly managed during ownership transitions. --- CHANGELOG.md | 2 + apps/channels/tests/test_ts_proxy_teardown.py | 232 ++++++++++++++ apps/proxy/live_proxy/client_manager.py | 16 + apps/proxy/live_proxy/input/manager.py | 60 +++- apps/proxy/live_proxy/server.py | 294 +++++++++++++----- .../live_proxy/services/channel_service.py | 99 +++++- apps/proxy/live_proxy/views.py | 21 +- 7 files changed, 625 insertions(+), 99 deletions(-) create mode 100644 apps/channels/tests/test_ts_proxy_teardown.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa324a..2622dfc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Live proxy channels could remain running with no clients after disconnect.** `stop_channel` called `log_system_event('channel_stop')` synchronously before local cleanup and Redis key deletion; Connect integrations or DB event writes on that path could block teardown indefinitely while the cleanup thread kept refreshing metadata TTL for a stale `stream_buffers` entry. Teardown now stops ffmpeg/output managers, releases ownership, removes local buffers and managers, deletes Redis keys, and only then logs the stop event asynchronously. Metadata TTL refresh skips channels mid-shutdown and non-owned channels; a stuck-stop watchdog forces cleanup if `stop_channel` does not return within ~30s. Stream buffers also ignore late `add_chunk()` writes once shutdown begins (`buffer.stopping` set at the start of `stream_manager.stop()`). +- **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` during pending shutdown or active teardown; `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) +- **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** - Memory could spike to multiple GB on large EPG sources when building a full in-memory catalog before fuzzy matching; single-channel matching now streams rows and bounds ML work to a small candidate set. diff --git a/apps/channels/tests/test_ts_proxy_teardown.py b/apps/channels/tests/test_ts_proxy_teardown.py new file mode 100644 index 00000000..14b4e898 --- /dev/null +++ b/apps/channels/tests/test_ts_proxy_teardown.py @@ -0,0 +1,232 @@ +"""Tests for multi-worker channel teardown coordination.""" +import time +from unittest.mock import MagicMock, patch + +from django.test import TestCase + +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState +from apps.proxy.live_proxy.input.manager import StreamManager +from apps.proxy.live_proxy.redis_keys import RedisKeys +from apps.proxy.live_proxy.server import ProxyServer +from apps.proxy.live_proxy.services.channel_service import ChannelService + + +CHANNEL_ID = "00000000-0000-0000-0000-000000000099" + + +def _mock_proxy_server(redis_client=None): + server = MagicMock() + server.redis_client = redis_client or MagicMock() + return server + + +class ChannelTeardownAvailabilityTests(TestCase): + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_teardown_active_when_stopping_key_exists(self, mock_get_instance): + redis = MagicMock() + redis.exists.side_effect = lambda key: key == RedisKeys.channel_stopping(CHANNEL_ID) + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID)) + + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_teardown_active_when_metadata_state_is_stopping(self, mock_get_instance): + redis = MagicMock() + redis.exists.return_value = False + redis.hget.return_value = ChannelState.STOPPING.encode() + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID)) + + @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_shutdown_pending_within_delay_window(self, mock_get_instance, mock_delay): + mock_delay.return_value = 5 + redis = MagicMock() + redis.exists.return_value = False + redis.get.return_value = str(time.time() - 2).encode() + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertTrue(ChannelService.is_shutdown_pending(CHANNEL_ID)) + self.assertTrue(ChannelService.is_channel_unavailable_for_new_clients(CHANNEL_ID)) + + @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_shutdown_pending_expired_after_delay(self, mock_get_instance, mock_delay): + mock_delay.return_value = 5 + redis = MagicMock() + redis.exists.return_value = False + redis.get.return_value = str(time.time() - 10).encode() + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertFalse(ChannelService.is_shutdown_pending(CHANNEL_ID)) + + +class LocalStreamActivityTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server.stream_buffers = {} + server.client_managers = {} + server.profile_managers = {} + server.profile_buffers = {} + server._live_stream_managers = {} + server._stopping_channels = set() + server.redis_client = MagicMock() + server.stop_all_output_formats = MagicMock() + server.stop_all_output_profiles = MagicMock() + return server + + def test_has_local_stream_activity_from_live_registry(self): + server = self._make_server() + server._live_stream_managers[CHANNEL_ID] = MagicMock() + self.assertTrue(server._has_local_stream_activity(CHANNEL_ID)) + + @patch.object(ProxyServer, "_join_stream_thread") + def test_stop_local_stream_activity_stops_live_manager(self, mock_join): + server = self._make_server() + manager = MagicMock() + server._live_stream_managers[CHANNEL_ID] = manager + + server._stop_local_stream_activity(CHANNEL_ID) + + manager.stop.assert_called_once() + mock_join.assert_called_once_with(CHANNEL_ID) + self.assertNotIn(CHANNEL_ID, server._live_stream_managers) + + +class OrphanMetadataCleanupTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server.stream_buffers = {} + server.client_managers = {} + server.profile_managers = {} + server.profile_buffers = {} + server._live_stream_managers = {} + server._stopping_channels = set() + server.redis_client = MagicMock() + return server + + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity") + @patch.object(ProxyServer, "_has_local_stream_activity", return_value=True) + def test_orphan_metadata_stops_local_processes_before_redis( + self, mock_has_local, mock_stop_local, mock_clean_redis + ): + server = self._make_server() + metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) + server.redis_client.keys.return_value = [metadata_key.encode()] + server.redis_client.hgetall.return_value = { + b"owner": b"", + b"state": b"unknown", + } + server.redis_client.exists.return_value = False + server.redis_client.scard.return_value = 0 + + server._check_orphaned_metadata() + + mock_has_local.assert_called_with(CHANNEL_ID) + mock_stop_local.assert_called_once_with(CHANNEL_ID) + mock_clean_redis.assert_called_once_with(CHANNEL_ID) + + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity") + @patch.object(ProxyServer, "_has_local_stream_activity", return_value=False) + def test_orphan_metadata_remote_channel_only_cleans_redis( + self, mock_has_local, mock_stop_local, mock_clean_redis + ): + server = self._make_server() + metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) + server.redis_client.keys.return_value = [metadata_key.encode()] + server.redis_client.hgetall.return_value = {b"owner": b"", b"state": b"unknown"} + server.redis_client.exists.return_value = False + server.redis_client.scard.return_value = 0 + + server._check_orphaned_metadata() + + mock_stop_local.assert_not_called() + mock_clean_redis.assert_called_once_with(CHANNEL_ID) + + +class OrphanChannelCleanupTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server.stream_buffers = {} + server.client_managers = {} + server.profile_managers = {} + server.profile_buffers = {} + server._live_stream_managers = {} + server._stopping_channels = set() + server.redis_client = MagicMock() + server.get_channel_owner = MagicMock(return_value=None) + return server + + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity") + @patch.object(ProxyServer, "_has_local_stream_activity", return_value=True) + def test_orphan_channel_stops_local_before_redis( + self, mock_has_local, mock_stop_local, mock_clean_redis + ): + server = self._make_server() + metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) + server.redis_client.keys.return_value = [metadata_key.encode()] + server.redis_client.scard.return_value = 0 + + server._check_orphaned_channels() + + mock_stop_local.assert_called_once_with(CHANNEL_ID) + mock_clean_redis.assert_called_once_with(CHANNEL_ID) + + +class StreamManagerOwnershipTests(TestCase): + def test_still_owner_false_when_different_worker(self): + buffer = MagicMock() + buffer.redis_client.get.side_effect = lambda key: ( + None if "stopping" in key else b"worker-b" + ) + buffer.redis_client.exists.return_value = False + manager = StreamManager( + CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" + ) + + self.assertFalse(manager._still_owner()) + + def test_still_owner_true_when_owner_lock_expired_but_not_stopping(self): + buffer = MagicMock() + buffer.redis_client.get.return_value = None + buffer.redis_client.exists.return_value = False + manager = StreamManager( + CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" + ) + + self.assertTrue(manager._still_owner()) + + def test_still_owner_false_when_channel_stopping_key_set(self): + buffer = MagicMock() + buffer.redis_client.exists.return_value = True + manager = StreamManager( + CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" + ) + + self.assertFalse(manager._still_owner()) + + def test_update_bytes_skipped_after_ownership_lost(self): + buffer = MagicMock() + buffer.redis_client.get.return_value = b"other-worker" + buffer.redis_client.exists.return_value = False + manager = StreamManager( + CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" + ) + manager.bytes_processed = 1000 + + manager._update_bytes_processed(500) + + buffer.redis_client.hincrby.assert_not_called() diff --git a/apps/proxy/live_proxy/client_manager.py b/apps/proxy/live_proxy/client_manager.py index 3ab1861c..3771d55c 100644 --- a/apps/proxy/live_proxy/client_manager.py +++ b/apps/proxy/live_proxy/client_manager.py @@ -442,3 +442,19 @@ class ClientManager: ) return stale_ids + + @staticmethod + def clear_all_clients(redis_client, channel_id): + """Remove every client entry from Redis for a channel.""" + client_set_key = RedisKeys.clients(channel_id) + client_ids = redis_client.smembers(client_set_key) + if not client_ids: + return 0 + + pipe = redis_client.pipeline(transaction=False) + for cid in client_ids: + cid_str = cid.decode() if isinstance(cid, bytes) else cid + pipe.delete(RedisKeys.client_metadata(channel_id, cid_str)) + pipe.delete(client_set_key) + pipe.execute() + return len(client_ids) diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index b560923b..06b12644 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -182,6 +182,47 @@ class StreamManager: logger.warning(f"Timeout waiting for existing processes to close for channel {self.channel_id} after {timeout}s") return False + def _still_owner(self): + """Return True while this worker should keep the upstream connection open.""" + if self.stopping or self.stop_requested: + return False + + if not self.worker_id: + return True + + redis_client = getattr(self.buffer, 'redis_client', None) + if not redis_client: + return True + + try: + stop_key = RedisKeys.channel_stopping(self.channel_id) + if redis_client.exists(stop_key): + return False + + owner_key = RedisKeys.channel_owner(self.channel_id) + current_owner = redis_client.get(owner_key) + if not current_owner: + # Owner lock TTL may lapse briefly between cleanup-thread renewals. + # Keep running unless coordinated teardown has started (checked above). + return True + if isinstance(current_owner, bytes): + current_owner = current_owner.decode() + return current_owner == self.worker_id + except Exception as e: + logger.debug(f"Ownership check failed for channel {self.channel_id}: {e}") + return False + + def _ensure_owner_or_stop(self): + if self._still_owner(): + return True + + logger.warning( + f"Stream manager for channel {self.channel_id} lost ownership " + f"(worker {self.worker_id}) - stopping upstream" + ) + self.stop() + return False + def run(self): """Main execution loop using HTTP streaming with improved connection handling and stream switching""" # Add a stop flag to the class properties @@ -203,6 +244,8 @@ class StreamManager: # Main stream switching loop - we'll try different streams if needed while self.running and stream_switch_attempts <= max_stream_switches: close_old_connections() + if not self._ensure_owner_or_stop(): + break # Check for stuck switching state if self.url_switching and time.time() - self.url_switch_start_time > self.url_switch_timeout: logger.warning(f"URL switching state appears stuck for channel {self.channel_id} " @@ -255,6 +298,8 @@ class StreamManager: continue # Connection retry loop for current URL while self.running and self.retry_count < self.max_retries and not url_failed and not self.needs_stream_switch: + if not self._ensure_owner_or_stop(): + break logger.info(f"Connection attempt {self.retry_count + 1}/{self.max_retries} for URL: {self.url} for channel {self.channel_id}") @@ -382,6 +427,12 @@ class StreamManager: except Exception as e: logger.error(f"Stream error: {e}", exc_info=True) finally: + try: + from ..server import ProxyServer + ProxyServer.get_instance()._live_stream_managers.pop(self.channel_id, None) + except Exception: + pass + # Enhanced cleanup in the finally block self.connected = False @@ -1020,6 +1071,9 @@ class StreamManager: def _update_bytes_processed(self, chunk_size): """Update the total bytes processed in Redis metadata""" + if not self._still_owner(): + return + try: # Update local counter self.bytes_processed += chunk_size @@ -1584,6 +1638,10 @@ class StreamManager: self.connected = False return False + if not self._still_owner(): + self.stop() + return False + # Track chunk size before adding to buffer chunk_size = len(chunk) self._update_bytes_processed(chunk_size) @@ -1592,7 +1650,7 @@ class StreamManager: success = self.buffer.add_chunk(chunk) # Update last data timestamp in Redis if successful - if success and hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: + if success and self._still_owner() and hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: last_data_key = RedisKeys.last_data(self.buffer.channel_id) self.buffer.redis_client.set(last_data_key, str(time.time()), ex=60) diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 7145fcd4..a348c876 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -72,6 +72,8 @@ class ProxyServer: self._channel_names = {} self._stopping_channels = set() # channels with an active stop_channel call in progress self._stopping_since = {} # channel_id -> time.time() when stop_channel began + # Managers kept until the stream OS thread exits (may outlive stream_managers dict) + self._live_stream_managers = {} # Generate a unique worker ID pid = os.getpid() @@ -295,22 +297,36 @@ class ProxyServer: json.dumps(switch_result) ) elif event_type == EventType.CHANNEL_STOP: - logger.info(f"Received {EventType.CHANNEL_STOP} event for channel {channel_id}") - # First mark channel as stopping in Redis - if self.redis_client: - # Set stopping state in metadata - metadata_key = RedisKeys.channel_metadata(channel_id) - if self.redis_client.exists(metadata_key): - self.redis_client.hset(metadata_key, mapping={ - "state": ChannelState.STOPPING, - "state_changed_at": str(time.time()) - }) + requester_worker_id = data.get("requester_worker_id") + logger.info( + f"Received {EventType.CHANNEL_STOP} event for channel {channel_id} " + f"from worker {requester_worker_id}" + ) - # If we have local resources for this channel, clean them up - if channel_id in self.stream_buffers or channel_id in self.client_managers: - # Use existing stop_channel method - logger.info(f"Stopping local resources for channel {channel_id}") + # Initiating worker already runs local stop_channel() + if requester_worker_id and requester_worker_id == self.worker_id: + logger.debug( + f"Ignoring {EventType.CHANNEL_STOP} for {channel_id}; " + f"this worker initiated teardown" + ) + elif ( + channel_id in self.stream_managers + or channel_id in self._live_stream_managers + ): + logger.info( + f"Owner worker stopping local upstream for channel {channel_id}" + ) self.stop_channel(channel_id) + elif ( + channel_id in self.stream_buffers + or channel_id in self.client_managers + or channel_id in self.profile_managers + or channel_id in self.output_managers + ): + logger.info( + f"Non-owner worker cleaning local resources for channel {channel_id}" + ) + self._cleanup_local_resources(channel_id) # Acknowledge stop by publishing a response stop_response = { @@ -491,8 +507,16 @@ class ProxyServer: current = self.redis_client.get(lock_key) if current is None: - # Key expired — re-acquire if we have the stream_manager + # Key expired, re-acquire if we have the stream_manager, but never + # during coordinated teardown (multi-worker reconnect-during-stop race). if channel_id in self.stream_managers: + if self._channel_unavailable_for_new_clients(channel_id): + logger.info( + f"Refusing ownership re-acquisition for {channel_id}; " + f"teardown or pending shutdown active" + ) + return False + acquired = self.redis_client.set(lock_key, self.worker_id, nx=True, ex=ttl) if acquired: logger.warning(f"Re-acquired expired ownership for channel {channel_id}") @@ -515,6 +539,13 @@ class ProxyServer: def initialize_channel(self, url, channel_id, user_agent=None, transcode=False, stream_id=None): """Initialize a channel without redundant active key""" try: + if self._channel_unavailable_for_new_clients(channel_id): + logger.warning( + f"Refusing to initialize channel {channel_id}; " + f"teardown or pending shutdown active" + ) + return False + if self.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) if self.redis_client.exists(metadata_key): @@ -721,6 +752,7 @@ class ProxyServer: self.client_managers[channel_id] = client_manager # Start stream manager thread only for the owner + self._live_stream_managers[channel_id] = stream_manager thread = threading.Thread(target=stream_manager.run, daemon=True) thread.name = f"stream-{channel_id}" thread.start() @@ -970,10 +1002,34 @@ class ProxyServer: self.redis_client.delete(disconnect_key) return - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) except Exception as e: logger.error(f"Error handling client disconnect for channel {channel_id}: {e}") + def _channel_unavailable_for_new_clients(self, channel_id): + """True when new clients or ownership re-acquisition should be refused.""" + from .services.channel_service import ChannelService + return ChannelService.is_channel_unavailable_for_new_clients(channel_id) + + def _channel_teardown_active(self, channel_id): + """True when a coordinated stop is in progress (Redis-visible to all workers).""" + from .services.channel_service import ChannelService + return ChannelService.is_channel_teardown_active(channel_id) + + def _coordinated_stop_channel(self, channel_id): + """Stop a channel with Redis markers and pubsub visible to all uWSGI workers.""" + from .services.channel_service import ChannelService + return ChannelService.stop_channel(channel_id) + + def _force_clear_channel_clients(self, channel_id): + """Remove all client entries from Redis for a wedged channel.""" + if not self.redis_client: + return 0 + removed = ClientManager.clear_all_clients(self.redis_client, channel_id) + if removed: + logger.warning(f"Force-cleared {removed} client(s) from channel {channel_id}") + return removed + # ------------------------------------------------------------------ # Output format lifecycle # ------------------------------------------------------------------ @@ -1308,40 +1364,90 @@ class ProxyServer: gevent.spawn(_log_stop) - def _cleanup_stopped_channel_local(self, channel_id): - """Remove local managers/buffers for a channel that is shutting down.""" + def _get_stream_thread(self, channel_id): + thread_name = f"stream-{channel_id}" + for thread in threading.enumerate(): + if thread.name == thread_name: + return thread + return None + + def _has_local_stream_activity(self, channel_id): if channel_id in self.stream_managers: - del self.stream_managers[channel_id] - logger.info(f"Removed stream manager for channel {channel_id}") + return True + if channel_id in self._live_stream_managers: + return True + if channel_id in self.stream_buffers: + return True + if channel_id in self.client_managers: + return True + thread = self._get_stream_thread(channel_id) + return thread is not None and thread.is_alive() + + def _join_stream_thread(self, channel_id, timeout=2.0): + thread = self._get_stream_thread(channel_id) + if thread and thread.is_alive(): + logger.info(f"Waiting for stream thread to terminate for channel {channel_id}") + try: + thread.join(timeout=timeout) + if thread.is_alive(): + logger.warning( + f"Stream thread for channel {channel_id} did not terminate within {timeout}s" + ) + except RuntimeError: + logger.debug(f"Could not join stream thread for channel {channel_id}") + + def _resolve_stream_manager(self, channel_id): + manager = self.stream_managers.pop(channel_id, None) + if manager is None: + manager = self._live_stream_managers.pop(channel_id, None) + return manager + + def _stop_local_stream_activity(self, channel_id): + """Stop local ffmpeg/stream threads regardless of registry state.""" + stream_manager = self._resolve_stream_manager(channel_id) + if stream_manager is not None: + if hasattr(stream_manager, 'stop'): + try: + stream_manager.stop() + except Exception as e: + logger.error(f"Error stopping stream manager for channel {channel_id}: {e}") + logger.info(f"Stopped stream manager for channel {channel_id}") + elif self._get_stream_thread(channel_id): + logger.warning( + f"Found live stream thread for channel {channel_id} without a manager reference" + ) + if channel_id in self.stream_buffers: + self.stream_buffers[channel_id].stopping = True + + self._join_stream_thread(channel_id) if channel_id in self.stream_buffers: - buffer = self.stream_buffers[channel_id] + buffer = self.stream_buffers.pop(channel_id) if hasattr(buffer, 'stop'): try: buffer.stop() - logger.debug(f"Buffer for channel {channel_id} properly stopped") except Exception as e: - logger.error(f"Error stopping buffer: {e}") - - try: - if channel_id in self.stream_buffers: - del self.stream_buffers[channel_id] - logger.info(f"Removed stream buffer for channel {channel_id}") - except KeyError: - logger.debug(f"Buffer for channel {channel_id} already removed") + logger.error(f"Error stopping buffer for channel {channel_id}: {e}") + logger.info(f"Removed stream buffer for channel {channel_id}") if channel_id in self.client_managers: try: - client_manager = self.client_managers[channel_id] + client_manager = self.client_managers.pop(channel_id) if hasattr(client_manager, 'stop'): client_manager.stop() - del self.client_managers[channel_id] logger.info(f"Removed client manager for channel {channel_id}") except KeyError: logger.debug(f"Client manager for channel {channel_id} already removed") + self.stop_all_output_formats(channel_id) + self.stop_all_output_profiles(channel_id) self.profile_managers.pop(channel_id, None) self.profile_buffers.pop(channel_id, None) + self._live_stream_managers.pop(channel_id, None) + + def _cleanup_stopped_channel_local(self, channel_id): + """Remove local managers/buffers for a channel that is shutting down.""" + self._stop_local_stream_activity(channel_id) def _recover_stuck_channel_stops(self): """Force cleanup when stop_channel never finishes (e.g. blocked on logging).""" @@ -1363,14 +1469,11 @@ class ProxyServer: self._stopping_channels.discard(channel_id) self._stopping_since.pop(channel_id, None) - if (channel_id in self.stream_buffers - or channel_id in self.client_managers - or channel_id in self.stream_managers): - try: - self._cleanup_stopped_channel_local(channel_id) - self._clean_redis_keys(channel_id) - except Exception as e: - logger.error(f"Error during forced cleanup for channel {channel_id}: {e}") + try: + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) + except Exception as e: + logger.error(f"Error during forced cleanup for channel {channel_id}: {e}") def stop_channel(self, channel_id): """Stop a channel with proper ownership handling""" @@ -1386,7 +1489,10 @@ class ProxyServer: # First set a stopping key that clients will check if self.redis_client: stop_key = RedisKeys.channel_stopping(channel_id) - self.redis_client.setex(stop_key, 10, "true") + if self.redis_client.exists(stop_key): + self.redis_client.expire(stop_key, 60) + else: + self.redis_client.setex(stop_key, 60, "true") # Only stop the actual stream manager if we're the owner if self.am_i_owner(channel_id): @@ -1456,7 +1562,7 @@ class ProxyServer: for channel_id in channels_to_stop: logger.info(f"Auto-stopping inactive channel {channel_id}") - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) def _cleanup_channel(self, channel_id: str) -> None: """Remove channel resources""" @@ -1467,7 +1573,7 @@ class ProxyServer: def shutdown(self) -> None: """Stop all channels and cleanup""" for channel_id in list(self.stream_managers.keys()): - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) def _start_cleanup_thread(self): """Start background thread to maintain ownership and clean up resources""" @@ -1577,7 +1683,7 @@ class ProxyServer: f"Channel {channel_id} in {channel_state} state with 0 clients for {time_since_ready:.1f}s " f"(after reaching ready, shutdown_delay: {shutdown_delay}s) - stopping channel" ) - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) continue else: # Never reached ready - use grace_period timeout @@ -1589,7 +1695,7 @@ class ProxyServer: f"Channel {channel_id} stuck in {channel_state} state for {time_since_start:.1f}s " f"with no clients (timeout: {connecting_timeout}s) - stopping channel due to upstream issues" ) - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) continue elif connection_ready_time: # We have clients now, but check grace period for state transition @@ -1643,7 +1749,7 @@ class ProxyServer: elif current_time - disconnect_time > ConfigHelper.channel_shutdown_delay(): # We've had no clients for the shutdown delay period logger.warning(f"No clients for {current_time - disconnect_time:.1f}s, stopping channel {channel_id}") - self.stop_channel(channel_id) + self._coordinated_stop_channel(channel_id) else: # Still in shutdown delay period logger.debug(f"Channel {channel_id} shutdown timer: " @@ -1663,6 +1769,15 @@ class ProxyServer: # don't fight the shutdown by trying to re-acquire. if channel_id in self._stopping_channels: continue + + if self._channel_unavailable_for_new_clients(channel_id): + logger.info( + f"Channel {channel_id} teardown active with local stream_manager; " + f"finishing stop instead of re-acquiring ownership" + ) + self.stop_channel(channel_id) + continue + logger.warning( f"Ownership gap for {channel_id}: this worker has stream_manager " f"but am_i_owner returned False. Attempting re-acquisition." @@ -1815,13 +1930,12 @@ class ProxyServer: # Orphaned channel with no clients - clean it up logger.info(f"Cleaning up orphaned channel {channel_id}") - # If we have it locally, stop it properly to clean up processes - if channel_id in self.stream_managers or channel_id in self.client_managers: - logger.info(f"Orphaned channel {channel_id} is local - calling stop_channel") - self.stop_channel(channel_id) - else: - # Just clean up Redis keys for remote channels - self._clean_redis_keys(channel_id) + if self._has_local_stream_activity(channel_id): + logger.info( + f"Orphaned channel {channel_id} has local stream activity - stopping processes" + ) + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) except Exception as e: logger.error(f"Error processing channel key {key}: {e}") @@ -1850,11 +1964,9 @@ class ProxyServer: if not metadata: # Empty metadata - clean it up logger.warning(f"Found empty metadata for channel {channel_id} - cleaning up") - # If we have it locally, stop it properly - if channel_id in self.stream_managers or channel_id in self.client_managers: - self.stop_channel(channel_id) - else: - self._clean_redis_keys(channel_id) + if self._has_local_stream_activity(channel_id): + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) continue # Get owner @@ -1875,13 +1987,12 @@ class ProxyServer: state = metadata.get('state', 'unknown') logger.warning(f"Found orphaned metadata for channel {channel_id} (state: {state}, owner: {owner}, clients: {client_count}) - cleaning up") - # If we have it locally, stop it properly to clean up transcode/proxy processes - if channel_id in self.stream_managers or channel_id in self.client_managers: - logger.info(f"Channel {channel_id} is local - calling stop_channel to clean up processes") - self.stop_channel(channel_id) - else: - # Just clean up Redis keys for remote channels - self._clean_redis_keys(channel_id) + if self._has_local_stream_activity(channel_id): + logger.info( + f"Channel {channel_id} has local stream activity - stopping processes" + ) + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) elif not owner_alive and client_count > 0: # SCARD may include ghost entries from a dead worker's # expired metadata hashes. Validate before deciding. @@ -1897,16 +2008,25 @@ class ProxyServer: f"owner: {owner}) had {client_count} ghost client(s) " f"- cleaning up" ) - if channel_id in self.stream_managers or channel_id in self.client_managers: - self.stop_channel(channel_id) - else: - self._clean_redis_keys(channel_id) + if self._has_local_stream_activity(channel_id): + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) else: - logger.warning( - f"Orphaned channel {channel_id} still has " - f"{real_count} live client(s) after ghost removal " - f"- may need ownership takeover" - ) + if self._channel_teardown_active(channel_id): + logger.warning( + f"Orphaned channel {channel_id} still has " + f"{real_count} client(s) during teardown; forcing cleanup" + ) + self._force_clear_channel_clients(channel_id) + if self._has_local_stream_activity(channel_id): + self._stop_local_stream_activity(channel_id) + self._clean_redis_keys(channel_id) + else: + logger.warning( + f"Orphaned channel {channel_id} still has " + f"{real_count} live client(s) after ghost removal " + f"- may need ownership takeover" + ) except Exception as e: logger.error(f"Error processing metadata key {key}: {e}", exc_info=True) @@ -2023,14 +2143,28 @@ class ProxyServer: def _cleanup_local_resources(self, channel_id): """Clean up local resources for a channel without affecting Redis keys""" try: - if channel_id in self.stream_managers: - if hasattr(self.stream_managers[channel_id], 'stop'): - self.stream_managers[channel_id].stop() - del self.stream_managers[channel_id] - logger.info(f"Non-owner cleanup: Removed stream manager for channel {channel_id}") + stream_manager = self._resolve_stream_manager(channel_id) + if stream_manager is not None: + if hasattr(stream_manager, 'stop'): + stream_manager.stop() + logger.info(f"Non-owner cleanup: Stopped stream manager for channel {channel_id}") + elif self._get_stream_thread(channel_id): + logger.warning( + f"Non-owner cleanup: Found orphaned stream thread for channel {channel_id}" + ) + if channel_id in self.stream_buffers: + self.stream_buffers[channel_id].stopping = True + + self._join_stream_thread(channel_id) + self._live_stream_managers.pop(channel_id, None) if channel_id in self.stream_buffers: - del self.stream_buffers[channel_id] + buffer = self.stream_buffers.pop(channel_id) + if hasattr(buffer, 'stop'): + try: + buffer.stop() + except Exception as e: + logger.error(f"Non-owner cleanup: Error stopping buffer for {channel_id}: {e}") logger.info(f"Non-owner cleanup: Removed stream buffer for channel {channel_id}") if channel_id in self.client_managers: diff --git a/apps/proxy/live_proxy/services/channel_service.py b/apps/proxy/live_proxy/services/channel_service.py index 8cc4a030..6671e444 100644 --- a/apps/proxy/live_proxy/services/channel_service.py +++ b/apps/proxy/live_proxy/services/channel_service.py @@ -10,6 +10,7 @@ from apps.channels.models import Channel, Stream from ..server import ProxyServer from ..redis_keys import RedisKeys from ..constants import EventType, ChannelState, ChannelMetadataField +from ..config_helper import ConfigHelper from ..url_utils import get_stream_info_for_switch from core.utils import log_system_event from .log_parsers import LogParserFactory @@ -19,6 +20,82 @@ logger = logging.getLogger("live_proxy") class ChannelService: """Service class for channel operations""" + @staticmethod + def mark_channel_stopping(channel_id, broadcast=False): + """Mark a channel as stopping in Redis so all uWSGI workers converge on teardown.""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: + return False + + try: + metadata_key = RedisKeys.channel_metadata(channel_id) + if proxy_server.redis_client.exists(metadata_key): + proxy_server.redis_client.hset(metadata_key, mapping={ + ChannelMetadataField.STATE: ChannelState.STOPPING, + ChannelMetadataField.STATE_CHANGED_AT: str(time.time()), + }) + + stop_key = RedisKeys.channel_stopping(channel_id) + proxy_server.redis_client.setex(stop_key, 60, "true") + + if broadcast: + ChannelService._publish_channel_stop_event(channel_id) + return True + except Exception as e: + logger.error(f"Error marking channel {channel_id} as stopping: {e}") + return False + + @staticmethod + def is_shutdown_pending(channel_id): + """True while the post-disconnect shutdown delay has started but stop has not run.""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: + return False + + delay = ConfigHelper.channel_shutdown_delay() + if delay <= 0: + return False + + disconnect_value = proxy_server.redis_client.get( + RedisKeys.last_client_disconnect(channel_id) + ) + if not disconnect_value: + return False + + try: + disconnect_time = float(disconnect_value) + except (ValueError, TypeError): + return False + + return (time.time() - disconnect_time) < delay + + @staticmethod + def is_channel_teardown_active(channel_id): + """True when a coordinated channel stop is in progress (visible to all workers).""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: + return False + + if proxy_server.redis_client.exists(RedisKeys.channel_stopping(channel_id)): + return True + + metadata_key = RedisKeys.channel_metadata(channel_id) + state = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STATE) + if state: + state_str = state.decode() if isinstance(state, bytes) else state + if state_str == ChannelState.STOPPING: + return True + + return False + + @staticmethod + def is_channel_unavailable_for_new_clients(channel_id): + """Reject new stream requests while teardown is active or shutdown is pending.""" + return ( + ChannelService.is_channel_teardown_active(channel_id) + or ChannelService.is_shutdown_pending(channel_id) + ) + @staticmethod def initialize_channel(channel_id, stream_url, user_agent, transcode=False, stream_profile_value=None, stream_id=None, m3u_profile_id=None, channel_name=None, stream_name=None): """ @@ -39,6 +116,7 @@ class ChannelService: bool: Success status """ proxy_server = ProxyServer.get_instance() + if stream_id and proxy_server.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) # Check if metadata already exists @@ -263,29 +341,16 @@ class ChannelService: try: metadata = proxy_server.redis_client.hgetall(metadata_key) if metadata and 'state' in metadata: - state = metadata['state'] - channel_info = {"state": state} - - # Immediately mark as stopping in metadata so clients detect it faster - proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE, ChannelState.STOPPING) - proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE_CHANGED_AT, str(time.time())) + channel_info = {"state": metadata['state']} except Exception as e: logger.error(f"Error fetching channel state: {e}") - # Set stopping flag with higher TTL to ensure it persists + # Mark stopping in Redis and notify all workers before local teardown if proxy_server.redis_client: - stop_key = RedisKeys.channel_stopping(channel_id) - proxy_server.redis_client.setex(stop_key, 60, "true") # Higher TTL of 60 seconds - logger.info(f"Set channel stopping flag with 60s TTL for channel {channel_id}") - - # Broadcast stop event to all workers via PubSub - if proxy_server.redis_client: - ChannelService._publish_channel_stop_event(channel_id) - - # Also stop locally to ensure this worker cleans up right away + ChannelService.mark_channel_stopping(channel_id, broadcast=True) + logger.info(f"Marked channel {channel_id} stopping and broadcast stop to all workers") local_result = proxy_server.stop_channel(channel_id) else: - # No Redis, just stop locally local_result = proxy_server.stop_channel(channel_id) # Release the channel in the channel model if applicable diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index d60d9703..b547cefd 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -43,6 +43,15 @@ from apps.proxy.utils import check_user_stream_limits logger = get_logger() +def _channel_stopping_response(): + response = JsonResponse( + {"error": "Channel is stopping, retry shortly"}, + status=503, + ) + response["Retry-After"] = "1" + return response + + def _resolve_output_format(user, force=None, request=None): """Return the output format string to use for this client.""" _FORMAT_ALIASES = { @@ -123,6 +132,12 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): status=429 ) + if ChannelService.is_channel_unavailable_for_new_clients(channel_id): + logger.info( + f"[{client_id}] Channel {channel_id} unavailable. Teardown or pending shutdown" + ) + return _channel_stopping_response() + # Check if we need to reinitialize the channel needs_initialization = True channel_state = None @@ -144,7 +159,6 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ChannelState.BUFFERING, ChannelState.INITIALIZING, ChannelState.CONNECTING, - ChannelState.STOPPING, ]: needs_initialization = False logger.debug( @@ -160,6 +174,11 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.debug( f"[{client_id}] Channel {channel_id} is still initializing, client will wait" ) + elif channel_state == ChannelState.STOPPING: + logger.info( + f"[{client_id}] Channel {channel_id} is stopping, rejecting request" + ) + return _channel_stopping_response() # Terminal states - channel needs cleanup before reinitialization elif channel_state in [ ChannelState.ERROR, From 67239d921d2c51190caa7b54f123d1562fc04201 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 14 Jun 2026 21:41:50 -0500 Subject: [PATCH 31/81] fix(proxy): enhance channel teardown and client management - Improved channel teardown process to prevent lingering upstream connections and ensure proper resource cleanup. - Enhanced client management by implementing checks for ghost clients and ensuring accurate client disconnection handling. - Updated logging to provide clearer insights during channel initialization and teardown, including handling of unavailable channels. - Refined stream manager behavior to manage ownership transitions more effectively and prevent blocking during shutdown. --- CHANGELOG.md | 4 +- apps/channels/models.py | 28 +- apps/channels/tests/test_ts_proxy_teardown.py | 452 +++++++++++++++++- apps/proxy/live_proxy/client_manager.py | 116 +++-- apps/proxy/live_proxy/input/buffer.py | 42 +- apps/proxy/live_proxy/input/manager.py | 160 +++++-- apps/proxy/live_proxy/output/ts/generator.py | 107 ++++- apps/proxy/live_proxy/server.py | 279 ++++++----- .../live_proxy/services/channel_service.py | 33 +- apps/proxy/live_proxy/views.py | 26 + 10 files changed, 945 insertions(+), 302 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2622dfc2..4de12ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,9 +43,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Live proxy channels could remain running with no clients after disconnect.** `stop_channel` called `log_system_event('channel_stop')` synchronously before local cleanup and Redis key deletion; Connect integrations or DB event writes on that path could block teardown indefinitely while the cleanup thread kept refreshing metadata TTL for a stale `stream_buffers` entry. Teardown now stops ffmpeg/output managers, releases ownership, removes local buffers and managers, deletes Redis keys, and only then logs the stop event asynchronously. Metadata TTL refresh skips channels mid-shutdown and non-owned channels; a stuck-stop watchdog forces cleanup if `stop_channel` does not return within ~30s. Stream buffers also ignore late `add_chunk()` writes once shutdown begins (`buffer.stopping` set at the start of `stream_manager.stop()`). +- **Live proxy channels could remain running with no clients after disconnect.** `stop_channel` called `log_system_event('channel_stop')` synchronously before local cleanup and Redis key deletion; Connect integrations or DB event writes on that path could block teardown indefinitely while the cleanup thread kept refreshing metadata TTL for a stale `stream_buffers` entry. Teardown now signals shutdown (`buffer.stopping`), runs model `release_stream()` and Redis key deletion, releases ownership, stops ffmpeg/output managers and local buffers, and only then logs the stop event asynchronously. Metadata TTL refresh skips channels mid-shutdown and non-owned channels; a stuck-stop watchdog forces cleanup if `stop_channel` does not return within ~10s (or 2× `channel_shutdown_delay`, whichever is greater). Stream buffers ignore late `add_chunk()` writes once shutdown begins (`buffer.stopping` set at teardown start). - **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` during pending shutdown or active teardown; `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. +- **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. +- **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** - Memory could spike to multiple GB on large EPG sources when building a full in-memory catalog before fuzzy matching; single-channel matching now streams rows and bounds ML work to a small candidate set. diff --git a/apps/channels/models.py b/apps/channels/models.py index f900a817..ef1d0cff 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -626,11 +626,11 @@ class Channel(models.Model): def _release_stale_stream_assignment(self, redis_client, stream_id: int) -> None: """Release pool counters and remove stale channel/stream assignment keys.""" + profile_id = None profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") if profile_id_bytes: try: profile_id = int(profile_id_bytes) - release_profile_slot(profile_id, redis_client) except (ValueError, TypeError): logger.debug( "Invalid profile ID for stale assignment on stream %s: %s", @@ -638,6 +638,32 @@ class Channel(models.Model): profile_id_bytes, ) + if profile_id is None: + metadata_key = RedisKeys.channel_metadata(str(self.uuid)) + meta_profile_id = redis_client.hget( + metadata_key, ChannelMetadataField.M3U_PROFILE + ) + if meta_profile_id: + try: + profile_id = int(meta_profile_id) + except (ValueError, TypeError): + logger.debug( + "Invalid profile ID in metadata for stale assignment on " + "channel %s: %s", + self.uuid, + meta_profile_id, + ) + + if profile_id is not None: + release_profile_slot(profile_id, redis_client) + else: + logger.warning( + "Channel %s: releasing stale stream %s assignment without profile " + "info - profile_connections may leak", + self.uuid, + stream_id, + ) + redis_client.delete(f"channel_stream:{self.id}") redis_client.delete(f"stream_profile:{stream_id}") diff --git a/apps/channels/tests/test_ts_proxy_teardown.py b/apps/channels/tests/test_ts_proxy_teardown.py index 14b4e898..3bc92f18 100644 --- a/apps/channels/tests/test_ts_proxy_teardown.py +++ b/apps/channels/tests/test_ts_proxy_teardown.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch from django.test import TestCase from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState +from apps.proxy.live_proxy.input.buffer import StreamBuffer from apps.proxy.live_proxy.input.manager import StreamManager from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.server import ProxyServer @@ -14,6 +15,29 @@ from apps.proxy.live_proxy.services.channel_service import ChannelService CHANNEL_ID = "00000000-0000-0000-0000-000000000099" +def _configure_ownership_pipeline( + redis, + *, + stop_exists=0, + metadata_exists=1, + client_count=0, + owner=None, + disconnect=None, + state=None, +): + pipe = MagicMock() + redis.pipeline.return_value = pipe + pipe.execute.return_value = ( + stop_exists, + metadata_exists, + client_count, + owner, + disconnect, + state, + ) + return pipe + + def _mock_proxy_server(redis_client=None): server = MagicMock() server.redis_client = redis_client or MagicMock() @@ -79,11 +103,6 @@ class LocalStreamActivityTests(TestCase): server.stop_all_output_profiles = MagicMock() return server - def test_has_local_stream_activity_from_live_registry(self): - server = self._make_server() - server._live_stream_managers[CHANNEL_ID] = MagicMock() - self.assertTrue(server._has_local_stream_activity(CHANNEL_ID)) - @patch.object(ProxyServer, "_join_stream_thread") def test_stop_local_stream_activity_stops_live_manager(self, mock_join): server = self._make_server() @@ -114,9 +133,9 @@ class OrphanMetadataCleanupTests(TestCase): @patch.object(ProxyServer, "_clean_redis_keys") @patch.object(ProxyServer, "_stop_local_stream_activity") - @patch.object(ProxyServer, "_has_local_stream_activity", return_value=True) + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True) def test_orphan_metadata_stops_local_processes_before_redis( - self, mock_has_local, mock_stop_local, mock_clean_redis + self, mock_has_upstream, mock_stop_local, mock_clean_redis ): server = self._make_server() metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) @@ -130,15 +149,16 @@ class OrphanMetadataCleanupTests(TestCase): server._check_orphaned_metadata() - mock_has_local.assert_called_with(CHANNEL_ID) + mock_has_upstream.assert_called_with(CHANNEL_ID) mock_stop_local.assert_called_once_with(CHANNEL_ID) mock_clean_redis.assert_called_once_with(CHANNEL_ID) @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_broadcast_upstream_stop") @patch.object(ProxyServer, "_stop_local_stream_activity") - @patch.object(ProxyServer, "_has_local_stream_activity", return_value=False) - def test_orphan_metadata_remote_channel_only_cleans_redis( - self, mock_has_local, mock_stop_local, mock_clean_redis + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=False) + def test_orphan_metadata_remote_channel_broadcasts_stop( + self, mock_has_upstream, mock_stop_local, mock_broadcast, mock_clean_redis ): server = self._make_server() metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) @@ -149,6 +169,7 @@ class OrphanMetadataCleanupTests(TestCase): server._check_orphaned_metadata() + mock_broadcast.assert_called_once_with(CHANNEL_ID) mock_stop_local.assert_not_called() mock_clean_redis.assert_called_once_with(CHANNEL_ID) @@ -171,9 +192,9 @@ class OrphanChannelCleanupTests(TestCase): @patch.object(ProxyServer, "_clean_redis_keys") @patch.object(ProxyServer, "_stop_local_stream_activity") - @patch.object(ProxyServer, "_has_local_stream_activity", return_value=True) + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True) def test_orphan_channel_stops_local_before_redis( - self, mock_has_local, mock_stop_local, mock_clean_redis + self, mock_has_upstream, mock_stop_local, mock_clean_redis ): server = self._make_server() metadata_key = RedisKeys.channel_metadata(CHANNEL_ID) @@ -189,10 +210,10 @@ class OrphanChannelCleanupTests(TestCase): class StreamManagerOwnershipTests(TestCase): def test_still_owner_false_when_different_worker(self): buffer = MagicMock() - buffer.redis_client.get.side_effect = lambda key: ( - None if "stopping" in key else b"worker-b" + _configure_ownership_pipeline( + buffer.redis_client, + owner=b"worker-b", ) - buffer.redis_client.exists.return_value = False manager = StreamManager( CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" ) @@ -201,8 +222,12 @@ class StreamManagerOwnershipTests(TestCase): def test_still_owner_true_when_owner_lock_expired_but_not_stopping(self): buffer = MagicMock() - buffer.redis_client.get.return_value = None - buffer.redis_client.exists.return_value = False + _configure_ownership_pipeline( + buffer.redis_client, + client_count=0, + owner=None, + state=ChannelState.CONNECTING.encode(), + ) manager = StreamManager( CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" ) @@ -211,7 +236,10 @@ class StreamManagerOwnershipTests(TestCase): def test_still_owner_false_when_channel_stopping_key_set(self): buffer = MagicMock() - buffer.redis_client.exists.return_value = True + _configure_ownership_pipeline( + buffer.redis_client, + stop_exists=1, + ) manager = StreamManager( CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" ) @@ -220,8 +248,10 @@ class StreamManagerOwnershipTests(TestCase): def test_update_bytes_skipped_after_ownership_lost(self): buffer = MagicMock() - buffer.redis_client.get.return_value = b"other-worker" - buffer.redis_client.exists.return_value = False + _configure_ownership_pipeline( + buffer.redis_client, + owner=b"other-worker", + ) manager = StreamManager( CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a" ) @@ -230,3 +260,383 @@ class StreamManagerOwnershipTests(TestCase): manager._update_bytes_processed(500) buffer.redis_client.hincrby.assert_not_called() + + +class StreamBufferStopTests(TestCase): + def test_stop_discards_local_data_without_redis_writes(self): + redis = MagicMock() + buffer = StreamBuffer(channel_id=CHANNEL_ID, redis_client=redis) + buffer._write_buffer = bytearray(b"x" * 376) + + buffer.stop() + + self.assertTrue(buffer.stopping) + self.assertEqual(len(buffer._write_buffer), 0) + redis.incr.assert_not_called() + redis.setex.assert_not_called() + redis.delete.assert_not_called() + + +class StopChannelTeardownTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server.stream_buffers = {} + server.client_managers = {} + server.profile_managers = {} + server.profile_buffers = {} + server._live_stream_managers = {} + server._stopping_channels = set() + server._stopping_since = {} + server.redis_client = MagicMock() + server.redis_client.exists.return_value = False + server.am_i_owner = MagicMock(return_value=False) + server._collect_channel_stop_event_data = MagicMock(return_value=None) + server.release_ownership = MagicMock() + return server + + @patch.object(ProxyServer, "_spawn_channel_stop_event") + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity") + def test_stop_channel_cleans_redis_before_blocking_local_stop( + self, mock_stop_local, mock_clean_redis, mock_spawn_event + ): + server = self._make_server() + call_order = [] + + def stop_local(channel_id): + call_order.append("local") + + def clean_redis(channel_id): + call_order.append("redis") + + mock_stop_local.side_effect = stop_local + mock_clean_redis.side_effect = clean_redis + + server.stop_channel(CHANNEL_ID) + + self.assertEqual(call_order, ["redis", "local"]) + mock_spawn_event.assert_called_once_with(None) + + @patch.object(ProxyServer, "_spawn_channel_stop_event") + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity", side_effect=RuntimeError("boom")) + def test_stop_channel_cleans_redis_in_finally_when_local_stop_fails( + self, mock_stop_local, mock_clean_redis, mock_spawn_event + ): + server = self._make_server() + + result = server.stop_channel(CHANNEL_ID) + + self.assertFalse(result) + mock_clean_redis.assert_called_once_with(CHANNEL_ID) + mock_spawn_event.assert_not_called() + self.assertNotIn(CHANNEL_ID, server._stopping_channels) + + @patch.object(ProxyServer, "_spawn_channel_stop_event") + @patch.object(ProxyServer, "_clean_redis_keys") + @patch.object(ProxyServer, "_stop_local_stream_activity") + def test_stop_channel_owner_releases_after_redis_cleanup( + self, mock_stop_local, mock_clean_redis, mock_spawn_event + ): + server = self._make_server() + server.am_i_owner.return_value = True + stop_data = {"channel_id": CHANNEL_ID} + server._collect_channel_stop_event_data.return_value = stop_data + call_order = [] + + def stop_local(channel_id): + call_order.append("local") + + def release(channel_id): + call_order.append("release") + + def clean_redis(channel_id): + call_order.append("redis") + + mock_stop_local.side_effect = stop_local + server.release_ownership.side_effect = release + mock_clean_redis.side_effect = clean_redis + + server.stop_channel(CHANNEL_ID) + + self.assertEqual(call_order, ["redis", "release", "local"]) + server._collect_channel_stop_event_data.assert_called_once_with(CHANNEL_ID) + mock_spawn_event.assert_called_once_with(stop_data) + + +class CleanRedisKeysOrderTests(TestCase): + @patch("apps.proxy.live_proxy.server.Stream.objects.get") + @patch("apps.proxy.live_proxy.server.Channel.objects.get") + def test_clean_redis_keys_releases_profile_slot_before_live_keys_deleted( + self, mock_channel_get, mock_stream_get + ): + from apps.channels.models import Channel, Stream + + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.redis_client = MagicMock() + call_order = [] + + channel = MagicMock() + channel.release_stream.return_value = True + + def channel_get(uuid): + call_order.append("release") + return channel + + mock_channel_get.side_effect = channel_get + mock_stream_get.side_effect = Stream.DoesNotExist + + def scan(*args, **kwargs): + call_order.append("redis") + return (0, [b"live:channel:foo:buffer:index"]) + + server.redis_client.scan.side_effect = scan + + server._clean_redis_keys(CHANNEL_ID) + + self.assertEqual(call_order, ["release", "redis"]) + channel.release_stream.assert_called_once() + server.redis_client.delete.assert_called_once() + + +class LocalUpstreamActivityTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server.stream_buffers = {} + server.client_managers = {} + server._live_stream_managers = {} + return server + + def test_upstream_activity_excludes_reader_only_client_manager(self): + server = self._make_server() + server.client_managers[CHANNEL_ID] = MagicMock() + server.stream_buffers[CHANNEL_ID] = MagicMock() + self.assertFalse(server._has_local_upstream_activity(CHANNEL_ID)) + + def test_upstream_activity_from_live_registry(self): + server = self._make_server() + server._live_stream_managers[CHANNEL_ID] = MagicMock() + self.assertTrue(server._has_local_upstream_activity(CHANNEL_ID)) + + +class ClientDisconnectOwnershipTests(TestCase): + def test_last_client_triggers_stop_when_upstream_active_without_owner_lock(self): + from apps.proxy.live_proxy.client_manager import ClientManager + + proxy_server = MagicMock() + proxy_server.worker_id = "testhost:1" + proxy_server.am_i_owner.return_value = False + proxy_server._has_local_upstream_activity.return_value = True + proxy_server.extend_ownership.return_value = False + + redis = MagicMock() + redis.hget.return_value = b"viewer" + redis.scard.return_value = 0 + + manager = ClientManager( + channel_id=CHANNEL_ID, + redis_client=redis, + worker_id="testhost:1", + ) + manager.proxy_server = proxy_server + manager.clients = {"client-1"} + manager.client_set_key = RedisKeys.clients(CHANNEL_ID) + manager._notify_owner_of_activity = MagicMock() + manager._trigger_stats_update = MagicMock() + manager.get_total_client_count = MagicMock(return_value=0) + proxy_server._spawn_on_hub = MagicMock() + + manager.remove_client("client-1") + + proxy_server._spawn_on_hub.assert_called_once_with( + proxy_server.handle_client_disconnect, CHANNEL_ID + ) + + +class TeardownActiveLocalStopTests(TestCase): + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_teardown_active_when_local_stop_in_progress(self, mock_get_instance): + server = MagicMock() + server._stopping_channels = {CHANNEL_ID} + server.redis_client = MagicMock() + server.redis_client.exists.return_value = False + mock_get_instance.return_value = server + + self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID)) + + +class HandleClientDisconnectUpstreamFirstTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.client_managers = {CHANNEL_ID: MagicMock()} + server.stream_managers = {CHANNEL_ID: MagicMock()} + server._live_stream_managers = {} + server.profile_managers = {} + server.output_managers = {} + server.redis_client = MagicMock() + server.redis_client.scard.return_value = 0 + server._stopping_channels = set() + return server + + @patch.object(ProxyServer, "_coordinated_stop_channel") + @patch.object(ProxyServer, "_stop_upstream_before_redis_cleanup") + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True) + def test_last_client_uses_coordinated_stop_only( + self, _mock_upstream, mock_stop_upstream, mock_coordinated + ): + server = self._make_server() + + with patch( + "apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", + return_value=0, + ): + server.handle_client_disconnect(CHANNEL_ID) + + mock_stop_upstream.assert_not_called() + mock_coordinated.assert_called_once_with(CHANNEL_ID) + + +class InitWaitAbortTests(TestCase): + def _make_generator(self): + from apps.proxy.live_proxy.output.ts.generator import StreamGenerator + + return StreamGenerator( + CHANNEL_ID, + "client-1", + "127.0.0.1", + "test-agent", + channel_initializing=True, + ) + + def test_abort_when_client_removed_locally(self): + generator = self._make_generator() + server = MagicMock() + client_manager = MagicMock() + client_manager.clients = set() + server.client_managers = {CHANNEL_ID: client_manager} + server.redis_client = MagicMock() + + self.assertEqual(generator._init_wait_abort_reason(server, time.time()), "client_gone") + + def test_abort_when_connect_stalled_without_buffer(self): + from apps.proxy.config import TSConfig as Config + + generator = self._make_generator() + server = MagicMock() + client_manager = MagicMock() + client_manager.clients = {"client-1"} + server.client_managers = {CHANNEL_ID: client_manager} + buffer = MagicMock() + buffer.index = 0 + server.stream_buffers = {CHANNEL_ID: buffer} + server.redis_client = MagicMock() + server.redis_client.hget.return_value = b"connecting" + server.redis_client.get.return_value = None + + started = time.time() - (getattr(Config, "CONNECTION_TIMEOUT", 10) + 1) + self.assertEqual(generator._init_wait_abort_reason(server, started), "stalled") + + +class UpstreamStopBroadcastTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + server = ProxyServer() + server.worker_id = "testhost:1" + server.stream_managers = {} + server._live_stream_managers = {} + server.redis_client = MagicMock() + return server + + @patch.object(ProxyServer, "_stop_local_stream_activity") + @patch.object(ProxyServer, "_broadcast_upstream_stop") + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True) + def test_local_upstream_stops_locally_without_broadcast( + self, _mock_has, mock_broadcast, mock_stop_local + ): + server = self._make_server() + server._stop_upstream_before_redis_cleanup(CHANNEL_ID) + mock_stop_local.assert_called_once_with(CHANNEL_ID) + mock_broadcast.assert_not_called() + + @patch.object(ProxyServer, "_stop_local_stream_activity") + @patch.object(ProxyServer, "_broadcast_upstream_stop") + @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=False) + def test_orphan_cleanup_broadcasts_when_no_local_upstream( + self, _mock_has, mock_broadcast, mock_stop_local + ): + server = self._make_server() + server._stop_upstream_before_redis_cleanup(CHANNEL_ID) + mock_broadcast.assert_called_once_with(CHANNEL_ID) + mock_stop_local.assert_not_called() + + +class StreamManagerStillOwnerTests(TestCase): + def _make_manager(self, redis_client): + buffer = MagicMock() + buffer.redis_client = redis_client + buffer.channel_id = CHANNEL_ID + manager = StreamManager( + CHANNEL_ID, + "http://example/stream.ts", + buffer, + worker_id="testhost:1", + ) + return manager + + def test_stops_when_metadata_removed(self): + redis = MagicMock() + _configure_ownership_pipeline(redis, metadata_exists=0) + manager = self._make_manager(redis) + self.assertFalse(manager._still_owner()) + + def test_keeps_running_during_connecting_before_client_registered(self): + redis = MagicMock() + _configure_ownership_pipeline( + redis, + client_count=0, + owner=b"testhost:1", + state=ChannelState.CONNECTING.encode(), + ) + manager = self._make_manager(redis) + self.assertTrue(manager._still_owner()) + + def test_stops_after_disconnect_when_shutdown_delay_is_zero(self): + redis = MagicMock() + _configure_ownership_pipeline( + redis, + client_count=0, + owner=b"testhost:1", + disconnect=b"1700000000.0", + state=ChannelState.ACTIVE.encode(), + ) + manager = self._make_manager(redis) + with patch( + "apps.proxy.live_proxy.input.manager.ConfigHelper.channel_shutdown_delay", + return_value=0, + ): + self.assertFalse(manager._still_owner()) + + def test_keeps_running_during_shutdown_delay(self): + redis = MagicMock() + _configure_ownership_pipeline( + redis, + client_count=0, + owner=b"testhost:1", + disconnect=str(time.time()).encode(), + state=ChannelState.ACTIVE.encode(), + ) + manager = self._make_manager(redis) + with patch( + "apps.proxy.live_proxy.input.manager.ConfigHelper.channel_shutdown_delay", + return_value=5, + ): + self.assertTrue(manager._still_owner()) diff --git a/apps/proxy/live_proxy/client_manager.py b/apps/proxy/live_proxy/client_manager.py index 3771d55c..3767a25a 100644 --- a/apps/proxy/live_proxy/client_manager.py +++ b/apps/proxy/live_proxy/client_manager.py @@ -102,6 +102,7 @@ class ClientManager: break # Send heartbeat for all local clients + clients_to_remove = set() with self.lock: # Skip this cycle if we have no local clients if not self.clients: @@ -109,7 +110,6 @@ class ClientManager: # IMPROVED GHOST DETECTION: Check for stale clients before sending heartbeats current_time = time.time() - clients_to_remove = set() # First identify clients that should be removed for client_id in self.clients: @@ -132,12 +132,11 @@ class ClientManager: logger.debug(f"Client {client_id} inactive for {current_time - last_active_time:.1f}s, removing as ghost") clients_to_remove.add(client_id) - # Remove ghost clients in a separate step - for client_id in clients_to_remove: - self.remove_client(client_id) - if clients_to_remove: - logger.info(f"Removed {len(clients_to_remove)} ghost clients from channel {self.channel_id}") + logger.info( + f"Identified {len(clients_to_remove)} ghost clients " + f"for channel {self.channel_id}" + ) # Now send heartbeats only for remaining clients pipe = self.redis_client.pipeline() @@ -172,6 +171,9 @@ class ClientManager: if self.clients and not all(c in clients_to_remove for c in self.clients): self._notify_owner_of_activity() + for client_id in clients_to_remove: + self.remove_client(client_id) + except Exception as e: logger.error(f"Error in client heartbeat thread: {e}") @@ -305,6 +307,9 @@ class ClientManager: def remove_client(self, client_id): """Remove a client from this channel and Redis""" + client_username = "unknown" + remaining = None + with self.lock: if client_id in self.clients: self.clients.remove(client_id) @@ -315,61 +320,80 @@ class ClientManager: self.last_active_time = time.time() if self.redis_client: - # Get client data before removing the data client_key = f"live:channel:{self.channel_id}:clients:{client_id}" client_username = self.redis_client.hget(client_key, "username") or "unknown" if isinstance(client_username, bytes): client_username = client_username.decode("utf-8") - # Remove from channel's client set self.redis_client.srem(self.client_set_key, client_id) - - client_key = f"live:channel:{self.channel_id}:clients:{client_id}" self.redis_client.delete(client_key) - - # Check if this was the last client remaining = self.redis_client.scard(self.client_set_key) or 0 - if remaining == 0: - logger.warning(f"Last client removed: {client_id} - channel may shut down soon") - # Trigger disconnect time tracking even if we're not the owner - disconnect_key = RedisKeys.last_client_disconnect(self.channel_id) - self.redis_client.setex(disconnect_key, 60, str(time.time())) + local_count = len(self.clients) - self._notify_owner_of_activity() + if not self.redis_client: + return local_count - # Check if we're the owner - if so, handle locally; if not, publish event - am_i_owner = self.proxy_server and self.proxy_server.am_i_owner(self.channel_id) + if remaining == 0: + logger.warning(f"Last client removed: {client_id} - channel may shut down soon") + disconnect_key = RedisKeys.last_client_disconnect(self.channel_id) + self.redis_client.setex(disconnect_key, 60, str(time.time())) - if am_i_owner: - # We're the owner - handle the disconnect directly - logger.debug(f"Owner handling CLIENT_DISCONNECTED for client {client_id} locally (not publishing)") - if (remaining == 0 - or self.proxy_server.output_managers.get(self.channel_id) - or self.proxy_server.profile_managers.get(self.channel_id)): - import gevent - gevent.spawn(self.proxy_server.handle_client_disconnect, self.channel_id) - else: - # We're not the owner - publish event so owner can handle it - logger.debug(f"Non-owner publishing CLIENT_DISCONNECTED event for client {client_id} on channel {self.channel_id} from worker {self.worker_id}") - event_data = json.dumps({ - "event": EventType.CLIENT_DISCONNECTED, - "channel_id": self.channel_id, - "client_id": client_id, - "worker_id": self.worker_id or "unknown", - "timestamp": time.time(), - "remaining_clients": remaining, - "username": client_username - }) - self.redis_client.publish(RedisKeys.events_channel(self.channel_id), event_data) + self._notify_owner_of_activity() - # Trigger channel stats update via WebSocket - self._trigger_stats_update() + am_i_owner = self.proxy_server and self.proxy_server.am_i_owner(self.channel_id) - total_clients = self.get_total_client_count() - logger.info(f"Client disconnected: {client_id} (local: {len(self.clients)}, total: {total_clients})") + # Owner lock TTL can expire while local ffmpeg is still running on this worker. + if (not am_i_owner and self.proxy_server + and self.proxy_server._has_local_upstream_activity(self.channel_id)): + if self.proxy_server.extend_ownership(self.channel_id): + am_i_owner = True - return len(self.clients) + schedule_disconnect = False + if am_i_owner: + logger.debug( + f"Owner handling CLIENT_DISCONNECTED for client {client_id} locally (not publishing)" + ) + if (remaining == 0 + or self.proxy_server.output_managers.get(self.channel_id) + or self.proxy_server.profile_managers.get(self.channel_id)): + schedule_disconnect = True + elif (remaining == 0 and self.proxy_server + and self.proxy_server._has_local_upstream_activity(self.channel_id)): + logger.warning( + f"Last client removed while local upstream still active for " + f"{self.channel_id} without owner lock - stopping channel" + ) + schedule_disconnect = True + else: + logger.debug( + f"Non-owner publishing CLIENT_DISCONNECTED event for client {client_id} " + f"on channel {self.channel_id} from worker {self.worker_id}" + ) + event_data = json.dumps({ + "event": EventType.CLIENT_DISCONNECTED, + "channel_id": self.channel_id, + "client_id": client_id, + "worker_id": self.worker_id or "unknown", + "timestamp": time.time(), + "remaining_clients": remaining, + "username": client_username + }) + self.redis_client.publish(RedisKeys.events_channel(self.channel_id), event_data) + + if schedule_disconnect and self.proxy_server: + self.proxy_server._spawn_on_hub( + self.proxy_server.handle_client_disconnect, self.channel_id + ) + + self._trigger_stats_update() + + total_clients = self.get_total_client_count() + logger.info( + f"Client disconnected: {client_id} (local: {local_count}, total: {total_clients})" + ) + + return local_count def get_client_count(self): """Get local client count""" diff --git a/apps/proxy/live_proxy/input/buffer.py b/apps/proxy/live_proxy/input/buffer.py index b345ead3..3d3fb091 100644 --- a/apps/proxy/live_proxy/input/buffer.py +++ b/apps/proxy/live_proxy/input/buffer.py @@ -309,38 +309,16 @@ class StreamBuffer: self.fill_timers.clear() try: - # Flush any remaining data in the write buffer - if hasattr(self, '_write_buffer') and len(self._write_buffer) > 0: - # Ensure remaining data is aligned to TS packets - complete_size = (len(self._write_buffer) // 188) * 188 - - if complete_size > 0: - final_chunk = self._write_buffer[:complete_size] - - # Write final chunk to Redis - with self.lock: - if self.redis_client: - try: - chunk_index = self.redis_client.incr(self.buffer_index_key) - chunk_key = f"{self.buffer_prefix}{chunk_index}" - self.redis_client.setex(chunk_key, self.chunk_ttl, bytes(final_chunk)) - self.index = chunk_index - logger.info(f"Flushed final chunk of {len(final_chunk)} bytes to Redis") - except Exception as e: - logger.error(f"Error flushing final chunk: {e}") - - # Clear buffers - self._write_buffer = bytearray() - if hasattr(self, '_partial_packet'): - self._partial_packet = bytearray() - - # Clean up the chunk timestamps sorted set - if self.redis_client and self.chunk_timestamps_key: - try: - self.redis_client.delete(self.chunk_timestamps_key) - except Exception as e: - logger.error(f"Error deleting chunk timestamps key: {e}") - + with self.lock: + if hasattr(self, '_write_buffer') and len(self._write_buffer) > 0: + discarded = len(self._write_buffer) + self._write_buffer = bytearray() + if hasattr(self, '_partial_packet'): + self._partial_packet = bytearray() + logger.debug( + f"Discarded {discarded} bytes from local write buffer " + f"for channel {self.channel_id}" + ) except Exception as e: logger.error(f"Error during buffer stop: {e}") diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index 06b12644..ae744473 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -112,6 +112,11 @@ class StreamManager: # Add tracking for data throughput self.bytes_processed = 0 self.last_bytes_update = time.time() + + # Cached result of the pipelined Redis ownership audit (hot read path). + self._ownership_cache_valid_until = 0.0 + self._ownership_cached = True + self._OWNERSHIP_CHECK_INTERVAL = 1.0 self.bytes_update_interval = 5 # Update Redis every 5 seconds # Add stderr reader thread property @@ -182,7 +187,90 @@ class StreamManager: logger.warning(f"Timeout waiting for existing processes to close for channel {self.channel_id} after {timeout}s") return False - def _still_owner(self): + def _invalidate_ownership_cache(self): + self._ownership_cache_valid_until = 0.0 + + @staticmethod + def _decode_redis_value(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode() + return value + + def _disconnect_shutdown_ready(self, disconnect_value): + """True when last-client disconnect has passed the configured shutdown delay.""" + if not disconnect_value: + return False + + shutdown_delay = ConfigHelper.channel_shutdown_delay() + if shutdown_delay <= 0: + return True + + disconnect_value = self._decode_redis_value(disconnect_value) + try: + disconnect_time = float(disconnect_value) + except (ValueError, TypeError): + return False + return (time.time() - disconnect_time) >= shutdown_delay + + def _evaluate_ownership_from_redis(self, redis_client): + """Single pipelined Redis round-trip for the full ownership audit.""" + stop_key = RedisKeys.channel_stopping(self.channel_id) + metadata_key = RedisKeys.channel_metadata(self.channel_id) + clients_key = RedisKeys.clients(self.channel_id) + owner_key = RedisKeys.channel_owner(self.channel_id) + disconnect_key = RedisKeys.last_client_disconnect(self.channel_id) + + pipe = redis_client.pipeline(transaction=False) + pipe.exists(stop_key) + pipe.exists(metadata_key) + pipe.scard(clients_key) + pipe.get(owner_key) + pipe.get(disconnect_key) + pipe.hget(metadata_key, ChannelMetadataField.STATE) + ( + stop_exists, + metadata_exists, + client_count, + current_owner, + disconnect_value, + state_raw, + ) = pipe.execute() + + if stop_exists: + return False + + if not metadata_exists: + logger.warning( + f"Channel {self.channel_id} metadata removed from Redis - stopping upstream" + ) + return False + + client_count = client_count or 0 + state = self._decode_redis_value(state_raw) + current_owner = self._decode_redis_value(current_owner) + + if current_owner and current_owner != self.worker_id: + return False + + if not current_owner: + if client_count == 0 and state not in ChannelState.PRE_ACTIVE: + logger.warning( + f"Channel {self.channel_id} has no owner and no clients - stopping upstream" + ) + return False + return True + + if client_count == 0 and self._disconnect_shutdown_ready(disconnect_value): + logger.info( + f"Channel {self.channel_id} disconnect shutdown ready - stopping upstream" + ) + return False + + return True + + def _still_owner(self, *, force=False): """Return True while this worker should keep the upstream connection open.""" if self.stopping or self.stop_requested: return False @@ -194,26 +282,49 @@ class StreamManager: if not redis_client: return True - try: - stop_key = RedisKeys.channel_stopping(self.channel_id) - if redis_client.exists(stop_key): - return False + now = time.time() + if not force and now < self._ownership_cache_valid_until: + return self._ownership_cached - owner_key = RedisKeys.channel_owner(self.channel_id) - current_owner = redis_client.get(owner_key) - if not current_owner: - # Owner lock TTL may lapse briefly between cleanup-thread renewals. - # Keep running unless coordinated teardown has started (checked above). - return True - if isinstance(current_owner, bytes): - current_owner = current_owner.decode() - return current_owner == self.worker_id + try: + result = self._evaluate_ownership_from_redis(redis_client) + self._ownership_cached = result + self._ownership_cache_valid_until = now + self._OWNERSHIP_CHECK_INTERVAL + return result except Exception as e: logger.debug(f"Ownership check failed for channel {self.channel_id}: {e}") + return True + + def _upstream_may_continue(self): + """ + Per-chunk gate for the hot read path. + + Local stop flags are checked every chunk. The coordinated-teardown Redis + flag is checked every chunk (one EXISTS). The full ownership audit is + pipelined and cached for ~1s — enough for loop boundaries while avoiding + 6+ Redis round-trips per chunk during steady streaming. + """ + if self.stopping or self.stop_requested or not self.running: + return False + if self.buffer is not None and self.buffer.stopping: return False + redis_client = getattr(self.buffer, 'redis_client', None) + if redis_client and self.worker_id: + try: + if redis_client.exists(RedisKeys.channel_stopping(self.channel_id)): + self._ownership_cached = False + self._ownership_cache_valid_until = 0.0 + return False + except Exception as e: + logger.debug( + f"Channel stopping check failed for {self.channel_id}: {e}" + ) + + return self._still_owner() + def _ensure_owner_or_stop(self): - if self._still_owner(): + if self._still_owner(force=True): return True logger.warning( @@ -539,6 +650,7 @@ class StreamManager: logger.debug(f"Closing existing HTTP connections before establishing transcode connection for channel {self.channel_id}") self._close_connection() + close_old_connections() channel = get_stream_object(self.channel_id) # Use FFmpeg specifically for HLS streams @@ -1071,7 +1183,7 @@ class StreamManager: def _update_bytes_processed(self, chunk_size): """Update the total bytes processed in Redis metadata""" - if not self._still_owner(): + if not self._upstream_may_continue(): return try: @@ -1146,17 +1258,11 @@ class StreamManager: """Stop the stream manager and cancel all timers""" logger.info(f"Stopping stream manager for channel {self.channel_id}") - # Add at the beginning of your stop method self.stopping = True + self._invalidate_ownership_cache() if self.buffer is not None: self.buffer.stopping = True - # Release stream resources if we're the owner - if self.current_stream_id and hasattr(self, 'worker_id') and self.worker_id: - if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: - owner_key = RedisKeys.channel_owner(self.channel_id) - current_owner = self.buffer.redis_client.get(owner_key) - # Cancel all buffer check timers for timer in list(self._buffer_check_timers): try: @@ -1545,7 +1651,8 @@ class StreamManager: if hasattr(self, 'stderr_reader_thread') and self.stderr_reader_thread and self.stderr_reader_thread.is_alive(): try: logger.debug(f"Waiting for stderr reader thread to terminate for channel {self.channel_id}") - self.stderr_reader_thread.join(timeout=2.0) + stderr_join_timeout = 0.25 if self.stopping else 2.0 + self.stderr_reader_thread.join(timeout=stderr_join_timeout) if self.stderr_reader_thread.is_alive(): logger.warning(f"Stderr reader thread did not terminate within timeout for channel {self.channel_id}") except Exception as e: @@ -1638,7 +1745,7 @@ class StreamManager: self.connected = False return False - if not self._still_owner(): + if not self._upstream_may_continue(): self.stop() return False @@ -1649,8 +1756,7 @@ class StreamManager: # Add directly to buffer without TS-specific processing success = self.buffer.add_chunk(chunk) - # Update last data timestamp in Redis if successful - if success and self._still_owner() and hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: + if success and hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: last_data_key = RedisKeys.last_data(self.buffer.channel_id) self.buffer.redis_client.set(last_data_key, str(time.time()), ex=60) diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index 64bb4e1d..efde02ad 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -139,12 +139,75 @@ class StreamGenerator: finally: self._cleanup() + def _init_wait_abort_reason(self, proxy_server, initialization_start): + """Return a reason string when init wait should end early, else None.""" + from ...services.channel_service import ChannelService + + if ChannelService.is_channel_teardown_active(self.channel_id): + return "stopping" + + client_mgr = proxy_server.client_managers.get(self.channel_id) + if client_mgr is not None: + if self.client_id not in client_mgr.clients: + return "client_gone" + elif proxy_server.redis_client: + client_key = RedisKeys.client_metadata(self.channel_id, self.client_id) + if not proxy_server.redis_client.exists(client_key): + total = proxy_server.redis_client.scard(RedisKeys.clients(self.channel_id)) or 0 + if total == 0: + return "client_gone" + + elapsed = time.time() - initialization_start + connection_timeout = getattr(Config, 'CONNECTION_TIMEOUT', 10) + if elapsed < connection_timeout or not proxy_server.redis_client: + return None + + metadata_key = RedisKeys.channel_metadata(self.channel_id) + state_raw = proxy_server.redis_client.hget(metadata_key, 'state') + if not state_raw: + return None + + state = state_raw.decode() if isinstance(state_raw, bytes) else state_raw + if state not in ('connecting', 'initializing', 'buffering'): + return None + + buffer = proxy_server.stream_buffers.get(self.channel_id) + if buffer is not None and buffer.index > 0: + return None + + if proxy_server.redis_client.get(RedisKeys.last_data(self.channel_id)): + return None + + return "stalled" + def _wait_for_initialization(self): initialization_start = time.time() max_init_wait = ConfigHelper.client_wait_timeout() proxy_server = ProxyServer.get_instance() while time.time() - initialization_start < max_init_wait: + abort_reason = self._init_wait_abort_reason(proxy_server, initialization_start) + if abort_reason == "stopping": + logger.error( + f"[{self.client_id}] Channel {self.channel_id} teardown active during initialization" + ) + yield create_ts_packet('error', "Error: Channel is stopping") + return False + if abort_reason == "client_gone": + logger.info( + f"[{self.client_id}] Client disconnected during initialization wait, aborting" + ) + yield create_ts_packet('error', "Error: Client disconnected") + return False + if abort_reason == "stalled": + logger.warning( + f"[{self.client_id}] Channel {self.channel_id} stalled in connecting state " + f"with no buffer data after " + f"{getattr(Config, 'CONNECTION_TIMEOUT', 10)}s, aborting init wait" + ) + yield create_ts_packet('error', "Error: Connection stalled") + return False + if proxy_server.redis_client: metadata_key = RedisKeys.channel_metadata(self.channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) @@ -537,35 +600,31 @@ class StreamGenerator: stream_released = False if proxy_server.redis_client: try: - metadata_key = RedisKeys.channel_metadata(self.channel_id) - metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata: - stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) - if stream_id_bytes: - # Check if we're the last client - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() - # Only the last client or owner should release the stream - if client_count <= 1 and proxy_server.am_i_owner(self.channel_id): - try: - # Try Channel first (normal flow), fall back to Stream (preview flow) - try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - stream_released = obj.release_stream() - if stream_released: - logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") - else: - logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") - except Exception as e: - logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Pool slots are global; the last client on any worker must release. + if client_count <= 1: + try: + try: + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + stream_released = obj.release_stream() + if stream_released: + logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") + else: + logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") except Exception as e: logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") if self.channel_id in proxy_server.client_managers: client_manager = proxy_server.client_managers[self.channel_id] - local_clients = client_manager.remove_client(self.client_id) + if self.client_id in client_manager.clients: + local_clients = client_manager.remove_client(self.client_id) + else: + local_clients = client_manager.get_client_count() total_clients = client_manager.get_total_client_count() logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index a348c876..667daf2f 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -72,6 +72,7 @@ class ProxyServer: self._channel_names = {} self._stopping_channels = set() # channels with an active stop_channel call in progress self._stopping_since = {} # channel_id -> time.time() when stop_channel began + self._local_stop_locks = {} # Managers kept until the stream OS thread exits (may outlive stream_managers dict) self._live_stream_managers = {} @@ -141,6 +142,18 @@ class ProxyServer: logger.error(f"Redis command error: {e}") return None + def _spawn_on_hub(self, fn, *args, **kwargs): + """Schedule fn on the gevent hub from any thread without blocking the caller.""" + import gevent + + try: + hub = gevent.get_hub() + hub.loop.run_callback_threadsafe( + lambda: gevent.spawn(fn, *args, **kwargs) + ) + except Exception as e: + logger.error(f"Failed to schedule {getattr(fn, '__name__', fn)} on hub: {e}") + def _start_event_listener(self): """Listen for events from other workers""" if not self.redis_client: @@ -507,9 +520,9 @@ class ProxyServer: current = self.redis_client.get(lock_key) if current is None: - # Key expired, re-acquire if we have the stream_manager, but never + # Key expired, re-acquire if we still run local upstream, but never # during coordinated teardown (multi-worker reconnect-during-stop race). - if channel_id in self.stream_managers: + if channel_id in self.stream_managers or channel_id in self._live_stream_managers: if self._channel_unavailable_for_new_clients(channel_id): logger.info( f"Refusing ownership re-acquisition for {channel_id}; " @@ -546,6 +559,12 @@ class ProxyServer: ) return False + if self._has_local_upstream_activity(channel_id): + logger.warning( + f"Stopping lingering local upstream before initializing channel {channel_id}" + ) + self._stop_local_stream_activity(channel_id) + if self.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) if self.redis_client.exists(metadata_key): @@ -879,26 +898,7 @@ class ProxyServer: owner = metadata.get('owner', 'unknown') logger.info(f"Zombie channel details - state: {state}, owner: {owner}") - # Clean up Redis keys self._clean_redis_keys(channel_id) - - # Force release resources in the Channel model - try: - channel = Channel.objects.get(uuid=channel_id) - if not channel.release_stream(): - logger.warning(f"Failed to release stream for zombie channel {channel_id}") - else: - logger.info(f"Released stream allocation for zombie channel {channel_id}") - except Exception as e: - try: - stream = Stream.objects.get(stream_hash=channel_id) - if not stream.release_stream(): - logger.warning(f"Failed to release stream for zombie channel {channel_id}") - else: - logger.info(f"Released stream allocation for zombie channel {channel_id}") - except Exception as e: - logger.error(f"Error releasing stream for zombie channel {channel_id}: {e}") - return True except Exception as e: logger.error(f"Error cleaning zombie channel {channel_id}: {e}", exc_info=True) @@ -914,8 +914,10 @@ class ProxyServer: # to manage for this channel at all. if (channel_id not in self.client_managers and channel_id not in self.stream_managers + and channel_id not in self._live_stream_managers and channel_id not in self.profile_managers - and channel_id not in self.output_managers): + and channel_id not in self.output_managers + and not self._has_local_upstream_activity(channel_id)): return try: @@ -987,12 +989,12 @@ class ProxyServer: if total == 0: logger.debug(f"No clients left after disconnect event - stopping channel {channel_id}") - disconnect_key = RedisKeys.last_client_disconnect(channel_id) - self.redis_client.setex(disconnect_key, 60, str(time.time())) shutdown_delay = ConfigHelper.channel_shutdown_delay() + disconnect_key = RedisKeys.last_client_disconnect(channel_id) if shutdown_delay > 0: + self.redis_client.setex(disconnect_key, 60, str(time.time())) logger.info(f"Waiting {shutdown_delay}s before stopping channel...") gevent.sleep(shutdown_delay) @@ -1002,6 +1004,13 @@ class ProxyServer: self.redis_client.delete(disconnect_key) return + if shutdown_delay <= 0: + self.redis_client.setex(disconnect_key, 60, str(time.time())) + + # Coordinated stop runs local teardown + Redis cleanup once. + # Do not call _stop_upstream_before_redis_cleanup here — it races + # with stop_channel and can leave stop_channel blocked on stderr join + # before _clean_redis_keys ever runs (orphaned buffer:index keys). self._coordinated_stop_channel(channel_id) except Exception as e: logger.error(f"Error handling client disconnect for channel {channel_id}: {e}") @@ -1371,18 +1380,41 @@ class ProxyServer: return thread return None - def _has_local_stream_activity(self, channel_id): + def _has_local_upstream_activity(self, channel_id): + """True when this worker runs a local ffmpeg / stream-{uuid} OS thread.""" if channel_id in self.stream_managers: return True if channel_id in self._live_stream_managers: return True - if channel_id in self.stream_buffers: - return True - if channel_id in self.client_managers: - return True thread = self._get_stream_thread(channel_id) return thread is not None and thread.is_alive() + def _broadcast_upstream_stop(self, channel_id): + """Ask every uWSGI worker to stop local ffmpeg for this channel.""" + from .services.channel_service import ChannelService + + if not self.redis_client: + return + try: + ChannelService.mark_channel_stopping(channel_id, broadcast=True) + logger.info( + f"Broadcast upstream stop for channel {channel_id} to all workers" + ) + except Exception as e: + logger.error( + f"Error broadcasting upstream stop for channel {channel_id}: {e}" + ) + + def _stop_upstream_before_redis_cleanup(self, channel_id): + """Stop local ffmpeg before deleting Redis keys (prevents delete/recreate loops).""" + if self._has_local_upstream_activity(channel_id): + logger.info( + f"Channel {channel_id} has local upstream activity - stopping processes" + ) + self._stop_local_stream_activity(channel_id) + return + self._broadcast_upstream_stop(channel_id) + def _join_stream_thread(self, channel_id, timeout=2.0): thread = self._get_stream_thread(channel_id) if thread and thread.is_alive(): @@ -1402,8 +1434,41 @@ class ProxyServer: manager = self._live_stream_managers.pop(channel_id, None) return manager + def _signal_upstream_shutdown(self, channel_id): + """Halt upstream Redis writes immediately without blocking on ffmpeg join.""" + if channel_id in self.stream_buffers: + self.stream_buffers[channel_id].stopping = True + for registry in (self.stream_managers, self._live_stream_managers): + manager = registry.get(channel_id) + if manager is None: + continue + manager.stopping = True + manager.stop_requested = True + if getattr(manager, 'buffer', None) is not None: + manager.buffer.stopping = True + + def _get_local_stop_lock(self, channel_id): + lock = self._local_stop_locks.get(channel_id) + if lock is None: + lock = threading.Lock() + self._local_stop_locks[channel_id] = lock + return lock + def _stop_local_stream_activity(self, channel_id): """Stop local ffmpeg/stream threads regardless of registry state.""" + lock = self._get_local_stop_lock(channel_id) + if not lock.acquire(blocking=False): + logger.debug( + f"Local stop already in progress for channel {channel_id}, skipping duplicate" + ) + return + try: + self._stop_local_stream_activity_locked(channel_id) + finally: + lock.release() + self._local_stop_locks.pop(channel_id, None) + + def _stop_local_stream_activity_locked(self, channel_id): stream_manager = self._resolve_stream_manager(channel_id) if stream_manager is not None: if hasattr(stream_manager, 'stop'): @@ -1445,17 +1510,13 @@ class ProxyServer: self.profile_buffers.pop(channel_id, None) self._live_stream_managers.pop(channel_id, None) - def _cleanup_stopped_channel_local(self, channel_id): - """Remove local managers/buffers for a channel that is shutting down.""" - self._stop_local_stream_activity(channel_id) - def _recover_stuck_channel_stops(self): """Force cleanup when stop_channel never finishes (e.g. blocked on logging).""" if not self._stopping_channels: return now = time.time() - stuck_threshold = max(30, ConfigHelper.channel_shutdown_delay() * 2) + stuck_threshold = max(10, ConfigHelper.channel_shutdown_delay() * 2) for channel_id in list(self._stopping_channels): started = self._stopping_since.get(channel_id, now) @@ -1483,10 +1544,10 @@ class ProxyServer: self._stopping_channels.add(channel_id) self._stopping_since[channel_id] = time.time() stop_event_data = None + redis_cleaned = False try: logger.info(f"Stopping channel {channel_id}") - # First set a stopping key that clients will check if self.redis_client: stop_key = RedisKeys.channel_stopping(channel_id) if self.redis_client.exists(stop_key): @@ -1494,54 +1555,26 @@ class ProxyServer: else: self.redis_client.setex(stop_key, 60, "true") - # Only stop the actual stream manager if we're the owner - if self.am_i_owner(channel_id): - logger.info(f"This worker ({self.worker_id}) is the owner - closing provider connection") - if channel_id in self.stream_managers: - stream_manager = self.stream_managers[channel_id] - - # Signal thread to stop and close resources - if hasattr(stream_manager, 'stop'): - stream_manager.stop() - else: - stream_manager.running = False - if hasattr(stream_manager, '_close_socket'): - stream_manager._close_socket() - - # Wait for stream thread to finish - stream_thread_name = f"stream-{channel_id}" - stream_thread = None - - for thread in threading.enumerate(): - if thread.name == stream_thread_name: - stream_thread = thread - break - - if stream_thread and stream_thread.is_alive(): - logger.info(f"Waiting for stream thread to terminate") - try: - # Very short timeout to prevent hanging the app - stream_thread.join(timeout=2.0) - if stream_thread.is_alive(): - logger.warning(f"Stream thread did not terminate within timeout") - except RuntimeError: - logger.debug(f"Could not join stream thread (may be current thread)") - - # Stop all output format managers before releasing ownership - self.stop_all_output_formats(channel_id) - - # Stop all output profile transcodes before releasing ownership - self.stop_all_output_profiles(channel_id) - + was_owner = self.am_i_owner(channel_id) + if was_owner: + logger.info( + f"This worker ({self.worker_id}) is the owner - closing provider connection" + ) stop_event_data = self._collect_channel_stop_event_data(channel_id) - # Release ownership + # Stop new chunk writes before Redis cleanup; do not block on ffmpeg join yet. + self._signal_upstream_shutdown(channel_id) + + # Release profile slots and delete Redis keys before any blocking local stop. + # A concurrent disconnect + cleanup-thread stop used to wedge here behind a + # 2s stderr join, never reaching scan/delete (bare buffer:index with TTL -1). + self._clean_redis_keys(channel_id) + redis_cleaned = True + + if was_owner: self.release_ownership(channel_id) - # Always clean up local resources before Redis/logging so teardown - # cannot be blocked by connect integrations or DB event writes. - self._cleanup_stopped_channel_local(channel_id) - self._clean_redis_keys(channel_id) + self._stop_local_stream_activity(channel_id) self._spawn_channel_stop_event(stop_event_data) return True @@ -1549,6 +1582,13 @@ class ProxyServer: logger.error(f"Error stopping channel {channel_id}: {e}") return False finally: + if not redis_cleaned: + try: + self._clean_redis_keys(channel_id) + except Exception as e: + logger.error( + f"Error cleaning Redis keys for channel {channel_id} during finally: {e}" + ) self._stopping_channels.discard(channel_id) self._stopping_since.pop(channel_id, None) @@ -1595,7 +1635,11 @@ class ProxyServer: self._recover_stuck_channel_stops() # Create a unified list of all channels we have locally - all_local_channels = set(self.stream_managers.keys()) | set(self.client_managers.keys()) + all_local_channels = ( + set(self.stream_managers.keys()) + | set(self.client_managers.keys()) + | set(self._live_stream_managers.keys()) + ) # Single loop through all channels - process each exactly once for channel_id in list(all_local_channels): @@ -1764,7 +1808,8 @@ class ProxyServer: # === NON-OWNER CHANNEL HANDLING === # Safety: if we have a stream_manager, we ARE the real owner # but the Redis key may have expired. Try to re-acquire. - if channel_id in self.stream_managers: + if (channel_id in self.stream_managers + or channel_id in self._live_stream_managers): # Ownership was explicitly released by an active stop_channel call - # don't fight the shutdown by trying to re-acquire. if channel_id in self._stopping_channels: @@ -1930,11 +1975,7 @@ class ProxyServer: # Orphaned channel with no clients - clean it up logger.info(f"Cleaning up orphaned channel {channel_id}") - if self._has_local_stream_activity(channel_id): - logger.info( - f"Orphaned channel {channel_id} has local stream activity - stopping processes" - ) - self._stop_local_stream_activity(channel_id) + self._stop_upstream_before_redis_cleanup(channel_id) self._clean_redis_keys(channel_id) except Exception as e: logger.error(f"Error processing channel key {key}: {e}") @@ -1964,8 +2005,7 @@ class ProxyServer: if not metadata: # Empty metadata - clean it up logger.warning(f"Found empty metadata for channel {channel_id} - cleaning up") - if self._has_local_stream_activity(channel_id): - self._stop_local_stream_activity(channel_id) + self._stop_upstream_before_redis_cleanup(channel_id) self._clean_redis_keys(channel_id) continue @@ -1987,11 +2027,7 @@ class ProxyServer: state = metadata.get('state', 'unknown') logger.warning(f"Found orphaned metadata for channel {channel_id} (state: {state}, owner: {owner}, clients: {client_count}) - cleaning up") - if self._has_local_stream_activity(channel_id): - logger.info( - f"Channel {channel_id} has local stream activity - stopping processes" - ) - self._stop_local_stream_activity(channel_id) + self._stop_upstream_before_redis_cleanup(channel_id) self._clean_redis_keys(channel_id) elif not owner_alive and client_count > 0: # SCARD may include ghost entries from a dead worker's @@ -2008,8 +2044,7 @@ class ProxyServer: f"owner: {owner}) had {client_count} ghost client(s) " f"- cleaning up" ) - if self._has_local_stream_activity(channel_id): - self._stop_local_stream_activity(channel_id) + self._stop_upstream_before_redis_cleanup(channel_id) self._clean_redis_keys(channel_id) else: if self._channel_teardown_active(channel_id): @@ -2018,8 +2053,7 @@ class ProxyServer: f"{real_count} client(s) during teardown; forcing cleanup" ) self._force_clear_channel_clients(channel_id) - if self._has_local_stream_activity(channel_id): - self._stop_local_stream_activity(channel_id) + self._stop_upstream_before_redis_cleanup(channel_id) self._clean_redis_keys(channel_id) else: logger.warning( @@ -2036,7 +2070,11 @@ class ProxyServer: def _clean_redis_keys(self, channel_id): """Clean up all Redis keys for a channel more efficiently""" - # Release the channel, stream, and profile keys from the channel + total_deleted = 0 + + # Release the M3U profile slot while channel_stream / metadata still exist. + # Scanning live:channel keys first deletes metadata and breaks release_stream() + # fallback, leaving profile_connections counters stuck (e.g. profile_connections:70 = 1). try: channel = Channel.objects.get(uuid=channel_id) if not channel.release_stream(): @@ -2049,36 +2087,29 @@ class ProxyServer: except (Stream.DoesNotExist, Exception): logger.debug(f"No Channel or Stream found for {channel_id}") - if not self.redis_client: - return 0 + if self.redis_client: + try: + patterns = [ + f"live:channel:{channel_id}:*", + RedisKeys.events_channel(channel_id), + ] - try: - # Define key patterns to scan for - patterns = [ - f"live:channel:{channel_id}:*", # All channel keys - RedisKeys.events_channel(channel_id) # Event channel - ] + for pattern in patterns: + cursor = 0 + while True: + cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100) + if keys: + self.redis_client.delete(*keys) + total_deleted += len(keys) - total_deleted = 0 + if cursor == 0: + break - for pattern in patterns: - cursor = 0 - while True: - cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100) - if keys: - self.redis_client.delete(*keys) - total_deleted += len(keys) + logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}") + except Exception as e: + logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}") - # Exit when cursor returns to 0 - if cursor == 0: - break - - logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}") - return total_deleted - - except Exception as e: - logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}") - return 0 + return total_deleted def refresh_channel_registry(self): """Refresh TTL for active channels using standard keys""" diff --git a/apps/proxy/live_proxy/services/channel_service.py b/apps/proxy/live_proxy/services/channel_service.py index 6671e444..dcd0b5e1 100644 --- a/apps/proxy/live_proxy/services/channel_service.py +++ b/apps/proxy/live_proxy/services/channel_service.py @@ -73,6 +73,9 @@ class ChannelService: def is_channel_teardown_active(channel_id): """True when a coordinated channel stop is in progress (visible to all workers).""" proxy_server = ProxyServer.get_instance() + if channel_id in proxy_server._stopping_channels: + return True + if not proxy_server.redis_client: return False @@ -345,41 +348,19 @@ class ChannelService: except Exception as e: logger.error(f"Error fetching channel state: {e}") - # Mark stopping in Redis and notify all workers before local teardown + # Mark stopping in Redis and notify all workers before local teardown. + # stop_channel() releases profile slots via _clean_redis_keys() before Redis deletion. if proxy_server.redis_client: ChannelService.mark_channel_stopping(channel_id, broadcast=True) logger.info(f"Marked channel {channel_id} stopping and broadcast stop to all workers") - local_result = proxy_server.stop_channel(channel_id) - else: - local_result = proxy_server.stop_channel(channel_id) - - # Release the channel in the channel model if applicable - try: - channel = Channel.objects.get(uuid=channel_id) - model_released = channel.release_stream() - if model_released: - logger.info(f"Released channel {channel_id} stream allocation") - else: - logger.warning(f"Channel {channel_id}: release_stream found no keys to clean") - except (Channel.DoesNotExist, Exception): - logger.warning(f"Could not find Channel model for UUID {channel_id}, attempting stream hash") - try: - stream = Stream.objects.get(stream_hash=channel_id) - model_released = stream.release_stream() - if model_released: - logger.info(f"Released stream {channel_id} stream allocation") - else: - logger.warning(f"Stream {channel_id}: release_stream found no keys to clean") - except (Stream.DoesNotExist, Exception) as e: - logger.error(f"No Channel or Stream found for {channel_id}: {e}") - model_released = False + local_result = proxy_server.stop_channel(channel_id) return { 'status': 'success', 'message': 'Channel stop request sent', 'channel_id': channel_id, 'previous_state': channel_info, - 'model_released': model_released, + 'model_released': bool(local_result), 'local_stop_result': local_result } diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index b547cefd..2119b4bd 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -211,6 +211,12 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # Start initialization if needed if needs_initialization or not proxy_server.check_if_channel_exists(channel_id): + if ChannelService.is_channel_unavailable_for_new_clients(channel_id): + logger.info( + f"[{client_id}] Channel {channel_id} became unavailable before init, rejecting" + ) + return _channel_stopping_response() + logger.info(f"[{client_id}] Starting channel {channel_id} initialization") # Force cleanup of any previous instance if in terminal state if channel_state in [ @@ -433,6 +439,16 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ) # 502 Bad Gateway # Initialize channel with the stream's user agent (not the client's) + if ChannelService.is_channel_unavailable_for_new_clients(channel_id): + if connection_allocated: + if not channel.release_stream(): + logger.warning(f"[{client_id}] Failed to release stream before teardown reject") + connection_allocated = False + logger.info( + f"[{client_id}] Channel {channel_id} unavailable before init call, rejecting" + ) + return _channel_stopping_response() + success = ChannelService.initialize_channel( channel_id, stream_url, @@ -541,6 +557,16 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): f"[{client_id}] Successfully initialized channel {channel_id} locally" ) + if ChannelService.is_channel_unavailable_for_new_clients(channel_id): + if _client_pre_registered: + mgr = proxy_server.client_managers.get(channel_id) + if mgr: + mgr.remove_client(client_id) + logger.info( + f"[{client_id}] Channel {channel_id} became unavailable during setup, rejecting" + ) + return _channel_stopping_response() + # Register client output_profile = _resolve_output_profile(request, user) if output_profile: From 8f40f6065f5664fe27f48f31a0fd8fc596dc2a95 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 14 Jun 2026 21:55:57 -0500 Subject: [PATCH 32/81] fix(proxy): improve resource cleanup and connection management - Added calls to close_old_connections in various components to ensure proper resource cleanup and prevent connection leaks. --- CHANGELOG.md | 1 + apps/proxy/live_proxy/input/manager.py | 3 + .../proxy/live_proxy/output/fmp4/generator.py | 82 +++++++------- apps/proxy/live_proxy/output/ts/generator.py | 102 +++++++++--------- apps/proxy/live_proxy/server.py | 65 +++++------ core/utils.py | 5 + 6 files changed, 140 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de12ded..7e40e04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` during pending shutdown or active teardown; `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. +- **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `log_system_event()` (all system events), `_clean_redis_keys()`, channel init/start logging, client disconnect cleanup (TS and fMP4 generators), and profile lookup in `_establish_transcode_connection()` before spawning ffmpeg. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index ae744473..e8315006 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -690,6 +690,9 @@ class StreamManager: self.transcode_cmd = [arg for arg in self.transcode_cmd if self.user_agent not in arg and 'user-agent' not in arg.lower() and 'user_agent' not in arg.lower()] logger.debug(f"Removed user_agent parameters from UDP stream command for channel: {self.channel_id}") + # Profile lookup is done; release the pool slot before a long-lived ffmpeg process. + close_old_connections() + logger.debug(f"Starting transcode process: {self.transcode_cmd} for channel: {self.channel_id}") import os as _os diff --git a/apps/proxy/live_proxy/output/fmp4/generator.py b/apps/proxy/live_proxy/output/fmp4/generator.py index 65a8c5c7..e979008c 100644 --- a/apps/proxy/live_proxy/output/fmp4/generator.py +++ b/apps/proxy/live_proxy/output/fmp4/generator.py @@ -11,6 +11,7 @@ import time import gevent from apps.channels.models import Channel, Stream from core.utils import log_system_event +from django.db import close_old_connections from ...server import ProxyServer from ...redis_keys import RedisKeys from ...constants import ChannelMetadataField @@ -331,46 +332,49 @@ class FMP4StreamGenerator: # ------------------------------------------------------------------ def _cleanup(self): - elapsed = time.time() - self.stream_start_time - proxy_server = ProxyServer.get_instance() + try: + elapsed = time.time() - self.stream_start_time + proxy_server = ProxyServer.get_instance() - # Release stream allocation if last client (mirrors StreamGenerator) - if proxy_server.redis_client: - try: - meta_key = RedisKeys.channel_metadata(self.channel_id) - stream_id_bytes = proxy_server.redis_client.hget( - meta_key, ChannelMetadataField.STREAM_ID - ) - if stream_id_bytes: - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[ - self.channel_id - ].get_total_client_count() - if client_count <= 1 and proxy_server.am_i_owner(self.channel_id): - try: + # Release stream allocation if last client (mirrors StreamGenerator) + if proxy_server.redis_client: + try: + meta_key = RedisKeys.channel_metadata(self.channel_id) + stream_id_bytes = proxy_server.redis_client.hget( + meta_key, ChannelMetadataField.STREAM_ID + ) + if stream_id_bytes: + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[ + self.channel_id + ].get_total_client_count() + if client_count <= 1 and proxy_server.am_i_owner(self.channel_id): try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - obj.release_stream() - except Exception as e: - logger.error( - f"[{self.client_id}] Error releasing stream: {e}" - ) - except Exception as e: - logger.error(f"[{self.client_id}] Error in stream release check: {e}") + try: + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + obj.release_stream() + except Exception as e: + logger.error( + f"[{self.client_id}] Error releasing stream: {e}" + ) + except Exception as e: + logger.error(f"[{self.client_id}] Error in stream release check: {e}") - # Remove from MAIN ClientManager - this is what triggers handle_client_disconnect - # and the zero-clients → stop_channel path, same as TS clients. - local_clients = 0 - total_clients = 0 - if self.channel_id in proxy_server.client_managers: - client_manager = proxy_server.client_managers[self.channel_id] - local_clients = client_manager.remove_client(self.client_id) - total_clients = client_manager.get_total_client_count() + # Remove from MAIN ClientManager - this is what triggers handle_client_disconnect + # and the zero-clients → stop_channel path, same as TS clients. + local_clients = 0 + total_clients = 0 + if self.channel_id in proxy_server.client_managers: + client_manager = proxy_server.client_managers[self.channel_id] + local_clients = client_manager.remove_client(self.client_id) + total_clients = client_manager.get_total_client_count() - logger.info( - f"[{self.client_id}] fMP4 client disconnected after {elapsed:.2f}s " - f"({self.bytes_sent / 1024:.1f} KB sent, " - f"local: {local_clients}, total: {total_clients})" - ) + logger.info( + f"[{self.client_id}] fMP4 client disconnected after {elapsed:.2f}s " + f"({self.bytes_sent / 1024:.1f} KB sent, " + f"local: {local_clients}, total: {total_clients})" + ) + finally: + close_old_connections() diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index efde02ad..69dc341f 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -8,6 +8,7 @@ import gevent from apps.proxy.config import TSConfig as Config from apps.channels.models import Channel, Stream from core.utils import log_system_event +from django.db import close_old_connections from ...server import ProxyServer from ...utils import create_ts_packet, get_logger from ...redis_keys import RedisKeys @@ -590,59 +591,62 @@ class StreamGenerator: def _cleanup(self): """Clean up resources and report final statistics.""" - # Client cleanup - elapsed = time.time() - self.stream_start_time - local_clients = 0 - total_clients = 0 - proxy_server = ProxyServer.get_instance() + try: + # Client cleanup + elapsed = time.time() - self.stream_start_time + local_clients = 0 + total_clients = 0 + proxy_server = ProxyServer.get_instance() - # Release M3U profile stream allocation if this is the last client - stream_released = False - if proxy_server.redis_client: - try: - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() - # Pool slots are global; the last client on any worker must release. - if client_count <= 1: - try: + # Release M3U profile stream allocation if this is the last client + stream_released = False + if proxy_server.redis_client: + try: + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Pool slots are global; the last client on any worker must release. + if client_count <= 1: try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - stream_released = obj.release_stream() - if stream_released: - logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") - else: - logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") - except Exception as e: - logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") - except Exception as e: - logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") + try: + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + stream_released = obj.release_stream() + if stream_released: + logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") + else: + logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") + except Exception as e: + logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") - if self.channel_id in proxy_server.client_managers: - client_manager = proxy_server.client_managers[self.channel_id] - if self.client_id in client_manager.clients: - local_clients = client_manager.remove_client(self.client_id) - else: - local_clients = client_manager.get_client_count() - total_clients = client_manager.get_total_client_count() - logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") + if self.channel_id in proxy_server.client_managers: + client_manager = proxy_server.client_managers[self.channel_id] + if self.client_id in client_manager.clients: + local_clients = client_manager.remove_client(self.client_id) + else: + local_clients = client_manager.get_client_count() + total_clients = client_manager.get_total_client_count() + logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") - # Log client disconnect event - try: - log_system_event( - 'client_disconnect', - channel_id=self.channel_id, - channel_name=self.channel_name, - client_ip=self.client_ip, - client_id=self.client_id, - user_agent=self.client_user_agent[:100] if self.client_user_agent else None, - duration=round(elapsed, 2), - bytes_sent=self.bytes_sent, - username=self.user.username if self.user else None - ) - except Exception as e: - logger.error(f"Could not log client disconnect event: {e}") + # Log client disconnect event + try: + log_system_event( + 'client_disconnect', + channel_id=self.channel_id, + channel_name=self.channel_name, + client_ip=self.client_ip, + client_id=self.client_id, + user_agent=self.client_user_agent[:100] if self.client_user_agent else None, + duration=round(elapsed, 2), + bytes_sent=self.bytes_sent, + username=self.user.username if self.user else None + ) + except Exception as e: + logger.error(f"Could not log client disconnect event: {e}") + finally: + close_old_connections() def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None, buffer=None): diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 667daf2f..98346676 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -760,6 +760,8 @@ class ProxyServer: ) except Exception as e: logger.error(f"Could not log channel start event: {e}") + finally: + close_old_connections() # Create client manager with channel_id, redis_client AND worker_id (only if not already exists) if channel_id not in self.client_managers: @@ -2072,42 +2074,45 @@ class ProxyServer: """Clean up all Redis keys for a channel more efficiently""" total_deleted = 0 - # Release the M3U profile slot while channel_stream / metadata still exist. - # Scanning live:channel keys first deletes metadata and breaks release_stream() - # fallback, leaving profile_connections counters stuck (e.g. profile_connections:70 = 1). try: - channel = Channel.objects.get(uuid=channel_id) - if not channel.release_stream(): - logger.debug(f"Channel {channel_id}: release_stream found no keys to clean") - except (Channel.DoesNotExist, Exception): + # Release the M3U profile slot while channel_stream / metadata still exist. + # Scanning live:channel keys first deletes metadata and breaks release_stream() + # fallback, leaving profile_connections counters stuck (e.g. profile_connections:70 = 1). try: - stream = Stream.objects.get(stream_hash=channel_id) - if not stream.release_stream(): - logger.debug(f"Stream {channel_id}: release_stream found no keys to clean") - except (Stream.DoesNotExist, Exception): - logger.debug(f"No Channel or Stream found for {channel_id}") + channel = Channel.objects.get(uuid=channel_id) + if not channel.release_stream(): + logger.debug(f"Channel {channel_id}: release_stream found no keys to clean") + except (Channel.DoesNotExist, Exception): + try: + stream = Stream.objects.get(stream_hash=channel_id) + if not stream.release_stream(): + logger.debug(f"Stream {channel_id}: release_stream found no keys to clean") + except (Stream.DoesNotExist, Exception): + logger.debug(f"No Channel or Stream found for {channel_id}") - if self.redis_client: - try: - patterns = [ - f"live:channel:{channel_id}:*", - RedisKeys.events_channel(channel_id), - ] + if self.redis_client: + try: + patterns = [ + f"live:channel:{channel_id}:*", + RedisKeys.events_channel(channel_id), + ] - for pattern in patterns: - cursor = 0 - while True: - cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100) - if keys: - self.redis_client.delete(*keys) - total_deleted += len(keys) + for pattern in patterns: + cursor = 0 + while True: + cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100) + if keys: + self.redis_client.delete(*keys) + total_deleted += len(keys) - if cursor == 0: - break + if cursor == 0: + break - logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}") - except Exception as e: - logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}") + logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}") + except Exception as e: + logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}") + finally: + close_old_connections() return total_deleted diff --git a/core/utils.py b/core/utils.py index 08dda3dc..5931d985 100644 --- a/core/utils.py +++ b/core/utils.py @@ -715,6 +715,7 @@ def log_system_event(event_type, channel_id=None, channel_name=None, **details): stream_url='http://...', user='admin') """ from core.models import SystemEvent, CoreSettings + from django.db import close_old_connections try: # Create the event @@ -750,6 +751,10 @@ def log_system_event(event_type, channel_id=None, channel_name=None, **details): except Exception as e: # Don't let event logging break the main application logger.error(f"Failed to log system event {event_type}: {e}") + finally: + # geventpool keeps checked-out connections until close(); release promptly + # when logging from proxy greenlets/threads outside a normal request cycle. + close_old_connections() def _send_async(channel_layer, group, message): From db40421faa228b59e47125f8e40b188f2d103d7b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 16:50:08 -0500 Subject: [PATCH 33/81] fix(plugins): ensure proper database connection management in plugin actions - Added calls to `close_old_connections()` in `run_action()` and `stop_plugin()` methods to prevent connection leaks after plugin execution. - Updated documentation to clarify connection handling for plugins, emphasizing the importance of cleanup in long-running tasks and event hooks. --- CHANGELOG.md | 1 + Plugins.md | 12 ++ apps/plugins/loader.py | 120 +++++++++--------- apps/plugins/tests/__init__.py | 0 .../tests/test_run_action_db_cleanup.py | 64 ++++++++++ 5 files changed, 140 insertions(+), 57 deletions(-) create mode 100644 apps/plugins/tests/__init__.py create mode 100644 apps/plugins/tests/test_run_action_db_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e40e04a..01ab218f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. - **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `log_system_event()` (all system events), `_clean_redis_keys()`, channel init/start logging, client disconnect cleanup (TS and fMP4 generators), and profile lookup in `_establish_transcode_connection()` before spawning ffmpeg. +- **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** diff --git a/Plugins.md b/Plugins.md index d528f1e4..97e95ed2 100644 --- a/Plugins.md +++ b/Plugins.md @@ -307,6 +307,18 @@ Plugins are server-side Python code running within the Django application. You c Prefer Celery tasks (`.delay()`) to keep `run` fast and non-blocking. +### Database connections + +Dispatcharr uses `django-db-geventpool` with a bounded per-uWSGI-worker pool (`MAX_CONNS=8`). Each greenlet or OS thread that runs ORM code checks out a connection until Django closes it. + +`PluginManager.run_action()` and `stop_plugin()` always call `close_old_connections()` in a `finally` block after your plugin returns (success or error). That returns the current greenlet's checkout to the pool. **You do not need to call `close_old_connections()` yourself for normal inline ORM inside `run()` or `stop()`.** + +Still follow these rules: + +- **Heavy or long work:** dispatch a Celery task (`.delay()`) and return quickly from `run()`. Celery workers close connections after each task; blocking the uWSGI gevent hub with `time.sleep`, sync HTTP, or large CPU work can freeze the whole worker regardless of DB cleanup. +- **Background threads or greenlets you spawn:** each thread/greenlet that uses the ORM must call `close_old_connections()` (or `connection.close()`) in its own `finally` block when done. The wrapper only covers the thread/greenlet that called `run_action()`. +- **Connect event hooks:** actions with an `"events"` list run synchronously inside whatever caller triggered the event (for example a live-proxy system event). Keep event handlers short or defer to Celery. + ### Important: Don’t Ask Users for URL/User/Password Dispatcharr plugins run **inside** the Dispatcharr backend process. That means they already have direct access to the app’s models, tasks, and internal utilities. Plugins **should not** ask users for “Dispatcharr URL”, “Admin Username”, or “Admin Password” just to call the API. That is unnecessary and unsafe because: diff --git a/apps/plugins/loader.py b/apps/plugins/loader.py index a7ecb255..3c275f69 100644 --- a/apps/plugins/loader.py +++ b/apps/plugins/loader.py @@ -10,7 +10,7 @@ import types from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from django.db import transaction +from django.db import close_old_connections, transaction from .models import PluginConfig @@ -492,79 +492,85 @@ class PluginManager: return cfg.settings def run_action(self, key: str, action_id: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - lp = self.get_plugin(key) - if not lp or not lp.instance: - # Attempt a lightweight re-discovery in case the registry was rebuilt - self.discover_plugins(sync_db=False, force_reload=False, use_cache=False) + try: lp = self.get_plugin(key) if not lp or not lp.instance: - raise ValueError(f"Plugin '{key}' not found") + # Attempt a lightweight re-discovery in case the registry was rebuilt + self.discover_plugins(sync_db=False, force_reload=False, use_cache=False) + lp = self.get_plugin(key) + if not lp or not lp.instance: + raise ValueError(f"Plugin '{key}' not found") - cfg = PluginConfig.objects.get(key=key) - if not cfg.enabled: - raise PermissionError(f"Plugin '{key}' is disabled") - params = params or {} + cfg = PluginConfig.objects.get(key=key) + if not cfg.enabled: + raise PermissionError(f"Plugin '{key}' is disabled") + params = params or {} - context = self._build_context(lp, cfg) + context = self._build_context(lp, cfg) - # Run either via Celery if plugin provides a delayed method, or inline - run_method = getattr(lp.instance, "run", None) - if not callable(run_method): - raise ValueError(f"Plugin '{key}' has no runnable 'run' method") + run_method = getattr(lp.instance, "run", None) + if not callable(run_method): + raise ValueError(f"Plugin '{key}' has no runnable 'run' method") - try: - result = run_method(action_id, params, context) - except Exception: - logger.exception(f"Plugin '{key}' action '{action_id}' failed") - raise + try: + result = run_method(action_id, params, context) + except Exception: + logger.exception(f"Plugin '{key}' action '{action_id}' failed") + raise - # Normalize return - if isinstance(result, dict): - return result - return {"status": "ok", "result": result} + if isinstance(result, dict): + return result + return {"status": "ok", "result": result} + finally: + # Return geventpool checkouts for this greenlet/thread after every action, + # including Connect event hooks and manual UI runs. + close_old_connections() def stop_plugin(self, key: str, reason: Optional[str] = None) -> bool: - lp = self.get_plugin(key) - if not lp or not lp.instance: - return False try: - cfg = PluginConfig.objects.get(key=key) - except PluginConfig.DoesNotExist: - return False - if not cfg.enabled: - return False - - context = self._build_context(lp, cfg) - if reason: - context["reason"] = reason - - stop_method = getattr(lp.instance, "stop", None) - if callable(stop_method): + lp = self.get_plugin(key) + if not lp or not lp.instance: + return False try: - stop_method(context) - return True - except TypeError: + cfg = PluginConfig.objects.get(key=key) + except PluginConfig.DoesNotExist: + return False + if not cfg.enabled: + return False + + context = self._build_context(lp, cfg) + if reason: + context["reason"] = reason + + stop_method = getattr(lp.instance, "stop", None) + if callable(stop_method): try: - stop_method() + stop_method(context) return True + except TypeError: + try: + stop_method() + return True + except Exception: + logger.exception("Plugin '%s' stop() failed", key) + return False except Exception: logger.exception("Plugin '%s' stop() failed", key) return False - except Exception: - logger.exception("Plugin '%s' stop() failed", key) - return False - run_method = getattr(lp.instance, "run", None) - if callable(run_method): - actions = {a.get("id") for a in (lp.actions or []) if isinstance(a, dict)} - if "stop" in actions: - try: - run_method("stop", {}, context) - return True - except Exception: - logger.exception("Plugin '%s' stop action failed", key) - return False - return False + run_method = getattr(lp.instance, "run", None) + if callable(run_method): + actions = {a.get("id") for a in (lp.actions or []) if isinstance(a, dict)} + if "stop" in actions: + try: + run_method("stop", {}, context) + return True + except Exception: + logger.exception("Plugin '%s' stop action failed", key) + return False + return False + finally: + close_old_connections() def stop_all_plugins(self, reason: Optional[str] = None) -> int: stopped = 0 diff --git a/apps/plugins/tests/__init__.py b/apps/plugins/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/plugins/tests/test_run_action_db_cleanup.py b/apps/plugins/tests/test_run_action_db_cleanup.py new file mode 100644 index 00000000..d9a45c88 --- /dev/null +++ b/apps/plugins/tests/test_run_action_db_cleanup.py @@ -0,0 +1,64 @@ +"""PluginManager must release geventpool checkouts after every run/stop.""" + +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.plugins.loader import LoadedPlugin, PluginManager + + +class PluginRunActionDbCleanupTests(SimpleTestCase): + @contextmanager + def _manager_with_plugin(self, run_impl): + instance = MagicMock() + instance.run = run_impl + lp = LoadedPlugin( + key="test_plugin", + name="Test Plugin", + instance=instance, + actions=[{"id": "do_work"}], + ) + pm = PluginManager() + cfg = MagicMock(enabled=True, settings={}) + with patch.object(pm, "get_plugin", return_value=lp), patch( + "apps.plugins.loader.PluginConfig.objects.get", return_value=cfg + ): + yield pm + + @patch("apps.plugins.loader.close_old_connections") + def test_run_action_closes_connections_on_success(self, mock_close): + with self._manager_with_plugin(lambda *_a, **_k: {"status": "ok"}) as pm: + result = pm.run_action("test_plugin", "do_work") + + self.assertEqual(result, {"status": "ok"}) + mock_close.assert_called_once() + + @patch("apps.plugins.loader.close_old_connections") + def test_run_action_closes_connections_on_plugin_error(self, mock_close): + def _boom(*_a, **_k): + raise RuntimeError("plugin failed") + + with self._manager_with_plugin(_boom) as pm: + with self.assertRaises(RuntimeError): + pm.run_action("test_plugin", "do_work") + + mock_close.assert_called_once() + + @patch("apps.plugins.loader.close_old_connections") + def test_stop_plugin_closes_connections(self, mock_close): + instance = MagicMock() + instance.stop = MagicMock() + lp = LoadedPlugin( + key="test_plugin", + name="Test Plugin", + instance=instance, + ) + pm = PluginManager() + cfg = MagicMock(enabled=True, settings={}) + with patch.object(pm, "get_plugin", return_value=lp), patch( + "apps.plugins.loader.PluginConfig.objects.get", return_value=cfg + ): + self.assertTrue(pm.stop_plugin("test_plugin", reason="shutdown")) + + mock_close.assert_called_once() From c389462dde4b672accd7547217cc969e20519da8 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 17:14:48 -0500 Subject: [PATCH 34/81] fix(proxy): enhance connection management and event handling during streaming operations - Improved database connection management by ensuring `close_old_connections()` is called in various methods to prevent connection leaks. - Updated event dispatching to run asynchronously in gevent, preventing blocking during live-proxy and streaming paths. --- CHANGELOG.md | 3 +- Plugins.md | 2 +- apps/proxy/live_proxy/input/manager.py | 74 +++++++------- apps/proxy/live_proxy/output/ts/generator.py | 102 +++++++++---------- apps/proxy/live_proxy/server.py | 7 +- core/utils.py | 52 +++++++++- tests/test_log_system_event.py | 55 ++++++++++ 7 files changed, 194 insertions(+), 101 deletions(-) create mode 100644 tests/test_log_system_event.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 01ab218f..e8cacfda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` during pending shutdown or active teardown; `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. -- **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `log_system_event()` (all system events), `_clean_redis_keys()`, channel init/start logging, client disconnect cleanup (TS and fMP4 generators), and profile lookup in `_establish_transcode_connection()` before spawning ffmpeg. +- **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). +- **System events could block callers on Connect and plugin handlers.** `log_system_event()` ran Connect subscriptions and plugin `"events"` hooks synchronously after the DB write, so slow integrations could stall live-proxy and streaming paths even when inserting the event row was fast. Connect/plugin dispatch now runs on a separate gevent when the hub is available (synchronously in Celery workers without a hub). `log_system_event()` and `dispatch_event_system()` also call `close_old_connections()` in `finally` blocks so integration work does not leave pool slots checked out on the caller or dispatch greenlet. - **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. diff --git a/Plugins.md b/Plugins.md index 97e95ed2..4dc15a55 100644 --- a/Plugins.md +++ b/Plugins.md @@ -317,7 +317,7 @@ Still follow these rules: - **Heavy or long work:** dispatch a Celery task (`.delay()`) and return quickly from `run()`. Celery workers close connections after each task; blocking the uWSGI gevent hub with `time.sleep`, sync HTTP, or large CPU work can freeze the whole worker regardless of DB cleanup. - **Background threads or greenlets you spawn:** each thread/greenlet that uses the ORM must call `close_old_connections()` (or `connection.close()`) in its own `finally` block when done. The wrapper only covers the thread/greenlet that called `run_action()`. -- **Connect event hooks:** actions with an `"events"` list run synchronously inside whatever caller triggered the event (for example a live-proxy system event). Keep event handlers short or defer to Celery. +- **Connect event hooks:** actions with an `"events"` list are dispatched from `log_system_event()` on a separate gevent when uWSGI has an active hub (otherwise synchronously, e.g. Celery). Keep handlers short or defer heavy work to Celery. ### Important: Don’t Ask Users for URL/User/Password Dispatcharr plugins run **inside** the Dispatcharr backend process. That means they already have direct access to the app’s models, tasks, and internal utilities. diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index e8315006..7d01a88f 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -650,48 +650,48 @@ class StreamManager: logger.debug(f"Closing existing HTTP connections before establishing transcode connection for channel {self.channel_id}") self._close_connection() - close_old_connections() - channel = get_stream_object(self.channel_id) + try: + channel = get_stream_object(self.channel_id) - # Use FFmpeg specifically for HLS streams - if hasattr(self, 'force_ffmpeg') and self.force_ffmpeg: - from core.models import StreamProfile - try: - stream_profile = StreamProfile.objects.get(name='ffmpeg', locked=True) - logger.info("Using FFmpeg stream profile for unsupported proxy content (HLS/RTSP/UDP)") - except StreamProfile.DoesNotExist: - # Fall back to channel's profile if FFmpeg not found + # Use FFmpeg specifically for HLS streams + if hasattr(self, 'force_ffmpeg') and self.force_ffmpeg: + from core.models import StreamProfile + try: + stream_profile = StreamProfile.objects.get(name='ffmpeg', locked=True) + logger.info("Using FFmpeg stream profile for unsupported proxy content (HLS/RTSP/UDP)") + except StreamProfile.DoesNotExist: + # Fall back to channel's profile if FFmpeg not found + stream_profile = channel.get_stream_profile() + logger.warning(f"FFmpeg profile not found, using channel default profile for channel: {self.channel_id}") + else: stream_profile = channel.get_stream_profile() - logger.warning(f"FFmpeg profile not found, using channel default profile for channel: {self.channel_id}") - else: - stream_profile = channel.get_stream_profile() - # Build and start transcode command - self.transcode_cmd = stream_profile.build_command(self.url, self.user_agent) + # Build and start transcode command + self.transcode_cmd = stream_profile.build_command(self.url, self.user_agent) - # Store stream command for efficient log parser routing - self.stream_command = stream_profile.command - # Map actual commands to parser types for direct routing - command_to_parser = { - 'ffmpeg': 'ffmpeg', - 'cvlc': 'vlc', - 'vlc': 'vlc', - 'streamlink': 'streamlink' - } - self.parser_type = command_to_parser.get(self.stream_command.lower()) - if self.parser_type: - logger.debug(f"Using {self.parser_type} parser for log parsing (command: {self.stream_command})") - else: - logger.debug(f"Unknown stream command '{self.stream_command}', will use auto-detection for log parsing") + # Store stream command for efficient log parser routing + self.stream_command = stream_profile.command + # Map actual commands to parser types for direct routing + command_to_parser = { + 'ffmpeg': 'ffmpeg', + 'cvlc': 'vlc', + 'vlc': 'vlc', + 'streamlink': 'streamlink' + } + self.parser_type = command_to_parser.get(self.stream_command.lower()) + if self.parser_type: + logger.debug(f"Using {self.parser_type} parser for log parsing (command: {self.stream_command})") + else: + logger.debug(f"Unknown stream command '{self.stream_command}', will use auto-detection for log parsing") - # For UDP streams, remove any user_agent parameters from the command - if hasattr(self, 'stream_type') and self.stream_type == StreamType.UDP: - # Filter out any arguments that contain the user_agent value or related headers - self.transcode_cmd = [arg for arg in self.transcode_cmd if self.user_agent not in arg and 'user-agent' not in arg.lower() and 'user_agent' not in arg.lower()] - logger.debug(f"Removed user_agent parameters from UDP stream command for channel: {self.channel_id}") - - # Profile lookup is done; release the pool slot before a long-lived ffmpeg process. - close_old_connections() + # For UDP streams, remove any user_agent parameters from the command + if hasattr(self, 'stream_type') and self.stream_type == StreamType.UDP: + # Filter out any arguments that contain the user_agent value or related headers + self.transcode_cmd = [arg for arg in self.transcode_cmd if self.user_agent not in arg and 'user-agent' not in arg.lower() and 'user_agent' not in arg.lower()] + logger.debug(f"Removed user_agent parameters from UDP stream command for channel: {self.channel_id}") + finally: + # Release the pool slot before posix_spawn or before returning on profile errors. + close_old_connections() logger.debug(f"Starting transcode process: {self.transcode_cmd} for channel: {self.channel_id}") diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index 69dc341f..efde02ad 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -8,7 +8,6 @@ import gevent from apps.proxy.config import TSConfig as Config from apps.channels.models import Channel, Stream from core.utils import log_system_event -from django.db import close_old_connections from ...server import ProxyServer from ...utils import create_ts_packet, get_logger from ...redis_keys import RedisKeys @@ -591,62 +590,59 @@ class StreamGenerator: def _cleanup(self): """Clean up resources and report final statistics.""" - try: - # Client cleanup - elapsed = time.time() - self.stream_start_time - local_clients = 0 - total_clients = 0 - proxy_server = ProxyServer.get_instance() + # Client cleanup + elapsed = time.time() - self.stream_start_time + local_clients = 0 + total_clients = 0 + proxy_server = ProxyServer.get_instance() - # Release M3U profile stream allocation if this is the last client - stream_released = False - if proxy_server.redis_client: - try: - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() - # Pool slots are global; the last client on any worker must release. - if client_count <= 1: + # Release M3U profile stream allocation if this is the last client + stream_released = False + if proxy_server.redis_client: + try: + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Pool slots are global; the last client on any worker must release. + if client_count <= 1: + try: try: - try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - stream_released = obj.release_stream() - if stream_released: - logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") - else: - logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") - except Exception as e: - logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") - except Exception as e: - logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + stream_released = obj.release_stream() + if stream_released: + logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") + else: + logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") + except Exception as e: + logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") - if self.channel_id in proxy_server.client_managers: - client_manager = proxy_server.client_managers[self.channel_id] - if self.client_id in client_manager.clients: - local_clients = client_manager.remove_client(self.client_id) - else: - local_clients = client_manager.get_client_count() - total_clients = client_manager.get_total_client_count() - logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") + if self.channel_id in proxy_server.client_managers: + client_manager = proxy_server.client_managers[self.channel_id] + if self.client_id in client_manager.clients: + local_clients = client_manager.remove_client(self.client_id) + else: + local_clients = client_manager.get_client_count() + total_clients = client_manager.get_total_client_count() + logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") - # Log client disconnect event - try: - log_system_event( - 'client_disconnect', - channel_id=self.channel_id, - channel_name=self.channel_name, - client_ip=self.client_ip, - client_id=self.client_id, - user_agent=self.client_user_agent[:100] if self.client_user_agent else None, - duration=round(elapsed, 2), - bytes_sent=self.bytes_sent, - username=self.user.username if self.user else None - ) - except Exception as e: - logger.error(f"Could not log client disconnect event: {e}") - finally: - close_old_connections() + # Log client disconnect event + try: + log_system_event( + 'client_disconnect', + channel_id=self.channel_id, + channel_name=self.channel_name, + client_ip=self.client_ip, + client_id=self.client_id, + user_agent=self.client_user_agent[:100] if self.client_user_agent else None, + duration=round(elapsed, 2), + bytes_sent=self.bytes_sent, + username=self.user.username if self.user else None + ) + except Exception as e: + logger.error(f"Could not log client disconnect event: {e}") def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None, buffer=None): diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 98346676..3cb1fdae 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -760,7 +760,6 @@ class ProxyServer: ) except Exception as e: logger.error(f"Could not log channel start event: {e}") - finally: close_old_connections() # Create client manager with channel_id, redis_client AND worker_id (only if not already exists) @@ -1367,11 +1366,7 @@ class ProxyServer: return def _log_stop(): - try: - close_old_connections() - log_system_event('channel_stop', **stop_event_data) - except Exception as e: - logger.error(f"Could not log channel stop event: {e}") + log_system_event('channel_stop', **stop_event_data) gevent.spawn(_log_stop) diff --git a/core/utils.py b/core/utils.py index 5931d985..188d9cb2 100644 --- a/core/utils.py +++ b/core/utils.py @@ -623,6 +623,8 @@ def validate_flexible_url(value): raise ValidationError("Enter a valid URL.") def dispatch_event_system(event_type, channel_id=None, channel_name=None, **details): + from django.db import close_old_connections + try: from apps.connect.utils import trigger_event from apps.channels.models import Channel, Stream @@ -696,9 +698,48 @@ def dispatch_event_system(event_type, channel_id=None, channel_name=None, **deta trigger_event(event_type, payload) - except Exception as e: + except Exception: # Don't fail main path if connect dispatch fails pass + finally: + close_old_connections() + + +def _dispatch_system_event_integrations( + event_type, channel_id=None, channel_name=None, **details +): + """ + Run Connect subscriptions and plugin event hooks without blocking the caller. + + On gevent uWSGI workers, dispatch runs in a spawned greenlet so slow webhooks, + scripts, or plugin handlers cannot stall live-proxy teardown or streaming paths. + Celery prefork workers (gevent patched but no hub) run synchronously instead. + """ + + def _run(): + try: + dispatch_event_system( + event_type, + channel_id=channel_id, + channel_name=channel_name, + **details, + ) + except Exception as e: + logger.error( + "Failed to dispatch Connect/plugin handlers for event %s: %s", + event_type, + e, + ) + + if _should_use_sync_websocket_send(): + _run() + elif _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + def log_system_event(event_type, channel_id=None, channel_name=None, **details): """ @@ -726,8 +767,13 @@ def log_system_event(event_type, channel_id=None, channel_name=None, **details): details=details ) - # Trigger connect integrations for specific events - dispatch_event_system(event_type, channel_id=channel_id, channel_name=channel_name, **details) + # Connect integrations and plugin event hooks (non-blocking on gevent uWSGI) + _dispatch_system_event_integrations( + event_type, + channel_id=channel_id, + channel_name=channel_name, + **details, + ) # Get max events from settings (default 100) try: diff --git a/tests/test_log_system_event.py b/tests/test_log_system_event.py new file mode 100644 index 00000000..6ff92ff4 --- /dev/null +++ b/tests/test_log_system_event.py @@ -0,0 +1,55 @@ +"""Tests for log_system_event Connect/plugin dispatch and DB cleanup.""" + +from unittest.mock import patch + +from django.test import SimpleTestCase + + +class LogSystemEventDispatchTests(SimpleTestCase): + @patch("django.db.close_old_connections") + @patch("core.utils._dispatch_system_event_integrations") + @patch("core.models.SystemEvent.objects") + @patch("core.models.CoreSettings.objects") + def test_log_system_event_dispatches_integrations_and_closes_db( + self, mock_core_settings, mock_system_event, mock_dispatch, mock_close + ): + mock_system_event.count.return_value = 1 + mock_core_settings.filter.return_value.first.return_value = None + + from core.utils import log_system_event + + log_system_event("channel_start", channel_id="abc", channel_name="Test") + + mock_dispatch.assert_called_once_with( + "channel_start", + channel_id="abc", + channel_name="Test", + ) + mock_close.assert_called_once() + + @patch("django.db.close_old_connections") + @patch("apps.connect.utils.trigger_event") + def test_integration_dispatch_closes_db_on_sync_path( + self, mock_trigger, mock_close + ): + from core.utils import _dispatch_system_event_integrations + + with patch("core.utils._should_use_sync_websocket_send", return_value=True): + _dispatch_system_event_integrations("client_connect", channel_id="abc") + + mock_trigger.assert_called_once() + mock_close.assert_called_once() + + @patch("core.utils.dispatch_event_system") + def test_integration_dispatch_spawns_on_gevent_uwsgi( + self, mock_dispatch + ): + from core.utils import _dispatch_system_event_integrations + + with patch("core.utils._should_use_sync_websocket_send", return_value=False), patch( + "core.utils._is_gevent_monkey_patched", return_value=True + ), patch("gevent.spawn") as mock_spawn: + _dispatch_system_event_integrations("channel_stop", channel_id="abc") + + mock_spawn.assert_called_once() + mock_dispatch.assert_not_called() From 7989fa3673a58888befa9977d451626fac7d48a4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 17:32:38 -0500 Subject: [PATCH 35/81] fix(proxy): enhance database connection management during VOD playback and stats updates - Added calls to `close_old_connections()` in `stream_vod()` and `build_vod_stats_data()` to prevent connection leaks during long-lived streaming responses and background stats refreshes. - Improved connection handling in various components to ensure proper resource cleanup and prevent blocking issues. --- CHANGELOG.md | 1 + .../tests/test_ts_proxy_ghost_clients.py | 8 + .../channels/tests/test_ts_proxy_keepalive.py | 12 +- apps/channels/tests/test_ts_proxy_teardown.py | 12 +- apps/proxy/live_proxy/input/manager.py | 15 +- apps/proxy/live_proxy/server.py | 17 +- .../vod_proxy/tests/test_vod_db_cleanup.py | 146 ++++++++++++++++++ apps/proxy/vod_proxy/views.py | 6 + 8 files changed, 198 insertions(+), 19 deletions(-) create mode 100644 apps/proxy/vod_proxy/tests/test_vod_db_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e8cacfda..acce9b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). - **System events could block callers on Connect and plugin handlers.** `log_system_event()` ran Connect subscriptions and plugin `"events"` hooks synchronously after the DB write, so slow integrations could stall live-proxy and streaming paths even when inserting the event row was fast. Connect/plugin dispatch now runs on a separate gevent when the hub is available (synchronously in Celery workers without a hub). `log_system_event()` and `dispatch_event_system()` also call `close_old_connections()` in `finally` blocks so integration work does not leave pool slots checked out on the caller or dispatch greenlet. - **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. +- **VOD proxy could leak geventpool DB checkouts during playback and stats updates.** `stream_vod()` ran ORM lookups for content and M3U profiles, then returned a long-lived `StreamingHttpResponse` without releasing the checkout, so each movie/episode stream could hold a pool slot for its full duration. Background VOD stats refresh (`build_vod_stats_data()`, triggered on start/stop and by the admin stats API) also queried movie, episode, and profile rows from daemon threads with no cleanup. `stream_vod()` now calls `close_old_connections()` before handing off to the streaming generator, and `build_vod_stats_data()` releases its checkout in a `finally` block. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** diff --git a/apps/channels/tests/test_ts_proxy_ghost_clients.py b/apps/channels/tests/test_ts_proxy_ghost_clients.py index 6742cf4b..3cc7aa9e 100644 --- a/apps/channels/tests/test_ts_proxy_ghost_clients.py +++ b/apps/channels/tests/test_ts_proxy_ghost_clients.py @@ -247,6 +247,14 @@ class BasicStatsGhostClientTests(TestCase): redis.scard.return_value = len(client_ids) redis.smembers.return_value = client_ids redis.hget.return_value = None # individual field lookups + redis.hmget.return_value = [ + b'VLC/3.0', + b'127.0.0.1', + b'1773500000.0', + None, + b'mpegts', + None, + ] # Pipeline for remove_ghost_clients pipe = MagicMock() diff --git a/apps/channels/tests/test_ts_proxy_keepalive.py b/apps/channels/tests/test_ts_proxy_keepalive.py index b8fb524a..8645a158 100644 --- a/apps/channels/tests/test_ts_proxy_keepalive.py +++ b/apps/channels/tests/test_ts_proxy_keepalive.py @@ -218,7 +218,7 @@ class DoStatsUpdateTests(TestCase): mock_redis.scan.return_value = (0, []) with patch("apps.proxy.live_proxy.client_manager.send_websocket_update") as mock_ws, \ - patch("redis.Redis.from_url", return_value=mock_redis): + patch("core.utils.RedisClient.get_client", return_value=mock_redis): cm._do_stats_update() mock_ws.assert_called_once() @@ -231,25 +231,25 @@ class DoStatsUpdateTests(TestCase): """Redis failure must be swallowed (logged), not propagated.""" cm = self._make_client_manager() - with patch("redis.Redis.from_url", side_effect=Exception("Redis down")): + with patch("core.utils.RedisClient.get_client", side_effect=Exception("Redis down")): try: cm._do_stats_update() except Exception as e: self.fail(f"_do_stats_update raised an exception: {e}") - def test_do_stats_update_scans_channel_client_keys(self): - """Must scan for live:channel:*:clients pattern.""" + def test_do_stats_update_scans_channel_metadata_keys(self): + """Must scan for live:channel:*:metadata pattern.""" cm = self._make_client_manager() mock_redis = MagicMock() mock_redis.scan.return_value = (0, []) with patch("apps.proxy.live_proxy.client_manager.send_websocket_update"), \ - patch("redis.Redis.from_url", return_value=mock_redis): + patch("core.utils.RedisClient.get_client", return_value=mock_redis): cm._do_stats_update() scan_call = mock_redis.scan.call_args - self.assertIn("live:channel:*:clients", str(scan_call)) + self.assertIn("live:channel:*:metadata", str(scan_call)) # --------------------------------------------------------------------------- diff --git a/apps/channels/tests/test_ts_proxy_teardown.py b/apps/channels/tests/test_ts_proxy_teardown.py index 3bc92f18..593df034 100644 --- a/apps/channels/tests/test_ts_proxy_teardown.py +++ b/apps/channels/tests/test_ts_proxy_teardown.py @@ -390,17 +390,21 @@ class CleanRedisKeysOrderTests(TestCase): mock_channel_get.side_effect = channel_get mock_stream_get.side_effect = Stream.DoesNotExist - def scan(*args, **kwargs): + channel_key = f"live:channel:{CHANNEL_ID}:input:buffer:index".encode() + + def scan(cursor, match=None, count=100): call_order.append("redis") - return (0, [b"live:channel:foo:buffer:index"]) + if match == f"live:channel:{CHANNEL_ID}:*": + return (0, [channel_key]) + return (0, []) server.redis_client.scan.side_effect = scan server._clean_redis_keys(CHANNEL_ID) - self.assertEqual(call_order, ["release", "redis"]) + self.assertEqual(call_order, ["release", "redis", "redis"]) channel.release_stream.assert_called_once() - server.redis_client.delete.assert_called_once() + server.redis_client.delete.assert_called_once_with(channel_key) class LocalUpstreamActivityTests(TestCase): diff --git a/apps/proxy/live_proxy/input/manager.py b/apps/proxy/live_proxy/input/manager.py index 7d01a88f..3c7c147d 100644 --- a/apps/proxy/live_proxy/input/manager.py +++ b/apps/proxy/live_proxy/input/manager.py @@ -67,6 +67,7 @@ class StreamManager: # Add to your __init__ method self._buffer_check_timers = [] self.stopping = False + self.stop_requested = False # Add tracking for tried streams and current stream self.current_stream_id = stream_id @@ -572,7 +573,9 @@ class StreamManager: try: metadata_key = RedisKeys.channel_metadata(self.channel_id) owner_key = RedisKeys.channel_owner(self.channel_id) - current_owner = self.buffer.redis_client.get(owner_key) + current_owner = self._decode_redis_value( + self.buffer.redis_client.get(owner_key) + ) is_owner = ( current_owner @@ -583,12 +586,10 @@ class StreamManager: should_update = is_owner if not should_update and no_owner: - current_state_bytes = self.buffer.redis_client.hget( - metadata_key, ChannelMetadataField.STATE - ) - current_state = ( - current_state_bytes - if current_state_bytes else None + current_state = self._decode_redis_value( + self.buffer.redis_client.hget( + metadata_key, ChannelMetadataField.STATE + ) ) should_update = current_state in ChannelState.PRE_ACTIVE if not should_update and current_state: diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index 3cb1fdae..ddec2d02 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -1402,6 +1402,15 @@ class ProxyServer: f"Error broadcasting upstream stop for channel {channel_id}: {e}" ) + @staticmethod + def _channel_id_from_metadata_key(key): + if isinstance(key, bytes): + key = key.decode('utf-8', errors='replace') + parts = key.split(':') + if len(parts) >= 3: + return parts[2] + return None + def _stop_upstream_before_redis_cleanup(self, channel_id): """Stop local ffmpeg before deleting Redis keys (prevents delete/recreate loops).""" if self._has_local_upstream_activity(channel_id): @@ -1955,7 +1964,9 @@ class ProxyServer: for key in channel_keys: try: - channel_id = key.split(':')[2] + channel_id = self._channel_id_from_metadata_key(key) + if not channel_id: + continue # Check if this channel has an owner owner = self.get_channel_owner(channel_id) @@ -1995,7 +2006,9 @@ class ProxyServer: for key in channel_keys: try: - channel_id = key.split(':')[2] + channel_id = self._channel_id_from_metadata_key(key) + if not channel_id: + continue # Get metadata first metadata = self.redis_client.hgetall(key) diff --git a/apps/proxy/vod_proxy/tests/test_vod_db_cleanup.py b/apps/proxy/vod_proxy/tests/test_vod_db_cleanup.py new file mode 100644 index 00000000..5d009803 --- /dev/null +++ b/apps/proxy/vod_proxy/tests/test_vod_db_cleanup.py @@ -0,0 +1,146 @@ +"""VOD proxy must release geventpool checkouts after ORM on stream and stats paths.""" + +from unittest.mock import MagicMock, patch + +from django.http import StreamingHttpResponse +from django.test import RequestFactory, SimpleTestCase + + +class StreamVodDbCleanupTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + @patch("apps.proxy.vod_proxy.views.close_old_connections") + @patch("apps.proxy.vod_proxy.views.MultiWorkerVODConnectionManager") + @patch("apps.proxy.vod_proxy.views._transform_url", return_value="http://example.com/movie.mp4") + @patch("apps.proxy.vod_proxy.views._get_m3u_profile") + @patch("apps.proxy.vod_proxy.views._get_stream_url_from_relation", return_value="http://upstream/movie.mp4") + @patch("apps.proxy.vod_proxy.views._get_content_and_relation") + @patch("apps.proxy.vod_proxy.views.network_access_allowed", return_value=True) + def test_stream_vod_closes_db_before_streaming_response( + self, + _network_ok, + mock_content, + _stream_url, + mock_profile, + _transform, + mock_manager_cls, + mock_close, + ): + movie = MagicMock() + movie.name = "Test Movie" + relation = MagicMock() + relation.m3u_account.name = "Provider" + mock_content.return_value = (movie, relation) + + profile = MagicMock() + profile.id = 1 + profile.max_streams = 5 + mock_profile.return_value = (profile, 0) + + mock_manager = MagicMock() + mock_manager.stream_content_with_session.return_value = StreamingHttpResponse( + streaming_content=iter([b"data"]), + content_type="video/mp4", + ) + mock_manager_cls.get_instance.return_value = mock_manager + + request = self.factory.get( + "/proxy/vod/movie/uuid/session123/", + HTTP_USER_AGENT="test-agent", + ) + request.user = MagicMock(is_authenticated=False) + + from apps.proxy.vod_proxy.views import stream_vod + + response = stream_vod( + request, + content_type="movie", + content_id="uuid", + session_id="session123", + ) + + self.assertIsInstance(response, StreamingHttpResponse) + mock_close.assert_called_once() + mock_manager.stream_content_with_session.assert_called_once() + + +class BuildVodStatsDbCleanupTests(SimpleTestCase): + @patch("apps.proxy.vod_proxy.views.close_old_connections") + @patch("apps.proxy.vod_proxy.views.Movie") + def test_build_vod_stats_data_closes_db(self, mock_movie, mock_close): + redis_client = MagicMock() + redis_client.scan.side_effect = [ + (0, ["vod_persistent_connection:s1"]), + ] + redis_client.hgetall.return_value = { + "content_obj_type": "movie", + "content_uuid": "movie-uuid", + "content_name": "Test Movie", + "m3u_profile_id": "1", + "client_ip": "127.0.0.1", + "client_user_agent": "agent", + "connected_at": "1000.0", + "last_activity": "1001.0", + "active_streams": "1", + } + + movie_obj = MagicMock( + name="Test Movie", + logo=None, + year=2020, + rating=7.5, + genre="Action", + description="Desc", + tmdb_id="1", + imdb_id="tt1", + ) + mock_movie.objects.select_related.return_value.get.return_value = movie_obj + + with patch("apps.m3u.models.M3UAccountProfile") as mock_profile_model: + mock_profile_model.objects.select_related.return_value.get.return_value = MagicMock( + name="Profile 1", + m3u_account=MagicMock(name="Account", id=1), + ) + + from apps.proxy.vod_proxy.views import build_vod_stats_data + + stats = build_vod_stats_data(redis_client) + + self.assertEqual(stats["total_connections"], 1) + mock_close.assert_called_once() + + @patch("apps.proxy.vod_proxy.views.close_old_connections") + def test_build_vod_stats_data_closes_db_on_error(self, mock_close): + redis_client = MagicMock() + redis_client.scan.side_effect = RuntimeError("redis down") + + from apps.proxy.vod_proxy.views import build_vod_stats_data + + stats = build_vod_stats_data(redis_client) + + self.assertEqual(stats["total_connections"], 0) + mock_close.assert_called_once() + + +class VodStatsUpdateDbCleanupTests(SimpleTestCase): + @patch("core.utils.send_websocket_update") + @patch("apps.proxy.vod_proxy.views.build_vod_stats_data") + def test_do_vod_stats_update_uses_build_vod_stats_data(self, mock_build, mock_ws): + mock_build.return_value = { + "vod_connections": [], + "total_connections": 0, + "timestamp": 0, + } + + from apps.proxy.vod_proxy.multi_worker_connection_manager import ( + MultiWorkerVODConnectionManager, + ) + + manager = MultiWorkerVODConnectionManager.__new__(MultiWorkerVODConnectionManager) + manager.redis_client = MagicMock() + + manager._do_vod_stats_update() + + mock_build.assert_called_once_with(manager.redis_client) + mock_ws.assert_called_once() diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index f9740aa1..fef8fed0 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -8,6 +8,7 @@ import random import logging import requests from urllib.parse import urlencode +from django.db import close_old_connections from django.http import JsonResponse, Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt @@ -601,6 +602,9 @@ def stream_vod(request, content_type, content_id, session_id=None, profile_id=No # Get connection manager (Redis-backed for multi-worker support) connection_manager = MultiWorkerVODConnectionManager.get_instance() + # Release ORM checkout before returning a long-lived StreamingHttpResponse. + close_old_connections() + # Stream the content with session-based connection reuse logger.info("[VOD-STREAM] Calling connection manager to stream content") response = connection_manager.stream_content_with_session( @@ -1063,6 +1067,8 @@ def build_vod_stats_data(redis_client): except Exception as e: logger.error(f"Error building VOD stats: {e}") return {'vod_connections': [], 'total_connections': 0, 'timestamp': time.time()} + finally: + close_old_connections() @api_view(["GET"]) From 32442e0b6401662ce3cf0452fa1ce34fad90b2f3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 18:53:49 -0500 Subject: [PATCH 36/81] fix(proxy): enhance channel shutdown management and client reconnection handling - Improved the handling of channel shutdown delays to ensure they reset correctly upon client reconnections, preventing premature shutdowns. - Implemented logic to cancel pending shutdowns when clients reconnect, ensuring proper resource management and preventing client disconnect issues across uWSGI workers. - Enhanced logging for shutdown processes and client management to provide clearer insights during channel operations. --- CHANGELOG.md | 3 +- apps/channels/tests/test_ts_proxy_teardown.py | 163 +++++++++++++++++- apps/proxy/live_proxy/client_manager.py | 16 +- .../proxy/live_proxy/output/fmp4/generator.py | 6 +- apps/proxy/live_proxy/output/ts/generator.py | 3 +- apps/proxy/live_proxy/server.py | 109 ++++++++++-- .../live_proxy/services/channel_service.py | 101 ++++++++++- 7 files changed, 378 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acce9b4e..fd70a28e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,13 +44,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Live proxy channels could remain running with no clients after disconnect.** `stop_channel` called `log_system_event('channel_stop')` synchronously before local cleanup and Redis key deletion; Connect integrations or DB event writes on that path could block teardown indefinitely while the cleanup thread kept refreshing metadata TTL for a stale `stream_buffers` entry. Teardown now signals shutdown (`buffer.stopping`), runs model `release_stream()` and Redis key deletion, releases ownership, stops ffmpeg/output managers and local buffers, and only then logs the stop event asynchronously. Metadata TTL refresh skips channels mid-shutdown and non-owned channels; a stuck-stop watchdog forces cleanup if `stop_channel` does not return within ~10s (or 2× `channel_shutdown_delay`, whichever is greater). Stream buffers ignore late `add_chunk()` writes once shutdown begins (`buffer.stopping` set at teardown start). -- **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` during pending shutdown or active teardown; `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) +- **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` only during active teardown (not during the post-disconnect shutdown delay grace period); `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. - **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). - **System events could block callers on Connect and plugin handlers.** `log_system_event()` ran Connect subscriptions and plugin `"events"` hooks synchronously after the DB write, so slow integrations could stall live-proxy and streaming paths even when inserting the event row was fast. Connect/plugin dispatch now runs on a separate gevent when the hub is available (synchronously in Celery workers without a hub). `log_system_event()` and `dispatch_event_system()` also call `close_old_connections()` in `finally` blocks so integration work does not leave pool slots checked out on the caller or dispatch greenlet. - **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. - **VOD proxy could leak geventpool DB checkouts during playback and stats updates.** `stream_vod()` ran ORM lookups for content and M3U profiles, then returned a long-lived `StreamingHttpResponse` without releasing the checkout, so each movie/episode stream could hold a pool slot for its full duration. Background VOD stats refresh (`build_vod_stats_data()`, triggered on start/stop and by the admin stats API) also queried movie, episode, and profile rows from daemon threads with no cleanup. `stream_vod()` now calls `close_old_connections()` before handing off to the streaming generator, and `build_vod_stats_data()` releases its checkout in a `finally` block. +- **Channel shutdown delay did not reset after a reconnect within the grace period.** `handle_client_disconnect()` used a fixed `gevent.sleep(shutdown_delay)` from the first last-client disconnect. If a client reconnected and disconnected again during the delay, an earlier disconnect handler could still stop the channel on the original timer instead of waiting the full delay from the latest disconnect. Shutdown now polls Redis (`last_client_disconnect` timestamp and client count) so concurrent disconnect handlers and multi-worker reconnects always honour the latest disconnect time. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. - **Stale `channel_stream` Redis keys after a channel stopped could skip connection accounting on retune.** On `dev`, `get_stream()` reused any existing `channel_stream` / `stream_profile` assignment without reserving a new slot. If those keys were left behind after a stop, the next tune-in could reach the provider without incrementing Redis counters. `get_stream()` now releases stale assignments when proxy metadata shows the channel is inactive, then reserves fresh slots. - **EPG auto-match reliability fixes.** diff --git a/apps/channels/tests/test_ts_proxy_teardown.py b/apps/channels/tests/test_ts_proxy_teardown.py index 593df034..20fd92ea 100644 --- a/apps/channels/tests/test_ts_proxy_teardown.py +++ b/apps/channels/tests/test_ts_proxy_teardown.py @@ -41,6 +41,7 @@ def _configure_ownership_pipeline( def _mock_proxy_server(redis_client=None): server = MagicMock() server.redis_client = redis_client or MagicMock() + server._stopping_channels = set() return server @@ -72,7 +73,47 @@ class ChannelTeardownAvailabilityTests(TestCase): mock_get_instance.return_value = _mock_proxy_server(redis) self.assertTrue(ChannelService.is_shutdown_pending(CHANNEL_ID)) - self.assertTrue(ChannelService.is_channel_unavailable_for_new_clients(CHANNEL_ID)) + self.assertFalse(ChannelService.is_channel_unavailable_for_new_clients(CHANNEL_ID)) + + @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_cancel_pending_shutdown_clears_disconnect_key(self, mock_get_instance, mock_delay): + mock_delay.return_value = 30 + redis = MagicMock() + redis.exists.side_effect = lambda key: "last_client_disconnect" in key + redis.get.return_value = None + redis.hget.return_value = ChannelState.ACTIVE.encode() + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertTrue(ChannelService.cancel_pending_shutdown(CHANNEL_ID)) + redis.delete.assert_any_call(RedisKeys.last_client_disconnect(CHANNEL_ID)) + + @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_cancel_pending_shutdown_skips_during_active_stop(self, mock_get_instance, mock_delay): + mock_delay.return_value = 30 + redis = MagicMock() + redis.exists.return_value = True + server = _mock_proxy_server(redis) + server._stopping_channels = {CHANNEL_ID} + mock_get_instance.return_value = server + + self.assertFalse(ChannelService.cancel_pending_shutdown(CHANNEL_ID)) + redis.delete.assert_not_called() + + @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") + @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") + def test_cancel_pending_shutdown_skips_real_teardown_without_grace(self, mock_get_instance, mock_delay): + mock_delay.return_value = 30 + redis = MagicMock() + redis.exists.side_effect = lambda key: "stopping" in key + redis.get.return_value = None + redis.hget.return_value = ChannelState.STOPPING.encode() + mock_get_instance.return_value = _mock_proxy_server(redis) + + self.assertFalse(ChannelService.cancel_pending_shutdown(CHANNEL_ID)) + redis.hset.assert_not_called() + redis.delete.assert_not_called() @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay") @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance") @@ -86,6 +127,28 @@ class ChannelTeardownAvailabilityTests(TestCase): self.assertFalse(ChannelService.is_shutdown_pending(CHANNEL_ID)) +class ClientManagerAddClientTests(TestCase): + @patch("apps.proxy.live_proxy.services.channel_service.ChannelService.cancel_pending_shutdown", return_value=False) + @patch("apps.proxy.live_proxy.client_manager.send_websocket_update") + def test_add_client_stores_ip_and_user_agent_in_redis(self, _mock_ws, _mock_cancel): + from apps.proxy.live_proxy.client_manager import ClientManager + + redis = MagicMock() + cm = ClientManager(CHANNEL_ID, redis_client=redis, worker_id="worker-1") + cm.proxy_server = MagicMock() + + result = cm.add_client( + "client-1", + "10.0.2.163", + user_agent="VLC/3.0.21", + ) + + self.assertEqual(result, 1) + mapping = redis.hset.call_args[1]["mapping"] + self.assertEqual(mapping["ip_address"], "10.0.2.163") + self.assertEqual(mapping["user_agent"], "VLC/3.0.21") + + class LocalStreamActivityTests(TestCase): def _make_server(self): with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): @@ -509,6 +572,104 @@ class HandleClientDisconnectUpstreamFirstTests(TestCase): mock_coordinated.assert_called_once_with(CHANNEL_ID) +class ShutdownDelayWaitTests(TestCase): + def _make_server(self): + with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()): + with patch.object(ProxyServer, "_start_cleanup_thread"): + server = ProxyServer() + server.redis_client = MagicMock() + server.redis_client.scard.return_value = 0 + return server + + @patch("apps.proxy.live_proxy.server.gevent.sleep") + @patch("apps.proxy.live_proxy.server.time.time", return_value=1000.0) + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + def test_aborts_when_disconnect_key_deleted(self, _mock_delay, _mock_time, mock_sleep): + server = self._make_server() + server.redis_client.get.side_effect = [b"1000.0", None] + + result = server._wait_for_shutdown_delay(CHANNEL_ID) + + self.assertFalse(result) + self.assertGreaterEqual(mock_sleep.call_count, 1) + + @patch("apps.proxy.live_proxy.server.gevent.sleep") + @patch("apps.proxy.live_proxy.server.time.time", return_value=1000.0) + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + def test_aborts_when_clients_reconnect(self, _mock_delay, _mock_time, _mock_sleep): + server = self._make_server() + server.redis_client.get.return_value = b"1000.0" + server.redis_client.scard.side_effect = [0, 1] + + result = server._wait_for_shutdown_delay(CHANNEL_ID) + + self.assertFalse(result) + server.redis_client.delete.assert_called_with( + RedisKeys.last_client_disconnect(CHANNEL_ID) + ) + + @patch("apps.proxy.live_proxy.server.gevent.sleep") + @patch("apps.proxy.live_proxy.server.time.time") + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + def test_timer_resets_when_disconnect_timestamp_updated( + self, _mock_delay, mock_time, mock_sleep + ): + server = self._make_server() + disconnect_key = RedisKeys.last_client_disconnect(CHANNEL_ID) + current_time = [1000.0] + disconnect_timestamp = [1000.0] + poll_count = [0] + + mock_time.side_effect = lambda: current_time[0] + + def get_side_effect(key): + poll_count[0] += 1 + if poll_count[0] >= 3: + disconnect_timestamp[0] = 1020.0 + if key == disconnect_key: + return str(disconnect_timestamp[0]).encode() + return None + + server.redis_client.get.side_effect = get_side_effect + + def advance_sleep(duration): + current_time[0] += duration + + mock_sleep.side_effect = advance_sleep + + result = server._wait_for_shutdown_delay(CHANNEL_ID) + + self.assertTrue(result) + self.assertGreaterEqual(current_time[0], 1050.0) + + @patch.object(ProxyServer, "_coordinated_stop_channel") + @patch.object(ProxyServer, "_wait_for_shutdown_delay", return_value=True) + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + def test_handle_client_disconnect_uses_polling_wait( + self, _mock_delay, mock_wait, mock_coordinated + ): + server = HandleClientDisconnectUpstreamFirstTests()._make_server() + server.redis_client.get.return_value = b"1700000000.0" + + server.handle_client_disconnect(CHANNEL_ID) + + mock_wait.assert_called_once_with(CHANNEL_ID) + mock_coordinated.assert_called_once_with(CHANNEL_ID) + + @patch.object(ProxyServer, "_coordinated_stop_channel") + @patch.object(ProxyServer, "_wait_for_shutdown_delay", return_value=False) + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + def test_handle_client_disconnect_skips_stop_when_wait_aborted( + self, _mock_delay, _mock_wait, mock_coordinated + ): + server = HandleClientDisconnectUpstreamFirstTests()._make_server() + server.redis_client.get.return_value = b"1700000000.0" + + server.handle_client_disconnect(CHANNEL_ID) + + mock_coordinated.assert_not_called() + + class InitWaitAbortTests(TestCase): def _make_generator(self): from apps.proxy.live_proxy.output.ts.generator import StreamGenerator diff --git a/apps/proxy/live_proxy/client_manager.py b/apps/proxy/live_proxy/client_manager.py index 3767a25a..ff031e5b 100644 --- a/apps/proxy/live_proxy/client_manager.py +++ b/apps/proxy/live_proxy/client_manager.py @@ -256,6 +256,14 @@ class ClientManager: # Store in Redis if self.redis_client: + from .services.channel_service import ChannelService + + if ChannelService.cancel_pending_shutdown(self.channel_id): + logger.info( + f"Cancelled pending shutdown for channel {self.channel_id} " + f"(client {client_id} reconnected)" + ) + self.redis_client.hset(client_key, mapping=client_data) self.redis_client.expire(client_key, self.client_ttl) @@ -302,7 +310,10 @@ class ClientManager: return len(self.clients) except Exception as e: - logger.error(f"Error adding client {client_id}: {e}") + logger.error(f"Error adding client {client_id}: {e}", exc_info=True) + with self.lock: + self.clients.discard(client_id) + self._registered_clients.discard(client_id) return False def remove_client(self, client_id): @@ -337,7 +348,8 @@ class ClientManager: if remaining == 0: logger.warning(f"Last client removed: {client_id} - channel may shut down soon") disconnect_key = RedisKeys.last_client_disconnect(self.channel_id) - self.redis_client.setex(disconnect_key, 60, str(time.time())) + ttl = max(int(ConfigHelper.channel_shutdown_delay() * 2), 60) + self.redis_client.setex(disconnect_key, ttl, str(time.time())) self._notify_owner_of_activity() diff --git a/apps/proxy/live_proxy/output/fmp4/generator.py b/apps/proxy/live_proxy/output/fmp4/generator.py index e979008c..d967b5f9 100644 --- a/apps/proxy/live_proxy/output/fmp4/generator.py +++ b/apps/proxy/live_proxy/output/fmp4/generator.py @@ -348,7 +348,11 @@ class FMP4StreamGenerator: client_count = proxy_server.client_managers[ self.channel_id ].get_total_client_count() - if client_count <= 1 and proxy_server.am_i_owner(self.channel_id): + if ( + client_count <= 1 + and proxy_server.am_i_owner(self.channel_id) + and ConfigHelper.channel_shutdown_delay() <= 0 + ): try: try: obj = Channel.objects.get(uuid=self.channel_id) diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index efde02ad..2bd88725 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -603,7 +603,8 @@ class StreamGenerator: if self.channel_id in proxy_server.client_managers: client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() # Pool slots are global; the last client on any worker must release. - if client_count <= 1: + # During shutdown_delay, keep the slot until coordinated stop runs. + if client_count <= 1 and ConfigHelper.channel_shutdown_delay() <= 0: try: try: obj = Channel.objects.get(uuid=self.channel_id) diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index ddec2d02..8eca8572 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -905,6 +905,81 @@ class ProxyServer: logger.error(f"Error cleaning zombie channel {channel_id}: {e}", exc_info=True) return False + @staticmethod + def _shutdown_disconnect_ttl(): + delay = ConfigHelper.channel_shutdown_delay() + return max(int(delay * 2), 60) + + def _wait_for_shutdown_delay(self, channel_id): + """ + Wait until shutdown_delay has elapsed since the Redis disconnect + timestamp. Returns False if clients reconnect or the timer is cancelled. + Uses Redis state so concurrent disconnect handlers and multi-worker + reconnects always honour the latest last-client disconnect time. + """ + if not self.redis_client: + return True + + shutdown_delay = ConfigHelper.channel_shutdown_delay() + if shutdown_delay <= 0: + return True + + disconnect_key = RedisKeys.last_client_disconnect(channel_id) + client_set_key = RedisKeys.clients(channel_id) + poll_interval = 1.0 + + logger.info( + f"Waiting up to {shutdown_delay}s before stopping channel {channel_id}..." + ) + + while True: + total = self.redis_client.scard(client_set_key) or 0 + if total > 0: + logger.info( + f"New clients connected during shutdown delay for " + f"{channel_id} - aborting shutdown" + ) + self.redis_client.delete(disconnect_key) + return False + + disconnect_value = self.redis_client.get(disconnect_key) + if not disconnect_value: + logger.info( + f"Shutdown delay cancelled for {channel_id} - aborting shutdown" + ) + return False + + try: + if isinstance(disconnect_value, bytes): + disconnect_value = disconnect_value.decode() + disconnect_time = float(disconnect_value) + except (ValueError, TypeError): + logger.warning( + f"Invalid disconnect timestamp for {channel_id}, aborting wait" + ) + return False + + elapsed = time.time() - disconnect_time + if elapsed >= shutdown_delay: + total = self.redis_client.scard(client_set_key) or 0 + if total > 0: + logger.info( + f"Clients connected at end of shutdown delay for " + f"{channel_id} - aborting shutdown" + ) + self.redis_client.delete(disconnect_key) + return False + return True + + elapsed_display = max(0.0, elapsed) + remaining = max(0.0, shutdown_delay - elapsed) + logger.debug( + f"Channel {channel_id[:8]} shutdown timer: " + f"{elapsed_display:.1f}s of {shutdown_delay}s elapsed " + f"({remaining:.1f}s remaining)" + ) + gevent.sleep(min(poll_interval, remaining)) + def handle_client_disconnect(self, channel_id): """ Handle client disconnect event - check if channel should shut down and @@ -995,18 +1070,20 @@ class ProxyServer: disconnect_key = RedisKeys.last_client_disconnect(channel_id) if shutdown_delay > 0: - self.redis_client.setex(disconnect_key, 60, str(time.time())) - logger.info(f"Waiting {shutdown_delay}s before stopping channel...") - gevent.sleep(shutdown_delay) - - total = self.redis_client.scard(client_set_key) or 0 - if total > 0: - logger.info(f"New clients connected during shutdown delay - aborting shutdown") - self.redis_client.delete(disconnect_key) + if not self.redis_client.get(disconnect_key): + self.redis_client.setex( + disconnect_key, + self._shutdown_disconnect_ttl(), + str(time.time()), + ) + if not self._wait_for_shutdown_delay(channel_id): return - - if shutdown_delay <= 0: - self.redis_client.setex(disconnect_key, 60, str(time.time())) + else: + self.redis_client.setex( + disconnect_key, + self._shutdown_disconnect_ttl(), + str(time.time()), + ) # Coordinated stop runs local teardown + Redis cleanup once. # Do not call _stop_upstream_before_redis_cleanup here — it races @@ -1794,7 +1871,11 @@ class ProxyServer: if not disconnect_time: # First time seeing zero clients, set timestamp if self.redis_client: - self.redis_client.setex(disconnect_key, 60, str(current_time)) + self.redis_client.setex( + disconnect_key, + self._shutdown_disconnect_ttl(), + str(current_time), + ) logger.warning(f"No clients detected for channel {channel_id}, starting shutdown timer") elif current_time - disconnect_time > ConfigHelper.channel_shutdown_delay(): # We've had no clients for the shutdown delay period @@ -1808,7 +1889,9 @@ class ProxyServer: else: # There are clients or we're still connecting - clear any disconnect timestamp if self.redis_client: - self.redis_client.delete(f"live:channel:{channel_id}:last_client_disconnect_time") + self.redis_client.delete( + RedisKeys.last_client_disconnect(channel_id) + ) else: # === NON-OWNER CHANNEL HANDLING === diff --git a/apps/proxy/live_proxy/services/channel_service.py b/apps/proxy/live_proxy/services/channel_service.py index dcd0b5e1..400f55f4 100644 --- a/apps/proxy/live_proxy/services/channel_service.py +++ b/apps/proxy/live_proxy/services/channel_service.py @@ -93,10 +93,103 @@ class ChannelService: @staticmethod def is_channel_unavailable_for_new_clients(channel_id): - """Reject new stream requests while teardown is active or shutdown is pending.""" - return ( - ChannelService.is_channel_teardown_active(channel_id) - or ChannelService.is_shutdown_pending(channel_id) + """Reject new stream requests only while coordinated teardown is active.""" + return ChannelService.is_channel_teardown_active(channel_id) + + @staticmethod + def cancel_pending_shutdown(channel_id): + """ + Abort the post-disconnect grace timer when a client reconnects. + + Clears the disconnect timestamp and any leaked stopping markers. When + upstream is still active but the last client released its profile slot + during the grace window, re-reserve the slot from Redis metadata. + + Does not run during coordinated stop_channel() — clearing teardown + markers mid-stop would leave clients attached to upstream that is + about to be torn down. + """ + from django.db import close_old_connections + + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: + return False + + if channel_id in proxy_server._stopping_channels: + return False + + disconnect_key = RedisKeys.last_client_disconnect(channel_id) + had_pending = bool(proxy_server.redis_client.exists(disconnect_key)) + in_grace = had_pending or ChannelService.is_shutdown_pending(channel_id) + + if not in_grace: + return False + + try: + proxy_server.redis_client.delete(disconnect_key) + + metadata_key = RedisKeys.channel_metadata(channel_id) + state = proxy_server.redis_client.hget( + metadata_key, ChannelMetadataField.STATE + ) + if state: + state_str = state.decode() if isinstance(state, bytes) else state + if state_str == ChannelState.STOPPING: + proxy_server.redis_client.hset(metadata_key, mapping={ + ChannelMetadataField.STATE: ChannelState.ACTIVE, + ChannelMetadataField.STATE_CHANGED_AT: str(time.time()), + }) + + stop_key = RedisKeys.channel_stopping(channel_id) + if proxy_server.redis_client.exists(stop_key): + proxy_server.redis_client.delete(stop_key) + + if ChannelService._channel_proxy_is_active( + proxy_server.redis_client, channel_id + ): + from apps.channels.models import Channel + + channel = Channel.objects.filter(uuid=channel_id).first() + if channel and not proxy_server.redis_client.get( + f"channel_stream:{channel.id}" + ): + sid, pid, error, slot_reserved = channel.get_stream() + if error: + logger.warning( + f"Could not re-reserve stream for {channel_id} " + f"after shutdown cancel: {error}" + ) + elif slot_reserved and sid and pid: + proxy_server.redis_client.hset(metadata_key, mapping={ + ChannelMetadataField.STREAM_ID: str(sid), + ChannelMetadataField.M3U_PROFILE: str(pid), + }) + logger.info( + f"Re-reserved profile slot for {channel_id} " + f"(stream={sid}, profile={pid})" + ) + finally: + close_old_connections() + + return True + + @staticmethod + def _channel_proxy_is_active(redis_client, channel_id): + """True when live proxy metadata shows this channel is still running.""" + metadata_key = RedisKeys.channel_metadata(channel_id) + if not redis_client.exists(metadata_key): + return False + state = redis_client.hget(metadata_key, ChannelMetadataField.STATE) + if state is None: + return False + if isinstance(state, bytes): + state = state.decode() + return state in ( + ChannelState.ACTIVE, + ChannelState.WAITING_FOR_CLIENTS, + ChannelState.BUFFERING, + ChannelState.INITIALIZING, + ChannelState.CONNECTING, ) @staticmethod From 34c938b1cefff099616345a722c939d09eeaf1ce Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 19:50:50 -0500 Subject: [PATCH 37/81] feat(epg): implement staging and batch processing for EPG program updates - Introduced a temporary staging table for efficient batch processing of EPG program inserts, reducing memory and I/O contention during updates. - Enhanced the `parse_programs_for_source` function to stream parsed rows into the staging table before swapping them atomically into the main program data. - Added unit tests to validate the new staging and swapping logic, ensuring existing programs are preserved during failures and that batch processing works as intended. --- apps/epg/tasks.py | 451 ++++++++++++------ .../tests/test_parse_programs_for_source.py | 238 +++++++++ dispatcharr/celery.py | 1 + 3 files changed, 553 insertions(+), 137 deletions(-) create mode 100644 apps/epg/tests/test_parse_programs_for_source.py diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 248e9a5f..0e8eda42 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -561,9 +561,10 @@ def refresh_epg_data(source_id, force=False): gc.collect() return - # Build byte-offset index for preview lookups in the background so refresh isn't blocked by it - build_programme_index_task.delay(source.id) + # Build byte-offset index after programme data is committed so refresh + # does not compete for memory/IO during the programme swap. parse_programs_for_source(source) + build_programme_index_task.delay(source.id) elif source.source_type == 'schedules_direct': fetch_schedules_direct(source, force=force) @@ -590,6 +591,7 @@ def refresh_epg_data(source_id, force=False): source = None # Force garbage collection before releasing the lock gc.collect() + connection.close() lock_renewer.stop() release_task_lock('refresh_epg_data', source_id) @@ -1804,6 +1806,163 @@ def parse_programs_for_tvg_id(epg_id, force=False): release_task_lock('parse_epg_programs', epg_id) +_EPG_PROGRAM_STAGING_TABLE = 'epg_program_staging' +# Parse batches bound Python memory during XML iterparse; swap batches bound each +# DELETE/INSERT statement inside the single atomic swap transaction. +_EPG_PARSE_BATCH_SIZE = 2500 +_EPG_SWAP_BATCH_SIZE = 5000 + + +def _epg_program_staging_supported(): + return connection.vendor == 'postgresql' + + +def _prepare_epg_program_staging_table(): + """Create/truncate a session-scoped temp table for streaming EPG programme inserts.""" + if not _epg_program_staging_supported(): + return False + + with connection.cursor() as cursor: + cursor.execute( + f""" + CREATE TEMP TABLE IF NOT EXISTS {_EPG_PROGRAM_STAGING_TABLE} ( + epg_id bigint NOT NULL, + start_time timestamptz NOT NULL, + end_time timestamptz NOT NULL, + title varchar(255) NOT NULL, + sub_title text, + description text, + tvg_id varchar(255), + custom_properties jsonb + ) ON COMMIT PRESERVE ROWS + """ + ) + cursor.execute(f"TRUNCATE {_EPG_PROGRAM_STAGING_TABLE}") + return True + + +def _clear_epg_program_staging_table(): + if not _epg_program_staging_supported(): + return + with connection.cursor() as cursor: + cursor.execute(f"TRUNCATE {_EPG_PROGRAM_STAGING_TABLE}") + + +def _flush_epg_program_staging_batch(programs_batch): + """Insert a batch of unsaved ProgramData rows into the session staging table.""" + if not programs_batch or not _epg_program_staging_supported(): + return + + values_sql = [] + params = [] + for program in programs_batch: + values_sql.append("(%s, %s, %s, %s, %s, %s, %s, %s)") + custom_properties = program.custom_properties + if custom_properties is not None and not isinstance(custom_properties, str): + custom_properties = json.dumps(custom_properties) + params.extend([ + program.epg_id, + program.start_time, + program.end_time, + program.title, + program.sub_title, + program.description, + program.tvg_id, + custom_properties, + ]) + + with connection.cursor() as cursor: + cursor.execute( + f""" + INSERT INTO {_EPG_PROGRAM_STAGING_TABLE} ( + epg_id, start_time, end_time, title, sub_title, description, tvg_id, custom_properties + ) VALUES {', '.join(values_sql)} + """, + params, + ) + + +def _swap_staged_epg_programs(mapped_epg_ids, epg_source, batch_size=_EPG_SWAP_BATCH_SIZE): + """ + Atomically replace mapped programme rows with staged data. + Must be called inside transaction.atomic(). + + Staged rows are moved in batches (DELETE ... RETURNING + INSERT) so Postgres + does not need to materialize the entire catalogue in one statement. + """ + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + + deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] + logger.debug(f"Deleted {deleted_count} existing programs") + + unmapped_epg_ids = list( + EPGData.objects.filter(epg_source=epg_source) + .exclude(id__in=mapped_epg_ids) + .values_list('id', flat=True) + ) + if unmapped_epg_ids: + orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] + if orphaned_count > 0: + logger.info( + f"Cleaned up {orphaned_count} orphaned programs for " + f"{len(unmapped_epg_ids)} unmapped EPG entries" + ) + + if not _epg_program_staging_supported(): + raise RuntimeError('_swap_staged_epg_programs requires PostgreSQL staging support') + + program_table = ProgramData._meta.db_table + total_inserted = 0 + while True: + with connection.cursor() as cursor: + cursor.execute( + f""" + WITH moved AS ( + DELETE FROM {_EPG_PROGRAM_STAGING_TABLE} + WHERE ctid IN ( + SELECT ctid FROM {_EPG_PROGRAM_STAGING_TABLE} LIMIT %s + ) + RETURNING + epg_id, start_time, end_time, title, sub_title, + description, tvg_id, custom_properties + ) + INSERT INTO {program_table} ( + epg_id, start_time, end_time, title, sub_title, + description, tvg_id, custom_properties + ) + SELECT + epg_id, start_time, end_time, title, sub_title, + description, tvg_id, custom_properties + FROM moved + """, + [batch_size], + ) + moved_count = cursor.rowcount + if moved_count == 0: + break + total_inserted += moved_count + + logger.debug(f"Inserted {total_inserted} staged programs in batches of {batch_size}") + + return deleted_count + + +def _swap_parsed_epg_programs(mapped_epg_ids, epg_source, programs_to_create, batch_size=_EPG_SWAP_BATCH_SIZE): + """SQLite/dev fallback: atomic delete + bulk insert from an in-memory batch list.""" + with transaction.atomic(): + deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] + unmapped_epg_ids = list( + EPGData.objects.filter(epg_source=epg_source) + .exclude(id__in=mapped_epg_ids) + .values_list('id', flat=True) + ) + if unmapped_epg_ids: + ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete() + for i in range(0, len(programs_to_create), batch_size): + ProgramData.objects.bulk_create(programs_to_create[i:i + batch_size]) + return deleted_count + def parse_programs_for_source(epg_source, tvg_id=None): """ @@ -1910,106 +2069,164 @@ def parse_programs_for_source(epg_source, tvg_id=None): send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") return False - # SINGLE PASS PARSING: Parse the XML file once and collect all programs in memory - # We parse FIRST, then do an atomic delete+insert to avoid race conditions - # where clients might see empty/partial EPG data during the transition - all_programs_to_create = [] - programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} # Track count per channel + # Stream parsed rows into a session temp table, then swap in a short transaction. + # This bounds Python memory (batched staging inserts) and Postgres memory (no + # long-lived transaction spanning the entire XML parse). + programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} total_programs = 0 skipped_programs = 0 last_progress_update = 0 + parse_batch_size = _EPG_PARSE_BATCH_SIZE + swap_batch_size = _EPG_SWAP_BATCH_SIZE + programs_batch = [] + deleted_count = 0 + staging_prepared = False + use_staging = False + programs_accumulator = [] + + send_epg_update(epg_source.id, "parsing_programs", 10, message="Parsing programs...") try: - logger.debug(f"Opening file for single-pass parsing: {file_path}") + staging_prepared = _prepare_epg_program_staging_table() + use_staging = staging_prepared + + logger.debug(f"Opening file for streaming parse: {file_path}") source_file = _open_xmltv_file(file_path) + try: + program_parser = etree.iterparse( + source_file, + events=('end',), + tag='programme', + remove_blank_text=True, + recover=True, + ) - # Stream parse the file using lxml's iterparse - program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) + for _, elem in program_parser: + channel_id = elem.get('channel') - for _, elem in program_parser: - channel_id = elem.get('channel') + if channel_id not in mapped_tvg_ids: + skipped_programs += 1 + clear_element(elem) + continue - # Skip programmes for unmapped channels immediately - if channel_id not in mapped_tvg_ids: - skipped_programs += 1 - # Clear element to free memory - clear_element(elem) - continue + try: + start_time = parse_xmltv_time(elem.get('start')) + end_time = parse_xmltv_time(elem.get('stop')) + title = None + desc = None + sub_title = None - # This programme is for a mapped channel - process it - try: - start_time = parse_xmltv_time(elem.get('start')) - end_time = parse_xmltv_time(elem.get('stop')) - title = None - desc = None - sub_title = None + for child in elem: + if child.tag == 'title': + title = child.text or 'No Title' + elif child.tag == 'desc': + desc = child.text or '' + elif child.tag == 'sub-title': + sub_title = child.text or '' - # Efficiently process child elements - for child in elem: - if child.tag == 'title': - title = child.text or 'No Title' - elif child.tag == 'desc': - desc = child.text or '' - elif child.tag == 'sub-title': - sub_title = child.text or '' + if not title: + title = 'No Title' - if not title: - title = 'No Title' + custom_props = extract_custom_properties(elem) + custom_properties_json = custom_props if custom_props else None - # Extract custom properties - custom_props = extract_custom_properties(elem) - custom_properties_json = custom_props if custom_props else None + if desc: + has_season = (custom_properties_json or {}).get('season') is not None + has_episode = (custom_properties_json or {}).get('episode') is not None + if not has_season or not has_episode: + d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) + if d_season is not None and d_episode is not None: + if custom_properties_json is None: + custom_properties_json = {} + if not has_season: + custom_properties_json['season'] = d_season + if not has_episode: + custom_properties_json['episode'] = d_episode + custom_properties_json['season_episode_source'] = 'description' + desc = cleaned_desc - # Fallback: extract S/E from description when episode-num - # elements didn't provide them - if desc: - has_season = (custom_properties_json or {}).get('season') is not None - has_episode = (custom_properties_json or {}).get('episode') is not None - if not has_season or not has_episode: - d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) - if d_season is not None and d_episode is not None: - if custom_properties_json is None: - custom_properties_json = {} - if not has_season: - custom_properties_json['season'] = d_season - if not has_episode: - custom_properties_json['episode'] = d_episode - custom_properties_json['season_episode_source'] = 'description' - desc = cleaned_desc + epg_id = tvg_id_to_epg_id[channel_id] + programs_batch.append(ProgramData( + epg_id=epg_id, + start_time=start_time, + end_time=end_time, + title=title[:255], + description=desc, + sub_title=sub_title, + tvg_id=channel_id, + custom_properties=custom_properties_json, + )) + total_programs += 1 + programs_by_channel[channel_id] += 1 + clear_element(elem) - epg_id = tvg_id_to_epg_id[channel_id] - all_programs_to_create.append(ProgramData( - epg_id=epg_id, - start_time=start_time, - end_time=end_time, - title=title[:255], - description=desc, - sub_title=sub_title, - tvg_id=channel_id, - custom_properties=custom_properties_json - )) - total_programs += 1 - programs_by_channel[channel_id] += 1 + if len(programs_batch) >= parse_batch_size: + if use_staging: + _flush_epg_program_staging_batch(programs_batch) + programs_batch = [] + else: + programs_accumulator.extend(programs_batch) + programs_batch = [] - # Clear the element to free memory - clear_element(elem) + if total_programs - last_progress_update >= 5000: + last_progress_update = total_programs + progress = min( + 85, + 10 + int((total_programs / max(total_programs + 10000, 1)) * 75), + ) + send_epg_update( + epg_source.id, + "parsing_programs", + progress, + processed=total_programs, + channels=mapped_count, + message=f"Staging programs... {total_programs:,}", + ) - # Send progress update (estimate based on programs processed) - if total_programs - last_progress_update >= 5000: - last_progress_update = total_programs - # Cap at 70% during parsing phase (save 30% for DB operations) - progress = min(70, 10 + int((total_programs / max(total_programs + 10000, 1)) * 60)) - send_epg_update(epg_source.id, "parsing_programs", progress, - processed=total_programs, channels=mapped_count) + if total_programs % 5000 == 0: + gc.collect() - # Periodic garbage collection during parsing - if total_programs % 5000 == 0: - gc.collect() + except Exception as e: + logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) + clear_element(elem) + continue - except Exception as e: - logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) - clear_element(elem) - continue + if programs_batch: + if use_staging: + _flush_epg_program_staging_batch(programs_batch) + else: + programs_accumulator.extend(programs_batch) + programs_batch = [] + finally: + if source_file: + source_file.close() + source_file = None + + try: + send_epg_update(epg_source.id, "parsing_programs", 90, message="Updating database...") + if use_staging: + with transaction.atomic(): + deleted_count = _swap_staged_epg_programs( + mapped_epg_ids, epg_source, batch_size=swap_batch_size + ) + else: + deleted_count = _swap_parsed_epg_programs( + mapped_epg_ids, epg_source, programs_accumulator, batch_size=swap_batch_size + ) + programs_accumulator = [] + + logger.info( + f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs" + ) + except Exception as db_error: + logger.error(f"Database error during atomic update: {db_error}", exc_info=True) + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"Database error: {str(db_error)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update( + epg_source.id, "parsing_programs", 100, status="error", message=str(db_error) + ) + return False except etree.XMLSyntaxError as xml_error: logger.error(f"XML syntax error parsing program data: {xml_error}") @@ -2018,63 +2235,23 @@ def parse_programs_for_source(epg_source, tvg_id=None): epg_source.save(update_fields=['status', 'last_message']) send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(xml_error)) return False - except Exception as e: - logger.error(f"Error parsing XML for programs: {e}", exc_info=True) - raise - finally: - if source_file: - source_file.close() - source_file = None - - # Now perform atomic delete + bulk insert - # This ensures clients never see empty/partial EPG data - logger.info(f"Parsed {total_programs} programs, performing atomic database update...") - send_epg_update(epg_source.id, "parsing_programs", 75, message="Updating database...") - - batch_size = 1000 - try: - with transaction.atomic(): - # Kill any individual statement that hangs longer than 10 minutes. - # SET LOCAL automatically resets when this transaction ends (commit or rollback). - with connection.cursor() as cursor: - cursor.execute("SET LOCAL statement_timeout = '10min'") - # Delete existing programs for mapped EPGs - deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] - logger.debug(f"Deleted {deleted_count} existing programs") - - # Clean up orphaned programs for unmapped EPG entries - unmapped_epg_ids = list(EPGData.objects.filter( - epg_source=epg_source - ).exclude(id__in=mapped_epg_ids).values_list('id', flat=True)) - - if unmapped_epg_ids: - orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] - if orphaned_count > 0: - logger.info(f"Cleaned up {orphaned_count} orphaned programs for {len(unmapped_epg_ids)} unmapped EPG entries") - - # Bulk insert all new programs in batches within the same transaction - for i in range(0, len(all_programs_to_create), batch_size): - batch = all_programs_to_create[i:i + batch_size] - ProgramData.objects.bulk_create(batch) - - # Update progress during insertion - progress = 75 + int((i / len(all_programs_to_create)) * 20) if all_programs_to_create else 95 - if i % (batch_size * 5) == 0: - send_epg_update(epg_source.id, "parsing_programs", min(95, progress), - message=f"Inserting programs... {i}/{len(all_programs_to_create)}") - - logger.info(f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs") - - except Exception as db_error: - logger.error(f"Database error during atomic update: {db_error}", exc_info=True) + except Exception as parse_error: + logger.error(f"Error parsing programs from XML: {parse_error}", exc_info=True) epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"Database error: {str(db_error)}" + epg_source.last_message = f"Error parsing programs: {str(parse_error)}" epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(db_error)) + send_epg_update( + epg_source.id, "parsing_programs", 100, status="error", message=str(parse_error) + ) return False finally: - # Clear the large list to free memory - all_programs_to_create = None + programs_batch = None + programs_accumulator = None + if staging_prepared: + try: + _clear_epg_program_staging_table() + except Exception: + pass gc.collect() # Count channels that actually got programs @@ -2130,7 +2307,7 @@ def parse_programs_for_source(epg_source, tvg_id=None): source_file = None # Explicitly release any remaining large data structures - programs_to_create = None + programs_batch = None programs_by_channel = None mapped_epg_ids = None mapped_tvg_ids = None diff --git a/apps/epg/tests/test_parse_programs_for_source.py b/apps/epg/tests/test_parse_programs_for_source.py new file mode 100644 index 00000000..f49a5d70 --- /dev/null +++ b/apps/epg/tests/test_parse_programs_for_source.py @@ -0,0 +1,238 @@ +import os +import tempfile +from datetime import timedelta +from unittest.mock import patch + +from django.db import connection, transaction +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import Channel +from apps.epg.models import EPGSource, EPGData, ProgramData +from apps.epg.tasks import ( + parse_programs_for_source, + _flush_epg_program_staging_batch, + _swap_staged_epg_programs, + _EPG_PARSE_BATCH_SIZE, +) + + +def _programme_xml(channel_id, title, start, stop): + return ( + f' \n' + f' {title}\n' + f' \n' + ) + + +def _xmltv_file(programmes): + body = ( + '\n' + '\n' + f'{programmes}' + '\n' + ) + handle = tempfile.NamedTemporaryFile( + mode='w', + suffix='.xml', + delete=False, + encoding='utf-8', + ) + handle.write(body) + handle.close() + return handle.name + + +class ParseProgramsForSourceTests(TestCase): + def setUp(self): + self.source = EPGSource.objects.create( + name='XMLTV Parse Test', + source_type='xmltv', + ) + self.mapped_epg = EPGData.objects.create( + epg_source=self.source, + tvg_id='mapped.channel', + name='Mapped Channel', + ) + self.unmapped_epg = EPGData.objects.create( + epg_source=self.source, + tvg_id='unmapped.channel', + name='Unmapped Channel', + ) + Channel.objects.create( + channel_number=1, + name='Mapped Channel', + epg_data=self.mapped_epg, + ) + self.base_time = timezone.now().replace(minute=0, second=0, microsecond=0) + self.start = self.base_time.strftime('%Y%m%d%H%M%S +0000') + self.stop = (self.base_time + timedelta(hours=1)).strftime('%Y%m%d%H%M%S +0000') + + def tearDown(self): + if getattr(self, 'xml_path', None) and os.path.exists(self.xml_path): + os.unlink(self.xml_path) + + def _configure_source_file(self, programmes): + self.xml_path = _xmltv_file(programmes) + self.source.file_path = self.xml_path + self.source.save(update_fields=['file_path']) + + @patch('apps.epg.tasks.log_system_event') + @patch('apps.epg.tasks.send_epg_update') + def test_replaces_programs_for_mapped_channels(self, _send_update, _log_event): + old_start = self.base_time - timedelta(days=1) + ProgramData.objects.create( + epg=self.mapped_epg, + start_time=old_start, + end_time=old_start + timedelta(hours=1), + title='Old Programme', + tvg_id=self.mapped_epg.tvg_id, + ) + orphan_start = self.base_time - timedelta(days=1) + ProgramData.objects.create( + epg=self.unmapped_epg, + start_time=orphan_start, + end_time=orphan_start + timedelta(hours=1), + title='Orphan Programme', + tvg_id=self.unmapped_epg.tvg_id, + ) + + programmes = ( + _programme_xml('mapped.channel', 'New Show', self.start, self.stop) + + _programme_xml('unmapped.channel', 'Skipped Show', self.start, self.stop) + ) + self._configure_source_file(programmes) + + result = parse_programs_for_source(self.source) + + self.assertTrue(result) + mapped_programs = ProgramData.objects.filter(epg=self.mapped_epg) + self.assertEqual(mapped_programs.count(), 1) + self.assertEqual(mapped_programs.get().title, 'New Show') + self.assertFalse(ProgramData.objects.filter(epg=self.unmapped_epg).exists()) + + @patch('apps.epg.tasks.log_system_event') + @patch('apps.epg.tasks.send_epg_update') + def test_atomic_failure_rolls_back_and_preserves_existing_programs(self, _send_update, _log_event): + old_start = self.base_time - timedelta(days=1) + ProgramData.objects.create( + epg=self.mapped_epg, + start_time=old_start, + end_time=old_start + timedelta(hours=1), + title='Keep Me', + tvg_id=self.mapped_epg.tvg_id, + ) + + self._configure_source_file( + _programme_xml('mapped.channel', 'Replacement', self.start, self.stop) + ) + + swap_path = ( + 'apps.epg.tasks._swap_staged_epg_programs' + if connection.vendor == 'postgresql' + else 'apps.epg.tasks._swap_parsed_epg_programs' + ) + with patch(swap_path, side_effect=RuntimeError('simulated insert failure')): + result = parse_programs_for_source(self.source) + + self.assertFalse(result) + self.assertEqual(ProgramData.objects.filter(epg=self.mapped_epg).count(), 1) + self.assertEqual( + ProgramData.objects.get(epg=self.mapped_epg).title, + 'Keep Me', + ) + + @patch('apps.epg.tasks.log_system_event') + @patch('apps.epg.tasks.send_epg_update') + def test_streams_batches_without_holding_full_program_list(self, _send_update, _log_event): + if connection.vendor != 'postgresql': + self.skipTest('PostgreSQL staging batches are required for this assertion') + + programme_count = _EPG_PARSE_BATCH_SIZE * 2 + programmes = ''.join( + _programme_xml( + 'mapped.channel', + f'Show {idx}', + self.start, + self.stop, + ) + for idx in range(programme_count) + ) + self._configure_source_file(programmes) + flush_sizes = [] + original_flush = _flush_epg_program_staging_batch + + def tracking_flush(batch): + flush_sizes.append(len(batch)) + return original_flush(batch) + + with patch('apps.epg.tasks._flush_epg_program_staging_batch', side_effect=tracking_flush): + result = parse_programs_for_source(self.source) + + self.assertTrue(result) + self.assertEqual(ProgramData.objects.filter(epg=self.mapped_epg).count(), programme_count) + self.assertEqual(sum(flush_sizes), programme_count) + self.assertTrue(all(size <= _EPG_PARSE_BATCH_SIZE for size in flush_sizes)) + self.assertGreater(len(flush_sizes), 1) + + @patch('apps.epg.tasks.log_system_event') + @patch('apps.epg.tasks.send_epg_update') + def test_live_programs_remain_until_swap_commits(self, _send_update, _log_event): + if connection.vendor != 'postgresql': + self.skipTest('PostgreSQL staging swap is required for this assertion') + + old_start = self.base_time - timedelta(days=1) + ProgramData.objects.create( + epg=self.mapped_epg, + start_time=old_start, + end_time=old_start + timedelta(hours=1), + title='Old Programme', + tvg_id=self.mapped_epg.tvg_id, + ) + self._configure_source_file( + _programme_xml('mapped.channel', 'New Show', self.start, self.stop) + ) + + observed_titles_at_swap = [] + + def swap_with_visibility_check(mapped_epg_ids, epg_source, *args, **kwargs): + observed_titles_at_swap.append( + ProgramData.objects.get(epg=self.mapped_epg).title + ) + return _swap_staged_epg_programs(mapped_epg_ids, epg_source, *args, **kwargs) + + with patch('apps.epg.tasks._swap_staged_epg_programs', side_effect=swap_with_visibility_check): + result = parse_programs_for_source(self.source) + + self.assertTrue(result) + self.assertEqual(observed_titles_at_swap, ['Old Programme']) + self.assertEqual(ProgramData.objects.get(epg=self.mapped_epg).title, 'New Show') + + @patch('apps.epg.tasks.log_system_event') + @patch('apps.epg.tasks.send_epg_update') + def test_swap_delete_is_rolled_back_when_insert_fails(self, _send_update, _log_event): + if connection.vendor != 'postgresql': + self.skipTest('PostgreSQL staging swap is required for this assertion') + + old_start = self.base_time - timedelta(days=1) + ProgramData.objects.create( + epg=self.mapped_epg, + start_time=old_start, + end_time=old_start + timedelta(hours=1), + title='Keep Me', + tvg_id=self.mapped_epg.tvg_id, + ) + self._configure_source_file( + _programme_xml('mapped.channel', 'Replacement', self.start, self.stop) + ) + + def failing_swap(mapped_epg_ids, epg_source, *args, **kwargs): + with transaction.atomic(): + ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete() + raise RuntimeError('simulated insert failure') + + with patch('apps.epg.tasks._swap_staged_epg_programs', side_effect=failing_swap): + result = parse_programs_for_source(self.source) + + self.assertFalse(result) + self.assertEqual(ProgramData.objects.get(epg=self.mapped_epg).title, 'Keep Me') diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 7ce0ab29..bdb5e48d 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -94,6 +94,7 @@ def cleanup_task_memory(**kwargs): 'apps.epg.tasks.refresh_all_epg_data', 'apps.epg.tasks.parse_programs_for_source', 'apps.epg.tasks.parse_programs_for_tvg_id', + 'apps.epg.tasks.build_programme_index_task', 'apps.channels.tasks.match_epg_channels', 'apps.channels.tasks.match_selected_channels_epg', 'apps.channels.tasks.match_single_channel_epg', From 8fec2060ca12686f4f433412d9ed8cbaa32923d0 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 20:28:07 -0500 Subject: [PATCH 38/81] changelog: Update for last two prs. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd70a28e..9268fd73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **EPG programme parse now streams through PostgreSQL staging instead of holding the full catalogue in Python.** `parse_programs_for_source` writes parsed rows into a session-scoped temp table in batches (`_EPG_PARSE_BATCH_SIZE=2500`), then atomically swaps them into `ProgramData` with batched `DELETE ... RETURNING` + `INSERT` (`_EPG_SWAP_BATCH_SIZE=5000`) so Postgres never materializes the entire guide in one statement. Peak Celery memory during large XMLTV refreshes drops sharply compared with building a monolithic in-memory list before bulk insert. The byte-offset programme index build is deferred until after the swap completes so index construction no longer competes with parse for memory and I/O. `refresh_epg_data` closes its DB connection in `finally`, and `build_programme_index_task` is registered as a memory-intensive Celery task. SQLite/dev installs keep an in-memory fallback path. +- **Pooled PostgreSQL connections now rotate on a bounded lifetime.** psycopg3 client-side cache grows on handles kept open indefinitely by `django-db-geventpool`; recycling uWSGI workers would interrupt live streams. A thin custom backend (`dispatcharr.db.backends.postgresql_psycopg3`) closes and replaces pool connections after `DATABASE_POOL_CONN_MAX_LIFETIME` seconds (default 600, env-overridable; set `0` to disable) on checkout and return while preserving warm-pool reuse within the window. (Fixes #1343) - **Schedules Direct refresh only fetches guide data for mapped channels.** Schedule MD5 checks and schedule downloads now target mapped lineup stations instead of the entire lineup, and schedule MD5 cache for unmapped stations is pruned each refresh. Unmapped lineup entries no longer trigger wasted schedule API calls when their MD5 changes. - **EPG auto-match memory and throughput improvements.** - Single-channel matching streams active EPG rows and keeps only the best match plus the top 20 candidates in memory; ML validates at most 21 names per channel instead of embedding the full catalog. From c4bf21141c3eea67d9da391739d7b777c20974c3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 15 Jun 2026 20:48:59 -0500 Subject: [PATCH 39/81] changelog: Add for plugin pr and link issue. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9268fd73..a2621637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Plugin repo manifests support split download and metadata base URLs.** `download_base_url` and `metadata_base_url` are optional alternatives to `root_url` in the plugin repository manifest, so repo authors can serve metadata (manifests, icons) and release zips from different origins without absolute URLs everywhere. Manifest and icon URLs resolve via `metadata_base_url` then `root_url`; latest/download URLs resolve via `download_base_url` then `root_url`. When `metadata_base_url` is set and a plugin entry has `manifest_url` but no `icon_url`, the icon defaults to `logo.png` in the same directory as the per-plugin manifest. Manifests using only `root_url` behave identically to before. — Thanks [@sethwv](https://github.com/sethwv) - **Live and VOD connection slot logic now routes through `connection_pool`.** `Channel.get_stream()`, direct `Stream.get_stream()`, VOD profile selection, and the multi-worker VOD connection manager all call `reserve_profile_slot()` / `release_profile_slot()` instead of inline Redis INCR/DECR. VOD profile selection also checks `pool_has_capacity_for_profile()` before choosing a profile. - **Live XC upstream URLs use current credentials.** `_resolve_live_stream_url()` builds `/live/{user}/{pass}/{stream_id}.ts` from transformed account credentials and the stream's provider `stream_id`, so playback stays aligned with the active login after credential or profile changes instead of reusing a stale `stream.url` from sync. - **Channel stream switches check pooled capacity.** `get_stream_info_for_switch()` uses `profile_available_for_channel_switch()` when targeting a specific stream, and releases a reserved slot if URL assembly fails after `get_stream()` allocated one. @@ -49,7 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Client reconnect during channel teardown could wedge the channel across uWSGI workers.** The disconnect path called `proxy_server.stop_channel()` directly, so other workers were not notified and reconnecting clients could attach while ownership was released on the owner worker. The cleanup thread then re-acquired expired ownership and looped on orphaned clients while the upstream connection stayed open. All lifecycle stops now go through coordinated Redis teardown (`channel_stopping` flag, metadata `stopping` state, and `CHANNEL_STOP` pubsub). New stream requests are rejected with `503 Retry-After` only during active teardown (not during the post-disconnect shutdown delay grace period); `initialize_channel()` and ownership re-acquisition refuse to proceed in those states; non-owner workers clean local resources only on pubsub; and orphaned-client sweeps force cleanup when teardown is already active. (Fixes #1342) - **Orphaned upstream threads kept writing Redis after teardown.** Partial cleanup could remove a channel from `stream_managers` while the `stream-{uuid}` OS thread kept running, so orphan metadata sweeps only deleted Redis keys and chunks/metadata were immediately recreated (bare `total_bytes` hash with TTL -1). Stream managers now stop upstream when ownership is lost, orphan cleanup stops local ffmpeg/stream threads even when registry entries are gone (`_live_stream_managers` tracks managers until the thread exits), and forced recovery stops processes before Redis deletion instead of dropping dict entries without calling `stop()`. - **Rapid channel switching could leave bare `buffer:index` keys in Redis.** `buffer.stop()` flushed a final partial chunk via `INCR` during teardown, creating a persistent index key (TTL -1) after chunk keys expired; under load overlapping disconnect and cleanup-thread stops could race on `_stop_local_stream_activity`, blocking on ffmpeg stderr join before `_clean_redis_keys` ever ran. Teardown no longer writes to Redis from `buffer.stop()`, disconnect uses coordinated stop only (no duplicate upstream stop), model `release_stream()` runs before Redis keys are scanned/deleted (so `profile_connections` counters are not left stuck), ownership is released after Redis cleanup but before the blocking local ffmpeg stop, and a `finally` block guarantees Redis cleanup if local teardown raises. -- **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). +- **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). (Fixes #1345) - **System events could block callers on Connect and plugin handlers.** `log_system_event()` ran Connect subscriptions and plugin `"events"` hooks synchronously after the DB write, so slow integrations could stall live-proxy and streaming paths even when inserting the event row was fast. Connect/plugin dispatch now runs on a separate gevent when the hub is available (synchronously in Celery workers without a hub). `log_system_event()` and `dispatch_event_system()` also call `close_old_connections()` in `finally` blocks so integration work does not leave pool slots checked out on the caller or dispatch greenlet. - **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. - **VOD proxy could leak geventpool DB checkouts during playback and stats updates.** `stream_vod()` ran ORM lookups for content and M3U profiles, then returned a long-lived `StreamingHttpResponse` without releasing the checkout, so each movie/episode stream could hold a pool slot for its full duration. Background VOD stats refresh (`build_vod_stats_data()`, triggered on start/stop and by the admin stats API) also queried movie, episode, and profile rows from daemon threads with no cleanup. `stream_vod()` now calls `close_old_connections()` before handing off to the streaming generator, and `build_vod_stats_data()` releases its checkout in a `finally` block. From b2496dc4e7c280270e7f4288e7c85d443b545fad Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 11:25:17 -0500 Subject: [PATCH 40/81] refactor(trigger_event): streamline event dispatching and improve plugin action handling - Replaced the plugin listing mechanism with a more efficient handler iteration for event actions. - Enhanced logging to provide clearer insights into the dispatching process and plugin action execution. - Added error handling to ensure that failures in one plugin action do not block subsequent actions. - Updated tests to cover new behavior and ensure proper handling of enabled and disabled plugins. --- apps/connect/tests/test_trigger_event.py | 129 +++++++++++++----- apps/connect/utils.py | 59 +++++--- apps/plugins/loader.py | 42 +++++- .../tests/test_iter_actions_for_event.py | 51 +++++++ .../tests/test_run_action_db_cleanup.py | 17 +++ 5 files changed, 241 insertions(+), 57 deletions(-) create mode 100644 apps/plugins/tests/test_iter_actions_for_event.py diff --git a/apps/connect/tests/test_trigger_event.py b/apps/connect/tests/test_trigger_event.py index 8a1c011d..5977d01c 100644 --- a/apps/connect/tests/test_trigger_event.py +++ b/apps/connect/tests/test_trigger_event.py @@ -35,43 +35,56 @@ def _empty_subscription_chain(): class TriggerEventDispatchTests(SimpleTestCase): - def _run_trigger_event(self, plugins, event_name, payload): + def _run_trigger_event(self, handlers, event_name, payload, enabled_keys=None): pm = MagicMock() - pm.list_plugins.return_value = plugins + pm.iter_actions_for_event.return_value = handlers + if enabled_keys is None: + enabled_keys = [key for key, _ in handlers] + + enabled_qs = MagicMock() + enabled_qs.values_list.return_value = enabled_keys + with patch( "apps.connect.utils.PluginManager.get", return_value=pm ), patch( "apps.connect.utils.EventSubscription.objects.filter", return_value=_empty_subscription_chain(), - ): + ), patch( + "apps.plugins.models.PluginConfig" + ) as mock_cfg: + mock_cfg.objects.filter.return_value = enabled_qs from apps.connect.utils import trigger_event trigger_event(event_name, payload) return pm def test_disabled_plugin_does_not_abort_dispatch_for_later_enabled_plugin(self): - """The original bug: AttributeError on a disabled plugin's debug log - aborted the loop, so any plugin iterated AFTER a disabled one never - received events.""" - plugins = [ - { - "key": "disabled-plugin", - "name": "Disabled Plugin", - "enabled": False, - "actions": [], - }, - { - "key": "enabled-plugin", - "name": "Enabled Plugin", - "enabled": True, - "actions": [ - {"id": "on_event", "events": ["channel_start"]}, - ], - }, + """Enabled handlers still run when other plugins are disabled in DB.""" + handlers = [ + ("enabled-plugin", "on_event"), ] pm = self._run_trigger_event( - plugins, "channel_start", {"channel_name": "TEST"} + handlers, "channel_start", {"channel_name": "TEST"} + ) + + pm.run_action.assert_called_once_with( + "enabled-plugin", + "on_event", + {"event": "channel_start", "payload": {"channel_name": "TEST"}}, + ) + + def test_skips_handlers_for_disabled_plugins(self): + handlers = [ + ("disabled-plugin", "on_event"), + ("enabled-plugin", "on_event"), + ] + + pm = self._run_trigger_event( + handlers, + "channel_start", + {"channel_name": "TEST"}, + enabled_keys=["enabled-plugin"], ) pm.run_action.assert_called_once_with( @@ -81,22 +94,64 @@ class TriggerEventDispatchTests(SimpleTestCase): ) def test_action_without_matching_event_is_not_dispatched(self): - """Sanity check: actions whose `events` list doesn't include the - fired event, or which have no `events` key at all, are skipped.""" - plugins = [ - { - "key": "plugin", - "name": "Plugin", - "enabled": True, - "actions": [ - {"id": "on_event", "events": ["channel_stop"]}, - {"id": "manual_button"}, # no events key - ], - }, - ] - + """When no handlers are registered for the event, run_action is not called.""" pm = self._run_trigger_event( - plugins, "channel_start", {"channel_name": "TEST"} + [], "channel_start", {"channel_name": "TEST"} ) pm.run_action.assert_not_called() + + def test_no_plugin_config_query_when_no_handlers(self): + pm = MagicMock() + pm.iter_actions_for_event.return_value = [] + with patch( + "apps.connect.utils.PluginManager.get", return_value=pm + ), patch( + "apps.connect.utils.EventSubscription.objects.filter", + return_value=_empty_subscription_chain(), + ), patch( + "apps.plugins.models.PluginConfig" + ) as mock_cfg: + from apps.connect.utils import trigger_event + + trigger_event("channel_start", {"channel_name": "TEST"}) + + mock_cfg.objects.filter.assert_not_called() + + def test_plugin_action_failure_does_not_block_sibling_handlers(self): + handlers = [ + ("failing-plugin", "on_event"), + ("working-plugin", "on_event"), + ] + + pm = MagicMock() + pm.iter_actions_for_event.return_value = handlers + pm.run_action.side_effect = [RuntimeError("boom"), {"status": "ok"}] + + enabled_qs = MagicMock() + enabled_qs.values_list.return_value = ["failing-plugin", "working-plugin"] + + with patch( + "apps.connect.utils.PluginManager.get", return_value=pm + ), patch( + "apps.connect.utils.EventSubscription.objects.filter", + return_value=_empty_subscription_chain(), + ), patch( + "apps.plugins.models.PluginConfig" + ) as mock_cfg: + mock_cfg.objects.filter.return_value = enabled_qs + from apps.connect.utils import trigger_event + + trigger_event("channel_start", {"channel_name": "TEST"}) + + self.assertEqual(pm.run_action.call_count, 2) + pm.run_action.assert_any_call( + "failing-plugin", + "on_event", + {"event": "channel_start", "payload": {"channel_name": "TEST"}}, + ) + pm.run_action.assert_any_call( + "working-plugin", + "on_event", + {"event": "channel_start", "payload": {"channel_name": "TEST"}}, + ) diff --git a/apps/connect/utils.py b/apps/connect/utils.py index b71c38e5..d8ef59b7 100644 --- a/apps/connect/utils.py +++ b/apps/connect/utils.py @@ -1,5 +1,5 @@ # connect/utils.py -import logging, json +import logging from django.template import Template, Context from .models import EventSubscription, DeliveryLog, SUPPORTED_EVENTS from .handlers.webhook import WebhookHandler @@ -94,23 +94,44 @@ def trigger_event(event_name, payload): pm = PluginManager.get() pm.discover_plugins(sync_db=False, use_cache=True) - plugins = pm.list_plugins() + handlers = list(pm.iter_actions_for_event(event_name)) + if not handlers: + return - logger.debug(f"Checking {len(plugins)} plugins for event '{event_name}'") - for plugin in plugins: - if not plugin["enabled"]: - logger.debug(f"Skipping disabled plugin id={plugin['key']} name={plugin['name']}") + from apps.plugins.models import PluginConfig + + handler_keys = {key for key, _ in handlers} + enabled_keys = set( + PluginConfig.objects.filter(enabled=True, key__in=handler_keys).values_list( + "key", flat=True + ) + ) + + logger.debug( + "Dispatching event '%s' to %d plugin action(s) (%d enabled)", + event_name, + len(handlers), + len(enabled_keys), + ) + params = {"event": event_name, "payload": payload} + for key, action_id in handlers: + if key not in enabled_keys: + logger.debug( + "Skipping disabled plugin id=%s for event '%s'", key, event_name + ) continue - - logger.debug(json.dumps(plugin)) - for action in plugin["actions"]: - if "events" in action and event_name in action["events"]: - key = plugin["key"] - params = {"event": event_name, "payload": payload} - action_name = action.get("label") or action.get("id") - action_id = action.get("id") - logger.debug( - f"Triggering plugin action for event '{event_name}' on plugin id={key} action={action_name}" - ) - if action_id: - pm.run_action(key, action_id, params) + logger.debug( + "Triggering plugin action for event '%s' on plugin id=%s action=%s", + event_name, + key, + action_id, + ) + try: + pm.run_action(key, action_id, params) + except Exception: + logger.exception( + "Plugin action failed for event '%s' on plugin id=%s action=%s", + event_name, + key, + action_id, + ) diff --git a/apps/plugins/loader.py b/apps/plugins/loader.py index 3c275f69..d49da0f8 100644 --- a/apps/plugins/loader.py +++ b/apps/plugins/loader.py @@ -8,7 +8,7 @@ import sys import threading import types from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterator, List, Optional, Tuple from django.db import close_old_connections, transaction @@ -92,6 +92,29 @@ class PluginManager: key: lp.path for key, lp in self._registry.items() if lp and lp.path } + try: + return self._discover_plugins_impl( + sync_db=sync_db, + force_reload=force_reload, + previous_packages=previous_packages, + previous_aliases=previous_aliases, + previous_paths=previous_paths, + token=token, + ) + finally: + # Discovery runs outside Django's request/task cycle (boot, worker_ready). + close_old_connections() + + def _discover_plugins_impl( + self, + *, + sync_db: bool, + force_reload: bool, + previous_packages: Dict[str, str], + previous_aliases: Dict[str, str], + previous_paths: Dict[str, str], + token: int, + ) -> Dict[str, LoadedPlugin]: try: configs: Optional[Dict[str, PluginConfig]] = None try: @@ -247,6 +270,23 @@ class PluginManager: logger.exception("Deferring plugin DB sync; database not ready yet") return self._registry + def iter_actions_for_event(self, event_name: str) -> Iterator[Tuple[str, str]]: + """Yield (plugin_key, action_id) pairs from the in-memory registry.""" + with self._lock: + registry = list(self._registry.items()) + for key, lp in registry: + for action in lp.actions or []: + if not isinstance(action, dict): + continue + action_id = action.get("id") + events = action.get("events") + if ( + action_id + and isinstance(events, (list, tuple)) + and event_name in events + ): + yield key, action_id + def _load_plugin( self, key: str, diff --git a/apps/plugins/tests/test_iter_actions_for_event.py b/apps/plugins/tests/test_iter_actions_for_event.py new file mode 100644 index 00000000..05888157 --- /dev/null +++ b/apps/plugins/tests/test_iter_actions_for_event.py @@ -0,0 +1,51 @@ +from django.test import SimpleTestCase + +from apps.plugins.loader import LoadedPlugin, PluginManager + + +class IterActionsForEventTests(SimpleTestCase): + def test_yields_matching_handlers(self): + pm = PluginManager() + pm._registry = { + "alpha": LoadedPlugin( + key="alpha", + name="Alpha", + actions=[ + {"id": "on_start", "events": ["channel_start"]}, + {"id": "manual"}, + ], + ), + "beta": LoadedPlugin( + key="beta", + name="Beta", + actions=[ + {"id": "on_both", "events": ["channel_start", "channel_stop"]}, + ], + ), + } + + self.assertEqual( + list(pm.iter_actions_for_event("channel_start")), + [("alpha", "on_start"), ("beta", "on_both")], + ) + self.assertEqual(list(pm.iter_actions_for_event("channel_stop")), [("beta", "on_both")]) + + def test_ignores_string_events_value(self): + pm = PluginManager() + pm._registry = { + "bad": LoadedPlugin( + key="bad", + name="Bad", + actions=[{"id": "hook", "events": "client_connect"}], + ), + "good": LoadedPlugin( + key="good", + name="Good", + actions=[{"id": "hook", "events": ["client_connect"]}], + ), + } + + self.assertEqual( + list(pm.iter_actions_for_event("client_connect")), + [("good", "hook")], + ) diff --git a/apps/plugins/tests/test_run_action_db_cleanup.py b/apps/plugins/tests/test_run_action_db_cleanup.py index d9a45c88..6313473d 100644 --- a/apps/plugins/tests/test_run_action_db_cleanup.py +++ b/apps/plugins/tests/test_run_action_db_cleanup.py @@ -62,3 +62,20 @@ class PluginRunActionDbCleanupTests(SimpleTestCase): self.assertTrue(pm.stop_plugin("test_plugin", reason="shutdown")) mock_close.assert_called_once() + + +class PluginDiscoverDbCleanupTests(SimpleTestCase): + @patch("apps.plugins.loader.close_old_connections") + @patch.object(PluginManager, "_discover_plugins_impl", return_value={}) + def test_discover_plugins_closes_connections(self, _mock_impl, mock_close): + pm = PluginManager() + pm.discover_plugins(sync_db=False) + mock_close.assert_called_once() + + @patch("apps.plugins.loader.close_old_connections") + def test_discover_plugins_cache_hit_skips_close(self, mock_close): + pm = PluginManager() + pm._discovery_completed = True + pm._last_reload_token = pm._get_reload_token() + pm.discover_plugins(sync_db=False, use_cache=True) + mock_close.assert_not_called() From c1ff5e35aaa0a23bfe7a6ea1b57091dcac0c0920 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 11:26:12 -0500 Subject: [PATCH 41/81] feat(process_label): add process role labeling for PostgreSQL connections - Introduced a new module to determine the process role based on command-line arguments, enhancing PostgreSQL connection labeling for better monitoring. - Updated the database connection parameters to include the application name derived from the process role. - Added unit tests to validate the process role detection logic for different scenarios, including Celery and uWSGI. --- .../db/backends/postgresql_psycopg3/base.py | 3 ++ dispatcharr/db/process_label.py | 34 +++++++++++++++++++ tests/test_process_label.py | 18 ++++++++++ 3 files changed, 55 insertions(+) create mode 100644 dispatcharr/db/process_label.py create mode 100644 tests/test_process_label.py diff --git a/dispatcharr/db/backends/postgresql_psycopg3/base.py b/dispatcharr/db/backends/postgresql_psycopg3/base.py index f5a49ffd..098a01f4 100644 --- a/dispatcharr/db/backends/postgresql_psycopg3/base.py +++ b/dispatcharr/db/backends/postgresql_psycopg3/base.py @@ -37,7 +37,10 @@ class DatabaseWrapper(DatabaseWrapperMixin, OriginalDatabaseWrapper): INTRANS = psycopg.pq.TransactionStatus.INTRANS def get_connection_params(self) -> dict: + from dispatcharr.db.process_label import db_application_name + conn_params = super().get_connection_params() + conn_params["application_name"] = db_application_name() for attr in ("MAX_CONNS", "REUSE_CONNS", "CONN_MAX_LIFETIME"): if attr in self.settings_dict["OPTIONS"]: conn_params[attr] = self.settings_dict["OPTIONS"][attr] diff --git a/dispatcharr/db/process_label.py b/dispatcharr/db/process_label.py new file mode 100644 index 00000000..bd50b6a6 --- /dev/null +++ b/dispatcharr/db/process_label.py @@ -0,0 +1,34 @@ +"""Label Postgres connections by Dispatcharr process role (for pg_stat_activity).""" + +from __future__ import annotations + +import os +import sys + + +def get_process_role(argv: list[str] | None = None) -> str: + argv = argv if argv is not None else sys.argv + argv0 = os.path.basename(argv[0]) if argv else "" + cmdline = " ".join(argv) + + if "celery" in argv0 or any("celery" in arg for arg in argv): + if "beat" in cmdline: + return "celery-beat" + if "-Q" in argv: + try: + if "dvr" in argv[argv.index("-Q") + 1]: + return "celery-dvr" + except (IndexError, ValueError): + pass + return "celery-worker" + if "daphne" in argv0: + return "daphne" + if argv0 == "manage.py" and len(argv) > 1: + return f"manage-{argv[1]}" + if os.environ.get("GEVENT_SUPPORT"): + return "uwsgi" + return "django" + + +def db_application_name() -> str: + return f"Dispatcharr-{get_process_role()}-{os.getpid()}" diff --git a/tests/test_process_label.py b/tests/test_process_label.py new file mode 100644 index 00000000..4b12b0f9 --- /dev/null +++ b/tests/test_process_label.py @@ -0,0 +1,18 @@ +from unittest.mock import patch + +from django.test import SimpleTestCase + +from dispatcharr.db.process_label import get_process_role + + +class ProcessLabelTests(SimpleTestCase): + def test_celery_worker_not_labeled_as_uwsgi(self): + role = get_process_role( + ["/dispatcharrpy/bin/celery", "-A", "dispatcharr", "worker"] + ) + self.assertEqual(role, "celery-worker") + + def test_uwsgi_labeled_when_gevent_support(self): + with patch.dict("os.environ", {"GEVENT_SUPPORT": "True"}): + role = get_process_role(["uwsgi"]) + self.assertEqual(role, "uwsgi") From fe8309fd4596d334a899686f484ad12d511f9eab Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 11:28:43 -0500 Subject: [PATCH 42/81] changelog: update. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2621637..173fdc6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **VOD profile selection** uses `pool_has_capacity_for_profile()` so grouped accounts respect shared credential limits before a profile is chosen (non-grouped accounts behave as before). - **Live profile switches** move the shared credential counter when the new profile uses a different provider login; same-login switches leave the credential counter unchanged. - **Credential release keys** stored at reserve time allow counters to be released even if the M3U profile row is deleted afterward. +- **PostgreSQL `application_name` tagging by process role.** Pool connections now set `application_name` at connect time (e.g. `Dispatcharr-uwsgi-{pid}`, `Dispatcharr-celery-worker-{pid}`, `Dispatcharr-celery-dvr-{pid}`) so `pg_stat_activity` shows which Dispatcharr process owns each backend instead of a generic client label. ### Changed @@ -53,6 +54,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Live proxy could leak geventpool DB checkouts outside HTTP requests.** With `django-db-geventpool`, `connection.close()` returns handles to the per-worker pool (`MAX_CONNS=8`); without it, greenlets and OS threads keep connections checked out until the pool blocks on `pool.get()`. Proxy paths that touch the ORM outside Django's request cycle now call `close_old_connections()` after work completes: `_clean_redis_keys()`, profile lookup in `_establish_transcode_connection()` before spawning ffmpeg, and fMP4 client disconnect cleanup when releasing streams (TS disconnect relies on `log_system_event()`). (Fixes #1345) - **System events could block callers on Connect and plugin handlers.** `log_system_event()` ran Connect subscriptions and plugin `"events"` hooks synchronously after the DB write, so slow integrations could stall live-proxy and streaming paths even when inserting the event row was fast. Connect/plugin dispatch now runs on a separate gevent when the hub is available (synchronously in Celery workers without a hub). `log_system_event()` and `dispatch_event_system()` also call `close_old_connections()` in `finally` blocks so integration work does not leave pool slots checked out on the caller or dispatch greenlet. - **Plugins could leak geventpool DB checkouts after UI or Connect event runs.** Third-party plugins run inline on uWSGI greenlets (manual actions and Connect `"events"` hooks) with no guaranteed connection cleanup at the plugin boundary. `PluginManager.run_action()` and `stop_plugin()` now call `close_old_connections()` in a `finally` block so each action returns its pool slot whether the plugin succeeds or raises. +- **Connect events repeatedly queried the full plugin catalog during streaming.** `trigger_event()` called `list_plugins()` on every `client_connect` / `client_disconnect`, loading all `PluginConfig` rows and sometimes hitting plugin-repo work even when no plugin subscribed to the event. Dispatch now walks the in-memory registry via `iter_actions_for_event()`, returns immediately when no handlers exist, and runs a single batched `enabled` lookup for matching plugins only. Failed plugin actions are logged per handler instead of aborting the rest. +- **Plugin discovery left idle Postgres backends after worker boot.** `discover_plugins()` runs outside Django's request cycle during uWSGI and Celery startup; it now calls `close_old_connections()` in a `finally` block so bootstrap `PluginConfig` queries return their pool slot instead of appearing as long-lived idle connections in `pg_stat_activity`. - **VOD proxy could leak geventpool DB checkouts during playback and stats updates.** `stream_vod()` ran ORM lookups for content and M3U profiles, then returned a long-lived `StreamingHttpResponse` without releasing the checkout, so each movie/episode stream could hold a pool slot for its full duration. Background VOD stats refresh (`build_vod_stats_data()`, triggered on start/stop and by the admin stats API) also queried movie, episode, and profile rows from daemon threads with no cleanup. `stream_vod()` now calls `close_old_connections()` before handing off to the streaming generator, and `build_vod_stats_data()` releases its checkout in a `finally` block. - **Channel shutdown delay did not reset after a reconnect within the grace period.** `handle_client_disconnect()` used a fixed `gevent.sleep(shutdown_delay)` from the first last-client disconnect. If a client reconnected and disconnected again during the delay, an earlier disconnect handler could still stop the channel on the original timer instead of waiting the full delay from the latest disconnect. Shutdown now polls Redis (`last_client_disconnect` timestamp and client count) so concurrent disconnect handlers and multi-worker reconnects always honour the latest disconnect time. - **Zombie ffmpeg could survive owner-lock expiry and orphan Redis sweeps.** When the 30s owner lock lapsed under single-worker load, disconnect handling treated the worker as non-owner so coordinated stop never ran, while ffmpeg kept writing and the orphan sweeper only deleted Redis keys (recreating them immediately). Disconnect now re-acquires ownership when local upstream is still active, stops locally when the last client leaves without a lock, orphan cleanup stops local ffmpeg before Redis deletion via `_has_local_upstream_activity`, re-init stops lingering upstream before starting a duplicate thread, and `is_channel_teardown_active` includes channels mid-`stop_channel` on this worker so rapid reconnect gets 503 during teardown. From 56199aef812bbc12666b4572bd1c11b1478bc792 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 14:24:26 -0500 Subject: [PATCH 43/81] perf(m3u): Enhance memory management and performance during XC stream processing - Improved memory cleanup by invoking `gc.collect()` after batch processing in `process_m3u_batch_direct` and `collect_xc_streams`, ensuring timely release of resources. - Added tests to verify that garbage collection is triggered appropriately after batch operations. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 15 +++++++++++++-- apps/m3u/tests/test_memory_cleanup.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 173fdc6b..d04219d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **XC live refresh releases bulk catalog data sooner during batch processing.** After filtering the single `get_all_live_streams` response, the full provider catalog list is dropped before batch DB work. `process_m3u_batch_direct` (the path XC refreshes use) now runs `gc.collect()` after each batch and clears batch slice references as thread futures complete. Large structures are still `del`'d in the refresh `finally` block before Celery's `task_postrun` runs `cleanup_memory()`. - **EPG programme parse now streams through PostgreSQL staging instead of holding the full catalogue in Python.** `parse_programs_for_source` writes parsed rows into a session-scoped temp table in batches (`_EPG_PARSE_BATCH_SIZE=2500`), then atomically swaps them into `ProgramData` with batched `DELETE ... RETURNING` + `INSERT` (`_EPG_SWAP_BATCH_SIZE=5000`) so Postgres never materializes the entire guide in one statement. Peak Celery memory during large XMLTV refreshes drops sharply compared with building a monolithic in-memory list before bulk insert. The byte-offset programme index build is deferred until after the swap completes so index construction no longer competes with parse for memory and I/O. `refresh_epg_data` closes its DB connection in `finally`, and `build_programme_index_task` is registered as a memory-intensive Celery task. SQLite/dev installs keep an in-memory fallback path. - **Pooled PostgreSQL connections now rotate on a bounded lifetime.** psycopg3 client-side cache grows on handles kept open indefinitely by `django-db-geventpool`; recycling uWSGI workers would interrupt live streams. A thin custom backend (`dispatcharr.db.backends.postgresql_psycopg3`) closes and replaces pool connections after `DATABASE_POOL_CONN_MAX_LIFETIME` seconds (default 600, env-overridable; set `0` to disable) on checkout and return while preserving warm-pool reuse within the window. (Fixes #1343) - **Schedules Direct refresh only fetches guide data for mapped channels.** Schedule MD5 checks and schedule downloads now target mapped lineup stations instead of the entire lineup, and schedule MD5 cache for unmapped stations is pruned each refresh. Unmapped lineup entries no longer trigger wasted schedule API calls when their MD5 changes. diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index f2aa4d3d..8c69965e 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -790,6 +790,7 @@ def collect_xc_streams(account_id, enabled_groups): """Collect all XC streams in a single API call and filter by enabled groups.""" account = M3UAccount.objects.get(id=account_id) all_streams = [] + filtered_count = 0 # Create a mapping from category_id to group info for filtering enabled_category_ids = {} @@ -819,7 +820,6 @@ def collect_xc_streams(account_id, enabled_groups): logger.info(f"Retrieved {len(all_xc_streams)} total live streams from provider") # Filter streams based on enabled categories - filtered_count = 0 for stream in all_xc_streams: # Fall back to a generated name if the provider returns null/empty stream_name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" @@ -862,11 +862,17 @@ def collect_xc_streams(account_id, enabled_groups): all_streams.append(stream_data) filtered_count += 1 + # Drop the full provider catalog before returning; only filtered rows are needed. + del all_xc_streams + gc.collect() + except Exception as e: logger.error(f"Failed to fetch XC streams: {str(e)}") return [] - logger.info(f"Filtered {filtered_count} streams from {len(enabled_category_ids)} enabled categories") + logger.info( + f"Filtered {filtered_count} streams from {len(enabled_category_ids)} enabled categories" + ) return all_streams def process_xc_category_direct(account_id, batch, groups, hash_keys): @@ -1270,6 +1276,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): # Free batch data structures (reference-counted deallocation) del streams_to_create, streams_to_update, stream_hashes, existing_streams + gc.collect() return retval @@ -3338,6 +3345,8 @@ def _refresh_single_m3u_account_impl(account_id): except Exception as e: logger.error(f"Error in thread batch {batch_idx}: {str(e)}") completed_batches += 1 # Still count it to avoid hanging + finally: + batches[batch_idx] = None logger.info(f"Thread-based processing completed for account {account_id}") else: @@ -3449,6 +3458,8 @@ def _refresh_single_m3u_account_impl(account_id): except Exception as e: logger.error(f"Error in XC thread batch {batch_idx}: {str(e)}") completed_batches += 1 # Still count it to avoid hanging + finally: + batches[batch_idx] = None logger.info(f"XC thread-based processing completed for account {account_id}") diff --git a/apps/m3u/tests/test_memory_cleanup.py b/apps/m3u/tests/test_memory_cleanup.py index bd556e4c..bfc964d0 100644 --- a/apps/m3u/tests/test_memory_cleanup.py +++ b/apps/m3u/tests/test_memory_cleanup.py @@ -33,6 +33,24 @@ class ProcessM3UBatchCleanupTests(SimpleTestCase): process_m3u_batch_direct(1, [], {}, ["name", "url"]) mock_connections.close_all.assert_called() + @patch("apps.m3u.tasks.Stream") + @patch("apps.m3u.tasks.M3UAccount") + def test_batch_calls_gc_collect(self, mock_account_cls, mock_stream_cls): + """gc.collect() must run after each batch so XC refresh threads release promptly.""" + from apps.m3u.tasks import process_m3u_batch_direct + + mock_account = MagicMock() + mock_account.filters.order_by.return_value = [] + mock_account_cls.objects.get.return_value = mock_account + mock_stream_cls.objects.filter.return_value.select_related.return_value.only.return_value = ( + [] + ) + mock_stream_cls.generate_hash_key = MagicMock(return_value="hash123") + + with patch("gc.collect") as mock_gc, patch("django.db.connections"): + process_m3u_batch_direct(1, [], {}, ["name", "url"]) + mock_gc.assert_called() + class LockReleaseTests(SimpleTestCase): """Verify task lock is released on all exit paths.""" From 5d424adbe809843410012b414b2651e3b528b251 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 14:45:24 -0500 Subject: [PATCH 44/81] perf(vod): optimize VOD batch processing and memory management - Introduced `lookup_by_name_year` function to limit database scans for movies and series without TMDB/IMDB IDs, enhancing performance by scoping lookups to current batch names. - Updated `process_movie_batch` and `process_series_batch` to utilize the new lookup function, reducing memory usage and improving efficiency. - Registered `refresh_vod_content` and `batch_refresh_series_episodes` for post-task memory cleanup in Celery, ensuring better resource management during VOD content refreshes. --- CHANGELOG.md | 1 + apps/vod/tasks.py | 57 ++++++++++++++++++++++++++++--------------- dispatcharr/celery.py | 4 ++- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d04219d2..55481706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **XC live refresh releases bulk catalog data sooner during batch processing.** After filtering the single `get_all_live_streams` response, the full provider catalog list is dropped before batch DB work. `process_m3u_batch_direct` (the path XC refreshes use) now runs `gc.collect()` after each batch and clears batch slice references as thread futures complete. Large structures are still `del`'d in the refresh `finally` block before Celery's `task_postrun` runs `cleanup_memory()`. +- **VOD movie/series batch matching no longer scans the full no-ID catalog.** `process_movie_batch` and `process_series_batch` previously loaded every `Movie`/`Series` row without TMDB or IMDB IDs on each 1000-item chunk to resolve name+year duplicates. Lookup is now scoped to the names in the current batch via `lookup_by_name_year()`, which reduces memory and DB time per chunk. `refresh_vod_content` and `batch_refresh_series_episodes` are registered for Celery post-task memory cleanup (one GC pass at task end, not per chunk). - **EPG programme parse now streams through PostgreSQL staging instead of holding the full catalogue in Python.** `parse_programs_for_source` writes parsed rows into a session-scoped temp table in batches (`_EPG_PARSE_BATCH_SIZE=2500`), then atomically swaps them into `ProgramData` with batched `DELETE ... RETURNING` + `INSERT` (`_EPG_SWAP_BATCH_SIZE=5000`) so Postgres never materializes the entire guide in one statement. Peak Celery memory during large XMLTV refreshes drops sharply compared with building a monolithic in-memory list before bulk insert. The byte-offset programme index build is deferred until after the swap completes so index construction no longer competes with parse for memory and I/O. `refresh_epg_data` closes its DB connection in `finally`, and `build_programme_index_task` is registered as a memory-intensive Celery task. SQLite/dev installs keep an in-memory fallback path. - **Pooled PostgreSQL connections now rotate on a bounded lifetime.** psycopg3 client-side cache grows on handles kept open indefinitely by `django-db-geventpool`; recycling uWSGI workers would interrupt live streams. A thin custom backend (`dispatcharr.db.backends.postgresql_psycopg3`) closes and replaces pool connections after `DATABASE_POOL_CONN_MAX_LIFETIME` seconds (default 600, env-overridable; set `0` to disable) on checkout and return while preserving warm-pool reuse within the window. (Fixes #1343) - **Schedules Direct refresh only fetches guide data for mapped channels.** Schedule MD5 checks and schedule downloads now target mapped lineup stations instead of the entire lineup, and schedule MD5 cache for unmapped stations is pruned each refresh. Unmapped lineup entries no longer trigger wasted schedule API calls when their MD5 changes. diff --git a/apps/vod/tasks.py b/apps/vod/tasks.py index 367baff3..66566745 100644 --- a/apps/vod/tasks.py +++ b/apps/vod/tasks.py @@ -16,6 +16,27 @@ import re logger = logging.getLogger(__name__) +def lookup_by_name_year(model, name_year_pairs): + """Return {(name, year): row} for rows without TMDB/IMDB IDs. + + Scoped to the names in this batch instead of scanning the full table. + """ + if not name_year_pairs: + return {} + wanted = set(name_year_pairs) + names = {name for name, _ in wanted} + found = {} + for row in model.objects.filter( + tmdb_id__isnull=True, + imdb_id__isnull=True, + name__in=names, + ): + key = (row.name, row.year) + if key in wanted: + found[key] = row + return found + + @shared_task def refresh_vod_content(account_id): """Refresh VOD content for an M3U account with batch processing for improved performance""" @@ -176,6 +197,7 @@ def refresh_movies(client, account, categories_by_provider, relations, scan_star logger.info(f"Processing movie chunk {chunk_num}/{total_chunks} ({len(chunk)} movies)") process_movie_batch(account, chunk, categories_by_provider, relations, scan_start_time) + del all_movies_data logger.info(f"Completed processing all {total_movies} movies in {total_chunks} chunks") @@ -230,6 +252,7 @@ def refresh_series(client, account, categories_by_provider, relations, scan_star logger.info(f"Processing series chunk {chunk_num}/{total_chunks} ({len(chunk)} series)") process_series_batch(account, chunk, categories_by_provider, relations, scan_start_time) + del all_series_data logger.info(f"Completed processing all {total_series} series in {total_chunks} chunks") @@ -539,10 +562,12 @@ def process_movie_batch(account, batch, categories, relations, scan_start_time=N # Query by name+year for movies without external IDs name_year_keys = [k for k in movie_keys.keys() if k.startswith('name_')] if name_year_keys: - for movie in Movie.objects.filter(tmdb_id__isnull=True, imdb_id__isnull=True): - key = f"name_{movie.name}_{movie.year or 'None'}" - if key in name_year_keys: - existing_movies[key] = movie + name_year_pairs = [ + (movie_keys[k]['props']['name'], movie_keys[k]['props'].get('year')) + for k in name_year_keys + ] + for key_tuple, movie in lookup_by_name_year(Movie, name_year_pairs).items(): + existing_movies[f"name_{key_tuple[0]}_{key_tuple[1] or 'None'}"] = movie # Get existing relations stream_ids = [data['stream_id'] for data in movie_keys.values()] @@ -659,12 +684,7 @@ def process_movie_batch(account, batch, categories, relations, scan_start_time=N existing_by_tmdb = {m.tmdb_id: m for m in Movie.objects.filter(tmdb_id__in=tmdb_ids)} if tmdb_ids else {} existing_by_imdb = {m.imdb_id: m for m in Movie.objects.filter(imdb_id__in=imdb_ids)} if imdb_ids else {} - existing_by_name_year = {} - if name_year_pairs: - for movie in Movie.objects.filter(tmdb_id__isnull=True, imdb_id__isnull=True): - key = (movie.name, movie.year) - if key in name_year_pairs: - existing_by_name_year[key] = movie + existing_by_name_year = lookup_by_name_year(Movie, name_year_pairs) # Check each movie against the bulk query results movies_actually_created = [] @@ -911,10 +931,12 @@ def process_series_batch(account, batch, categories, relations, scan_start_time= # Query by name+year for series without external IDs name_year_keys = [k for k in series_keys.keys() if k.startswith('name_')] if name_year_keys: - for series in Series.objects.filter(tmdb_id__isnull=True, imdb_id__isnull=True): - key = f"name_{series.name}_{series.year or 'None'}" - if key in name_year_keys: - existing_series[key] = series + name_year_pairs = [ + (series_keys[k]['props']['name'], series_keys[k]['props'].get('year')) + for k in name_year_keys + ] + for key_tuple, series in lookup_by_name_year(Series, name_year_pairs).items(): + existing_series[f"name_{key_tuple[0]}_{key_tuple[1] or 'None'}"] = series # Get existing relations series_ids = [data['series_id'] for data in series_keys.values()] @@ -1023,12 +1045,7 @@ def process_series_batch(account, batch, categories, relations, scan_start_time= existing_by_tmdb = {s.tmdb_id: s for s in Series.objects.filter(tmdb_id__in=tmdb_ids)} if tmdb_ids else {} existing_by_imdb = {s.imdb_id: s for s in Series.objects.filter(imdb_id__in=imdb_ids)} if imdb_ids else {} - existing_by_name_year = {} - if name_year_pairs: - for series in Series.objects.filter(tmdb_id__isnull=True, imdb_id__isnull=True): - key = (series.name, series.year) - if key in name_year_pairs: - existing_by_name_year[key] = series + existing_by_name_year = lookup_by_name_year(Series, name_year_pairs) # Check each series against the bulk query results series_actually_created = [] diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index bdb5e48d..8154b52e 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -98,7 +98,9 @@ def cleanup_task_memory(**kwargs): 'apps.channels.tasks.match_epg_channels', 'apps.channels.tasks.match_selected_channels_epg', 'apps.channels.tasks.match_single_channel_epg', - 'core.tasks.rehash_streams' + 'core.tasks.rehash_streams', + 'apps.vod.tasks.refresh_vod_content', + 'apps.vod.tasks.batch_refresh_series_episodes', ] # Check if this is a memory-intensive task From a7b4db83fa4846d16a7f18bd33f434c46ac87bb1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 16 Jun 2026 16:21:22 -0500 Subject: [PATCH 45/81] feat(process_label): enhance uWSGI process role detection - Added a new function `_is_uwsgi_worker` to accurately identify when the application is running within a uWSGI worker. - Updated `get_process_role` to label processes as "uwsgi" based on the new function or command-line arguments, improving process role accuracy. - Expanded unit tests to cover various scenarios for uWSGI role detection, ensuring robust validation of the new logic. --- dispatcharr/db/process_label.py | 11 ++++++++++- docker/uwsgi.debug.ini | 1 - tests/test_process_label.py | 17 ++++++++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/dispatcharr/db/process_label.py b/dispatcharr/db/process_label.py index bd50b6a6..209ad291 100644 --- a/dispatcharr/db/process_label.py +++ b/dispatcharr/db/process_label.py @@ -6,6 +6,15 @@ import os import sys +def _is_uwsgi_worker() -> bool: + """True when running inside a uWSGI worker (not master or a non-uWSGI process).""" + try: + import uwsgi + except ImportError: + return False + return uwsgi.worker_id() > 0 + + def get_process_role(argv: list[str] | None = None) -> str: argv = argv if argv is not None else sys.argv argv0 = os.path.basename(argv[0]) if argv else "" @@ -25,7 +34,7 @@ def get_process_role(argv: list[str] | None = None) -> str: return "daphne" if argv0 == "manage.py" and len(argv) > 1: return f"manage-{argv[1]}" - if os.environ.get("GEVENT_SUPPORT"): + if _is_uwsgi_worker() or argv0 == "uwsgi": return "uwsgi" return "django" diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index d5f577b3..37a5253b 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -70,7 +70,6 @@ py-autoreload = 1 honour-stdin = true # Environment variables -env = GEVENT_SUPPORT=True env = PYTHONPATH=/app env = PYTHONUNBUFFERED=1 env = PYDEVD_DISABLE_FILE_VALIDATION=1 diff --git a/tests/test_process_label.py b/tests/test_process_label.py index 4b12b0f9..bb4c8429 100644 --- a/tests/test_process_label.py +++ b/tests/test_process_label.py @@ -12,7 +12,18 @@ class ProcessLabelTests(SimpleTestCase): ) self.assertEqual(role, "celery-worker") - def test_uwsgi_labeled_when_gevent_support(self): - with patch.dict("os.environ", {"GEVENT_SUPPORT": "True"}): - role = get_process_role(["uwsgi"]) + def test_uwsgi_labeled_from_argv(self): + role = get_process_role(["/dispatcharrpy/bin/uwsgi", "--ini", "/app/docker/uwsgi.ini"]) self.assertEqual(role, "uwsgi") + + def test_uwsgi_labeled_when_worker_module_present(self): + fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 2})() + with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): + role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) + self.assertEqual(role, "uwsgi") + + def test_uwsgi_master_not_labeled_as_uwsgi(self): + fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 0})() + with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): + role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) + self.assertEqual(role, "django") From 704fb1f5297d60ca7c358dddfa564cdfbd1e3281 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 16 Jun 2026 22:04:01 +0000 Subject: [PATCH 46/81] Release v0.27.0 --- CHANGELOG.md | 2 ++ version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55481706..1da516a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.27.0] - 2026-06-16 + ### Added - **Manual Server Groups for shared M3U connection limits.** The existing `ServerGroup` model and account FK are now wired into live and VOD playback. Accounts assigned to the same group share a credential-scoped Redis counter when their provider logins match (hashed fingerprint); unrelated logins in the same group keep separate counters. Enforcement uses each profile's `max_streams` - not a group-wide cap - and profiles with `max_streams=0` skip credential pooling for that profile while still rotating on their own per-profile counter. New `apps/m3u/connection_pool.py` centralizes reserve/release, profile rotation, and credential moves on profile switch. (Closes #1137) — Thanks [@Goldenfreddy0703](https://github.com/Goldenfreddy0703) diff --git a/version.py b/version.py index 362d955e..87a27e3c 100644 --- a/version.py +++ b/version.py @@ -1,5 +1,5 @@ """ Dispatcharr version information. """ -__version__ = '0.26.0' # Follow semantic versioning (MAJOR.MINOR.PATCH) +__version__ = '0.27.0' # Follow semantic versioning (MAJOR.MINOR.PATCH) __timestamp__ = None # Set during CI/CD build process From f99133837690e431b5fd8b7d5a111cddbf0bb7b2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 17 Jun 2026 17:22:28 -0500 Subject: [PATCH 47/81] fix(proxy): update for improved DB connection management - Enhanced the live proxy to release geventpool DB connections more effectively across various paths. - Implemented `close_old_connections()` in `stream_ts()`, `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, `get_connections_left()`, and the cleanup process in `StreamGenerator`, ensuring that clients do not hold onto pool slots unnecessarily during streaming operations. --- CHANGELOG.md | 4 + apps/proxy/live_proxy/output/ts/generator.py | 104 +++++++------ .../live_proxy/services/channel_service.py | 138 +++++++++-------- .../live_proxy/tests/test_live_db_cleanup.py | 142 ++++++++++++++++++ apps/proxy/live_proxy/url_utils.py | 9 ++ apps/proxy/live_proxy/views.py | 5 +- 6 files changed, 286 insertions(+), 116 deletions(-) create mode 100644 apps/proxy/live_proxy/tests/test_live_db_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1da516a1..1f1234c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. + ## [0.27.0] - 2026-06-16 ### Added diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index 2bd88725..3524a2a6 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -7,6 +7,7 @@ import time import gevent from apps.proxy.config import TSConfig as Config from apps.channels.models import Channel, Stream +from django.db import close_old_connections from core.utils import log_system_event from ...server import ProxyServer from ...utils import create_ts_packet, get_logger @@ -590,60 +591,63 @@ class StreamGenerator: def _cleanup(self): """Clean up resources and report final statistics.""" - # Client cleanup - elapsed = time.time() - self.stream_start_time - local_clients = 0 - total_clients = 0 - proxy_server = ProxyServer.get_instance() + try: + # Client cleanup + elapsed = time.time() - self.stream_start_time + local_clients = 0 + total_clients = 0 + proxy_server = ProxyServer.get_instance() - # Release M3U profile stream allocation if this is the last client - stream_released = False - if proxy_server.redis_client: - try: - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() - # Pool slots are global; the last client on any worker must release. - # During shutdown_delay, keep the slot until coordinated stop runs. - if client_count <= 1 and ConfigHelper.channel_shutdown_delay() <= 0: - try: + # Release M3U profile stream allocation if this is the last client + stream_released = False + if proxy_server.redis_client: + try: + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Pool slots are global; the last client on any worker must release. + # During shutdown_delay, keep the slot until coordinated stop runs. + if client_count <= 1 and ConfigHelper.channel_shutdown_delay() <= 0: try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - stream_released = obj.release_stream() - if stream_released: - logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") - else: - logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") - except Exception as e: - logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") - except Exception as e: - logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") + try: + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + stream_released = obj.release_stream() + if stream_released: + logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") + else: + logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") + except Exception as e: + logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") - if self.channel_id in proxy_server.client_managers: - client_manager = proxy_server.client_managers[self.channel_id] - if self.client_id in client_manager.clients: - local_clients = client_manager.remove_client(self.client_id) - else: - local_clients = client_manager.get_client_count() - total_clients = client_manager.get_total_client_count() - logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") + if self.channel_id in proxy_server.client_managers: + client_manager = proxy_server.client_managers[self.channel_id] + if self.client_id in client_manager.clients: + local_clients = client_manager.remove_client(self.client_id) + else: + local_clients = client_manager.get_client_count() + total_clients = client_manager.get_total_client_count() + logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") - # Log client disconnect event - try: - log_system_event( - 'client_disconnect', - channel_id=self.channel_id, - channel_name=self.channel_name, - client_ip=self.client_ip, - client_id=self.client_id, - user_agent=self.client_user_agent[:100] if self.client_user_agent else None, - duration=round(elapsed, 2), - bytes_sent=self.bytes_sent, - username=self.user.username if self.user else None - ) - except Exception as e: - logger.error(f"Could not log client disconnect event: {e}") + # Log client disconnect event + try: + log_system_event( + 'client_disconnect', + channel_id=self.channel_id, + channel_name=self.channel_name, + client_ip=self.client_ip, + client_id=self.client_id, + user_agent=self.client_user_agent[:100] if self.client_user_agent else None, + duration=round(elapsed, 2), + bytes_sent=self.bytes_sent, + username=self.user.username if self.user else None + ) + except Exception as e: + logger.error(f"Could not log client disconnect event: {e}") + finally: + close_old_connections() def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None, buffer=None): diff --git a/apps/proxy/live_proxy/services/channel_service.py b/apps/proxy/live_proxy/services/channel_service.py index 400f55f4..7ad4e71b 100644 --- a/apps/proxy/live_proxy/services/channel_service.py +++ b/apps/proxy/live_proxy/services/channel_service.py @@ -241,34 +241,38 @@ class ChannelService: # Store additional metadata if initialization was successful if success and proxy_server.redis_client: - metadata_key = RedisKeys.channel_metadata(channel_id) - update_data = {} - if stream_profile_value: - update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value - if stream_id: - update_data[ChannelMetadataField.STREAM_ID] = str(stream_id) - if m3u_profile_id: - update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - - # Store channel name and stream name so stats workers don't need DB calls try: - if not channel_name: - from apps.channels.models import Channel - channel_name = Channel.objects.filter(uuid=channel_id).values_list('name', flat=True).first() - if channel_name: - update_data[ChannelMetadataField.CHANNEL_NAME] = channel_name - else: - # No channel name means stream preview mode, use stream name as display fallback - if stream_id and not stream_name: - from apps.channels.models import Stream - stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() - if stream_name: - update_data[ChannelMetadataField.STREAM_NAME] = stream_name - except Exception as e: - logger.warning(f"Failed to store channel/stream names in Redis for {channel_id}: {e}") + metadata_key = RedisKeys.channel_metadata(channel_id) + update_data = {} + if stream_profile_value: + update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value + if stream_id: + update_data[ChannelMetadataField.STREAM_ID] = str(stream_id) + if m3u_profile_id: + update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - if update_data: - proxy_server.redis_client.hset(metadata_key, mapping=update_data) + # Store channel name and stream name so stats workers don't need DB calls + try: + if not channel_name: + from apps.channels.models import Channel + channel_name = Channel.objects.filter(uuid=channel_id).values_list('name', flat=True).first() + if channel_name: + update_data[ChannelMetadataField.CHANNEL_NAME] = channel_name + else: + # No channel name means stream preview mode, use stream name as display fallback + if stream_id and not stream_name: + from apps.channels.models import Stream + stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() + if stream_name: + update_data[ChannelMetadataField.STREAM_NAME] = stream_name + except Exception as e: + logger.warning(f"Failed to store channel/stream names in Redis for {channel_id}: {e}") + + if update_data: + proxy_server.redis_client.hset(metadata_key, mapping=update_data) + finally: + from django.db import close_old_connections + close_old_connections() return success @@ -736,53 +740,57 @@ class ChannelService: @staticmethod def _update_channel_metadata(channel_id, url, user_agent=None, stream_id=None, m3u_profile_id=None, stream_name=None): """Update channel metadata in Redis""" - proxy_server = ProxyServer.get_instance() + try: + proxy_server = ProxyServer.get_instance() - if not proxy_server.redis_client: - return False + if not proxy_server.redis_client: + return False - metadata_key = RedisKeys.channel_metadata(channel_id) + metadata_key = RedisKeys.channel_metadata(channel_id) - # First check if the key exists and what type it is - key_type = proxy_server.redis_client.type(metadata_key) - logger.debug(f"Redis key {metadata_key} is of type: {key_type}") + # First check if the key exists and what type it is + key_type = proxy_server.redis_client.type(metadata_key) + logger.debug(f"Redis key {metadata_key} is of type: {key_type}") - # Build metadata update dict - metadata = {ChannelMetadataField.URL: url} - if user_agent: - metadata[ChannelMetadataField.USER_AGENT] = user_agent - if stream_id: - metadata[ChannelMetadataField.STREAM_ID] = str(stream_id) - if not stream_name: - try: - from apps.channels.models import Stream - stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() - except Exception as e: - logger.warning(f"Failed to update stream name in Redis for stream {stream_id}: {e}") - if stream_name: - metadata[ChannelMetadataField.STREAM_NAME] = stream_name - if m3u_profile_id: - metadata[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) + # Build metadata update dict + metadata = {ChannelMetadataField.URL: url} + if user_agent: + metadata[ChannelMetadataField.USER_AGENT] = user_agent + if stream_id: + metadata[ChannelMetadataField.STREAM_ID] = str(stream_id) + if not stream_name: + try: + from apps.channels.models import Stream + stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() + except Exception as e: + logger.warning(f"Failed to update stream name in Redis for stream {stream_id}: {e}") + if stream_name: + metadata[ChannelMetadataField.STREAM_NAME] = stream_name + if m3u_profile_id: + metadata[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - # Also update the stream switch time field - metadata[ChannelMetadataField.STREAM_SWITCH_TIME] = str(time.time()) + # Also update the stream switch time field + metadata[ChannelMetadataField.STREAM_SWITCH_TIME] = str(time.time()) - # Use the appropriate method based on the key type - if key_type == 'hash': - proxy_server.redis_client.hset(metadata_key, mapping=metadata) - elif key_type == 'none': # Key doesn't exist yet - proxy_server.redis_client.hset(metadata_key, mapping=metadata) - else: - # If key exists with wrong type, delete it and recreate - proxy_server.redis_client.delete(metadata_key) - proxy_server.redis_client.hset(metadata_key, mapping=metadata) + # Use the appropriate method based on the key type + if key_type == 'hash': + proxy_server.redis_client.hset(metadata_key, mapping=metadata) + elif key_type == 'none': # Key doesn't exist yet + proxy_server.redis_client.hset(metadata_key, mapping=metadata) + else: + # If key exists with wrong type, delete it and recreate + proxy_server.redis_client.delete(metadata_key) + proxy_server.redis_client.hset(metadata_key, mapping=metadata) - # Set switch request flag to ensure all workers see it - switch_key = RedisKeys.switch_request(channel_id) - proxy_server.redis_client.setex(switch_key, 30, url) # 30 second TTL + # Set switch request flag to ensure all workers see it + switch_key = RedisKeys.switch_request(channel_id) + proxy_server.redis_client.setex(switch_key, 30, url) # 30 second TTL - logger.debug(f"Updated metadata for channel {channel_id} in Redis") - return True + logger.debug(f"Updated metadata for channel {channel_id} in Redis") + return True + finally: + from django.db import close_old_connections + close_old_connections() @staticmethod def _publish_stream_switch_event(channel_id, new_url, user_agent=None, stream_id=None, m3u_profile_id=None): diff --git a/apps/proxy/live_proxy/tests/test_live_db_cleanup.py b/apps/proxy/live_proxy/tests/test_live_db_cleanup.py new file mode 100644 index 00000000..f2015a41 --- /dev/null +++ b/apps/proxy/live_proxy/tests/test_live_db_cleanup.py @@ -0,0 +1,142 @@ +"""Live proxy must release geventpool checkouts after ORM on stream and URL paths.""" + +from unittest.mock import MagicMock, patch + +from django.http import StreamingHttpResponse +from django.test import RequestFactory, SimpleTestCase + + +class StreamTsDbCleanupTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + @patch("apps.proxy.live_proxy.views.close_old_connections") + @patch("apps.proxy.live_proxy.views.create_stream_generator") + @patch("apps.proxy.live_proxy.views._resolve_output_format", return_value="mpegts") + @patch("apps.proxy.live_proxy.views._resolve_output_profile", return_value=None) + @patch("apps.proxy.live_proxy.views.ChannelService.is_channel_unavailable_for_new_clients", return_value=False) + @patch("apps.proxy.live_proxy.views.get_stream_object") + @patch("apps.proxy.live_proxy.views.network_access_allowed", return_value=True) + @patch("apps.proxy.live_proxy.views.ProxyServer") + def test_stream_ts_closes_db_before_streaming_response( + self, + mock_proxy_cls, + _network_ok, + mock_get_stream_object, + _unavailable, + _output_profile, + _output_format, + mock_create_generator, + mock_close, + ): + channel = MagicMock() + channel.id = 1 + channel.uuid = "channel-uuid" + channel.name = "Test Channel" + mock_get_stream_object.return_value = channel + + client_manager = MagicMock() + proxy_server = MagicMock() + proxy_server.redis_client = MagicMock() + proxy_server.redis_client.exists.return_value = True + proxy_server.redis_client.hgetall.return_value = {"state": "active"} + proxy_server.stream_buffers = {"channel-uuid": MagicMock()} + proxy_server.client_managers = {"channel-uuid": client_manager} + proxy_server.check_if_channel_exists.return_value = True + proxy_server.get_buffer.return_value = MagicMock() + mock_proxy_cls.get_instance.return_value = proxy_server + + def _generate(): + yield b"chunk" + + mock_create_generator.return_value = lambda: _generate() + + request = self.factory.get("/proxy/live/channel-uuid/") + request.user = MagicMock(is_authenticated=False) + + from apps.proxy.live_proxy.views import stream_ts + + response = stream_ts(request, "channel-uuid") + + self.assertIsInstance(response, StreamingHttpResponse) + mock_close.assert_called_once() + + +class UrlUtilsDbCleanupTests(SimpleTestCase): + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_stream_object") + def test_generate_stream_url_closes_db(self, mock_get_object, mock_close): + channel = MagicMock() + channel.get_stream.return_value = (None, None, "no streams", False) + mock_get_object.return_value = channel + + from apps.proxy.live_proxy.url_utils import generate_stream_url + + result = generate_stream_url("channel-uuid") + + self.assertIsNone(result[0]) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_stream_object") + def test_get_alternate_streams_closes_db(self, mock_get_object, mock_close): + channel = MagicMock() + channel.streams.all.return_value.order_by.return_value.exists.return_value = False + mock_get_object.return_value = channel + + from apps.proxy.live_proxy.url_utils import get_alternate_streams + + result = get_alternate_streams("channel-uuid", current_stream_id=1) + + self.assertEqual(result, []) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_object_or_404") + def test_get_stream_info_for_switch_closes_db_on_error(self, mock_get_404, mock_close): + mock_get_404.side_effect = RuntimeError("db error") + + from apps.proxy.live_proxy.url_utils import get_stream_info_for_switch + + result = get_stream_info_for_switch("channel-uuid", target_stream_id=99) + + self.assertIn("error", result) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.M3UAccountProfile.objects.get") + def test_get_connections_left_closes_db(self, mock_get, mock_close): + mock_get.side_effect = Exception("not found") + + from apps.proxy.live_proxy.url_utils import get_connections_left + + result = get_connections_left(999) + + self.assertEqual(result, 0) + mock_close.assert_called_once() + + +class TsGeneratorDbCleanupTests(SimpleTestCase): + @patch("apps.proxy.live_proxy.output.ts.generator.close_old_connections") + @patch("apps.proxy.live_proxy.output.ts.generator.ProxyServer.get_instance") + def test_ts_cleanup_closes_db(self, mock_proxy_cls, mock_close): + proxy_server = MagicMock() + proxy_server.redis_client = None + proxy_server.client_managers = {} + mock_proxy_cls.return_value = proxy_server + + from apps.proxy.live_proxy.output.ts.generator import StreamGenerator + + gen = StreamGenerator.__new__(StreamGenerator) + gen.channel_id = "channel-uuid" + gen.client_id = "client-1" + gen.stream_start_time = 0 + gen.channel_name = "Test" + gen.client_ip = "127.0.0.1" + gen.client_user_agent = "agent" + gen.bytes_sent = 0 + gen.user = None + + gen._cleanup() + + mock_close.assert_called_once() diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 7cdc92ef..2b32f7e6 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -5,6 +5,7 @@ Utilities for handling stream URLs and transformations. import logging import regex from typing import Optional, Tuple, List +from django.db import close_old_connections from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile @@ -156,6 +157,8 @@ def generate_stream_url( except Exception as e: logger.error(f"Error generating stream URL: {e}") return None, None, False, None, False, str(e) + finally: + close_old_connections() def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -300,6 +303,8 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] channel.release_stream() logger.error(f"Error getting stream info for switch: {e}", exc_info=True) return {'error': f'Error: {str(e)}'} + finally: + close_old_connections() def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = None) -> List[dict]: """ @@ -422,6 +427,8 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No except Exception as e: logger.error(f"Error getting alternate streams for channel {channel_id}: {e}", exc_info=True) return [] + finally: + close_old_connections() def validate_stream_url(url, user_agent=None, timeout=(5, 5)): """ @@ -592,3 +599,5 @@ def get_connections_left(m3u_profile_id: int) -> int: except Exception as e: logger.error(f"Error getting connections left for M3U profile {m3u_profile_id}: {e}") return 0 + finally: + close_old_connections() diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 2119b4bd..001cfc14 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -3,6 +3,7 @@ import time import random import re import pathlib +from django.db import close_old_connections from django.http import StreamingHttpResponse, JsonResponse, HttpResponseRedirect, HttpResponse from django.views.decorators.csrf import csrf_exempt from django.shortcuts import get_object_or_404 @@ -617,7 +618,9 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ) content_type = "video/mp2t" - # Return the StreamingHttpResponse from the main function + # Release ORM checkout before returning a long-lived StreamingHttpResponse. + close_old_connections() + response = StreamingHttpResponse( streaming_content=generate(), content_type=content_type ) From cfe58db222ea489cbc9ebc210ab4bd5f175aa287 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 10:12:12 -0500 Subject: [PATCH 48/81] fix(xc): normalize server URLs for live playback and stream URLs (Fixes #1363) - Implemented `normalize_server_url()` to standardize account server URLs, ensuring that on-demand live URLs are built correctly without including API endpoints or query parameters. - Updated `get_transformed_credentials()` and stream URL generation in `M3UMovieRelation` and `M3UEpisodeRelation` to utilize the new normalization function, improving URL handling for Xtream Codes accounts. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 5 +- apps/m3u/tests/test_xc_live_url.py | 139 +++++++++++++++++++++++++++++ apps/vod/models.py | 21 +++-- core/xtream_codes.py | 38 ++++---- 5 files changed, 176 insertions(+), 28 deletions(-) create mode 100644 apps/m3u/tests/test_xc_live_url.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f1234c5..1712d447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. ## [0.27.0] - 2026-06-16 diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 8c69965e..2d5ac1ee 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -2816,12 +2816,13 @@ def get_transformed_credentials(account, profile=None): logger.warning(f"Could not get primary profile for account {account.name}: {e}") profile = None - base_url = account.server_url + from core.xtream_codes import normalize_server_url + + base_url = normalize_server_url(account.server_url) base_username = account.username base_password = account.password # Build a complete URL with credentials (similar to how IPTV URLs are structured) # Format: http://server.com:port/live/username/password/1234.ts if base_url and base_username and base_password: - # Remove trailing slash from server URL if present clean_server_url = base_url.rstrip('/') # Build the complete URL with embedded credentials diff --git a/apps/m3u/tests/test_xc_live_url.py b/apps/m3u/tests/test_xc_live_url.py new file mode 100644 index 00000000..8353241d --- /dev/null +++ b/apps/m3u/tests/test_xc_live_url.py @@ -0,0 +1,139 @@ +"""Tests for XC stream URL normalization and on-demand URL building.""" + +from django.test import TestCase + +from apps.channels.models import Stream +from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.m3u.tasks import get_transformed_credentials +from apps.proxy.live_proxy.url_utils import _resolve_live_stream_url +from apps.vod.models import Episode, M3UEpisodeRelation, M3UMovieRelation, Movie, Series +from core.xtream_codes import normalize_server_url + + +class NormalizeServerUrlTests(TestCase): + def test_preserves_sub_path(self): + url = "https://myserver.fun/server1" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_strips_player_api_php_and_query_params(self): + url = "https://myserver.fun/server1/player_api.php?username=foo&password=bar" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_strips_trailing_slash(self): + url = "https://myserver.fun/server1/" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_nested_sub_path_with_php_endpoint(self): + url = "http://server/Pluto/gb/player_api.php" + self.assertEqual(normalize_server_url(url), "http://server/Pluto/gb") + + +class GetTransformedCredentialsTests(TestCase): + def test_returns_normalized_server_url(self): + account = M3UAccount.objects.create( + name="Sub-path XC", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + + server_url, username, password = get_transformed_credentials(account, profile) + + self.assertEqual(server_url, "https://myserver.fun/server1") + self.assertEqual(username, "alice") + self.assertEqual(password, "secret") + + +class ResolveLiveStreamUrlTests(TestCase): + def test_builds_url_from_normalized_base_not_raw_account_url(self): + account = M3UAccount.objects.create( + name="Live sub-path", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + stream = Stream.objects.create( + name="Test Channel", + m3u_account=account, + stream_id="12345", + url="https://myserver.fun/server1/live/olduser/oldpass/12345.ts", + ) + + url = _resolve_live_stream_url(stream, account, profile) + + self.assertEqual( + url, + "https://myserver.fun/server1/live/alice/secret/12345.ts", + ) + + def test_std_account_uses_stored_stream_url(self): + account = M3UAccount.objects.create( + name="STD account", + account_type="STD", + server_url="https://example.com/list.m3u", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + stream = Stream.objects.create( + name="STD Stream", + m3u_account=account, + url="https://provider.example/stream/abc123", + ) + + url = _resolve_live_stream_url(stream, account, profile) + + self.assertEqual(url, "https://provider.example/stream/abc123") + + +class VodStreamUrlTests(TestCase): + def setUp(self): + self.account = M3UAccount.objects.create( + name="VOD sub-path", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + + def test_movie_relation_builds_normalized_url(self): + movie = Movie.objects.create(name="Test Movie") + relation = M3UMovieRelation.objects.create( + m3u_account=self.account, + movie=movie, + stream_id="999", + container_extension="mkv", + ) + + url = relation.get_stream_url() + + self.assertEqual( + url, + "https://myserver.fun/server1/movie/alice/secret/999.mkv", + ) + + def test_episode_relation_builds_normalized_url(self): + series = Series.objects.create(name="Test Series") + episode = Episode.objects.create( + series=series, + name="Pilot", + season_number=1, + episode_number=1, + ) + relation = M3UEpisodeRelation.objects.create( + m3u_account=self.account, + episode=episode, + stream_id="888", + container_extension="mp4", + ) + + url = relation.get_stream_url() + + self.assertEqual( + url, + "https://myserver.fun/server1/series/alice/secret/888.mp4", + ) diff --git a/apps/vod/models.py b/apps/vod/models.py index dd7d1753..f609be81 100644 --- a/apps/vod/models.py +++ b/apps/vod/models.py @@ -243,12 +243,12 @@ class M3UMovieRelation(models.Model): def get_stream_url(self): """Get the full stream URL for this movie from this provider""" - # Build URL dynamically for XtreamCodes accounts if self.m3u_account.account_type == 'XC': - from core.xtream_codes import Client as XCClient - # Use XC client's URL normalization to handle malformed URLs - # (e.g., URLs with /player_api.php or query parameters) - normalized_url = XCClient(self.m3u_account.server_url, '', '')._normalize_url(self.m3u_account.server_url) + from core.xtream_codes import normalize_server_url + + normalized_url = normalize_server_url(self.m3u_account.server_url) + if not normalized_url: + return None username = self.m3u_account.username password = self.m3u_account.password return f"{normalized_url}/movie/{username}/{password}/{self.stream_id}.{self.container_extension or 'mp4'}" @@ -292,13 +292,12 @@ class M3UEpisodeRelation(models.Model): def get_stream_url(self): """Get the full stream URL for this episode from this provider""" - from core.xtream_codes import Client as XtreamCodesClient - if self.m3u_account.account_type == 'XC': - # For XtreamCodes accounts, build the URL dynamically - # Use XC client's URL normalization to handle malformed URLs - # (e.g., URLs with /player_api.php or query parameters) - normalized_url = XtreamCodesClient(self.m3u_account.server_url, '', '')._normalize_url(self.m3u_account.server_url) + from core.xtream_codes import normalize_server_url + + normalized_url = normalize_server_url(self.m3u_account.server_url) + if not normalized_url: + return None username = self.m3u_account.username password = self.m3u_account.password return f"{normalized_url}/series/{username}/{password}/{self.stream_id}.{self.container_extension or 'mp4'}" diff --git a/core/xtream_codes.py b/core/xtream_codes.py index 195d4428..bcbe22e2 100644 --- a/core/xtream_codes.py +++ b/core/xtream_codes.py @@ -5,6 +5,26 @@ import json logger = logging.getLogger(__name__) + +def normalize_server_url(url): + """Normalize server URL: strip XC API endpoints and query params, preserve base path.""" + if not url: + return url + + from urllib.parse import urlparse, urlunparse + + parsed = urlparse(url.strip()) + path = parsed.path.rstrip('/') + + # XC API endpoints are always .php files; legitimate base paths never are. + # Stripping the trailing segment when it ends in .php handles any pasted API URL. + last_segment = path.rsplit('/', 1)[-1] + if last_segment.endswith('.php'): + path = path[:-(len(last_segment) + 1)] if '/' in path else '' + + return urlunparse((parsed.scheme, parsed.netloc, path, '', '', '')) + + class Client: """Xtream Codes API Client with robust error handling""" @@ -43,22 +63,10 @@ class Client: self.server_info = None def _normalize_url(self, url): - """Normalize server URL: strip XC API endpoints and query params, preserve base path.""" - if not url: + normalized = normalize_server_url(url) + if not normalized: raise ValueError("Server URL cannot be empty") - - from urllib.parse import urlparse, urlunparse - - parsed = urlparse(url.strip()) - path = parsed.path.rstrip('/') - - # XC API endpoints are always .php files; legitimate base paths never are. - # Stripping the trailing segment when it ends in .php handles any pasted API URL. - last_segment = path.rsplit('/', 1)[-1] - if last_segment.endswith('.php'): - path = path[:-(len(last_segment) + 1)] if '/' in path else '' - - return urlunparse((parsed.scheme, parsed.netloc, path, '', '', '')) + return normalized def _make_request(self, endpoint, params=None): """Make request with detailed error handling""" From 46695588f73dc27f17930b52e7d84f4ac73860d7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 14:59:09 -0500 Subject: [PATCH 49/81] fix(m3u): enhance custom properties handling and DB connection management - Implemented `ensure_custom_properties_dict()` to normalize custom properties across various models and serializers, addressing issues with legacy JSON-encoded strings. - Updated M3U account and channel group models to ensure custom properties are consistently stored as dictionaries during save operations. - Enhanced Celery task management by ensuring old DB connections are closed before and after tasks, improving reliability and preventing errors during account refresh operations. (Fixes #1338) --- CHANGELOG.md | 2 + apps/channels/compact_numbering.py | 11 +- apps/channels/models.py | 9 +- apps/channels/serializers.py | 11 - apps/m3u/api_views.py | 10 +- apps/m3u/forms.py | 8 +- apps/m3u/models.py | 11 + apps/m3u/serializers.py | 20 +- apps/m3u/tasks.py | 273 ++++++++++++++------- apps/m3u/tests/test_refresh_db_recovery.py | 74 ++++++ core/utils.py | 42 +++- dispatcharr/celery.py | 20 +- tests/test_custom_properties_as_dict.py | 74 ++++++ 13 files changed, 446 insertions(+), 119 deletions(-) create mode 100644 apps/m3u/tests/test_refresh_db_recovery.py create mode 100644 tests/test_custom_properties_as_dict.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1712d447..74a5f6a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) +- **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/channels/compact_numbering.py b/apps/channels/compact_numbering.py index fd29a153..27a26cb6 100644 --- a/apps/channels/compact_numbering.py +++ b/apps/channels/compact_numbering.py @@ -25,7 +25,8 @@ import logging from django.db import transaction from .models import Channel, ChannelOverride, ChannelGroupM3UAccount -from apps.m3u.tasks import _custom_properties_as_dict, _next_available_number +from apps.m3u.tasks import _next_available_number +from core.utils import ensure_custom_properties_dict from core.utils import ( acquire_task_lock, natural_sort_key, @@ -37,7 +38,7 @@ logger = logging.getLogger(__name__) def is_compact_group(group_relation): """Return True if the given ChannelGroupM3UAccount is in compact mode.""" - cp = _custom_properties_as_dict(group_relation.custom_properties) + cp = ensure_custom_properties_dict(group_relation.custom_properties) return bool(cp.get("compact_numbering")) @@ -72,7 +73,7 @@ def get_group_relation_for_channel(channel): for rel in ChannelGroupM3UAccount.objects.filter( m3u_account_id=channel.auto_created_by_id ): - cp = _custom_properties_as_dict(rel.custom_properties) + cp = ensure_custom_properties_dict(rel.custom_properties) if str(cp.get("group_override", "")) == target: return rel return None @@ -179,7 +180,7 @@ def assign_compact_numbers_for_channels(channel_ids): for rel in ChannelGroupM3UAccount.objects.filter( m3u_account_id__in={k[0] for k in unresolved} ): - cp = _custom_properties_as_dict(rel.custom_properties) + cp = ensure_custom_properties_dict(rel.custom_properties) target = cp.get("group_override") if not target: continue @@ -295,7 +296,7 @@ def _repack_inner(group_relation): account_id = group_relation.m3u_account_id group_id = group_relation.channel_group_id - cp = _custom_properties_as_dict(group_relation.custom_properties) + cp = ensure_custom_properties_dict(group_relation.custom_properties) sort_order = cp.get("channel_sort_order") or "" sort_reverse = bool(cp.get("channel_sort_reverse")) diff --git a/apps/channels/models.py b/apps/channels/models.py index ef1d0cff..747d0b89 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -2,7 +2,7 @@ from django.db import models from django.core.exceptions import ValidationError from django.conf import settings from core.models import StreamProfile, CoreSettings -from core.utils import RedisClient +from core.utils import RedisClient, custom_properties_as_dict from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState import logging @@ -1119,6 +1119,13 @@ class ChannelGroupM3UAccount(models.Model): def __str__(self): return f"{self.channel_group.name} - {self.m3u_account.name} (Enabled: {self.enabled})" + def save(self, *args, **kwargs): + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + super().save(*args, **kwargs) + class Logo(models.Model): name = models.CharField(max_length=255) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 9b7d3a2a..2e80d840 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -221,17 +221,6 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): return data - def to_internal_value(self, data): - # Accept both dict and JSON string for custom_properties (for backward compatibility) - val = data.get("custom_properties") - if isinstance(val, str): - try: - data["custom_properties"] = json.loads(val) - except Exception: - pass - - return super().to_internal_value(data) - def validate(self, attrs): # Partial PATCHes only carry submitted fields; fill missing # start/end from the instance so the validator catches a PATCH diff --git a/apps/m3u/api_views.py b/apps/m3u/api_views.py index a49ddd6f..1117a226 100644 --- a/apps/m3u/api_views.py +++ b/apps/m3u/api_views.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile from core.models import UserAgent -from core.utils import safe_upload_path +from core.utils import safe_upload_path, ensure_custom_properties_dict from apps.channels.models import ChannelGroupM3UAccount from core.serializers import UserAgentSerializer from apps.vod.models import M3UVODCategoryRelation @@ -536,7 +536,9 @@ class M3UAccountViewSet(viewsets.ModelViewSet): auto_channel_sync=setting.get("auto_channel_sync", False), auto_sync_channel_start=setting.get("auto_sync_channel_start"), auto_sync_channel_end=setting.get("auto_sync_channel_end"), - custom_properties=setting.get("custom_properties", {}), + custom_properties=ensure_custom_properties_dict( + setting.get("custom_properties") + ), ) for setting in group_settings if setting.get("channel_group") @@ -561,7 +563,9 @@ class M3UAccountViewSet(viewsets.ModelViewSet): category_id=setting["id"], m3u_account=account, enabled=setting.get("enabled", True), - custom_properties=setting.get("custom_properties", {}), + custom_properties=ensure_custom_properties_dict( + setting.get("custom_properties") + ), ) for setting in category_settings if setting.get("id") diff --git a/apps/m3u/forms.py b/apps/m3u/forms.py index cf6586c3..af4d6072 100644 --- a/apps/m3u/forms.py +++ b/apps/m3u/forms.py @@ -1,6 +1,7 @@ # apps/m3u/forms.py from django import forms from .models import M3UAccount, M3UFilter +from core.utils import ensure_custom_properties_dict import re class M3UAccountForm(forms.ModelForm): @@ -28,7 +29,9 @@ class M3UAccountForm(forms.ModelForm): # Set initial value for enable_vod from custom_properties if self.instance and self.instance.custom_properties: - custom_props = self.instance.custom_properties or {} + custom_props = self.instance.custom_properties + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) self.fields['enable_vod'].initial = custom_props.get('enable_vod', False) def save(self, commit=True): @@ -37,8 +40,9 @@ class M3UAccountForm(forms.ModelForm): # Handle enable_vod field enable_vod = self.cleaned_data.get('enable_vod', False) - # Parse existing custom_properties custom_props = instance.custom_properties or {} + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) # Update VOD preference custom_props['enable_vod'] = enable_vod diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 53c18d38..7d4fe65a 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -7,6 +7,7 @@ from django.dispatch import receiver from apps.channels.models import StreamProfile from django_celery_beat.models import PeriodicTask from core.models import CoreSettings, UserAgent +from core.utils import custom_properties_as_dict CUSTOM_M3U_ACCOUNT_NAME = "custom" @@ -124,6 +125,11 @@ class M3UAccount(models.Model): return user_agent def save(self, *args, **kwargs): + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + # Prevent auto_now behavior by handling updated_at manually if "update_fields" in kwargs and "updated_at" not in kwargs["update_fields"]: # Don't modify updated_at for regular updates @@ -263,6 +269,11 @@ class M3UAccountProfile(models.Model): def save(self, *args, **kwargs): """Auto-sync exp_date from custom_properties for XC accounts on every save. For non-XC accounts, exp_date is set directly and left untouched here.""" + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + parsed = self._parse_exp_date_from_custom_properties() if parsed is not None: # XC account with exp_date in custom_properties — always sync diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 485a3687..c1dc6f92 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -1,4 +1,4 @@ -from core.utils import validate_flexible_url +from core.utils import validate_flexible_url, ensure_custom_properties_dict from rest_framework import serializers, status from rest_framework.response import Response from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile @@ -270,11 +270,15 @@ class M3UAccountSerializer(serializers.ModelSerializer): # overwrite their corresponding keys; clients should set those via # the typed top-level fields rather than the custom_properties # payload. - incoming_custom = validated_data.get("custom_properties") or {} - custom_props = { - **(instance.custom_properties or {}), - **incoming_custom, - } + incoming_custom = {} + if "custom_properties" in validated_data: + incoming_custom = validated_data["custom_properties"] or {} + if not isinstance(incoming_custom, dict): + incoming_custom = ensure_custom_properties_dict(incoming_custom) + existing_custom = instance.custom_properties or {} + if not isinstance(existing_custom, dict): + existing_custom = ensure_custom_properties_dict(existing_custom) + custom_props = {**existing_custom, **incoming_custom} if enable_vod is not None: custom_props["enable_vod"] = enable_vod @@ -346,7 +350,9 @@ class M3UAccountSerializer(serializers.ModelSerializer): auto_enable_new_groups_series = validated_data.pop("auto_enable_new_groups_series", True) # Parse existing custom_properties or create new - custom_props = validated_data.get("custom_properties", {}) + custom_props = validated_data.get("custom_properties") or {} + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) # Set preferences (default to True for auto_enable_new_groups) custom_props["enable_vod"] = enable_vod diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 2d5ac1ee..be2a612f 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -7,29 +7,23 @@ import os import gc import gzip, zipfile from concurrent.futures import ThreadPoolExecutor, as_completed -from celery.app.control import Inspect -from celery.result import AsyncResult -from celery import shared_task, current_app, group +from celery import shared_task from django.conf import settings -from django.core.cache import cache from django.db import models, transaction from .models import M3UAccount from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer from django.utils import timezone import time import json from core.utils import ( - RedisClient, acquire_task_lock, release_task_lock, TaskLockRenewer, natural_sort_key, log_system_event, + ensure_custom_properties_dict, ) from core.models import CoreSettings, UserAgent -from asgiref.sync import async_to_sync from core.xtream_codes import Client as XCClient from core.utils import send_websocket_update from .utils import normalize_stream_url @@ -39,6 +33,104 @@ logger = logging.getLogger(__name__) BATCH_SIZE = 1500 # Optimized batch size for threading m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") +_NON_TERMINAL_REFRESH_STATUSES = frozenset({ + M3UAccount.Status.FETCHING, + M3UAccount.Status.PARSING, +}) + + +def _release_task_db_connection(): + """Return the Celery worker's DB connection to a clean state after ORM errors.""" + from django.db import close_old_connections + close_old_connections() + + +def _db_query_with_retry(fn, *, label="DB query", max_retries=2): + """ + Run an ORM read with one connection reset + retry on transient failures. + + Poisoned Celery worker connections often surface as OperationalError or as + ``IndexError: list index out of range`` inside Django's row converters. + """ + from django.db import InterfaceError, OperationalError + + transient_errors = (OperationalError, InterfaceError, IndexError) + for attempt in range(max_retries): + try: + return fn() + except transient_errors as exc: + if attempt + 1 >= max_retries: + raise + logger.warning( + "%s failed (%s), resetting DB connection (%s/%s)", + label, + exc, + attempt + 1, + max_retries, + ) + _release_task_db_connection() + + +def _get_active_m3u_account(account_id): + return _db_query_with_retry( + lambda: M3UAccount.objects.get(id=account_id, is_active=True), + label=f"load active M3U account {account_id}", + ) + + +def _set_m3u_account_status( + account_id, + status, + last_message=None, + *, + notify_error=False, + ws_action="parsing", + ws_error=None, +): + """Update account status using a fresh connection (safe after DB failures).""" + _release_task_db_connection() + update = {"status": status} + if last_message is not None: + update["last_message"] = last_message + try: + M3UAccount.objects.filter(id=account_id).update(**update) + if notify_error: + send_m3u_update( + account_id, + ws_action, + 100, + status="error", + error=ws_error or last_message, + ) + except Exception as e: + logger.error( + f"Failed to set account {account_id} status to {status}: {e}" + ) + + +def _ensure_m3u_refresh_terminal_status(account_id): + """Mark refresh as failed when the task exits while still in progress.""" + _release_task_db_connection() + try: + current_status = ( + M3UAccount.objects.filter(id=account_id) + .values_list("status", flat=True) + .first() + ) + if current_status in _NON_TERMINAL_REFRESH_STATUSES: + message = "Refresh did not complete successfully" + M3UAccount.objects.filter(id=account_id).update( + status=M3UAccount.Status.ERROR, + last_message=message, + ) + send_m3u_update( + account_id, "parsing", 100, status="error", error=message + ) + except Exception as e: + logger.debug( + f"Could not verify terminal refresh status for account {account_id}: {e}" + ) + _EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') @@ -630,7 +722,7 @@ def process_groups(account, groups, scan_start_time=None): logger.info(f"Currently {len(existing_groups)} existing groups") # Check if we should auto-enable new groups based on account settings - account_custom_props = account.custom_properties or {} + account_custom_props = ensure_custom_properties_dict(account.custom_properties) auto_enable_new_groups_live = account_custom_props.get("auto_enable_new_groups_live", True) # Separate existing groups from groups that need to be created @@ -673,7 +765,9 @@ def process_groups(account, groups, scan_start_time=None): existing_rel = existing_relationships[group.name] # Get existing custom properties (now JSONB, no need to parse) - existing_custom_props = existing_rel.custom_properties or {} + existing_custom_props = ensure_custom_properties_dict( + existing_rel.custom_properties + ) # Get the new xc_id from groups data new_xc_id = custom_props.get("xc_id") @@ -696,6 +790,8 @@ def process_groups(account, groups, scan_start_time=None): logger.debug(f"Updated xc_id for group '{group.name}' from '{existing_xc_id}' to '{new_xc_id}' - account {account.id}") else: # Update last_seen even if xc_id hasn't changed + if isinstance(existing_rel.custom_properties, str): + existing_rel.custom_properties = existing_custom_props existing_rel.last_seen = scan_start_time existing_rel.is_stale = False relations_to_update.append(existing_rel) @@ -1755,33 +1851,6 @@ def _range_exhausted_error(mode, start_number, end_number, fallback_start): return f"Channel number range {range_start}-{int(end_number)} is full" -def _custom_properties_as_dict(value): - """ - Normalize a JSONField-backed custom_properties value into a dict. - - Historical data has rows where the field holds a JSON-encoded string - instead of a dict. Django's JSONField serializes whatever it gets, so - `.get()` on one of those rows raises AttributeError and aborts the - entire sync. Treat string values as JSON to parse, and fall back to an - empty dict for anything that isn't a dict after parsing. - """ - import json - - if isinstance(value, dict): - return value - if isinstance(value, str): - try: - parsed = json.loads(value) - except (ValueError, TypeError): - logger.warning( - "custom_properties stored as non-JSON string; ignoring: %r", - value[:100], - ) - return {} - return parsed if isinstance(parsed, dict) else {} - return {} - - def _classify_sync_failure(exc): """ Map an exception raised during per-stream sync to a coarse typed @@ -1890,7 +1959,7 @@ def sync_auto_channels(account_id, scan_start_time=None): custom_epg_id = None # New option: select specific EPG source (takes priority over force_dummy_epg) channel_numbering_mode = "fixed" # Default mode channel_numbering_fallback = 1 # Default fallback for provider mode - group_custom_props = _custom_properties_as_dict( + group_custom_props = ensure_custom_properties_dict( group_relation.custom_properties ) if group_custom_props: @@ -2734,7 +2803,7 @@ def sync_auto_channels(account_id, scan_start_time=None): # "preserve_customized" keeps those with a ChannelOverride row; # "never" disables cleanup. Hidden channels are preserved across all # modes so event/PPV channels that come and go are not silently lost. - cleanup_mode = (account.custom_properties or {}).get( + cleanup_mode = ensure_custom_properties_dict(account.custom_properties).get( "orphan_channel_cleanup", "always" ) if cleanup_mode != "never": @@ -2948,12 +3017,19 @@ def refresh_account_profiles(account_id): if profile_client.authenticate(): # Get account information specific to this profile's credentials profile_account_info = profile_client.get_account_info() + if not isinstance(profile_account_info, dict): + raise TypeError( + f"Unexpected account info type: {type(profile_account_info).__name__}" + ) # Merge with existing custom_properties if they exist - existing_props = profile.custom_properties or {} - existing_props.update(profile_account_info) - profile.custom_properties = existing_props - profile.save(update_fields=['custom_properties']) + profile.custom_properties = { + **ensure_custom_properties_dict( + profile.custom_properties + ), + **profile_account_info, + } + profile.save(update_fields=['custom_properties', 'exp_date']) profiles_updated += 1 logger.info(f"Updated account information for profile '{profile.name}' ({profiles_updated}/{profiles.count()})") @@ -2964,6 +3040,7 @@ def refresh_account_profiles(account_id): except Exception as profile_error: profiles_failed += 1 logger.error(f"Failed to update account information for profile '{profile.name}': {str(profile_error)}") + _release_task_db_connection() # Continue with other profiles even if one fails result_msg = f"Profile refresh complete for account {account.name}: {profiles_updated} updated, {profiles_failed} failed" @@ -2978,6 +3055,8 @@ def refresh_account_profiles(account_id): error_msg = f"Error refreshing profiles for account {account_id}: {str(e)}" logger.error(error_msg) return error_msg + finally: + _release_task_db_connection() @shared_task @@ -3032,10 +3111,11 @@ def refresh_account_info(profile_id): account_info = client.get_account_info() # Update only this specific profile with the new account info - if not profile.custom_properties: - profile.custom_properties = {} - profile.custom_properties.update(account_info) - profile.save() + profile.custom_properties = { + **ensure_custom_properties_dict(profile.custom_properties), + **account_info, + } + profile.save(update_fields=['custom_properties', 'exp_date']) # Send success notification to frontend via websocket send_websocket_update( @@ -3098,10 +3178,26 @@ def refresh_single_m3u_account(account_id): lock_renewer = TaskLockRenewer("refresh_single_m3u_account", account_id) lock_renewer.start() + _release_task_db_connection() + try: return _refresh_single_m3u_account_impl(account_id) + except Exception as e: + logger.error( + f"refresh_single_m3u_account failed for account {account_id}: {e}", + exc_info=True, + ) + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + f"Error processing M3U: {str(e)[:500]}", + notify_error=True, + ws_error=str(e)[:500], + ) + raise finally: - # Guaranteed cleanup on all exit paths (success, exception, early return) + _ensure_m3u_refresh_terminal_status(account_id) + _release_task_db_connection() lock_renewer.stop() release_task_lock("refresh_single_m3u_account", account_id) @@ -3116,22 +3212,25 @@ def _refresh_single_m3u_account_impl(account_id): streams_deleted = 0 try: - account = M3UAccount.objects.get(id=account_id, is_active=True) + account = _get_active_m3u_account(account_id) if not account.is_active: logger.debug(f"Account {account_id} is not active, skipping.") return - # Set status to fetching - account.status = M3UAccount.Status.FETCHING - account.save(update_fields=['status']) + # Set status to fetching and replace stale completion messages. + _set_m3u_account_status( + account_id, + M3UAccount.Status.FETCHING, + "Refresh in progress...", + ) + account = _get_active_m3u_account(account_id) filters = list(account.filters.all()) # Check if VOD is enabled for this account - vod_enabled = False - if account.custom_properties: - custom_props = account.custom_properties or {} - vod_enabled = custom_props.get('enable_vod', False) + vod_enabled = ensure_custom_properties_dict(account.custom_properties).get( + 'enable_vod', False + ) except M3UAccount.DoesNotExist: # The M3U account doesn't exist, so delete the periodic task if it exists @@ -3197,6 +3296,16 @@ def _refresh_single_m3u_account_impl(account_id): logger.error( f"Failed to refresh M3U groups for account {account_id}: {result}" ) + error_msg = ( + "Failed to refresh M3U groups - download failed or other error" + ) + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, + ) return "Failed to update m3u account - download failed or other error" extinf_data, groups = result @@ -3211,23 +3320,23 @@ def _refresh_single_m3u_account_impl(account_id): # For XC accounts, empty extinf_data is normal at this stage if not extinf_data and not is_xc_account: logger.error(f"No streams found for non-XC account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No streams found in M3U source" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "parsing", 100, status="error", error="No streams found" + error_msg = "No streams found in M3U source" + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) except Exception as e: logger.error(f"Exception in refresh_m3u_groups: {str(e)}", exc_info=True) - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error refreshing M3U groups: {str(e)}" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( + error_msg = f"Error refreshing M3U groups: {str(e)[:500]}" + _set_m3u_account_status( account_id, - "parsing", - 100, - status="error", - error=f"Error refreshing M3U groups: {str(e)}", + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) return "Failed to update m3u account" @@ -3241,15 +3350,13 @@ def _refresh_single_m3u_account_impl(account_id): # Modified validation logic for different account types if (not groups) or (not is_xc_account and not extinf_data): logger.error(f"No data to process for account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No data available for processing" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( + error_msg = "No data available for processing" + _set_m3u_account_status( account_id, - "parsing", - 100, - status="error", - error="No data available for processing", + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) return "Failed to update m3u account, no data available" @@ -3365,7 +3472,7 @@ def _refresh_single_m3u_account_impl(account_id): group_id = rel.channel_group.id # Load the custom properties with the xc_id - custom_props = rel.custom_properties or {} + custom_props = ensure_custom_properties_dict(rel.custom_properties) if "xc_id" in custom_props: filtered_groups[group_name] = { "xc_id": custom_props["xc_id"], @@ -3590,13 +3697,7 @@ def _refresh_single_m3u_account_impl(account_id): except Exception as e: logger.error(f"Error processing M3U for account {account_id}: {str(e)}") - try: - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error processing M3U: {str(e)}" - account.save(update_fields=["status", "last_message"]) - except Exception: - logger.debug(f"Failed to update account {account_id} status during error handling") - raise # Re-raise the exception for Celery to handle + raise finally: # Free large data structures regardless of success or failure if 'existing_groups' in locals(): @@ -3623,6 +3724,8 @@ def _refresh_single_m3u_account_impl(account_id): except OSError: pass + _release_task_db_connection() + return f"Dispatched jobs complete." diff --git a/apps/m3u/tests/test_refresh_db_recovery.py b/apps/m3u/tests/test_refresh_db_recovery.py new file mode 100644 index 00000000..42db8bd5 --- /dev/null +++ b/apps/m3u/tests/test_refresh_db_recovery.py @@ -0,0 +1,74 @@ +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.m3u.tasks import ( + _db_query_with_retry, + _get_active_m3u_account, + _release_task_db_connection, + refresh_single_m3u_account, +) + + +class DbQueryWithRetryTests(SimpleTestCase): + def test_retries_after_index_error_from_poisoned_connection(self): + fn = MagicMock(side_effect=[IndexError("list index out of range"), "ok"]) + + with patch( + "apps.m3u.tasks._release_task_db_connection" + ) as mock_release: + result = _db_query_with_retry(fn, label="test query") + + self.assertEqual(result, "ok") + self.assertEqual(fn.call_count, 2) + mock_release.assert_called_once() + + def test_raises_after_exhausting_retries(self): + fn = MagicMock(side_effect=IndexError("list index out of range")) + + with patch("apps.m3u.tasks._release_task_db_connection"): + with self.assertRaises(IndexError): + _db_query_with_retry(fn, label="test query", max_retries=2) + + self.assertEqual(fn.call_count, 2) + + +class RefreshTaskDbStartupTests(SimpleTestCase): + @patch("apps.m3u.tasks._ensure_m3u_refresh_terminal_status") + @patch("apps.m3u.tasks._refresh_single_m3u_account_impl") + @patch("apps.m3u.tasks.release_task_lock") + @patch("apps.m3u.tasks.acquire_task_lock", return_value=True) + @patch("apps.m3u.tasks.TaskLockRenewer") + @patch("apps.m3u.tasks._release_task_db_connection") + def test_refresh_releases_db_connection_before_impl( + self, + mock_release, + _mock_renewer, + _mock_acquire, + _mock_release_lock, + mock_impl, + _mock_ensure_terminal, + ): + call_order = [] + + def track_release(): + call_order.append("release") + + mock_release.side_effect = track_release + mock_impl.side_effect = lambda *_a, **_k: call_order.append("impl") or "done" + + result = refresh_single_m3u_account(140) + + self.assertEqual(result, "done") + self.assertEqual(call_order[:2], ["release", "impl"]) + + @patch("apps.m3u.tasks.M3UAccount") + def test_get_active_m3u_account_uses_retry_helper(self, mock_model): + mock_model.objects.get.return_value = MagicMock(is_active=True) + + with patch("apps.m3u.tasks._db_query_with_retry") as mock_retry: + mock_retry.side_effect = lambda fn, **_: fn() + account = _get_active_m3u_account(140) + + mock_retry.assert_called_once() + self.assertTrue(account.is_active) diff --git a/core/utils.py b/core/utils.py index 188d9cb2..29c613b3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,3 +1,4 @@ +import json import redis import logging import time @@ -7,7 +8,6 @@ from pathlib import Path import re from django.conf import settings from redis.exceptions import ConnectionError, TimeoutError -from django.core.cache import cache from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.core.validators import URLValidator @@ -71,6 +71,46 @@ def natural_sort_key(text): return [convert(c) for c in re.split('([0-9]+)', text)] + +def custom_properties_as_dict(value): + """ + Normalize a JSONField-backed custom_properties value into a dict. + + Historical rows (TextField era and early JSONField migration) may store a + JSON-encoded string instead of an object. API clients can also submit a + string value because JSONField accepts any JSON type. Call this before + reading or merging custom_properties. + """ + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except (ValueError, TypeError): + logger.warning( + "custom_properties stored as non-JSON string; ignoring: %r", + value[:100], + ) + return {} + return parsed if isinstance(parsed, dict) else {} + if value is None: + return {} + return {} + + +def ensure_custom_properties_dict(value): + """ + Return a dict for read/merge/bulk-write paths. Dict values pass through + without re-parsing. Use model ``save()`` (not this) as the canonical + normalizer for ORM writes that go through ``save()``. + """ + if isinstance(value, dict): + return value + if value is None: + return {} + return custom_properties_as_dict(value) + + class RedisClient: _client = None _buffer = None diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 8154b52e..18be81eb 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -2,7 +2,7 @@ import os from celery import Celery import logging -from celery.signals import task_postrun, worker_ready +from celery.signals import task_postrun, task_prerun, worker_ready logger = logging.getLogger(__name__) @@ -68,18 +68,30 @@ app.conf.task_routes = { 'apps.channels.tasks.run_recording': {'queue': 'dvr'}, } + +@task_prerun.connect +def reset_db_connection_before_task(**kwargs): + """Discard stale DB connections before each task (Celery workers are long-lived).""" + from django.db import close_old_connections + + try: + close_old_connections() + except Exception: + pass + + # Add memory cleanup after task completion @task_postrun.connect # Use the imported signal def cleanup_task_memory(**kwargs): """Clean up memory and database connections after each task completes""" - from django.db import connection + from django.db import close_old_connections # Get task name from kwargs task_name = kwargs.get('task').name if kwargs.get('task') else '' - # Close database connection for this Celery worker process + # Return all DB connections to the pool in a clean state try: - connection.close() + close_old_connections() except Exception: pass diff --git a/tests/test_custom_properties_as_dict.py b/tests/test_custom_properties_as_dict.py new file mode 100644 index 00000000..bd863fdc --- /dev/null +++ b/tests/test_custom_properties_as_dict.py @@ -0,0 +1,74 @@ +import uuid + +from django.test import SimpleTestCase, TestCase + +from core.utils import custom_properties_as_dict, ensure_custom_properties_dict +from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.channels.models import ChannelGroupM3UAccount, ChannelGroup + + +class CustomPropertiesAsDictTests(SimpleTestCase): + def test_dict_passthrough(self): + value = {"enable_vod": True} + self.assertIs(custom_properties_as_dict(value), value) + + def test_json_string_parsed(self): + self.assertEqual( + custom_properties_as_dict('{"enable_vod": true}'), + {"enable_vod": True}, + ) + + def test_non_json_string_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict("not-json"), {}) + + def test_json_array_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict("[1, 2]"), {}) + + def test_none_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict(None), {}) + + +class EnsureCustomPropertiesDictTests(SimpleTestCase): + def test_dict_passthrough_without_reparse(self): + value = {"enable_vod": True} + self.assertIs(ensure_custom_properties_dict(value), value) + + def test_none_returns_empty_dict(self): + self.assertEqual(ensure_custom_properties_dict(None), {}) + + +class CustomPropertiesSaveNormalizationTests(TestCase): + def test_m3u_account_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}", + custom_properties='{"enable_vod": true}', + ) + account.refresh_from_db() + self.assertEqual(account.custom_properties, {"enable_vod": True}) + + def test_profile_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}" + ) + profile = M3UAccountProfile.objects.get( + m3u_account=account, is_default=True + ) + profile.custom_properties = '{"notes": "hello"}' + profile.save(update_fields=["custom_properties"]) + profile.refresh_from_db() + self.assertEqual(profile.custom_properties, {"notes": "hello"}) + + def test_group_relation_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}" + ) + group = ChannelGroup.objects.create( + name=f"Sports {uuid.uuid4().hex[:8]}" + ) + rel = ChannelGroupM3UAccount.objects.create( + m3u_account=account, + channel_group=group, + custom_properties='{"force_dummy_epg": true}', + ) + rel.refresh_from_db() + self.assertEqual(rel.custom_properties, {"force_dummy_epg": True}) From 460677aeea84f6c8c9b25ea4f30c09cbeca16d0d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 15:24:23 -0500 Subject: [PATCH 50/81] refactor(channels): optimize stream retrieval in ChannelSerializer and add tests for query efficiency - Updated `get_streams` method in `ChannelSerializer` to use prefetched `channelstream_set`, improving performance by reducing database queries. - Added `ChannelListIncludeStreamsQueryTests` to ensure that the number of queries remains stable as channels grow, verifying efficient stream inclusion in API responses. --- apps/channels/serializers.py | 11 ++-- apps/channels/tests/test_channel_api.py | 69 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 2e80d840..61bbef8d 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -551,10 +551,13 @@ class ChannelSerializer(serializers.ModelSerializer): return LogoSerializer(obj.logo).data def get_streams(self, obj): - """Retrieve ordered stream IDs for GET requests.""" - return StreamSerializer( - obj.streams.all().order_by("channelstream__order"), many=True - ).data + """Retrieve ordered streams for GET requests using prefetched channelstream_set.""" + ordered_streams = [ + cs.stream + for cs in obj.channelstream_set.all() + if cs.stream_id is not None + ] + return StreamSerializer(ordered_streams, many=True).data def create(self, validated_data): streams = validated_data.pop("streams", []) diff --git a/apps/channels/tests/test_channel_api.py b/apps/channels/tests/test_channel_api.py index 363f5925..5d60a8da 100644 --- a/apps/channels/tests/test_channel_api.py +++ b/apps/channels/tests/test_channel_api.py @@ -504,3 +504,72 @@ class SeriesRuleAPITests(TestCase): def test_bulk_remove_requires_tvg_id_or_title(self): resp = self.client.post(self.bulk_remove_url, {}, format="json") self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + + +class ChannelListIncludeStreamsQueryTests(TestCase): + """include_streams=true must not issue one stream query per channel.""" + + def setUp(self): + from apps.channels.models import ChannelStream, Stream + from apps.m3u.models import M3UAccount + + self.user = User.objects.create_user(username="list_admin", password="x") + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + + self.account = M3UAccount.objects.create( + name="list-test-account", + account_type="XC", + username="user", + password="pass", + ) + self.group = ChannelGroup.objects.create(name=f"List Group {self.id}") + + def _add_channel_with_stream(self, number): + from apps.channels.models import ChannelStream, Stream + + channel = Channel.objects.create( + channel_number=float(number), + name=f"Channel {number}", + channel_group=self.group, + ) + stream = Stream.objects.create( + name=f"Stream {number}", + url=f"http://example.com/{number}.ts", + m3u_account=self.account, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + return channel + + def _query_count_for_list(self): + from django.db import connection + from django.test.utils import CaptureQueriesContext + + with CaptureQueriesContext(connection) as ctx: + response = self.client.get( + "/api/channels/channels/", + {"page": 1, "page_size": 50, "include_streams": "true"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + return len(ctx.captured_queries) + + def test_include_streams_query_count_stable_as_channels_grow(self): + self._add_channel_with_stream(1) + self._add_channel_with_stream(2) + self._add_channel_with_stream(3) + q_small = self._query_count_for_list() + + self._add_channel_with_stream(4) + self._add_channel_with_stream(5) + self._add_channel_with_stream(6) + self._add_channel_with_stream(7) + q_large = self._query_count_for_list() + + self.assertEqual( + q_small, + q_large, + "include_streams list should use prefetched channelstream_set, " + "not one streams M2M query per channel", + ) From c2ac08fdfdc13142a694f10ba9bfa4fee661cd9f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 15:53:55 -0500 Subject: [PATCH 51/81] enhancement(epg): Performance improvements and API enhancements for EPG refreshes. - Updated EPG import logic to reject dummy sources without loading the large `programme_index`, optimizing memory usage during API calls. --- CHANGELOG.md | 5 ++ apps/epg/api_views.py | 22 ++++--- apps/epg/tests/test_epg_import_api.py | 95 +++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 apps/epg/tests/test_epg_import_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a5f6a0..9c0b1729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. + ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. +- **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index b268ef65..fbc419c5 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -1119,18 +1119,24 @@ class EPGImportAPIView(APIView): epg_id = request.data.get("id", None) force = bool(request.data.get("force", False)) - # Check if this is a dummy EPG source - try: + # Reject dummy sources without loading programme_index (multi-MB JSON). + if epg_id is not None: from .models import EPGSource - epg_source = EPGSource.objects.get(id=epg_id) - if epg_source.source_type == 'dummy': - logger.info(f"EPGImportAPIView: Skipping refresh for dummy EPG source {epg_id}") + + if EPGSource.objects.filter( + id=epg_id, source_type="dummy" + ).exists(): + logger.info( + "EPGImportAPIView: Skipping refresh for dummy EPG source %s", + epg_id, + ) return Response( - {"success": False, "message": "Dummy EPG sources do not require refreshing."}, + { + "success": False, + "message": "Dummy EPG sources do not require refreshing.", + }, status=status.HTTP_400_BAD_REQUEST, ) - except EPGSource.DoesNotExist: - pass # Let the task handle the missing source refresh_epg_data.delay(epg_id, force=force) # Trigger Celery task logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") diff --git a/apps/epg/tests/test_epg_import_api.py b/apps/epg/tests/test_epg_import_api.py new file mode 100644 index 00000000..c58f2a45 --- /dev/null +++ b/apps/epg/tests/test_epg_import_api.py @@ -0,0 +1,95 @@ +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.db import connection +from django.test.utils import CaptureQueriesContext +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIClient + +from apps.epg.models import EPGSource + +User = get_user_model() + +IMPORT_URL = "/api/epg/import/" + + +class EPGImportAPITests(TestCase): + def setUp(self): + self.user = User.objects.create_user( + username="epg_import_admin", password="testpass123" + ) + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_dummy_source_rejected_without_dispatch(self, mock_delay): + source = EPGSource.objects.create( + name="Dummy EPG", + source_type="dummy", + ) + + response = self.client.post( + IMPORT_URL, {"id": source.id}, format="json" + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(response.data["success"]) + mock_delay.assert_not_called() + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_xmltv_dispatches_without_loading_programme_index( + self, mock_delay + ): + source = EPGSource.objects.create( + name="Large Index XMLTV", + source_type="xmltv", + url="http://example.com/epg.xml", + programme_index={ + "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)}, + "interleaved_channels": [], + }, + ) + mock_delay.reset_mock() + + with CaptureQueriesContext(connection) as ctx: + response = self.client.post( + IMPORT_URL, {"id": source.id}, format="json" + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertTrue(response.data["success"]) + mock_delay.assert_called_once_with(source.id, force=False) + for query in ctx.captured_queries: + self.assertNotIn( + "programme_index", + query["sql"].lower(), + "import trigger should not read programme_index", + ) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_missing_source_still_dispatches(self, mock_delay): + response = self.client.post(IMPORT_URL, {"id": 99999}, format="json") + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + mock_delay.assert_called_once_with(99999, force=False) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_honours_force_flag(self, mock_delay): + source = EPGSource.objects.create( + name="Force XMLTV", + source_type="xmltv", + url="http://example.com/epg.xml", + ) + mock_delay.reset_mock() + + response = self.client.post( + IMPORT_URL, + {"id": source.id, "force": True}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + mock_delay.assert_called_once_with(source.id, force=True) From 04394b7eac686db1eef98433e022a22f62800914 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 16:39:52 -0500 Subject: [PATCH 52/81] fix(epg): enhance DB connection management and error handling during EPG refreshes - Implemented robust DB connection management in `refresh_epg_data` to handle transient errors, ensuring reliable status updates for EPG sources. - Added retry logic for database queries and improved error handling to prevent task failures from unhandled exceptions. - Updated EPG source status handling to ensure accurate reporting of errors and terminal states during refresh operations. --- CHANGELOG.md | 1 + apps/epg/tasks.py | 275 ++++++++++++++------- apps/epg/tests/test_refresh_db_recovery.py | 106 ++++++++ 3 files changed, 291 insertions(+), 91 deletions(-) create mode 100644 apps/epg/tests/test_refresh_db_recovery.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c0b1729..5d8aeeb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) +- **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. - **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0e8eda42..9661c5b9 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -39,6 +39,105 @@ from core.utils import ( logger = logging.getLogger(__name__) +_NON_TERMINAL_REFRESH_STATUSES = frozenset({ + EPGSource.STATUS_FETCHING, + EPGSource.STATUS_PARSING, +}) + + +def _release_task_db_connection(): + """Return the Celery worker's DB connection to a clean state after ORM errors.""" + from django.db import close_old_connections + close_old_connections() + + +def _db_query_with_retry(fn, *, label="DB query", max_retries=2): + """ + Run an ORM read with one connection reset + retry on transient failures. + + Poisoned Celery worker connections often surface as OperationalError or as + ``IndexError: list index out of range`` inside Django's row converters. + """ + from django.db import InterfaceError, OperationalError + + transient_errors = (OperationalError, InterfaceError, IndexError) + for attempt in range(max_retries): + try: + return fn() + except transient_errors as exc: + if attempt + 1 >= max_retries: + raise + logger.warning( + "%s failed (%s), resetting DB connection (%s/%s)", + label, + exc, + attempt + 1, + max_retries, + ) + _release_task_db_connection() + + +def _get_epg_source(source_id): + return _db_query_with_retry( + lambda: EPGSource.objects.get(id=source_id), + label=f"load EPG source {source_id}", + ) + + +def _set_epg_source_status( + source_id, + status, + last_message=None, + *, + notify_error=False, + ws_action="refresh", + ws_error=None, +): + """Update source status using a fresh connection (safe after DB failures).""" + _release_task_db_connection() + update = {"status": status} + if last_message is not None: + update["last_message"] = last_message + try: + EPGSource.objects.filter(id=source_id).update(**update) + if notify_error: + send_epg_update( + source_id, + ws_action, + 100, + status="error", + error=ws_error or last_message, + ) + except Exception as e: + logger.error( + f"Failed to set EPG source {source_id} status to {status}: {e}" + ) + + +def _ensure_epg_refresh_terminal_status(source_id): + """Mark refresh as failed when the task exits while still in progress.""" + _release_task_db_connection() + try: + current_status = ( + EPGSource.objects.filter(id=source_id) + .values_list("status", flat=True) + .first() + ) + if current_status in _NON_TERMINAL_REFRESH_STATUSES: + message = "Refresh did not complete successfully" + EPGSource.objects.filter(id=source_id).update( + status=EPGSource.STATUS_ERROR, + last_message=message, + ) + send_epg_update( + source_id, "refresh", 100, status="error", error=message + ) + except Exception as e: + logger.debug( + f"Could not verify terminal refresh status for EPG source {source_id}: {e}" + ) + + SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' SD_DAYS_TO_FETCH = 20 SD_PROGRAM_BATCH_SIZE = 5000 @@ -498,104 +597,90 @@ def refresh_epg_data(source_id, force=False): lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) lock_renewer.start() - source = None + _release_task_db_connection() + try: - # Try to get the EPG source - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - # The EPG source doesn't exist, so delete the periodic task if it exists - logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") - - # Call the shared function to delete the task - if delete_epg_refresh_task_by_id(source_id): - logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") - else: - logger.info(f"No orphaned task found for EPG source {source_id}") - - # Release the lock and exit - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return f"EPG source {source_id} does not exist, task cleaned up" - - # The source exists but is not active, just skip processing - if not source.is_active: - logger.info(f"EPG source {source_id} is not active. Skipping.") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - # Skip refresh for dummy EPG sources - they don't need refreshing - if source.source_type == 'dummy': - logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - gc.collect() - return - - # Continue with the normal processing... - logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") - if source.source_type == 'xmltv': - # Invalidate the byte-offset index before downloading the new file - # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) - fetch_success = fetch_xmltv(source) - if not fetch_success: - logger.error(f"Failed to fetch XMLTV for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - parse_channels_success = parse_channels_only(source) - if not parse_channels_success: - logger.error(f"Failed to parse channels for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - # Build byte-offset index after programme data is committed so refresh - # does not compete for memory/IO during the programme swap. - parse_programs_for_source(source) - build_programme_index_task.delay(source.id) - - elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source, force=force) - - source.save(update_fields=['updated_at']) - # After successful EPG refresh, evaluate DVR series rules to schedule new episodes - try: - from apps.channels.tasks import evaluate_series_rules - evaluate_series_rules.delay() - except Exception: - pass + return _refresh_epg_data_impl(source_id, force=force) except Exception as e: - logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) - try: - if source: - source.status = 'error' - source.last_message = f"Error refreshing EPG data: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) - except Exception as inner_e: - logger.error(f"Error updating source status: {inner_e}") + logger.error( + f"Error in refresh_epg_data for source {source_id}: {e}", + exc_info=True, + ) + _set_epg_source_status( + source_id, + EPGSource.STATUS_ERROR, + f"Error refreshing EPG data: {str(e)[:500]}", + notify_error=True, + ws_error=str(e)[:500], + ) finally: - # Clear references to ensure proper garbage collection - source = None - # Force garbage collection before releasing the lock + _ensure_epg_refresh_terminal_status(source_id) + _release_task_db_connection() gc.collect() - connection.close() lock_renewer.stop() release_task_lock('refresh_epg_data', source_id) +def _refresh_epg_data_impl(source_id, force=False): + try: + source = _get_epg_source(source_id) + except EPGSource.DoesNotExist: + logger.warning( + f"EPG source with ID {source_id} not found, but task was triggered. " + "Cleaning up orphaned task." + ) + + if delete_epg_refresh_task_by_id(source_id): + logger.info( + f"Successfully cleaned up orphaned task for EPG source {source_id}" + ) + else: + logger.info(f"No orphaned task found for EPG source {source_id}") + + return f"EPG source {source_id} does not exist, task cleaned up" + + if not source.is_active: + logger.info(f"EPG source {source_id} is not active. Skipping.") + return + + if source.source_type == 'dummy': + logger.info( + f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})" + ) + return + + logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") + if source.source_type == 'xmltv': + # Invalidate the byte-offset index before downloading the new file + # so stale offsets are never used during the refresh window. + EPGSource.objects.filter(id=source.id).update(programme_index=None) + if not fetch_xmltv(source): + logger.error(f"Failed to fetch XMLTV for source {source.name}") + return + + if not parse_channels_only(source): + logger.error(f"Failed to parse channels for source {source.name}") + return + + # Build byte-offset index after programme data is committed so refresh + # does not compete for memory/IO during the programme swap. + if not parse_programs_for_source(source): + logger.error(f"Failed to parse programs for source {source.name}") + return + + build_programme_index_task.delay(source.id) + + elif source.source_type == 'schedules_direct': + fetch_schedules_direct(source, force=force) + + EPGSource.objects.filter(id=source.id).update(updated_at=timezone.now()) + try: + from apps.channels.tasks import evaluate_series_rules + evaluate_series_rules.delay() + except Exception: + pass + + def fetch_xmltv(source): # Handle cases with local file but no URL if not source.url and source.file_path and os.path.exists(source.file_path): @@ -1167,7 +1252,15 @@ def parse_channels_only(source): # Update status to error source.status = 'error' source.last_message = f"No URL provided, cannot fetch EPG data" - source.save(update_fields=['updated_at']) + source.save(update_fields=['status', 'last_message']) + send_epg_update( + source.id, + "parsing_channels", + 100, + status="error", + error="No URL provided", + ) + return False # Initialize process variable for memory tracking only in debug mode try: diff --git a/apps/epg/tests/test_refresh_db_recovery.py b/apps/epg/tests/test_refresh_db_recovery.py new file mode 100644 index 00000000..11885b34 --- /dev/null +++ b/apps/epg/tests/test_refresh_db_recovery.py @@ -0,0 +1,106 @@ +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.epg.tasks import ( + _db_query_with_retry, + _ensure_epg_refresh_terminal_status, + _get_epg_source, + _release_task_db_connection, + refresh_epg_data, +) + + +class DbQueryWithRetryTests(SimpleTestCase): + def test_retries_after_index_error_from_poisoned_connection(self): + fn = MagicMock(side_effect=[IndexError("list index out of range"), "ok"]) + + with patch( + "apps.epg.tasks._release_task_db_connection" + ) as mock_release: + result = _db_query_with_retry(fn, label="test query") + + self.assertEqual(result, "ok") + self.assertEqual(fn.call_count, 2) + mock_release.assert_called_once() + + def test_raises_after_exhausting_retries(self): + fn = MagicMock(side_effect=IndexError("list index out of range")) + + with patch("apps.epg.tasks._release_task_db_connection"): + with self.assertRaises(IndexError): + _db_query_with_retry(fn, label="test query", max_retries=2) + + self.assertEqual(fn.call_count, 2) + + +class RefreshTaskDbStartupTests(SimpleTestCase): + @patch("apps.epg.tasks._ensure_epg_refresh_terminal_status") + @patch("apps.epg.tasks._refresh_epg_data_impl") + @patch("apps.epg.tasks.release_task_lock") + @patch("apps.epg.tasks.acquire_task_lock", return_value=True) + @patch("apps.epg.tasks.TaskLockRenewer") + @patch("apps.epg.tasks._release_task_db_connection") + def test_refresh_releases_db_connection_before_impl( + self, + mock_release, + _mock_renewer, + _mock_acquire, + _mock_release_lock, + mock_impl, + _mock_ensure_terminal, + ): + call_order = [] + + def track_release(): + call_order.append("release") + + mock_release.side_effect = track_release + mock_impl.side_effect = lambda *_a, **_k: call_order.append("impl") or "done" + + result = refresh_epg_data(42) + + self.assertEqual(result, "done") + self.assertEqual(call_order[:2], ["release", "impl"]) + + @patch("apps.epg.tasks.EPGSource") + def test_get_epg_source_uses_retry_helper(self, mock_model): + mock_source = MagicMock(id=42) + mock_model.objects.get.return_value = mock_source + + with patch("apps.epg.tasks._db_query_with_retry") as mock_retry: + mock_retry.side_effect = lambda fn, **_: fn() + source = _get_epg_source(42) + + mock_retry.assert_called_once() + mock_model.objects.get.assert_called_once_with(id=42) + self.assertIs(source, mock_source) + + +class EnsureEpgTerminalStatusTests(SimpleTestCase): + @patch("apps.epg.tasks.send_epg_update") + @patch("apps.epg.tasks._release_task_db_connection") + def test_marks_stuck_fetching_as_error(self, _mock_release, mock_ws): + with patch("apps.epg.tasks.EPGSource") as mock_model: + mock_model.STATUS_ERROR = "error" + qs = MagicMock() + mock_model.objects.filter.return_value = qs + qs.values_list.return_value.first.return_value = "fetching" + + _ensure_epg_refresh_terminal_status(7) + + qs.update.assert_called_once() + mock_ws.assert_called_once() + + @patch("apps.epg.tasks.send_epg_update") + @patch("apps.epg.tasks._release_task_db_connection") + def test_leaves_success_unchanged(self, _mock_release, mock_ws): + with patch("apps.epg.tasks.EPGSource") as mock_model: + qs = MagicMock() + mock_model.objects.filter.return_value = qs + qs.values_list.return_value.first.return_value = "success" + + _ensure_epg_refresh_terminal_status(7) + + qs.update.assert_not_called() + mock_ws.assert_not_called() From 7139c88c350adf25414e07decc5feadc11946777 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 17:12:43 -0500 Subject: [PATCH 53/81] enhancement(epg): improve custom dummy EPG generation performance - Optimized `generate_epg()` to reduce database queries by prefetching ordered channel streams and implementing a new method for custom dummy epg name matching. - Enhanced handling of dummy EPG sources to minimize SQL round-trips, improving performance for large guides with numerous custom-dummy channels. --- CHANGELOG.md | 4 ++ apps/output/views.py | 87 ++++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d8aeeb6..0517493d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. +### Performance + +- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. + ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) diff --git a/apps/output/views.py b/apps/output/views.py index 8bcc37dc..dba35951 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -338,6 +338,29 @@ def generate_m3u(request, profile_name=None, user=None): return response +def _ordered_channel_streams(channel): + """Return a channel's streams ordered by channelstream join order.""" + prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') + if prefetched is not None: + return list(prefetched) + return list(channel.streams.all().order_by('channelstream__order')) + + +def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): + """Name used for custom dummy EPG regex matching (channel or stream title). + + Returns (name, stream_lookup_failed). stream_lookup_failed is True only when + name_source is 'stream' but the configured index is missing or out of range. + """ + if custom_props.get('name_source') != 'stream': + return effective_name, False + stream_index = custom_props.get('stream_index', 1) - 1 + streams = _ordered_channel_streams(channel) + if 0 <= stream_index < len(streams): + return streams[stream_index].name, False + return effective_name, True + + def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): """ Generate dummy programs using custom fallback templates when patterns don't match. @@ -1397,9 +1420,11 @@ def generate_epg(request, profile_name=None, user=None): with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") + .prefetch_related( + Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + ) ) - # For dummy EPG, use either the specified value or default to 3 days dummy_days = num_days if num_days > 0 else 3 @@ -1440,6 +1465,8 @@ def generate_epg(request, profile_name=None, user=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw + dummy_epg_ids_for_program_check = set() + # Process channels for the section for channel in channels: effective_name = channel.effective_name @@ -1465,23 +1492,17 @@ def generate_epg(request, profile_name=None, user=None): # Check if this is a custom dummy EPG with channel logo URL template if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + if channel.effective_epg_data_id: + dummy_epg_ids_for_program_check.add(channel.effective_epg_data_id) epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties channel_logo_url_template = custom_props.get('channel_logo_url', '') if channel_logo_url_template: - # Determine which name to use for pattern matching (same logic as program generation) - pattern_match_name = effective_name - name_source = custom_props.get('name_source') - - if name_source == 'stream': - stream_index = custom_props.get('stream_index', 1) - 1 - channel_streams = channel.streams.all().order_by('channelstream__order') - - if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): - stream = list(channel_streams)[stream_index] - pattern_match_name = stream.name + pattern_match_name, _ = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) # Try to extract groups from the channel/stream name and build the logo URL title_pattern = custom_props.get('title_pattern', '') @@ -1538,10 +1559,17 @@ def generate_epg(request, profile_name=None, user=None): yield channel_xml xml_lines = [] # Clear to save memory + dummy_epg_with_programs = set() + if dummy_epg_ids_for_program_check: + dummy_epg_with_programs = set( + ProgramData.objects.filter(epg_id__in=dummy_epg_ids_for_program_check) + .values_list('epg_id', flat=True) + .distinct() + ) + # Pre-pass: categorize channels into dummy and real EPG groups dummy_program_list = [] # (channel_id, pattern_match_name, epg_source_or_None) real_epg_map = {} # epg_data_id -> [channel_id, ...] - dummy_epg_checked = {} # epg_data_id -> bool (has stored programs) for channel in channels: effective_name = channel.effective_name @@ -1569,26 +1597,31 @@ def generate_epg(request, profile_name=None, user=None): epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties - name_source = custom_props.get('name_source') - - if name_source == 'stream': + pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + if ( + custom_props.get('name_source') == 'stream' + and not stream_lookup_failed + and pattern_match_name != effective_name + ): stream_index = custom_props.get('stream_index', 1) - 1 - channel_streams = channel.streams.all().order_by('channelstream__order') - - if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): - stream = list(channel_streams)[stream_index] - pattern_match_name = stream.name - logger.debug(f"Using stream name for parsing: {pattern_match_name} (stream index: {stream_index})") - else: - logger.warning(f"Stream index {stream_index} not found for channel {effective_name}, falling back to channel name") + logger.debug( + f"Using stream name for parsing: {pattern_match_name} " + f"(stream index: {stream_index})" + ) + elif stream_lookup_failed: + stream_index = custom_props.get('stream_index', 1) - 1 + logger.warning( + f"Stream index {stream_index} not found for channel " + f"{effective_name}, falling back to channel name" + ) if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) else: if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if effective_epg_data_id not in dummy_epg_checked: - dummy_epg_checked[effective_epg_data_id] = effective_epg_data.programs.exists() - if dummy_epg_checked[effective_epg_data_id]: + if effective_epg_data_id in dummy_epg_with_programs: real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) else: dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) From a29704af55015597e4f5215b300a4b584d42f892 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:40:10 -0500 Subject: [PATCH 54/81] fix(channels): account for group_override in the auto-sync range-conflict check The range-conflict warning classified each channel in the configured range as either this config's own auto-sync output or a real conflict, comparing each occupant against the source group. When a group sets a group_override, auto-sync creates its channels in the override target group, so the config's own channels failed that comparison and were misclassified as a conflict. - Add effectiveSyncGroupId to resolve the group the sync's channels actually land in (the group_override target when set, otherwise the source group). - Compare occupants against that effective target, so the config's own output is recognized while genuine conflicts (manual channels, channels from another account, channels in a different group, user-pinned numbers) still warn. - Add Vitest coverage for the helper, the override case, and an over-suppression guard, plus a backend test asserting the numbers-in-range endpoint reports the override target group. --- apps/m3u/tests/test_sync_correctness.py | 28 +++++ .../src/components/forms/LiveGroupFilter.jsx | 13 +- .../src/utils/forms/LiveGroupFilterUtils.js | 12 ++ .../__tests__/LiveGroupFilterUtils.test.js | 111 ++++++++++++++++++ 4 files changed, 161 insertions(+), 3 deletions(-) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 3cc24942..8ae09fa8 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -644,6 +644,34 @@ class NumbersInRangeLookupTests(TestCase): ) self.assertFalse(occupant["has_channel_number_override"]) + def test_group_override_channel_reports_target_group(self): + # When auto-sync routes channels into a different group via + # group_override, the occupant's channel_group_id is the override + # target, not the source group being configured. The frontend relies + # on this to recognize override-routed channels as the config's own + # output (effectiveSyncGroupId), so the warning does not flag them. + account = _make_account() + source = _make_group(name="SourceGrp") + target = _make_group(name="TargetGrp") + Channel.objects.create( + name="Routed", + channel_number=3210, + channel_group=target, + auto_created=True, + auto_created_by=account, + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=3210&end=3210" + ) + + occupant = response.data["occupants"][0] + self.assertEqual(occupant["channel_group_id"], target.id) + self.assertNotEqual(occupant["channel_group_id"], source.id) + self.assertTrue(occupant["auto_created"]) + self.assertEqual(occupant["auto_created_by_account_id"], account.id) + def test_manual_channel_exposed_with_auto_created_false(self): # Manual channels are always a real collision worth surfacing. # The response must flag them with auto_created=False and a null diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index 626ec55e..5ed2be94 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -34,6 +34,7 @@ import { getRegexOptions, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, isGroupVisible, rangeFor, } from '../../utils/forms/LiveGroupFilterUtils.js'; @@ -131,7 +132,12 @@ const LiveGroupFilter = ({ // (in-memory range overlap with sibling groups) sources so the sweep // can refresh form-overlap synchronously without firing HTTP for // groups that did not change. - const scheduleConflictScan = (groupId, rawStart, rawEnd) => { + const scheduleConflictScan = ( + groupId, + rawStart, + rawEnd, + expectedGroupId = groupId + ) => { if (conflictTimersRef.current[groupId]) { clearTimeout(conflictTimersRef.current[groupId]); } @@ -156,7 +162,7 @@ const LiveGroupFilter = ({ ? result.occupants : []; const unexpected = occupants.filter( - (o) => !isExpectedOccupantForGroup(o, groupId, playlist) + (o) => !isExpectedOccupantForGroup(o, expectedGroupId, playlist) ); setConflictSource(groupId, 'occupant', unexpected.length > 0); } catch (e) { @@ -221,7 +227,8 @@ const LiveGroupFilter = ({ scheduleConflictScan( g.channel_group, range.startRaw, - g.auto_sync_channel_end + g.auto_sync_channel_end, + effectiveSyncGroupId(g) ); } } diff --git a/frontend/src/utils/forms/LiveGroupFilterUtils.js b/frontend/src/utils/forms/LiveGroupFilterUtils.js index a1372887..3cb62bf5 100644 --- a/frontend/src/utils/forms/LiveGroupFilterUtils.js +++ b/frontend/src/utils/forms/LiveGroupFilterUtils.js @@ -57,6 +57,18 @@ export const isExpectedOccupantForGroup = ( ); }; +// The group the sync's own channels actually land in. A group_override +// routes auto-created channels into a different ChannelGroup, so the +// conflict check must recognize occupants of that target group as this +// config's own output rather than flagging them against the source group. +export const effectiveSyncGroupId = (group) => { + const override = group?.custom_properties?.group_override; + if (override !== undefined && override !== null && override !== '') { + return Number(override); + } + return group?.channel_group; +}; + export const rangeFor = (g) => { if (!g.enabled || !g.auto_channel_sync) return null; const mode = g.custom_properties?.channel_numbering_mode || 'fixed'; diff --git a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js index feb878e1..86874f23 100644 --- a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js +++ b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js @@ -4,6 +4,7 @@ import { getChannelsInRange, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, rangeFor, abortTimers, getRegexOptions, @@ -227,6 +228,116 @@ describe('LiveGroupFilterUtils', () => { }); }); + // ── effectiveSyncGroupId ─────────────────────────────────────────────────── + describe('effectiveSyncGroupId', () => { + it('returns the source channel_group when there is no override', () => { + expect(effectiveSyncGroupId(makeGroup({ channel_group: 7 }))).toBe(7); + }); + + it('returns the group_override target when set', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('coerces a string-stored group_override to a number', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '9' }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('falls back to the source group when group_override is blank', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '' }, + }); + expect(effectiveSyncGroupId(group)).toBe(7); + }); + + // Regression guard for the group-override range-conflict false positive: + // the auto-sync's own channels land in the override target group, so + // comparing against the source group (pre-fix) flags them as a conflict, + // while comparing against the effective target recognizes them as this + // config's own output. + it("makes group-override occupants count as this group's own", () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const occupant = makeOccupant({ channel_group_id: 9 }); + // Pre-fix comparison (source group) treats own channels as a conflict. + expect( + isExpectedOccupantForGroup( + occupant, + group.channel_group, + makePlaylist() + ) + ).toBe(false); + // Comparing against the effective target recognizes them as expected. + expect( + isExpectedOccupantForGroup( + occupant, + effectiveSyncGroupId(group), + makePlaylist() + ) + ).toBe(true); + }); + + // Guards against over-suppression: resolving the effective target group + // must still surface genuine collisions in an override config's range. + // Only the config's own output (auto-created, this account, in the + // target group, unpinned) is excluded. + it('still flags genuine collisions in a group-override config', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const target = effectiveSyncGroupId(group); + // Manual channel sitting in the range. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 9, auto_created: false }), + target, + makePlaylist() + ) + ).toBe(false); + // Auto-created by a different account. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + auto_created_by_account_id: 999, + }), + target, + makePlaylist() + ) + ).toBe(false); + // A channel in a different group than the override target. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 123 }), + target, + makePlaylist() + ) + ).toBe(false); + // A user-pinned channel number. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + has_channel_number_override: true, + }), + target, + makePlaylist() + ) + ).toBe(false); + }); + }); + // ── rangeFor ────────────────────────────────────────────────────────────── describe('rangeFor', () => { it('returns null when group is disabled', () => { From 3868f02c45e118acc9965745115bf4289eea2b24 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 10:57:06 -0500 Subject: [PATCH 55/81] enhancement(epg): optimize EPG export performance and database interactions - Streamlined `generate_epg()` to incrementally stream EPG data without loading the entire guide, improving memory efficiency. - Reduced database load by deferring the fetching of large `programme_index` blobs, enhancing response times for EPG generation. - Introduced a composite index on `(epg_id, id)` in `ProgramData` to optimize query performance during EPG exports. - Updated tests to ensure proper functionality and performance of new features. --- CHANGELOG.md | 6 + .../0025_programdata_epg_id_index.py | 40 ++ apps/epg/models.py | 5 + apps/output/epg_chunk_cache.py | 230 +++++++ apps/output/test_epg_chunk_cache.py | 187 ++++++ apps/output/tests.py | 292 +++++++-- apps/output/views.py | 573 ++++++++++-------- core/tests.py | 13 +- core/utils.py | 19 + 9 files changed, 1065 insertions(+), 300 deletions(-) create mode 100644 apps/epg/migrations/0025_programdata_epg_id_index.py create mode 100644 apps/output/epg_chunk_cache.py create mode 100644 apps/output/test_epg_chunk_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0517493d..3315456b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. + ### Changed - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. diff --git a/apps/epg/migrations/0025_programdata_epg_id_index.py b/apps/epg/migrations/0025_programdata_epg_id_index.py new file mode 100644 index 00000000..1a350b20 --- /dev/null +++ b/apps/epg/migrations/0025_programdata_epg_id_index.py @@ -0,0 +1,40 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class AddIndexConcurrentlyIfPostgres(AddIndexConcurrently): + """Create the index CONCURRENTLY on PostgreSQL (no table lock on large + tables), falling back to a normal blocking AddIndex on other backends + such as the sqlite dev/test fallback.""" + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_forwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_forwards( + self, app_label, schema_editor, from_state, to_state + ) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_backwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_backwards( + self, app_label, schema_editor, from_state, to_state + ) + + +class Migration(migrations.Migration): + # CREATE INDEX CONCURRENTLY cannot run inside a transaction. + atomic = False + + dependencies = [ + ('epg', '0024_remove_epgsource_api_key_epgsource_password_and_more'), + ] + + operations = [ + AddIndexConcurrentlyIfPostgres( + model_name='programdata', + index=models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 025945db..04c6072c 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -153,6 +153,11 @@ class ProgramData(models.Model): program_id = models.CharField(max_length=64, null=True, blank=True, help_text='Schedules Direct programID (e.g. EP123456789). Null for XMLTV sources.') custom_properties = models.JSONField(default=dict, blank=True, null=True) + class Meta: + indexes = [ + models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ] + def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" diff --git a/apps/output/epg_chunk_cache.py b/apps/output/epg_chunk_cache.py new file mode 100644 index 00000000..92ea6eeb --- /dev/null +++ b/apps/output/epg_chunk_cache.py @@ -0,0 +1,230 @@ +"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" + +import logging +import time + +from django.http import StreamingHttpResponse + +logger = logging.getLogger(__name__) + +STATUS_BUILDING = "building" +STATUS_READY = "ready" +STATUS_ERROR = "error" + +DEFAULT_CACHE_TTL = 300 +DEFAULT_LOCK_TTL = 120 +DEFAULT_POLL_INTERVAL = 0.05 +DEFAULT_MAX_FOLLOWER_WAIT = 600 + + +def _chunks_key(base_key): + return f"{base_key}:chunks" + + +def _ready_key(base_key): + return f"{base_key}:ready" + + +def _status_key(base_key): + return f"{base_key}:status" + + +def _lock_key(base_key): + return f"{base_key}:lock" + + +def _decode_chunk(chunk): + if chunk is None: + return None + if isinstance(chunk, bytes): + return chunk.decode("utf-8") + return chunk + + +def _encode_chunk(chunk): + if isinstance(chunk, bytes): + return chunk + return chunk.encode("utf-8") + + +def _poll_wait(interval): + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.sleep(interval) + return + except ImportError: + pass + time.sleep(interval) + + +def _get_redis(): + from django_redis import get_redis_connection + + return get_redis_connection("default") + + +def _get_status(redis, base_key): + raw = redis.get(_status_key(base_key)) + if raw is None: + return None + return _decode_chunk(raw) + + +def _clear_build_keys(redis, base_key): + redis.delete( + _chunks_key(base_key), + _status_key(base_key), + _ready_key(base_key), + _lock_key(base_key), + ) + + +def _try_acquire_lock(redis, base_key, lock_ttl): + return bool(redis.set(_lock_key(base_key), "1", nx=True, ex=lock_ttl)) + + +def _refresh_build_ttl(redis, base_key, lock_ttl): + redis.expire(_lock_key(base_key), lock_ttl) + redis.expire(_status_key(base_key), lock_ttl) + redis.expire(_chunks_key(base_key), lock_ttl) + + +def _stream_ready(redis, base_key): + offset = 0 + chunks_key = _chunks_key(base_key) + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is None: + break + yield _decode_chunk(chunk) + offset += 1 + + +def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): + """Leader: stream to client and append each chunk to Redis.""" + chunks_key = _chunks_key(base_key) + status_key = _status_key(base_key) + try: + from django.core.cache import cache as django_cache + + django_cache.delete(base_key) # legacy monolithic entry + redis.delete(chunks_key, _ready_key(base_key)) + redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + for chunk in source(): + redis.rpush(chunks_key, _encode_chunk(chunk)) + _refresh_build_ttl(redis, base_key, lock_ttl) + yield chunk + redis.set(status_key, STATUS_READY) + redis.set(_ready_key(base_key), "1") + redis.expire(chunks_key, cache_ttl) + redis.expire(status_key, cache_ttl) + redis.expire(_ready_key(base_key), cache_ttl) + logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + except Exception: + logger.exception("EPG cache build failed for %s", base_key) + redis.delete(chunks_key) + redis.set(status_key, STATUS_ERROR, ex=60) + raise + finally: + redis.delete(_lock_key(base_key)) + + +def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, max_follower_wait): + """Follower: read chunks as the leader writes them.""" + offset = 0 + deadline = time.monotonic() + max_follower_wait + idle_polls = 0 + chunks_key = _chunks_key(base_key) + lock_key = _lock_key(base_key) + + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is not None: + idle_polls = 0 + yield _decode_chunk(chunk) + offset += 1 + continue + + status = _get_status(redis, base_key) + if status == STATUS_READY: + break + + if status == STATUS_ERROR: + _clear_build_keys(redis, base_key) + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + raise RuntimeError("EPG cache build failed") + + if time.monotonic() >= deadline: + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + logger.warning("EPG cache follower timed out after partial read for %s", base_key) + break + + lock_active = bool(redis.exists(lock_key)) + if status != STATUS_BUILDING and not lock_active: + idle_polls += 1 + if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): + if _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache leader lost; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + else: + idle_polls = 0 + + _poll_wait(poll_interval) + + +def stream_epg_response( + cache_key, + source, + *, + cache_ttl=DEFAULT_CACHE_TTL, + lock_ttl=DEFAULT_LOCK_TTL, + poll_interval=DEFAULT_POLL_INTERVAL, + max_follower_wait=DEFAULT_MAX_FOLLOWER_WAIT, + redis=None, +): + """ + Stream XMLTV EPG output with single-flight Redis chunk caching. + + ``source`` must be a callable returning a chunk iterator. Only the leader + invokes it; followers read chunks already written to Redis. + """ + if redis is None: + redis = _get_redis() + + if redis.get(_ready_key(cache_key)): + logger.debug("Serving EPG from chunk cache") + stream = _stream_ready(redis, cache_key) + else: + status = _get_status(redis, cache_key) + if status == STATUS_ERROR: + _clear_build_keys(redis, cache_key) + + if _try_acquire_lock(redis, cache_key, lock_ttl): + logger.debug("Building EPG (cache leader)") + stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) + else: + logger.debug("Following in-flight EPG build") + stream = _stream_follow( + redis, + cache_key, + source, + cache_ttl, + lock_ttl, + poll_interval, + max_follower_wait, + ) + + response = StreamingHttpResponse(stream, content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_epg_chunk_cache.py new file mode 100644 index 00000000..6ca217af --- /dev/null +++ b/apps/output/test_epg_chunk_cache.py @@ -0,0 +1,187 @@ +import threading +import time +from unittest import TestCase + +from apps.output.epg_chunk_cache import ( + STATUS_BUILDING, + STATUS_READY, + _chunks_key, + _lock_key, + _ready_key, + _status_key, + stream_epg_response, +) + + +class FakeRedis: + """Minimal Redis stand-in for chunk-cache unit tests.""" + + def __init__(self): + self._strings = {} + self._lists = {} + self._expires_at = {} + + def _purge_expired(self): + now = time.monotonic() + expired = [key for key, deadline in self._expires_at.items() if deadline <= now] + for key in expired: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def get(self, key): + self._purge_expired() + return self._strings.get(key) + + def set(self, key, value, nx=False, ex=None): + self._purge_expired() + if nx and key in self._strings: + return None + self._strings[key] = value + if ex is not None: + self._expires_at[key] = time.monotonic() + ex + return True + + def delete(self, *keys): + for key in keys: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def exists(self, key): + self._purge_expired() + return key in self._strings or key in self._lists + + def expire(self, key, ttl): + if key in self._strings or key in self._lists: + self._expires_at[key] = time.monotonic() + ttl + return True + + def rpush(self, key, value): + self._lists.setdefault(key, []).append(value) + + def lindex(self, key, offset): + items = self._lists.get(key, []) + if offset < len(items): + return items[offset] + return None + + def llen(self, key): + return len(self._lists.get(key, [])) + + +def _consume(response): + return b"".join(response.streaming_content).decode("utf-8") + + +class EPGChunkCacheTests(TestCase): + def test_leader_caches_chunks_and_sets_ready(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, [1]) + self.assertEqual(redis.get(_ready_key("epg:test")), "1") + self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) + self.assertFalse(redis.exists(_lock_key("epg:test"))) + + def test_cache_hit_skips_source(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + _consume(stream_epg_response("epg:test", source, redis=redis)) + calls.clear() + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, []) + + def test_follower_reads_leader_chunks_without_rebuilding(self): + redis = FakeRedis() + base = "epg:follow" + leader_started = threading.Event() + rebuild_calls = [] + + def slow_source(): + rebuild_calls.append(1) + leader_started.set() + yield "a" + time.sleep(0.05) + yield "b" + + def forbidden_source(): + rebuild_calls.append(2) + yield "SHOULD_NOT_RUN" + + def leader(): + _consume( + stream_epg_response( + base, + slow_source, + redis=redis, + poll_interval=0.01, + ) + ) + + leader_thread = threading.Thread(target=leader) + leader_thread.start() + leader_started.wait(timeout=5) + follower_body = _consume( + stream_epg_response( + base, + forbidden_source, + redis=redis, + poll_interval=0.01, + ) + ) + leader_thread.join(timeout=5) + + self.assertEqual(follower_body, "ab") + self.assertEqual(rebuild_calls, [1]) + + def test_only_one_leader_when_two_clients_start_together(self): + redis = FakeRedis() + build_calls = [] + barrier = threading.Barrier(2) + results = {} + + def source(): + build_calls.append(threading.current_thread().name) + yield "x" + + def worker(): + barrier.wait() + results[threading.current_thread().name] = _consume( + stream_epg_response( + "epg:race", + source, + redis=redis, + poll_interval=0.01, + ) + ) + + threads = [ + threading.Thread(target=worker, name="t1"), + threading.Thread(target=worker, name="t2"), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + self.assertEqual(results["t1"], "x") + self.assertEqual(results["t2"], "x") + self.assertEqual(len(build_calls), 1) diff --git a/apps/output/tests.py b/apps/output/tests.py index b1c7d589..6e2ab395 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,31 +1,114 @@ -from django.test import TestCase, Client +from django.test import TestCase, Client, SimpleTestCase from django.urls import reverse -from apps.channels.models import Channel, ChannelGroup +from unittest.mock import patch +from uuid import uuid4 +from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource import xml.etree.ElementTree as ET +from datetime import timedelta + + +def _response_text(response): + """Read body from HttpResponse or StreamingHttpResponse.""" + if getattr(response, "streaming", False): + return b"".join(response.streaming_content).decode() + return response.content.decode() + + +def _epg_response_without_redis(cache_key, source, **kwargs): + """Test helper: stream EPG directly without Redis chunk caching.""" + from django.http import StreamingHttpResponse + + response = StreamingHttpResponse(source(), content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response + + +class OutputEndpointTestMixin: + """Isolate HTTP endpoint tests from network ACL, logging, DB teardown, and Redis.""" -class OutputM3UTest(TestCase): def setUp(self): + super().setUp() + self._network_patch = patch( + "apps.output.views.network_access_allowed", + return_value=True, + ) + self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._log_event_patch = patch("apps.output.views.log_system_event") + self._close_db_patch = patch("django.db.close_old_connections") + self._epg_cache_patch = patch( + "apps.output.views.stream_epg_response", + side_effect=_epg_response_without_redis, + ) + self._network_patch.start() + self._epg_teardown_patch.start() + self._log_event_patch.start() + self._close_db_patch.start() + self._epg_cache_patch.start() + + def tearDown(self): + from django.core.cache import cache + + cache.clear() + self._epg_cache_patch.stop() + self._close_db_patch.stop() + self._log_event_patch.stop() + self._epg_teardown_patch.stop() + self._network_patch.stop() + super().tearDown() + + def _create_isolated_profile(self, prefix): + """New profiles auto-include every channel via signal; clear that for tests.""" + profile = ChannelProfile.objects.create(name=f"{prefix}-{uuid4().hex[:8]}") + ChannelProfileMembership.objects.filter(channel_profile=profile).delete() + return profile + + def _add_channel_to_profile(self, profile, group, **kwargs): + channel = Channel.objects.create(channel_group=group, **kwargs) + ChannelProfileMembership.objects.create( + channel_profile=profile, + channel=channel, + enabled=True, + ) + return channel + + +class OutputM3UTest(OutputEndpointTestMixin, TestCase): + def setUp(self): + super().setUp() self.client = Client() - + self.group = ChannelGroup.objects.create(name=f"M3U Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("m3u") + self._add_channel_to_profile( + self.profile, + self.group, + channel_number=1.0, + name="Test M3U Channel", + ) + + def _m3u_url(self): + return reverse("output:m3u_endpoint", kwargs={"profile_name": self.profile.name}) + def test_generate_m3u_response(self): """ Test that the M3U endpoint returns a valid M3U file. """ - url = reverse('output:generate_m3u') - response = self.client.get(url) + response = self.client.get(self._m3u_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) self.assertIn("#EXTM3U", content) def test_generate_m3u_response_post_empty_body(self): """ Test that a POST request with an empty body returns 200 OK. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data=None, content_type='application/x-www-form-urlencoded') - content = response.content.decode() + response = self.client.post( + self._m3u_url(), + data=None, + content_type="application/x-www-form-urlencoded", + ) + content = _response_text(response) self.assertEqual(response.status_code, 200, "POST with empty body should return 200 OK") self.assertIn("#EXTM3U", content) @@ -34,35 +117,40 @@ class OutputM3UTest(TestCase): """ Test that a POST request with a non-empty body returns 403 Forbidden. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data={'evilstring': 'muhahaha'}) + response = self.client.post(self._m3u_url(), data={"evilstring": "muhahaha"}) self.assertEqual(response.status_code, 403, "POST with body should return 403 Forbidden") - self.assertIn("POST requests with body are not allowed, body is:", response.content.decode()) + self.assertIn("POST requests with body are not allowed", _response_text(response)) -class OutputEPGXMLEscapingTest(TestCase): +class OutputEPGXMLEscapingTest(OutputEndpointTestMixin, TestCase): """Test XML escaping of channel_id attributes in EPG generation""" def setUp(self): + super().setUp() self.client = Client() - self.group = ChannelGroup.objects.create(name="Test Group") + self.group = ChannelGroup.objects.create(name=f"Test Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("epg-xml") + + def _add_channel(self, **kwargs): + return self._add_channel_to_profile(self.profile, self.group, **kwargs) + + def _epg_url(self, query="tvg_id_source=tvg_id&days=0&prev_days=0"): + base = reverse("output:epg_endpoint", kwargs={"profile_name": self.profile.name}) + return f"{base}?{query}" def test_channel_id_with_ampersand(self): """Test channel ID with ampersand is properly escaped""" - channel = Channel.objects.create( + self._add_channel( channel_number=1.0, name="Test Channel", tvg_id="News & Sports", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) # Should contain escaped ampersand self.assertIn('id="News & Sports"', content) @@ -76,17 +164,15 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_angle_brackets(self): """Test channel ID with < and > characters""" - channel = Channel.objects.create( + self._add_channel( channel_number=2.0, name="HD Channel", tvg_id="Channel ", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Channel <HD>"', content) try: @@ -96,23 +182,28 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_all_special_chars(self): """Test channel ID with all XML special characters""" - channel = Channel.objects.create( + expected_id = 'Test & "Special" ' + self._add_channel( channel_number=3.0, name="Complex Channel", - tvg_id='Test & "Special" ', - channel_group=self.group + tvg_id=expected_id, ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Test & "Special" <Chars>"', content) try: tree = ET.fromstring(content) - # Verify we can find the channel with correct ID in parsed tree - channel_elem = tree.find('.//channel[@id="Test & \\"Special\\" "]') + channel_elem = next( + ( + elem + for elem in tree.findall(".//channel") + if elem.get("id") == expected_id + ), + None, + ) self.assertIsNotNone(channel_elem) except ET.ParseError as e: self.fail(f"Generated EPG with all special chars is not valid XML: {e}") @@ -121,25 +212,144 @@ class OutputEPGXMLEscapingTest(TestCase): """Test that programme elements also have escaped channel attributes""" epg_source = EPGSource.objects.create(name="Test EPG", source_type="dummy") epg_data = EPGData.objects.create(name="Test EPG Data", epg_source=epg_source) - channel = Channel.objects.create( + self._add_channel( channel_number=4.0, name="Program Test", tvg_id="News & Sports", epg_data=epg_data, - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) # Check programme elements have escaped channel attributes self.assertIn('channel="News & Sports"', content) try: tree = ET.fromstring(content) - programmes = tree.findall('.//programme[@channel="News & Sports"]') + programmes = [ + programme + for programme in tree.findall(".//programme") + if programme.get("channel") == "News & Sports" + ] self.assertGreater(len(programmes), 0) except ET.ParseError as e: self.fail(f"Generated EPG with programme elements is not valid XML: {e}") + + def test_programmes_emitted_in_start_time_order(self): + """Programmes for a channel are emitted in start_time order, not insert order.""" + from django.utils import timezone + from apps.epg.models import ProgramData + + epg_source = EPGSource.objects.create(name="Real EPG", source_type="xmltv") + epg_data = EPGData.objects.create(name="Station", epg_source=epg_source, tvg_id="station1") + self._add_channel( + channel_number=149.0, + name="Food Network", + tvg_id="station1", + epg_data=epg_data, + ) + now = timezone.now() + # Insert out of chronological order so id order != start_time order. + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=3), + end_time=now + timedelta(days=3, hours=1), + title="Third", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=1), + end_time=now + timedelta(days=1, hours=1), + title="First", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=2), + end_time=now + timedelta(days=2, hours=1), + title="Second", + tvg_id="station1", + ) + + content = _response_text(self.client.get(self._epg_url("tvg_id_source=tvg_id&days=7"))) + + self.assertLess(content.find('First'), content.find('Second')) + self.assertLess(content.find('Second'), content.find('Third')) + + +class OutputEPGCustomDummyTest(TestCase): + """Custom dummy EPG must not fall back to default when pattern matched but event is outside window.""" + + def setUp(self): + self.group = ChannelGroup.objects.create(name="Sports Group") + + def test_custom_dummy_outside_window_fills_with_ended_programmes(self): + from django.utils import timezone + from apps.output.views import generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + channel_name = ( + "NHL 01: Washington Capitals vs Philadelphia Flyers @ April 16 07:30 PM ET" + ) + now = timezone.now() + lookback = now - timedelta(days=7) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=7, + epg_source=epg_source, + export_lookback=lookback, + export_cutoff=now + timedelta(days=7), + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all(p['end_time'] >= lookback for p in programs), + "All programmes should fall inside the export window", + ) + self.assertTrue( + any('Ended' in p['description'] for p in programs), + "Past events outside the window should still show ended filler", + ) + for program in programs: + start = program['start_time'] + self.assertEqual(start.second, 0) + self.assertEqual(start.microsecond, 0) + self.assertIn( + start.minute, (0, 30), + "Filler programmes should start on half-hour boundaries", + ) + self.assertGreaterEqual(programs[0]['start_time'], lookback) + + +class OutputEPGHelperTest(SimpleTestCase): + def test_ceil_to_half_hour_on_boundary(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=30, second=0, microsecond=0) + self.assertEqual(_ceil_to_half_hour(dt), dt) + + def test_ceil_to_half_hour_rounds_up(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=17, second=42, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) diff --git a/apps/output/views.py b/apps/output/views.py index dba35951..a0875c7f 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,9 +25,55 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib +from apps.output.epg_chunk_cache import stream_epg_response logger = logging.getLogger(__name__) +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -112,6 +158,10 @@ def generate_m3u(request, profile_name=None, user=None): # Check if this is a POST request and the body is not empty (which we don't want to allow) logger.debug("Generating M3U for profile: %s, user: %s, method: %s", profile_name, user.username if user else "Anonymous", request.method) + if request.method == "POST" and request.body: + if request.body.decode() != '{}': + return HttpResponseForbidden("POST requests with body are not allowed.") + # Check cache for recent identical request (helps with double-GET from browsers) from django.core.cache import cache cache_params = f"{profile_name or 'all'}:{user.username if user else 'anonymous'}:{request.GET.urlencode()}" @@ -123,10 +173,6 @@ def generate_m3u(request, profile_name=None, user=None): response = HttpResponse(cached_content, content_type="audio/x-mpegurl") response["Content-Disposition"] = 'attachment; filename="channels.m3u"' return response - # Check if this is a POST request with data (which we don't want to allow) - if request.method == "POST" and request.body: - if request.body.decode() != '{}': - return HttpResponseForbidden("POST requests with body are not allowed.") if user is not None: if user.user_level < 10: @@ -409,7 +455,15 @@ def generate_fallback_programs(channel_id, channel_name, now, num_days, program_ return programs -def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4, epg_source=None): +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): """ Generate dummy EPG programs for channels. @@ -435,29 +489,26 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: custom_programs = generate_custom_dummy_programs( channel_id, channel_name, now, num_days, - epg_source.custom_properties + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, ) - # If custom generation succeeded, return those programs - # If it returned empty (pattern didn't match), check for custom fallback templates - if custom_programs: + if custom_programs is not None: return custom_programs - else: - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # Check if custom fallback templates are provided - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # If custom fallback templates exist, use them instead of default - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - else: - logger.info(f"No custom fallback templates found, using default dummy EPG") + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") # Default humorous program descriptions based on time of day time_descriptions = { @@ -531,7 +582,15 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length return programs -def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, custom_properties): +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): """ Generate programs using custom dummy EPG regex patterns. @@ -616,7 +675,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust if not title_pattern: logger.warning(f"No title_pattern in custom_properties, falling back to default") - return [] # Return empty, will use default + return None logger.debug(f"Title pattern from DB: {repr(title_pattern)}") @@ -633,7 +692,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust except Exception as e: logger.error(f"Invalid title regex pattern after conversion: {e}") logger.error(f"Pattern was: {repr(title_pattern)}") - return [] + return None time_regex = None if time_pattern: @@ -665,7 +724,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust title_match = title_regex.search(channel_name) if not title_match: logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return [] # Return empty, will use default + return None groups = title_match.groupdict() logger.debug(f"Title pattern matched. Groups: {groups}") @@ -917,57 +976,93 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust iterations = num_days for day in range(iterations): - # Start from current time (like standard dummy) instead of midnight - # This ensures programs appear in the guide's current viewing window - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - - if time_info: - # We have an extracted event time - this is when the MAIN event starts - # The extracted time is in the SOURCE timezone (e.g., 8PM ET) - # We need to convert it to UTC for storage - - # Determine which date to use - if date_info: - # Use the extracted date from the channel title - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'] - ).date() - logger.debug(f"Using extracted date: {current_date}") - else: - # No date extracted, use day offset from current time in SOURCE timezone - # This ensures we calculate "today" in the event's timezone, not UTC - # For example: 8:30 PM Central (1:30 AM UTC next day) for a 10 PM ET event - # should use today's date in ET, not tomorrow's date in UTC - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - # Create a naive datetime (no timezone info) representing the event in source timezone + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() event_start_naive = datetime.combine( current_date, datetime.min.time().replace( hour=time_info['hour'], - minute=time_info['minute'] - ) + minute=time_info['minute'], + ), ) - - # Use pytz to localize the naive datetime to the source timezone - # This automatically handles DST! try: - event_start_local = source_tz.localize(event_start_naive) - # Convert to UTC - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) except Exception as e: logger.error(f"Error localizing time to {source_tz}: {e}") - # Fallback: treat as UTC event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + # Pre-generate the main event title and description for reuse if title_template: main_event_title = format_template(title_template, all_groups) @@ -994,15 +1089,13 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) - is_event_day = (day == 0) + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window if is_event_day and not event_happened: - # This is THE day the event happens - # Fill programs BEFORE the event current_time = day_start - while current_time < event_start_utc: + while current_time < event_start_utc and current_time < day_end: program_start_utc = current_time program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) @@ -1084,8 +1177,8 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust event_happened = True - # Fill programs AFTER the event until end of day - current_time = event_end_utc + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) while current_time < day_end: program_start_utc = current_time @@ -1325,18 +1418,10 @@ def generate_dummy_epg( def generate_epg(request, profile_name=None, user=None): """ - Dynamically generate an XMLTV (EPG) file using streaming response to handle keep-alives. + Dynamically generate an XMLTV (EPG) file using a streaming response. Since the EPG data is stored independently of Channels, we group programmes by their associated EPGData record. - This version filters data based on the 'days' parameter and sends keep-alives during processing. """ - # Check cache for recent identical request (helps with double-GET from browsers) - from django.core.cache import cache - # Resolve all effective parameter values once here so they are reused for both - # the cache key and inside epg_generator() via closure. - # The cache key is built from resolved values only — not from the raw query string — - # so equivalent requests (e.g. days=7 via URL param vs. user default of 7) share - # the same cache entry regardless of how the value was supplied. user_custom = (user.custom_properties or {}) if user else {} try: num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) @@ -1356,21 +1441,13 @@ def generate_epg(request, profile_name=None, user=None): ) content_cache_key = f"epg_content:{cache_params}" - cached_content = cache.get(content_cache_key) - if cached_content: - logger.debug("Serving EPG from cache") - response = HttpResponse(cached_content, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response - def epg_generator(): """Generator function that yields EPG data with keep-alives during processing.""" - xml_lines = [] - xml_lines.append('') - xml_lines.append( - '' + yield '\n' + yield ( + '\n' ) # Get channels based on user/profile @@ -1416,14 +1493,19 @@ def generate_epg(request, profile_name=None, user=None): # Resolve effective values at SQL level and exclude hidden channels # so output ordering/display honors user overrides. from apps.channels.managers import with_effective_values - channels = ( + channels = list( with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) ) ) + channel_count = len(channels) # For dummy EPG, use either the specified value or default to 3 days dummy_days = num_days if num_days > 0 else 3 @@ -1465,12 +1547,14 @@ def generate_epg(request, profile_name=None, user=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - dummy_epg_ids_for_program_check = set() + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] - # Process channels for the section for channel in channels: effective_name = channel.effective_name effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id effective_logo = channel.effective_logo_obj effective_number = channel.effective_channel_number @@ -1492,8 +1576,6 @@ def generate_epg(request, profile_name=None, user=None): # Check if this is a custom dummy EPG with channel logo URL template if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if channel.effective_epg_data_id: - dummy_epg_ids_for_program_check.add(channel.effective_epg_data_id) epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties @@ -1548,51 +1630,16 @@ def generate_epg(request, profile_name=None, user=None): tvg_logo = direct_logo else: tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - display_name = effective_name - xml_lines.append(f' ') - xml_lines.append(f' {html.escape(display_name)}') - xml_lines.append(f' ') - xml_lines.append(" ") + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") - # Send all channel definitions - channel_xml = '\n'.join(xml_lines) + '\n' - yield channel_xml - xml_lines = [] # Clear to save memory + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] - dummy_epg_with_programs = set() - if dummy_epg_ids_for_program_check: - dummy_epg_with_programs = set( - ProgramData.objects.filter(epg_id__in=dummy_epg_ids_for_program_check) - .values_list('epg_id', flat=True) - .distinct() - ) - - # Pre-pass: categorize channels into dummy and real EPG groups - dummy_program_list = [] # (channel_id, pattern_match_name, epg_source_or_None) - real_epg_map = {} # epg_data_id -> [channel_id, ...] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_number = channel.effective_channel_number - - # Determine channel_id (same logic as channel section) - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - display_name = effective_epg_data.name if effective_epg_data else effective_name pattern_match_name = effective_name - - # Check if we should use stream name instead of channel name if effective_epg_data and effective_epg_data.epg_source: epg_source = effective_epg_data.epg_source if epg_source.custom_properties: @@ -1619,48 +1666,19 @@ def generate_epg(request, profile_name=None, user=None): if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) else: - if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if effective_epg_data_id in dummy_epg_with_programs: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - else: - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - continue - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - # Emit dummy programmes - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source - ) - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - yield f' \n' - yield f" {html.escape(program['title'])}\n" - if program.get('sub_title'): - yield f" {html.escape(program['sub_title'])}\n" - yield f" {html.escape(program['description'])}\n" - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - yield f" {html.escape(cat)}\n" - if 'date' in custom_data: - yield f" {html.escape(custom_data['date'])}\n" - if custom_data.get('live', False): - yield f" \n" - if custom_data.get('new', False): - yield f" \n" - if 'icon' in custom_data: - yield f' \n' - yield f" \n" + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - # Emit real programmes: single bulk query, chunked to avoid server-side cursor issues. all_epg_ids = list(real_epg_map.keys()) if all_epg_ids: if num_days > 0: @@ -1682,27 +1700,44 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None - is_multi = False - multi_buffer = [] + pending = [] program_batch = [] - batch_size = 1000 - chunk_size = 5000 - # Keyset pagination: track last (epg_id, id) instead of OFFSET - # to avoid skipping/duplicating rows if the table changes mid-stream. + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE last_epg_id = 0 last_id = 0 _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + while True: program_chunk = list( programs_base_qs.filter(epg_id__gte=last_epg_id) .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] ) - if not program_chunk: break - # Advance keyset cursor to last row in this chunk last_row = program_chunk[-1] last_epg_id = last_row['epg_id'] last_id = last_row['id'] @@ -1710,31 +1745,19 @@ def generate_epg(request, profile_name=None, user=None): for prog in program_chunk: epg_id = prog['epg_id'] - # When epg_id changes, flush multi-channel buffer for previous group if epg_id != current_epg_id: - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - multi_buffer = [] - + yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] - is_multi = len(channel_ids_for_epg) > 1 - # Build programme XML for primary channel_id primary_cid = channel_ids_for_epg[0] - start_str = prog['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = prog['end_time'].strftime("%Y%m%d%H%M%S %z") + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') @@ -1932,67 +1955,101 @@ def generate_epg(request, profile_name=None, user=None): program_xml.append(" ") xml_text = '\n'.join(program_xml) - program_batch.append(xml_text) + pending.append((prog['start_time'], prog['id'], xml_text)) - if is_multi: - multi_buffer.append(xml_text) + del program_chunk - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - - # Final flush of multi-channel buffer - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) + yield from flush_pending() if program_batch: yield '\n'.join(program_batch) + '\n' - # Send final closing tag and completion message + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + yield "\n" - # Log system event for EPG download after streaming completes (with deduplication based on client) client_id, client_ip, user_agent = get_client_identifier(request) event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - if not cache.get(event_cache_key): - # `len()` reuses the queryset's iteration cache populated above; - # `count()` would issue a separate SELECT COUNT(*). - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=len(channels), - client_ip=client_ip, - user_agent=user_agent, - ) - cache.set(event_cache_key, True, 2) # Prevent duplicate events for 2 seconds - # Wrapper generator that collects content for caching - def caching_generator(): - collected_content = [] - for chunk in epg_generator(): - collected_content.append(chunk) - yield chunk - # After streaming completes, cache the full content - full_content = ''.join(collected_content) - cache.set(content_cache_key, full_content, 300) - logger.debug("Cached EPG content (%d bytes)", len(full_content)) + def _log_epg_download(): + from django.core.cache import cache as event_cache - response = StreamingHttpResponse( - streaming_content=caching_generator(), - content_type="application/xml" - ) - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_epg_response(content_cache_key, build_epg_stream) def xc_get_user(request): diff --git a/core/tests.py b/core/tests.py index 7b2f1986..33475ae7 100644 --- a/core/tests.py +++ b/core/tests.py @@ -1,6 +1,6 @@ from unittest.mock import patch, MagicMock -from django.test import TestCase +from django.test import TestCase, SimpleTestCase from apps.epg.models import EPGSource from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -301,3 +301,14 @@ class DropDBCommandTlsTest(TestCase): host='localhost', port=5432, autocommit=True, ) + + +class MallocTrimTests(SimpleTestCase): + def test_trim_is_noop_when_libc_has_no_malloc_trim(self): + from core.utils import trim_c_allocator_heap + + fake_libc = MagicMock(spec=[]) + with patch('ctypes.util.find_library', return_value='libc.so.6'), patch( + 'ctypes.CDLL', return_value=fake_libc + ): + self.assertFalse(trim_c_allocator_heap()) diff --git a/core/utils.py b/core/utils.py index 29c613b3..89228088 100644 --- a/core/utils.py +++ b/core/utils.py @@ -546,6 +546,25 @@ def monitor_memory_usage(func): return result return wrapper +def trim_c_allocator_heap(): + """Return unused C heap pages to the OS where supported (glibc malloc_trim).""" + try: + import ctypes + import ctypes.util + + libc_name = ctypes.util.find_library("c") + if not libc_name: + return False + libc = ctypes.CDLL(libc_name) + if not hasattr(libc, "malloc_trim"): + return False + libc.malloc_trim(0) + return True + except Exception: + logger.debug("malloc_trim unavailable or failed", exc_info=True) + return False + + def cleanup_memory(log_usage=False, force_collection=True): """ Comprehensive memory cleanup function to reduce memory footprint From bccee9ebc1b36a140103bf3b715e91b8cb1b7def Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 11:50:52 -0500 Subject: [PATCH 56/81] refactor(epg): extract EPG generation logic into dedicated module - Moved all XMLTV output logic from `apps/output/views.py` to `apps/output/epg.py`, streamlining the codebase and maintaining the existing HTTP endpoint functionality. - Updated related tests and references to ensure proper integration with the new module structure. --- CHANGELOG.md | 2 + apps/output/epg.py | 1745 +++++++++++++++++ ...hunk_cache.py => streaming_chunk_cache.py} | 36 +- ...cache.py => test_streaming_chunk_cache.py} | 30 +- apps/output/tests.py | 11 +- apps/output/views.py | 1714 +--------------- 6 files changed, 1790 insertions(+), 1748 deletions(-) create mode 100644 apps/output/epg.py rename apps/output/{epg_chunk_cache.py => streaming_chunk_cache.py} (81%) rename apps/output/{test_epg_chunk_cache.py => test_streaming_chunk_cache.py} (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3315456b..840cb349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. + - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. ### Performance diff --git a/apps/output/epg.py b/apps/output/epg.py new file mode 100644 index 00000000..354d6e5b --- /dev/null +++ b/apps/output/epg.py @@ -0,0 +1,1745 @@ +"""XMLTV (EPG) output generation. + +Consolidates the EPG export logic that backs the `/epg` endpoint and the XC +XMLTV endpoint: real programme streaming, dummy/custom dummy program +generation, and the streaming XMLTV builder. HTTP endpoints live in views.py +and call into this module; Redis chunk caching lives in streaming_chunk_cache.py. +""" + +import html +import logging +from datetime import datetime, timedelta + +import regex + +from django.db.models import Prefetch +from django.http import Http404 +from django.urls import reverse +from django.utils import timezone as django_timezone + +from apps.channels.models import Channel, ChannelProfile, Stream +from apps.channels.utils import format_channel_number +from apps.epg.models import ProgramData +from apps.output.streaming_chunk_cache import stream_cached_response +from core.utils import build_absolute_uri_with_port, log_system_event + +logger = logging.getLogger(__name__) + +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + +def _ordered_channel_streams(channel): + """Return a channel's streams ordered by channelstream join order.""" + prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') + if prefetched is not None: + return list(prefetched) + return list(channel.streams.all().order_by('channelstream__order')) + + +def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): + """Name used for custom dummy EPG regex matching (channel or stream title). + + Returns (name, stream_lookup_failed). stream_lookup_failed is True only when + name_source is 'stream' but the configured index is missing or out of range. + """ + if custom_props.get('name_source') != 'stream': + return effective_name, False + stream_index = custom_props.get('stream_index', 1) - 1 + streams = _ordered_channel_streams(channel) + if 0 <= stream_index < len(streams): + return streams[stream_index].name, False + return effective_name, True + + +def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): + """ + Generate dummy programs using custom fallback templates when patterns don't match. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel name to use as fallback in templates + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + fallback_title: Custom fallback title template (empty string if not provided) + fallback_description: Custom fallback description template (empty string if not provided) + + Returns: + List of program dictionaries + """ + programs = [] + + # Use custom fallback title or channel name as default + title = fallback_title if fallback_title else channel_name + + # Use custom fallback description or a simple default message + if fallback_description: + description = fallback_description + else: + description = f"EPG information is currently unavailable for {channel_name}" + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": title, + "description": description, + }) + + return programs + + +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): + """ + Generate dummy EPG programs for channels. + + If epg_source is provided and it's a custom dummy EPG with patterns, + use those patterns to generate programs from the channel title. + Otherwise, generate default dummy programs. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title/name + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + epg_source: Optional EPGSource for custom dummy EPG with patterns + + Returns: + List of program dictionaries + """ + # Get current time rounded to hour + now = django_timezone.now() + now = now.replace(minute=0, second=0, microsecond=0) + + # Check if this is a custom dummy EPG with regex patterns + if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: + custom_programs = generate_custom_dummy_programs( + channel_id, channel_name, now, num_days, + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, + ) + if custom_programs is not None: + return custom_programs + + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") + + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") + + # Default humorous program descriptions based on time of day + time_descriptions = { + (0, 4): [ + f"Late Night with {channel_name} - Where insomniacs unite!", + f"The 'Why Am I Still Awake?' Show on {channel_name}", + f"Counting Sheep - A {channel_name} production for the sleepless", + ], + (4, 8): [ + f"Dawn Patrol - Rise and shine with {channel_name}!", + f"Early Bird Special - Coffee not included", + f"Morning Zombies - Before coffee viewing on {channel_name}", + ], + (8, 12): [ + f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", + f"The 'I Should Be Working' Hour on {channel_name}", + f"Productivity Killer - {channel_name}'s daytime programming", + ], + (12, 16): [ + f"Lunchtime Laziness with {channel_name}", + f"The Afternoon Slump - Brought to you by {channel_name}", + f"Post-Lunch Food Coma Theater on {channel_name}", + ], + (16, 20): [ + f"Rush Hour - {channel_name}'s alternative to traffic", + f"The 'What's For Dinner?' Debate on {channel_name}", + f"Evening Escapism - {channel_name}'s remedy for reality", + ], + (20, 24): [ + f"Prime Time Placeholder - {channel_name}'s finest not-programming", + f"The 'Netflix Was Too Complicated' Show on {channel_name}", + f"Family Argument Avoider - Courtesy of {channel_name}", + ], + } + + programs = [] + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + # Get the hour for selecting a description + hour = start_time.hour + + # Find the appropriate time slot for description + for time_range, descriptions in time_descriptions.items(): + start_range, end_range = time_range + if start_range <= hour < end_range: + # Pick a description using the sum of the hour and day as seed + # This makes it somewhat random but consistent for the same timeslot + description = descriptions[(hour + day) % len(descriptions)] + break + else: + # Fallback description if somehow no range matches + description = f"Placeholder program for {channel_name} - EPG data went on vacation" + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": channel_name, + "description": description, + }) + + return programs + + +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): + """ + Generate programs using custom dummy EPG regex patterns. + + Extracts information from channel title using regex patterns and generates + programs based on the extracted data. + + TIMEZONE HANDLING: + ------------------ + The timezone parameter specifies the timezone of the event times in your channel + titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). + DST (Daylight Saving Time) is handled automatically by pytz. + + Examples: + - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" + - Set timezone = "US/Eastern" + - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) + - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title to parse + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + custom_properties: Dict with title_pattern, time_pattern, templates, etc. + - timezone: Timezone name (e.g., 'US/Eastern') + + Returns: + List of program dictionaries with start_time/end_time in UTC + """ + import pytz + + logger.info(f"Generating custom dummy programs for channel: {channel_name}") + + # Extract patterns from custom properties + title_pattern = custom_properties.get('title_pattern', '') + time_pattern = custom_properties.get('time_pattern', '') + date_pattern = custom_properties.get('date_pattern', '') + + # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') + timezone_value = custom_properties.get('timezone', 'UTC') + output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone + program_duration = custom_properties.get('program_duration', 180) # Minutes + title_template = custom_properties.get('title_template', '') + subtitle_template = custom_properties.get('subtitle_template', '') + description_template = custom_properties.get('description_template', '') + + # Templates for upcoming/ended programs + upcoming_title_template = custom_properties.get('upcoming_title_template', '') + upcoming_description_template = custom_properties.get('upcoming_description_template', '') + ended_title_template = custom_properties.get('ended_title_template', '') + ended_description_template = custom_properties.get('ended_description_template', '') + + # Image URL templates + channel_logo_url_template = custom_properties.get('channel_logo_url', '') + program_poster_url_template = custom_properties.get('program_poster_url', '') + + # EPG metadata options + category_string = custom_properties.get('category', '') + # Split comma-separated categories and strip whitespace, filter out empty strings + categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] + include_date = custom_properties.get('include_date', True) + include_live = custom_properties.get('include_live', False) + include_new = custom_properties.get('include_new', False) + + # Parse timezone name + try: + source_tz = pytz.timezone(timezone_value) + logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") + source_tz = pytz.utc + + # Parse output timezone if provided (for display purposes) + output_tz = None + if output_timezone_value: + try: + output_tz = pytz.timezone(output_timezone_value) + logger.debug(f"Using output timezone for display: {output_timezone_value}") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") + output_tz = None + + if not title_pattern: + logger.warning(f"No title_pattern in custom_properties, falling back to default") + return None + + logger.debug(f"Title pattern from DB: {repr(title_pattern)}") + + # Convert PCRE/JavaScript named groups (?) to Python format (?P) + # This handles patterns created with JavaScript regex syntax + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) + logger.debug(f"Converted title pattern: {repr(title_pattern)}") + + # Compile regex patterns using the enhanced regex module + # (supports variable-width lookbehinds like JavaScript) + try: + title_regex = regex.compile(title_pattern) + except Exception as e: + logger.error(f"Invalid title regex pattern after conversion: {e}") + logger.error(f"Pattern was: {repr(title_pattern)}") + return None + + time_regex = None + if time_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) + logger.debug(f"Converted time pattern: {repr(time_pattern)}") + try: + time_regex = regex.compile(time_pattern) + except Exception as e: + logger.warning(f"Invalid time regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(time_pattern)}") + + # Compile date regex if provided + date_regex = None + if date_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) + logger.debug(f"Converted date pattern: {repr(date_pattern)}") + try: + date_regex = regex.compile(date_pattern) + except Exception as e: + logger.warning(f"Invalid date regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(date_pattern)}") + + # Try to match the channel name with the title pattern + # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string + title_match = title_regex.search(channel_name) + if not title_match: + logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") + return None + + groups = title_match.groupdict() + logger.debug(f"Title pattern matched. Groups: {groups}") + + # Helper function to format template with matched groups + def format_template(template, groups, url_encode=False): + """Replace {groupname} placeholders with matched group values + + Args: + template: Template string with {groupname} placeholders + groups: Dict of group names to values + url_encode: If True, URL encode the group values for safe use in URLs + """ + if not template: + return '' + result = template + for key, value in groups.items(): + if url_encode and value: + # URL encode the value to handle spaces and special characters + from urllib.parse import quote + encoded_value = quote(str(value), safe='') + result = result.replace(f'{{{key}}}', encoded_value) + else: + result = result.replace(f'{{{key}}}', str(value) if value else '') + return result + + # Extract time from title if time pattern exists + time_info = None + time_groups = {} + if time_regex: + time_match = time_regex.search(channel_name) + if time_match: + time_groups = time_match.groupdict() + try: + hour = int(time_groups.get('hour')) + # Handle optional minute group - could be None if not captured + minute_value = time_groups.get('minute') + minute = int(minute_value) if minute_value is not None else 0 + ampm = time_groups.get('ampm') + ampm = ampm.lower() if ampm else None + + # Determine if this is 12-hour or 24-hour format + if ampm in ('am', 'pm'): + # 12-hour format: convert to 24-hour + if ampm == 'pm' and hour != 12: + hour += 12 + elif ampm == 'am' and hour == 12: + hour = 0 + logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") + else: + # 24-hour format: hour is already in 24-hour format + # Validate that it's actually a 24-hour time (0-23) + if hour > 23: + logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") + hour = hour % 24 # Wrap around just in case + logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") + + time_info = {'hour': hour, 'minute': minute} + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing time: {e}") + + # Extract date from title if date pattern exists + date_info = None + date_groups = {} + if date_regex: + date_match = date_regex.search(channel_name) + if date_match: + date_groups = date_match.groupdict() + try: + # Support various date group names: month, day, year + month_str = date_groups.get('month', '') + day_str = date_groups.get('day', '') + year_str = date_groups.get('year', '') + + # Parse day - default to current day if empty or invalid + day = int(day_str) if day_str else now.day + + # Parse year - default to current year if empty or invalid (matches frontend behavior) + year = int(year_str) if year_str else now.year + + # Parse month - can be numeric (1-12) or text (Jan, January, etc.) + month = None + if month_str: + if month_str.isdigit(): + month = int(month_str) + else: + # Try to parse text month names + import calendar + month_str_lower = month_str.lower() + # Check full month names + for i, month_name in enumerate(calendar.month_name): + if month_name.lower() == month_str_lower: + month = i + break + # Check abbreviated month names if not found + if month is None: + for i, month_abbr in enumerate(calendar.month_abbr): + if month_abbr.lower() == month_str_lower: + month = i + break + + # Default to current month if not extracted or invalid + if month is None: + month = now.month + + if month and 1 <= month <= 12 and 1 <= day <= 31: + date_info = {'year': year, 'month': month, 'day': day} + logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") + else: + logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing date: {e}") + + # Merge title groups, time groups, and date groups for template formatting + all_groups = {**groups, **time_groups, **date_groups} + + # Add normalized versions of all groups for cleaner URLs + # These remove all non-alphanumeric characters and convert to lowercase + for key, value in list(all_groups.items()): + if value: + # Remove all non-alphanumeric characters (except spaces temporarily) + # then replace spaces with nothing, and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + all_groups[f'{key}_normalize'] = normalized + + # Format channel logo URL if template provided (with URL encoding) + channel_logo_url = None + if channel_logo_url_template: + channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted channel logo URL: {channel_logo_url}") + + # Format program poster URL if template provided (with URL encoding) + program_poster_url = None + if program_poster_url_template: + program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted program poster URL: {program_poster_url}") + + # Add formatted time strings for better display (handles minutes intelligently) + if time_info: + hour_24 = time_info['hour'] + minute = time_info['minute'] + + # Determine the base date to use for placeholders + # If date was extracted, use it; otherwise use current date + if date_info: + base_date = datetime(date_info['year'], date_info['month'], date_info['day']) + else: + base_date = datetime.now() + + # If output_timezone is specified, convert the display time to that timezone + if output_tz: + # Create a datetime in the source timezone using the base date + temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + # Convert to output timezone + temp_date_output = temp_date.astimezone(output_tz) + # Extract converted hour and minute for display + hour_24 = temp_date_output.hour + minute = temp_date_output.minute + logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") + + # Add date placeholders based on the OUTPUT timezone + # This ensures {date}, {month}, {day}, {year} reflect the converted timezone + all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_output.month) + all_groups['day'] = str(temp_date_output.day) + all_groups['year'] = str(temp_date_output.year) + logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") + else: + # No output timezone conversion - use source timezone for date + # Create temp date to get proper date in source timezone using the base date + temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_source.month) + all_groups['day'] = str(temp_date_source.day) + all_groups['year'] = str(temp_date_source.year) + + # Format 24-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime24'] = f"{hour_24}:{minute:02d}" + else: + all_groups['starttime24'] = f"{hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {starttime} placeholder + # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) + ampm = 'AM' if hour_24 < 12 else 'PM' + hour_12 = hour_24 + if hour_24 == 0: + hour_12 = 12 + elif hour_24 > 12: + hour_12 = hour_24 - 12 + + # Format 12-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" + else: + all_groups['starttime'] = f"{hour_12} {ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" + + # Calculate end time based on program duration + # Create a datetime for calculations + temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) + temp_end = temp_start + timedelta(minutes=program_duration) + + # Extract end time components (already in correct timezone if output_tz was applied above) + end_hour_24 = temp_end.hour + end_minute = temp_end.minute + + # Format 24-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" + else: + all_groups['endtime24'] = f"{end_hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {endtime} placeholder + end_ampm = 'AM' if end_hour_24 < 12 else 'PM' + end_hour_12 = end_hour_24 + if end_hour_24 == 0: + end_hour_12 = 12 + elif end_hour_24 > 12: + end_hour_12 = end_hour_24 - 12 + + # Format 12-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + else: + all_groups['endtime'] = f"{end_hour_12} {end_ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + + # Generate programs + programs = [] + + # If we have extracted time AND date, the event happens on a SPECIFIC date + # If we have time but NO date, generate for multiple days (existing behavior) + # All other days and times show "Upcoming" before or "Ended" after + event_happened = False + + # Determine how many iterations we need + if date_info and time_info: + # Specific date extracted - only generate for that one date + iterations = 1 + logger.debug(f"Date extracted, generating single event for specific date") + else: + # No specific date - use num_days (existing behavior) + iterations = num_days + + for day in range(iterations): + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + + # Pre-generate the main event title and description for reuse + if title_template: + main_event_title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + main_event_title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + main_event_subtitle = format_template(subtitle_template, all_groups) + else: + main_event_subtitle = None + + if description_template: + main_event_description = format_template(description_template, all_groups) + else: + main_event_description = main_event_title + + + + # Determine if this day is before, during, or after the event + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window + + if is_event_day and not event_happened: + current_time = day_start + + while current_time < event_start_utc and current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) + + # Use custom upcoming templates if provided, otherwise use defaults + if upcoming_title_template: + upcoming_title = format_template(upcoming_title_template, all_groups) + else: + upcoming_title = main_event_title + + if upcoming_description_template: + upcoming_description = format_template(upcoming_description_template, all_groups) + else: + upcoming_description = f"Upcoming: {main_event_description}" + + # Build custom_properties for upcoming programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": upcoming_title, + "sub_title": None, # No subtitle for filler programs + "description": upcoming_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + + # Add the MAIN EVENT at the extracted time + # Build custom_properties for main event (includes category and live) + main_event_custom_properties = {} + + # Add categories if provided + if categories: + main_event_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = event_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + main_event_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + main_event_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + main_event_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + main_event_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": event_start_utc, + "end_time": event_end_utc, + "title": main_event_title, + "sub_title": main_event_subtitle, + "description": main_event_description, + "custom_properties": main_event_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + event_happened = True + + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom ended templates if provided, otherwise use defaults + if ended_title_template: + ended_title = format_template(ended_title_template, all_groups) + else: + ended_title = main_event_title + + if ended_description_template: + ended_description = format_template(ended_description_template, all_groups) + else: + ended_description = f"Ended: {main_event_description}" + + # Build custom_properties for ended programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": ended_title, + "sub_title": None, # No subtitle for filler programs + "description": ended_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + else: + # This day is either before the event (future days) or after the event happened + # Fill entire day with appropriate message + current_time = day_start + + # If event already happened, all programs show "Ended" + # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" + is_ended = event_happened + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom templates based on whether event has ended or is upcoming + if is_ended: + if ended_title_template: + program_title = format_template(ended_title_template, all_groups) + else: + program_title = main_event_title + + if ended_description_template: + program_description = format_template(ended_description_template, all_groups) + else: + program_description = f"Ended: {main_event_description}" + else: + if upcoming_title_template: + program_title = format_template(upcoming_title_template, all_groups) + else: + program_title = main_event_title + + if upcoming_description_template: + program_description = format_template(upcoming_description_template, all_groups) + else: + program_description = f"Upcoming: {main_event_description}" + + # Build custom_properties (only date for upcoming/ended filler programs) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": program_title, + "sub_title": None, # No subtitle for filler programs + "description": program_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, + }) + + current_time += timedelta(minutes=program_duration) + else: + # No extracted time - fill entire day with regular intervals + # day_start and day_end are already in UTC, so no conversion needed + programs_per_day = max(1, int(24 / (program_duration / 60))) + + for program_num in range(programs_per_day): + program_start_utc = day_start + timedelta(minutes=program_num * program_duration) + program_end_utc = program_start_utc + timedelta(minutes=program_duration) + + if title_template: + title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + subtitle = format_template(subtitle_template, all_groups) + else: + subtitle = None + + if description_template: + description = format_template(description_template, all_groups) + else: + description = title + + # Build custom_properties for this program + program_custom_properties = {} + + # Add categories if provided + if categories: + program_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + program_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + program_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": title, + "sub_title": subtitle, + "description": description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") + return programs + + +def generate_dummy_epg( + channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 +): + """ + Generate dummy EPG programs for channels without EPG data. + Creates program blocks for a specified number of days. + + Args: + channel_id: The channel ID to use in the program entries + channel_name: The name of the channel to use in program titles + xml_lines: Optional list to append lines to, otherwise returns new list + num_days: Number of days to generate EPG data for (default: 1) + program_length_hours: Length of each program block in hours (default: 4) + + Returns: + List of XML lines for the dummy EPG entries + """ + if xml_lines is None: + xml_lines = [] + + for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): + # Format times in XMLTV format + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + + # Create program entry with escaped channel name + xml_lines.append( + f' ' + ) + xml_lines.append(f" {html.escape(program['title'])}") + + # Add subtitle if available + if program.get('sub_title'): + xml_lines.append(f" {html.escape(program['sub_title'])}") + + xml_lines.append(f" {html.escape(program['description'])}") + + # Add custom_properties if present + custom_data = program.get('custom_properties', {}) + + # Categories + if 'categories' in custom_data: + for cat in custom_data['categories']: + xml_lines.append(f" {html.escape(cat)}") + + # Date tag + if 'date' in custom_data: + xml_lines.append(f" {html.escape(custom_data['date'])}") + + # Live tag + if custom_data.get('live', False): + xml_lines.append(f" ") + + # New tag + if custom_data.get('new', False): + xml_lines.append(f" ") + + xml_lines.append(f" ") + + return xml_lines + + +def generate_epg(request, profile_name=None, user=None): + """ + Dynamically generate an XMLTV (EPG) file using a streaming response. + Since the EPG data is stored independently of Channels, we group programmes + by their associated EPGData record. + """ + user_custom = (user.custom_properties or {}) if user else {} + try: + num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) + num_days = max(0, min(num_days, 365)) + except (ValueError, TypeError): + num_days = 0 + try: + prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) + prev_days = max(0, min(prev_days, 30)) + except (ValueError, TypeError): + prev_days = 0 + use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' + tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() + cache_params = ( + f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" + f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" + ) + content_cache_key = f"epg_content:{cache_params}" + + def epg_generator(): + """Generator function that yields EPG data with keep-alives during processing.""" + + yield '\n' + yield ( + '\n' + ) + + # Get channels based on user/profile + if user is not None: + if user.user_level < 10: + user_profile_count = user.channel_profiles.count() + + # If user has ALL profiles or NO profiles, give unrestricted access + if user_profile_count == 0: + # No profile filtering - user sees all channels based on user_level + filters = {"user_level__lte": user.user_level} + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') + else: + # User has specific limited profiles assigned + filters = { + "channelprofilemembership__enabled": True, + "user_level__lte": user.user_level, + "channelprofilemembership__channel_profile__in": user.channel_profiles.all() + } + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() + else: + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') + else: + if profile_name is not None: + try: + channel_profile = ChannelProfile.objects.get(name=profile_name) + except ChannelProfile.DoesNotExist: + logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) + raise Http404(f"Channel profile '{profile_name}' not found") + base_qs = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True, + ).select_related('logo', 'epg_data__epg_source') + else: + base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') + + # Resolve effective values at SQL level and exclude hidden channels + # so output ordering/display honors user overrides. + from apps.channels.managers import with_effective_values + channels = list( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") + .prefetch_related( + Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + ) + ) + channel_count = len(channels) + + # For dummy EPG, use either the specified value or default to 3 days + dummy_days = num_days if num_days > 0 else 3 + + # Calculate cutoff dates for EPG data filtering + now = django_timezone.now() + cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None + lookback_cutoff = now - timedelta(days=prev_days) + + # Build collision-free channel number mapping for XC clients (if user is authenticated) + # XC clients require integer channel numbers, so we need to ensure no conflicts + channel_num_map = {} + if user is not None: + # This is an XC client - build collision-free mapping + used_numbers = set() + + # First pass: assign integers for channels that already have integer numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) + channel_num_map[channel.id] = num + used_numbers.add(num) + + # Second pass: assign integers for channels with float numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): + candidate = int(effective_num) + while candidate in used_numbers: + candidate += 1 + channel_num_map[channel.id] = candidate + used_numbers.add(candidate) + + # Host/port/scheme are constant per request; precompute logo URL prefix once. + _base_url = build_absolute_uri_with_port(request, "") + _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) + _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") + _logo_url_prefix = _base_url + _logo_prefix_raw + "/" + _logo_url_suffix = "/" + _logo_suffix_raw + + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] + + for channel in channels: + effective_name = channel.effective_name + effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id + effective_logo = channel.effective_logo_obj + effective_number = channel.effective_channel_number + + # user is set only for XC clients, which require integer channel numbers + if user is not None: + formatted_channel_number = channel_num_map[channel.id] + else: + formatted_channel_number = format_channel_number(effective_number) + + # Determine the channel ID based on the selected source + if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: + channel_id = channel.effective_tvg_id + elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: + channel_id = channel.effective_tvc_guide_stationid + else: + channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) + + tvg_logo = "" + + # Check if this is a custom dummy EPG with channel logo URL template + if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + channel_logo_url_template = custom_props.get('channel_logo_url', '') + + if channel_logo_url_template: + pattern_match_name, _ = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + + # Try to extract groups from the channel/stream name and build the logo URL + title_pattern = custom_props.get('title_pattern', '') + if title_pattern: + try: + # Convert PCRE/JavaScript named groups to Python format + title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) + title_regex = regex.compile(title_pattern) + title_match = title_regex.search(pattern_match_name) + + if title_match: + groups = title_match.groupdict() + + # Add normalized versions of all groups for cleaner URLs + for key, value in list(groups.items()): + if value: + # Remove all non-alphanumeric characters and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + groups[f'{key}_normalize'] = normalized + + # Format the logo URL template with the matched groups (with URL encoding) + from urllib.parse import quote + for key, value in groups.items(): + if value: + encoded_value = quote(str(value), safe='') + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) + else: + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') + tvg_logo = channel_logo_url_template + logger.debug(f"Built channel logo URL from template: {tvg_logo}") + except Exception as e: + logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") + + # If no custom dummy logo, use regular logo logic + if not tvg_logo and effective_logo: + if use_cached_logos: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + else: + # Use direct URL if available, otherwise fall back to cached version + direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None + if direct_logo: + tvg_logo = direct_logo + else: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") + + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] + + pattern_match_name = effective_name + if effective_epg_data and effective_epg_data.epg_source: + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + if ( + custom_props.get('name_source') == 'stream' + and not stream_lookup_failed + and pattern_match_name != effective_name + ): + stream_index = custom_props.get('stream_index', 1) - 1 + logger.debug( + f"Using stream name for parsing: {pattern_match_name} " + f"(stream index: {stream_index})" + ) + elif stream_lookup_failed: + stream_index = custom_props.get('stream_index', 1) - 1 + logger.warning( + f"Stream index {stream_index} not found for channel " + f"{effective_name}, falling back to channel name" + ) + + if not effective_epg_data: + dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) + else: + real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) + + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE + + all_epg_ids = list(real_epg_map.keys()) + if all_epg_ids: + if num_days > 0: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + start_time__lt=cutoff_date, + ) + else: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + ) + + programs_base_qs = programs_qs.order_by('epg_id', 'id').values( + 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', + 'description', 'custom_properties', + ) + + current_epg_id = None + channel_ids_for_epg = None + pending = [] + program_batch = [] + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE + last_epg_id = 0 + last_id = 0 + _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + + while True: + program_chunk = list( + programs_base_qs.filter(epg_id__gte=last_epg_id) + .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] + ) + if not program_chunk: + break + + last_row = program_chunk[-1] + last_epg_id = last_row['epg_id'] + last_id = last_row['id'] + + for prog in program_chunk: + epg_id = prog['epg_id'] + + if epg_id != current_epg_id: + yield from flush_pending() + current_epg_id = epg_id + channel_ids_for_epg = real_epg_map[epg_id] + + primary_cid = channel_ids_for_epg[0] + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" + + program_xml = [f' '] + program_xml.append(f' {html.escape(prog["title"])}') + + if prog['sub_title']: + program_xml.append(f" {html.escape(prog['sub_title'])}") + + if prog['description']: + program_xml.append(f" {html.escape(prog['description'])}") + + custom_data = prog['custom_properties'] or {} + if custom_data: + + if "categories" in custom_data and custom_data["categories"]: + for category in custom_data["categories"]: + program_xml.append(f" {html.escape(category)}") + + if "keywords" in custom_data and custom_data["keywords"]: + for keyword in custom_data["keywords"]: + program_xml.append(f" {html.escape(keyword)}") + + # onscreen_episode takes priority over episode for the onscreen system + if "onscreen_episode" in custom_data: + program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') + elif "episode" in custom_data: + program_xml.append(f' E{custom_data["episode"]}') + + # Handle dd_progid format + if 'dd_progid' in custom_data: + program_xml.append(f' {html.escape(custom_data["dd_progid"])}') + + # Handle external database IDs + for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: + if f'{system}_id' in custom_data: + program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') + + # Add season and episode numbers in xmltv_ns format if available + if "season" in custom_data and "episode" in custom_data: + season = ( + int(custom_data["season"]) - 1 + if str(custom_data["season"]).isdigit() + else 0 + ) + episode = ( + int(custom_data["episode"]) - 1 + if str(custom_data["episode"]).isdigit() + else 0 + ) + program_xml.append(f' {season}.{episode}.') + + if "language" in custom_data: + program_xml.append(f' {html.escape(custom_data["language"])}') + + if "original_language" in custom_data: + program_xml.append(f' {html.escape(custom_data["original_language"])}') + + if "length" in custom_data and isinstance(custom_data["length"], dict): + length_value = custom_data["length"].get("value", "") + length_units = custom_data["length"].get("units", "minutes") + program_xml.append(f' {html.escape(str(length_value))}') + + if "video" in custom_data and isinstance(custom_data["video"], dict): + program_xml.append(" ") + + if "audio" in custom_data and isinstance(custom_data["audio"], dict): + program_xml.append(" ") + + if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): + for subtitle in custom_data["subtitles"]: + if isinstance(subtitle, dict): + subtitle_type = subtitle.get("type", "") + type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" + program_xml.append(f" ") + if "language" in subtitle: + program_xml.append(f" {html.escape(subtitle['language'])}") + program_xml.append(" ") + + if "rating" in custom_data: + rating_system = custom_data.get("rating_system", "TV Parental Guidelines") + program_xml.append(f' ') + program_xml.append(f' {html.escape(custom_data["rating"])}') + program_xml.append(f" ") + + if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): + for star_rating in custom_data["star_ratings"]: + if isinstance(star_rating, dict) and "value" in star_rating: + system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" + program_xml.append(f" ") + program_xml.append(f" {html.escape(star_rating['value'])}") + program_xml.append(" ") + + if "reviews" in custom_data and isinstance(custom_data["reviews"], list): + for review in custom_data["reviews"]: + if isinstance(review, dict) and "content" in review: + review_type = review.get("type", "text") + attrs = [f'type="{html.escape(review_type)}"'] + if "source" in review: + attrs.append(f'source="{html.escape(review["source"])}"') + if "reviewer" in review: + attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') + attr_str = " ".join(attrs) + program_xml.append(f' {html.escape(review["content"])}') + + if "images" in custom_data and isinstance(custom_data["images"], list): + for image in custom_data["images"]: + if isinstance(image, dict) and "url" in image: + attrs = [] + for attr in ['type', 'size', 'orient', 'system']: + if attr in image: + attrs.append(f'{attr}="{html.escape(image[attr])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f' {html.escape(image["url"])}') + + # Add enhanced credits handling + if "credits" in custom_data: + program_xml.append(" ") + credits = custom_data["credits"] + + for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if role in credits: + people = credits[role] + if isinstance(people, list): + for person in people: + program_xml.append(f" <{role}>{html.escape(person)}") + else: + program_xml.append(f" <{role}>{html.escape(people)}") + + # Handle actors separately to include role and guest attributes + if "actor" in credits: + actors = credits["actor"] + if isinstance(actors, list): + for actor in actors: + if isinstance(actor, dict): + name = actor.get("name", "") + role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" + guest_attr = ' guest="yes"' if actor.get("guest") else "" + program_xml.append(f" {html.escape(name)}") + else: + program_xml.append(f" {html.escape(actor)}") + else: + program_xml.append(f" {html.escape(actors)}") + + program_xml.append(" ") + + if "date" in custom_data: + program_xml.append(f' {html.escape(custom_data["date"])}') + + if "country" in custom_data: + program_xml.append(f' {html.escape(custom_data["country"])}') + + if "icon" in custom_data: + program_xml.append(f' ') + elif "sd_icon" in custom_data: + program_xml.append(f' ') + + # Add special flags as proper tags with enhanced handling + if custom_data.get("previously_shown", False): + prev_shown_details = custom_data.get("previously_shown_details", {}) + attrs = [] + if "start" in prev_shown_details: + attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') + if "channel" in prev_shown_details: + attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f" ") + + if custom_data.get("premiere", False): + premiere_text = custom_data.get("premiere_text", "") + if premiere_text: + program_xml.append(f" {html.escape(premiere_text)}") + else: + program_xml.append(" ") + + if custom_data.get("last_chance", False): + last_chance_text = custom_data.get("last_chance_text", "") + if last_chance_text: + program_xml.append(f" {html.escape(last_chance_text)}") + else: + program_xml.append(" ") + + if custom_data.get("new", False): + program_xml.append(" ") + + if custom_data.get('live', False): + program_xml.append(' ') + + program_xml.append(" ") + + xml_text = '\n'.join(program_xml) + pending.append((prog['start_time'], prog['id'], xml_text)) + + del program_chunk + + yield from flush_pending() + + if program_batch: + yield '\n'.join(program_batch) + '\n' + + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + + yield "\n" + + from apps.output.views import get_client_identifier + + client_id, client_ip, user_agent = get_client_identifier(request) + event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" + + def _log_epg_download(): + from django.core.cache import cache as event_cache + + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_cached_response( + content_cache_key, + build_epg_stream, + content_type="application/xml", + filename="Dispatcharr.xml", + ) diff --git a/apps/output/epg_chunk_cache.py b/apps/output/streaming_chunk_cache.py similarity index 81% rename from apps/output/epg_chunk_cache.py rename to apps/output/streaming_chunk_cache.py index 92ea6eeb..f86bb060 100644 --- a/apps/output/epg_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -1,4 +1,4 @@ -"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" +"""Single-flight Redis chunk cache for large streaming HTTP responses.""" import logging import time @@ -111,7 +111,7 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): try: from django.core.cache import cache as django_cache - django_cache.delete(base_key) # legacy monolithic entry + django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) for chunk in source(): @@ -123,9 +123,9 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): redis.expire(chunks_key, cache_ttl) redis.expire(status_key, cache_ttl) redis.expire(_ready_key(base_key), cache_ttl) - logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + logger.debug("Cached response in %s chunks", redis.llen(chunks_key)) except Exception: - logger.exception("EPG cache build failed for %s", base_key) + logger.exception("Chunk cache build failed for %s", base_key) redis.delete(chunks_key) redis.set(status_key, STATUS_ERROR, ex=60) raise @@ -158,14 +158,14 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - raise RuntimeError("EPG cache build failed") + raise RuntimeError("Chunk cache build failed") if time.monotonic() >= deadline: if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + logger.warning("Chunk cache follower timed out; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - logger.warning("EPG cache follower timed out after partial read for %s", base_key) + logger.warning("Chunk cache follower timed out after partial read for %s", base_key) break lock_active = bool(redis.exists(lock_key)) @@ -173,7 +173,7 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, idle_polls += 1 if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): if _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache leader lost; rebuilding %s", base_key) + logger.warning("Chunk cache leader lost; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return else: @@ -182,10 +182,12 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, _poll_wait(poll_interval) -def stream_epg_response( +def stream_cached_response( cache_key, source, *, + content_type="application/xml", + filename=None, cache_ttl=DEFAULT_CACHE_TTL, lock_ttl=DEFAULT_LOCK_TTL, poll_interval=DEFAULT_POLL_INTERVAL, @@ -193,16 +195,17 @@ def stream_epg_response( redis=None, ): """ - Stream XMLTV EPG output with single-flight Redis chunk caching. + Stream a large response with single-flight Redis chunk caching. ``source`` must be a callable returning a chunk iterator. Only the leader - invokes it; followers read chunks already written to Redis. + invokes it; concurrent followers replay chunks already written to Redis, so + the expensive ``source`` runs at most once per ``cache_key``. """ if redis is None: redis = _get_redis() if redis.get(_ready_key(cache_key)): - logger.debug("Serving EPG from chunk cache") + logger.debug("Serving response from chunk cache") stream = _stream_ready(redis, cache_key) else: status = _get_status(redis, cache_key) @@ -210,10 +213,10 @@ def stream_epg_response( _clear_build_keys(redis, cache_key) if _try_acquire_lock(redis, cache_key, lock_ttl): - logger.debug("Building EPG (cache leader)") + logger.debug("Building response (cache leader)") stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) else: - logger.debug("Following in-flight EPG build") + logger.debug("Following in-flight cache build") stream = _stream_follow( redis, cache_key, @@ -224,7 +227,8 @@ def stream_epg_response( max_follower_wait, ) - response = StreamingHttpResponse(stream, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response = StreamingHttpResponse(stream, content_type=content_type) + if filename: + response["Content-Disposition"] = f'attachment; filename="{filename}"' response["Cache-Control"] = "no-cache" return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_streaming_chunk_cache.py similarity index 84% rename from apps/output/test_epg_chunk_cache.py rename to apps/output/test_streaming_chunk_cache.py index 6ca217af..599501b7 100644 --- a/apps/output/test_epg_chunk_cache.py +++ b/apps/output/test_streaming_chunk_cache.py @@ -2,14 +2,14 @@ import threading import time from unittest import TestCase -from apps.output.epg_chunk_cache import ( +from apps.output.streaming_chunk_cache import ( STATUS_BUILDING, STATUS_READY, _chunks_key, _lock_key, _ready_key, _status_key, - stream_epg_response, + stream_cached_response, ) @@ -74,7 +74,7 @@ def _consume(response): return b"".join(response.streaming_content).decode("utf-8") -class EPGChunkCacheTests(TestCase): +class StreamingChunkCacheTests(TestCase): def test_leader_caches_chunks_and_sets_ready(self): redis = FakeRedis() calls = [] @@ -84,14 +84,14 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, [1]) - self.assertEqual(redis.get(_ready_key("epg:test")), "1") - self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) - self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) - self.assertFalse(redis.exists(_lock_key("epg:test"))) + self.assertEqual(redis.get(_ready_key("cache:test")), "1") + self.assertEqual(redis.get(_status_key("cache:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("cache:test")), 2) + self.assertFalse(redis.exists(_lock_key("cache:test"))) def test_cache_hit_skips_source(self): redis = FakeRedis() @@ -102,16 +102,16 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - _consume(stream_epg_response("epg:test", source, redis=redis)) + _consume(stream_cached_response("cache:test", source, redis=redis)) calls.clear() - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, []) def test_follower_reads_leader_chunks_without_rebuilding(self): redis = FakeRedis() - base = "epg:follow" + base = "cache:follow" leader_started = threading.Event() rebuild_calls = [] @@ -128,7 +128,7 @@ class EPGChunkCacheTests(TestCase): def leader(): _consume( - stream_epg_response( + stream_cached_response( base, slow_source, redis=redis, @@ -140,7 +140,7 @@ class EPGChunkCacheTests(TestCase): leader_thread.start() leader_started.wait(timeout=5) follower_body = _consume( - stream_epg_response( + stream_cached_response( base, forbidden_source, redis=redis, @@ -165,8 +165,8 @@ class EPGChunkCacheTests(TestCase): def worker(): barrier.wait() results[threading.current_thread().name] = _consume( - stream_epg_response( - "epg:race", + stream_cached_response( + "cache:race", source, redis=redis, poll_interval=0.01, diff --git a/apps/output/tests.py b/apps/output/tests.py index 6e2ab395..5f4fd5f3 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -34,16 +34,18 @@ class OutputEndpointTestMixin: "apps.output.views.network_access_allowed", return_value=True, ) - self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._epg_teardown_patch = patch("apps.output.epg._epg_export_teardown") self._log_event_patch = patch("apps.output.views.log_system_event") + self._epg_log_event_patch = patch("apps.output.epg.log_system_event") self._close_db_patch = patch("django.db.close_old_connections") self._epg_cache_patch = patch( - "apps.output.views.stream_epg_response", + "apps.output.epg.stream_cached_response", side_effect=_epg_response_without_redis, ) self._network_patch.start() self._epg_teardown_patch.start() self._log_event_patch.start() + self._epg_log_event_patch.start() self._close_db_patch.start() self._epg_cache_patch.start() @@ -53,6 +55,7 @@ class OutputEndpointTestMixin: cache.clear() self._epg_cache_patch.stop() self._close_db_patch.stop() + self._epg_log_event_patch.stop() self._log_event_patch.stop() self._epg_teardown_patch.stop() self._network_patch.stop() @@ -339,14 +342,14 @@ class OutputEPGCustomDummyTest(TestCase): class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=30, second=0, microsecond=0) self.assertEqual(_ceil_to_half_hour(dt), dt) def test_ceil_to_half_hour_rounds_up(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=17, second=42, microsecond=123456) aligned = _ceil_to_half_hour(dt) diff --git a/apps/output/views.py b/apps/output/views.py index a0875c7f..4224fb35 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,54 +25,10 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib -from apps.output.epg_chunk_cache import stream_epg_response +from apps.output.epg import generate_epg, generate_dummy_programs logger = logging.getLogger(__name__) -_EPG_CHANNEL_XML_BATCH_SIZE = 200 -_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 -_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 - - -def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): - if end_time < lookback_cutoff: - return False - if cutoff_date is not None and start_time >= cutoff_date: - return False - return True - - -def _ceil_to_half_hour(dt): - """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) - - -def _epg_export_teardown(): - from django.db import close_old_connections - - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() - def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -384,1674 +340,6 @@ def generate_m3u(request, profile_name=None, user=None): return response -def _ordered_channel_streams(channel): - """Return a channel's streams ordered by channelstream join order.""" - prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') - if prefetched is not None: - return list(prefetched) - return list(channel.streams.all().order_by('channelstream__order')) - - -def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): - """Name used for custom dummy EPG regex matching (channel or stream title). - - Returns (name, stream_lookup_failed). stream_lookup_failed is True only when - name_source is 'stream' but the configured index is missing or out of range. - """ - if custom_props.get('name_source') != 'stream': - return effective_name, False - stream_index = custom_props.get('stream_index', 1) - 1 - streams = _ordered_channel_streams(channel) - if 0 <= stream_index < len(streams): - return streams[stream_index].name, False - return effective_name, True - - -def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): - """ - Generate dummy programs using custom fallback templates when patterns don't match. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel name to use as fallback in templates - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - fallback_title: Custom fallback title template (empty string if not provided) - fallback_description: Custom fallback description template (empty string if not provided) - - Returns: - List of program dictionaries - """ - programs = [] - - # Use custom fallback title or channel name as default - title = fallback_title if fallback_title else channel_name - - # Use custom fallback description or a simple default message - if fallback_description: - description = fallback_description - else: - description = f"EPG information is currently unavailable for {channel_name}" - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": title, - "description": description, - }) - - return programs - - -def generate_dummy_programs( - channel_id, - channel_name, - num_days=1, - program_length_hours=4, - epg_source=None, - export_lookback=None, - export_cutoff=None, -): - """ - Generate dummy EPG programs for channels. - - If epg_source is provided and it's a custom dummy EPG with patterns, - use those patterns to generate programs from the channel title. - Otherwise, generate default dummy programs. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title/name - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - epg_source: Optional EPGSource for custom dummy EPG with patterns - - Returns: - List of program dictionaries - """ - # Get current time rounded to hour - now = django_timezone.now() - now = now.replace(minute=0, second=0, microsecond=0) - - # Check if this is a custom dummy EPG with regex patterns - if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: - custom_programs = generate_custom_dummy_programs( - channel_id, channel_name, now, num_days, - epg_source.custom_properties, - export_lookback=export_lookback, - export_cutoff=export_cutoff, - ) - if custom_programs is not None: - return custom_programs - - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() - - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - logger.info(f"No custom fallback templates found, using default dummy EPG") - - # Default humorous program descriptions based on time of day - time_descriptions = { - (0, 4): [ - f"Late Night with {channel_name} - Where insomniacs unite!", - f"The 'Why Am I Still Awake?' Show on {channel_name}", - f"Counting Sheep - A {channel_name} production for the sleepless", - ], - (4, 8): [ - f"Dawn Patrol - Rise and shine with {channel_name}!", - f"Early Bird Special - Coffee not included", - f"Morning Zombies - Before coffee viewing on {channel_name}", - ], - (8, 12): [ - f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", - f"The 'I Should Be Working' Hour on {channel_name}", - f"Productivity Killer - {channel_name}'s daytime programming", - ], - (12, 16): [ - f"Lunchtime Laziness with {channel_name}", - f"The Afternoon Slump - Brought to you by {channel_name}", - f"Post-Lunch Food Coma Theater on {channel_name}", - ], - (16, 20): [ - f"Rush Hour - {channel_name}'s alternative to traffic", - f"The 'What's For Dinner?' Debate on {channel_name}", - f"Evening Escapism - {channel_name}'s remedy for reality", - ], - (20, 24): [ - f"Prime Time Placeholder - {channel_name}'s finest not-programming", - f"The 'Netflix Was Too Complicated' Show on {channel_name}", - f"Family Argument Avoider - Courtesy of {channel_name}", - ], - } - - programs = [] - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - # Get the hour for selecting a description - hour = start_time.hour - - # Find the appropriate time slot for description - for time_range, descriptions in time_descriptions.items(): - start_range, end_range = time_range - if start_range <= hour < end_range: - # Pick a description using the sum of the hour and day as seed - # This makes it somewhat random but consistent for the same timeslot - description = descriptions[(hour + day) % len(descriptions)] - break - else: - # Fallback description if somehow no range matches - description = f"Placeholder program for {channel_name} - EPG data went on vacation" - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": channel_name, - "description": description, - }) - - return programs - - -def generate_custom_dummy_programs( - channel_id, - channel_name, - now, - num_days, - custom_properties, - export_lookback=None, - export_cutoff=None, -): - """ - Generate programs using custom dummy EPG regex patterns. - - Extracts information from channel title using regex patterns and generates - programs based on the extracted data. - - TIMEZONE HANDLING: - ------------------ - The timezone parameter specifies the timezone of the event times in your channel - titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). - DST (Daylight Saving Time) is handled automatically by pytz. - - Examples: - - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" - - Set timezone = "US/Eastern" - - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) - - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title to parse - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - custom_properties: Dict with title_pattern, time_pattern, templates, etc. - - timezone: Timezone name (e.g., 'US/Eastern') - - Returns: - List of program dictionaries with start_time/end_time in UTC - """ - import pytz - - logger.info(f"Generating custom dummy programs for channel: {channel_name}") - - # Extract patterns from custom properties - title_pattern = custom_properties.get('title_pattern', '') - time_pattern = custom_properties.get('time_pattern', '') - date_pattern = custom_properties.get('date_pattern', '') - - # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') - timezone_value = custom_properties.get('timezone', 'UTC') - output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone - program_duration = custom_properties.get('program_duration', 180) # Minutes - title_template = custom_properties.get('title_template', '') - subtitle_template = custom_properties.get('subtitle_template', '') - description_template = custom_properties.get('description_template', '') - - # Templates for upcoming/ended programs - upcoming_title_template = custom_properties.get('upcoming_title_template', '') - upcoming_description_template = custom_properties.get('upcoming_description_template', '') - ended_title_template = custom_properties.get('ended_title_template', '') - ended_description_template = custom_properties.get('ended_description_template', '') - - # Image URL templates - channel_logo_url_template = custom_properties.get('channel_logo_url', '') - program_poster_url_template = custom_properties.get('program_poster_url', '') - - # EPG metadata options - category_string = custom_properties.get('category', '') - # Split comma-separated categories and strip whitespace, filter out empty strings - categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] - include_date = custom_properties.get('include_date', True) - include_live = custom_properties.get('include_live', False) - include_new = custom_properties.get('include_new', False) - - # Parse timezone name - try: - source_tz = pytz.timezone(timezone_value) - logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") - source_tz = pytz.utc - - # Parse output timezone if provided (for display purposes) - output_tz = None - if output_timezone_value: - try: - output_tz = pytz.timezone(output_timezone_value) - logger.debug(f"Using output timezone for display: {output_timezone_value}") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") - output_tz = None - - if not title_pattern: - logger.warning(f"No title_pattern in custom_properties, falling back to default") - return None - - logger.debug(f"Title pattern from DB: {repr(title_pattern)}") - - # Convert PCRE/JavaScript named groups (?) to Python format (?P) - # This handles patterns created with JavaScript regex syntax - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) - logger.debug(f"Converted title pattern: {repr(title_pattern)}") - - # Compile regex patterns using the enhanced regex module - # (supports variable-width lookbehinds like JavaScript) - try: - title_regex = regex.compile(title_pattern) - except Exception as e: - logger.error(f"Invalid title regex pattern after conversion: {e}") - logger.error(f"Pattern was: {repr(title_pattern)}") - return None - - time_regex = None - if time_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) - logger.debug(f"Converted time pattern: {repr(time_pattern)}") - try: - time_regex = regex.compile(time_pattern) - except Exception as e: - logger.warning(f"Invalid time regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(time_pattern)}") - - # Compile date regex if provided - date_regex = None - if date_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) - logger.debug(f"Converted date pattern: {repr(date_pattern)}") - try: - date_regex = regex.compile(date_pattern) - except Exception as e: - logger.warning(f"Invalid date regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(date_pattern)}") - - # Try to match the channel name with the title pattern - # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string - title_match = title_regex.search(channel_name) - if not title_match: - logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return None - - groups = title_match.groupdict() - logger.debug(f"Title pattern matched. Groups: {groups}") - - # Helper function to format template with matched groups - def format_template(template, groups, url_encode=False): - """Replace {groupname} placeholders with matched group values - - Args: - template: Template string with {groupname} placeholders - groups: Dict of group names to values - url_encode: If True, URL encode the group values for safe use in URLs - """ - if not template: - return '' - result = template - for key, value in groups.items(): - if url_encode and value: - # URL encode the value to handle spaces and special characters - from urllib.parse import quote - encoded_value = quote(str(value), safe='') - result = result.replace(f'{{{key}}}', encoded_value) - else: - result = result.replace(f'{{{key}}}', str(value) if value else '') - return result - - # Extract time from title if time pattern exists - time_info = None - time_groups = {} - if time_regex: - time_match = time_regex.search(channel_name) - if time_match: - time_groups = time_match.groupdict() - try: - hour = int(time_groups.get('hour')) - # Handle optional minute group - could be None if not captured - minute_value = time_groups.get('minute') - minute = int(minute_value) if minute_value is not None else 0 - ampm = time_groups.get('ampm') - ampm = ampm.lower() if ampm else None - - # Determine if this is 12-hour or 24-hour format - if ampm in ('am', 'pm'): - # 12-hour format: convert to 24-hour - if ampm == 'pm' and hour != 12: - hour += 12 - elif ampm == 'am' and hour == 12: - hour = 0 - logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") - else: - # 24-hour format: hour is already in 24-hour format - # Validate that it's actually a 24-hour time (0-23) - if hour > 23: - logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") - hour = hour % 24 # Wrap around just in case - logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") - - time_info = {'hour': hour, 'minute': minute} - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing time: {e}") - - # Extract date from title if date pattern exists - date_info = None - date_groups = {} - if date_regex: - date_match = date_regex.search(channel_name) - if date_match: - date_groups = date_match.groupdict() - try: - # Support various date group names: month, day, year - month_str = date_groups.get('month', '') - day_str = date_groups.get('day', '') - year_str = date_groups.get('year', '') - - # Parse day - default to current day if empty or invalid - day = int(day_str) if day_str else now.day - - # Parse year - default to current year if empty or invalid (matches frontend behavior) - year = int(year_str) if year_str else now.year - - # Parse month - can be numeric (1-12) or text (Jan, January, etc.) - month = None - if month_str: - if month_str.isdigit(): - month = int(month_str) - else: - # Try to parse text month names - import calendar - month_str_lower = month_str.lower() - # Check full month names - for i, month_name in enumerate(calendar.month_name): - if month_name.lower() == month_str_lower: - month = i - break - # Check abbreviated month names if not found - if month is None: - for i, month_abbr in enumerate(calendar.month_abbr): - if month_abbr.lower() == month_str_lower: - month = i - break - - # Default to current month if not extracted or invalid - if month is None: - month = now.month - - if month and 1 <= month <= 12 and 1 <= day <= 31: - date_info = {'year': year, 'month': month, 'day': day} - logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") - else: - logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing date: {e}") - - # Merge title groups, time groups, and date groups for template formatting - all_groups = {**groups, **time_groups, **date_groups} - - # Add normalized versions of all groups for cleaner URLs - # These remove all non-alphanumeric characters and convert to lowercase - for key, value in list(all_groups.items()): - if value: - # Remove all non-alphanumeric characters (except spaces temporarily) - # then replace spaces with nothing, and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - all_groups[f'{key}_normalize'] = normalized - - # Format channel logo URL if template provided (with URL encoding) - channel_logo_url = None - if channel_logo_url_template: - channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted channel logo URL: {channel_logo_url}") - - # Format program poster URL if template provided (with URL encoding) - program_poster_url = None - if program_poster_url_template: - program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted program poster URL: {program_poster_url}") - - # Add formatted time strings for better display (handles minutes intelligently) - if time_info: - hour_24 = time_info['hour'] - minute = time_info['minute'] - - # Determine the base date to use for placeholders - # If date was extracted, use it; otherwise use current date - if date_info: - base_date = datetime(date_info['year'], date_info['month'], date_info['day']) - else: - base_date = datetime.now() - - # If output_timezone is specified, convert the display time to that timezone - if output_tz: - # Create a datetime in the source timezone using the base date - temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - # Convert to output timezone - temp_date_output = temp_date.astimezone(output_tz) - # Extract converted hour and minute for display - hour_24 = temp_date_output.hour - minute = temp_date_output.minute - logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") - - # Add date placeholders based on the OUTPUT timezone - # This ensures {date}, {month}, {day}, {year} reflect the converted timezone - all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_output.month) - all_groups['day'] = str(temp_date_output.day) - all_groups['year'] = str(temp_date_output.year) - logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") - else: - # No output timezone conversion - use source timezone for date - # Create temp date to get proper date in source timezone using the base date - temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_source.month) - all_groups['day'] = str(temp_date_source.day) - all_groups['year'] = str(temp_date_source.year) - - # Format 24-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime24'] = f"{hour_24}:{minute:02d}" - else: - all_groups['starttime24'] = f"{hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {starttime} placeholder - # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) - ampm = 'AM' if hour_24 < 12 else 'PM' - hour_12 = hour_24 - if hour_24 == 0: - hour_12 = 12 - elif hour_24 > 12: - hour_12 = hour_24 - 12 - - # Format 12-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" - else: - all_groups['starttime'] = f"{hour_12} {ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" - - # Calculate end time based on program duration - # Create a datetime for calculations - temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) - temp_end = temp_start + timedelta(minutes=program_duration) - - # Extract end time components (already in correct timezone if output_tz was applied above) - end_hour_24 = temp_end.hour - end_minute = temp_end.minute - - # Format 24-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" - else: - all_groups['endtime24'] = f"{end_hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {endtime} placeholder - end_ampm = 'AM' if end_hour_24 < 12 else 'PM' - end_hour_12 = end_hour_24 - if end_hour_24 == 0: - end_hour_12 = 12 - elif end_hour_24 > 12: - end_hour_12 = end_hour_24 - 12 - - # Format 12-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - else: - all_groups['endtime'] = f"{end_hour_12} {end_ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - - # Generate programs - programs = [] - - # If we have extracted time AND date, the event happens on a SPECIFIC date - # If we have time but NO date, generate for multiple days (existing behavior) - # All other days and times show "Upcoming" before or "Ended" after - event_happened = False - - # Determine how many iterations we need - if date_info and time_info: - # Specific date extracted - only generate for that one date - iterations = 1 - logger.debug(f"Date extracted, generating single event for specific date") - else: - # No specific date - use num_days (existing behavior) - iterations = num_days - - for day in range(iterations): - event_overlaps_window = True - if date_info and time_info: - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'], - ).date() - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - event_overlaps_window = _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ) - if not event_overlaps_window: - logger.debug( - "Custom dummy event outside export window; filling window only: %s", - channel_name, - ) - event_happened = event_end_utc < lookback - day_start = _ceil_to_half_hour(lookback) - if export_cutoff is not None: - day_end = export_cutoff - else: - day_end = now + timedelta(days=num_days if num_days > 0 else 3) - else: - day_start = source_tz.localize( - datetime.combine(current_date, datetime.min.time()) - ).astimezone(pytz.utc) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - else: - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - - if day_start >= day_end: - continue - - if time_info: - if not date_info: - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_local = source_tz.localize(event_start_naive) - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - if not _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ): - continue - else: - logger.debug(f"Using extracted date: {current_date}") - - # Pre-generate the main event title and description for reuse - if title_template: - main_event_title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - main_event_title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - main_event_subtitle = format_template(subtitle_template, all_groups) - else: - main_event_subtitle = None - - if description_template: - main_event_description = format_template(description_template, all_groups) - else: - main_event_description = main_event_title - - - - # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) when it falls inside the window - is_event_day = (day == 0) and event_overlaps_window - - if is_event_day and not event_happened: - current_time = day_start - - while current_time < event_start_utc and current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) - - # Use custom upcoming templates if provided, otherwise use defaults - if upcoming_title_template: - upcoming_title = format_template(upcoming_title_template, all_groups) - else: - upcoming_title = main_event_title - - if upcoming_description_template: - upcoming_description = format_template(upcoming_description_template, all_groups) - else: - upcoming_description = f"Upcoming: {main_event_description}" - - # Build custom_properties for upcoming programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": upcoming_title, - "sub_title": None, # No subtitle for filler programs - "description": upcoming_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - - # Add the MAIN EVENT at the extracted time - # Build custom_properties for main event (includes category and live) - main_event_custom_properties = {} - - # Add categories if provided - if categories: - main_event_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = event_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - main_event_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - main_event_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - main_event_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - main_event_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": event_start_utc, - "end_time": event_end_utc, - "title": main_event_title, - "sub_title": main_event_subtitle, - "description": main_event_description, - "custom_properties": main_event_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - event_happened = True - - # Fill programs AFTER the event until end of export day window - current_time = max(event_end_utc, day_start) - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom ended templates if provided, otherwise use defaults - if ended_title_template: - ended_title = format_template(ended_title_template, all_groups) - else: - ended_title = main_event_title - - if ended_description_template: - ended_description = format_template(ended_description_template, all_groups) - else: - ended_description = f"Ended: {main_event_description}" - - # Build custom_properties for ended programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": ended_title, - "sub_title": None, # No subtitle for filler programs - "description": ended_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - else: - # This day is either before the event (future days) or after the event happened - # Fill entire day with appropriate message - current_time = day_start - - # If event already happened, all programs show "Ended" - # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" - is_ended = event_happened - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom templates based on whether event has ended or is upcoming - if is_ended: - if ended_title_template: - program_title = format_template(ended_title_template, all_groups) - else: - program_title = main_event_title - - if ended_description_template: - program_description = format_template(ended_description_template, all_groups) - else: - program_description = f"Ended: {main_event_description}" - else: - if upcoming_title_template: - program_title = format_template(upcoming_title_template, all_groups) - else: - program_title = main_event_title - - if upcoming_description_template: - program_description = format_template(upcoming_description_template, all_groups) - else: - program_description = f"Upcoming: {main_event_description}" - - # Build custom_properties (only date for upcoming/ended filler programs) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": program_title, - "sub_title": None, # No subtitle for filler programs - "description": program_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, - }) - - current_time += timedelta(minutes=program_duration) - else: - # No extracted time - fill entire day with regular intervals - # day_start and day_end are already in UTC, so no conversion needed - programs_per_day = max(1, int(24 / (program_duration / 60))) - - for program_num in range(programs_per_day): - program_start_utc = day_start + timedelta(minutes=program_num * program_duration) - program_end_utc = program_start_utc + timedelta(minutes=program_duration) - - if title_template: - title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - subtitle = format_template(subtitle_template, all_groups) - else: - subtitle = None - - if description_template: - description = format_template(description_template, all_groups) - else: - description = title - - # Build custom_properties for this program - program_custom_properties = {} - - # Add categories if provided - if categories: - program_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - program_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - program_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": title, - "sub_title": subtitle, - "description": description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") - return programs - - -def generate_dummy_epg( - channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 -): - """ - Generate dummy EPG programs for channels without EPG data. - Creates program blocks for a specified number of days. - - Args: - channel_id: The channel ID to use in the program entries - channel_name: The name of the channel to use in program titles - xml_lines: Optional list to append lines to, otherwise returns new list - num_days: Number of days to generate EPG data for (default: 1) - program_length_hours: Length of each program block in hours (default: 4) - - Returns: - List of XML lines for the dummy EPG entries - """ - if xml_lines is None: - xml_lines = [] - - for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): - # Format times in XMLTV format - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - - # Create program entry with escaped channel name - xml_lines.append( - f' ' - ) - xml_lines.append(f" {html.escape(program['title'])}") - - # Add subtitle if available - if program.get('sub_title'): - xml_lines.append(f" {html.escape(program['sub_title'])}") - - xml_lines.append(f" {html.escape(program['description'])}") - - # Add custom_properties if present - custom_data = program.get('custom_properties', {}) - - # Categories - if 'categories' in custom_data: - for cat in custom_data['categories']: - xml_lines.append(f" {html.escape(cat)}") - - # Date tag - if 'date' in custom_data: - xml_lines.append(f" {html.escape(custom_data['date'])}") - - # Live tag - if custom_data.get('live', False): - xml_lines.append(f" ") - - # New tag - if custom_data.get('new', False): - xml_lines.append(f" ") - - xml_lines.append(f" ") - - return xml_lines - - -def generate_epg(request, profile_name=None, user=None): - """ - Dynamically generate an XMLTV (EPG) file using a streaming response. - Since the EPG data is stored independently of Channels, we group programmes - by their associated EPGData record. - """ - user_custom = (user.custom_properties or {}) if user else {} - try: - num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) - num_days = max(0, min(num_days, 365)) - except (ValueError, TypeError): - num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 - use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' - tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() - cache_params = ( - f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" - f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" - ) - content_cache_key = f"epg_content:{cache_params}" - - def epg_generator(): - """Generator function that yields EPG data with keep-alives during processing.""" - - yield '\n' - yield ( - '\n' - ) - - # Get channels based on user/profile - if user is not None: - if user.user_level < 10: - user_profile_count = user.channel_profiles.count() - - # If user has ALL profiles or NO profiles, give unrestricted access - if user_profile_count == 0: - # No profile filtering - user sees all channels based on user_level - filters = {"user_level__lte": user.user_level} - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') - else: - # User has specific limited profiles assigned - filters = { - "channelprofilemembership__enabled": True, - "user_level__lte": user.user_level, - "channelprofilemembership__channel_profile__in": user.channel_profiles.all() - } - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() - else: - base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') - else: - if profile_name is not None: - try: - channel_profile = ChannelProfile.objects.get(name=profile_name) - except ChannelProfile.DoesNotExist: - logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) - raise Http404(f"Channel profile '{profile_name}' not found") - base_qs = Channel.objects.filter( - channelprofilemembership__channel_profile=channel_profile, - channelprofilemembership__enabled=True, - ).select_related('logo', 'epg_data__epg_source') - else: - base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') - - # Resolve effective values at SQL level and exclude hidden channels - # so output ordering/display honors user overrides. - from apps.channels.managers import with_effective_values - channels = list( - with_effective_values(base_qs, select_related_fks=True) - .exclude(hidden_from_output=True) - .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") - .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) - ) - ) - channel_count = len(channels) - - # For dummy EPG, use either the specified value or default to 3 days - dummy_days = num_days if num_days > 0 else 3 - - # Calculate cutoff dates for EPG data filtering - now = django_timezone.now() - cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None - lookback_cutoff = now - timedelta(days=prev_days) - - # Build collision-free channel number mapping for XC clients (if user is authenticated) - # XC clients require integer channel numbers, so we need to ensure no conflicts - channel_num_map = {} - if user is not None: - # This is an XC client - build collision-free mapping - used_numbers = set() - - # First pass: assign integers for channels that already have integer numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num == int(effective_num): - num = int(effective_num) - channel_num_map[channel.id] = num - used_numbers.add(num) - - # Second pass: assign integers for channels with float numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num != int(effective_num): - candidate = int(effective_num) - while candidate in used_numbers: - candidate += 1 - channel_num_map[channel.id] = candidate - used_numbers.add(candidate) - - # Host/port/scheme are constant per request; precompute logo URL prefix once. - _base_url = build_absolute_uri_with_port(request, "") - _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) - _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") - _logo_url_prefix = _base_url + _logo_prefix_raw + "/" - _logo_url_suffix = "/" + _logo_suffix_raw - - dummy_program_list = [] - real_epg_map = {} - channel_xml_batch = [] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_logo = channel.effective_logo_obj - effective_number = channel.effective_channel_number - - # user is set only for XC clients, which require integer channel numbers - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - - # Determine the channel ID based on the selected source - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - tvg_logo = "" - - # Check if this is a custom dummy EPG with channel logo URL template - if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - channel_logo_url_template = custom_props.get('channel_logo_url', '') - - if channel_logo_url_template: - pattern_match_name, _ = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - - # Try to extract groups from the channel/stream name and build the logo URL - title_pattern = custom_props.get('title_pattern', '') - if title_pattern: - try: - # Convert PCRE/JavaScript named groups to Python format - title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) - title_regex = regex.compile(title_pattern) - title_match = title_regex.search(pattern_match_name) - - if title_match: - groups = title_match.groupdict() - - # Add normalized versions of all groups for cleaner URLs - for key, value in list(groups.items()): - if value: - # Remove all non-alphanumeric characters and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - groups[f'{key}_normalize'] = normalized - - # Format the logo URL template with the matched groups (with URL encoding) - from urllib.parse import quote - for key, value in groups.items(): - if value: - encoded_value = quote(str(value), safe='') - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) - else: - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') - tvg_logo = channel_logo_url_template - logger.debug(f"Built channel logo URL from template: {tvg_logo}") - except Exception as e: - logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") - - # If no custom dummy logo, use regular logo logic - if not tvg_logo and effective_logo: - if use_cached_logos: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - else: - # Use direct URL if available, otherwise fall back to cached version - direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None - if direct_logo: - tvg_logo = direct_logo - else: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - channel_xml_batch.append(f' ') - channel_xml_batch.append(f' {html.escape(effective_name)}') - channel_xml_batch.append(f' ') - channel_xml_batch.append(" ") - - if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: - yield '\n'.join(channel_xml_batch) + '\n' - channel_xml_batch = [] - - pattern_match_name = effective_name - if effective_epg_data and effective_epg_data.epg_source: - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - if ( - custom_props.get('name_source') == 'stream' - and not stream_lookup_failed - and pattern_match_name != effective_name - ): - stream_index = custom_props.get('stream_index', 1) - 1 - logger.debug( - f"Using stream name for parsing: {pattern_match_name} " - f"(stream index: {stream_index})" - ) - elif stream_lookup_failed: - stream_index = custom_props.get('stream_index', 1) - 1 - logger.warning( - f"Stream index {stream_index} not found for channel " - f"{effective_name}, falling back to channel name" - ) - - if not effective_epg_data: - dummy_program_list.append((channel_id, pattern_match_name, None)) - elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - else: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - - if channel_xml_batch: - yield '\n'.join(channel_xml_batch) + '\n' - - del channels - del channel_num_map - - batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - - all_epg_ids = list(real_epg_map.keys()) - if all_epg_ids: - if num_days > 0: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - start_time__lt=cutoff_date, - ) - else: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - ) - - programs_base_qs = programs_qs.order_by('epg_id', 'id').values( - 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', - 'description', 'custom_properties', - ) - - current_epg_id = None - channel_ids_for_epg = None - pending = [] - program_batch = [] - chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE - last_epg_id = 0 - last_id = 0 - _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") - - def flush_pending(): - nonlocal program_batch, pending - if not pending: - return - pending.sort(key=lambda row: (row[0], row[1])) - escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None - ) - for _, _, xml_text in pending: - program_batch.append(xml_text) - if escaped_primary: - for cid in channel_ids_for_epg[1:]: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{html.escape(cid)}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - pending.clear() - - while True: - program_chunk = list( - programs_base_qs.filter(epg_id__gte=last_epg_id) - .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] - ) - if not program_chunk: - break - - last_row = program_chunk[-1] - last_epg_id = last_row['epg_id'] - last_id = last_row['id'] - - for prog in program_chunk: - epg_id = prog['epg_id'] - - if epg_id != current_epg_id: - yield from flush_pending() - current_epg_id = epg_id - channel_ids_for_epg = real_epg_map[epg_id] - - primary_cid = channel_ids_for_epg[0] - # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format - # directly instead of strftime("%Y%m%d%H%M%S %z"), which is - # ~10x slower and dominates XML build over 750k rows. - st = prog['start_time'] - et = prog['end_time'] - start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" - stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - - program_xml = [f' '] - program_xml.append(f' {html.escape(prog["title"])}') - - if prog['sub_title']: - program_xml.append(f" {html.escape(prog['sub_title'])}") - - if prog['description']: - program_xml.append(f" {html.escape(prog['description'])}") - - custom_data = prog['custom_properties'] or {} - if custom_data: - - if "categories" in custom_data and custom_data["categories"]: - for category in custom_data["categories"]: - program_xml.append(f" {html.escape(category)}") - - if "keywords" in custom_data and custom_data["keywords"]: - for keyword in custom_data["keywords"]: - program_xml.append(f" {html.escape(keyword)}") - - # onscreen_episode takes priority over episode for the onscreen system - if "onscreen_episode" in custom_data: - program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') - elif "episode" in custom_data: - program_xml.append(f' E{custom_data["episode"]}') - - # Handle dd_progid format - if 'dd_progid' in custom_data: - program_xml.append(f' {html.escape(custom_data["dd_progid"])}') - - # Handle external database IDs - for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: - if f'{system}_id' in custom_data: - program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') - - # Add season and episode numbers in xmltv_ns format if available - if "season" in custom_data and "episode" in custom_data: - season = ( - int(custom_data["season"]) - 1 - if str(custom_data["season"]).isdigit() - else 0 - ) - episode = ( - int(custom_data["episode"]) - 1 - if str(custom_data["episode"]).isdigit() - else 0 - ) - program_xml.append(f' {season}.{episode}.') - - if "language" in custom_data: - program_xml.append(f' {html.escape(custom_data["language"])}') - - if "original_language" in custom_data: - program_xml.append(f' {html.escape(custom_data["original_language"])}') - - if "length" in custom_data and isinstance(custom_data["length"], dict): - length_value = custom_data["length"].get("value", "") - length_units = custom_data["length"].get("units", "minutes") - program_xml.append(f' {html.escape(str(length_value))}') - - if "video" in custom_data and isinstance(custom_data["video"], dict): - program_xml.append(" ") - - if "audio" in custom_data and isinstance(custom_data["audio"], dict): - program_xml.append(" ") - - if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): - for subtitle in custom_data["subtitles"]: - if isinstance(subtitle, dict): - subtitle_type = subtitle.get("type", "") - type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" - program_xml.append(f" ") - if "language" in subtitle: - program_xml.append(f" {html.escape(subtitle['language'])}") - program_xml.append(" ") - - if "rating" in custom_data: - rating_system = custom_data.get("rating_system", "TV Parental Guidelines") - program_xml.append(f' ') - program_xml.append(f' {html.escape(custom_data["rating"])}') - program_xml.append(f" ") - - if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): - for star_rating in custom_data["star_ratings"]: - if isinstance(star_rating, dict) and "value" in star_rating: - system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" - program_xml.append(f" ") - program_xml.append(f" {html.escape(star_rating['value'])}") - program_xml.append(" ") - - if "reviews" in custom_data and isinstance(custom_data["reviews"], list): - for review in custom_data["reviews"]: - if isinstance(review, dict) and "content" in review: - review_type = review.get("type", "text") - attrs = [f'type="{html.escape(review_type)}"'] - if "source" in review: - attrs.append(f'source="{html.escape(review["source"])}"') - if "reviewer" in review: - attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') - attr_str = " ".join(attrs) - program_xml.append(f' {html.escape(review["content"])}') - - if "images" in custom_data and isinstance(custom_data["images"], list): - for image in custom_data["images"]: - if isinstance(image, dict) and "url" in image: - attrs = [] - for attr in ['type', 'size', 'orient', 'system']: - if attr in image: - attrs.append(f'{attr}="{html.escape(image[attr])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f' {html.escape(image["url"])}') - - # Add enhanced credits handling - if "credits" in custom_data: - program_xml.append(" ") - credits = custom_data["credits"] - - for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if role in credits: - people = credits[role] - if isinstance(people, list): - for person in people: - program_xml.append(f" <{role}>{html.escape(person)}") - else: - program_xml.append(f" <{role}>{html.escape(people)}") - - # Handle actors separately to include role and guest attributes - if "actor" in credits: - actors = credits["actor"] - if isinstance(actors, list): - for actor in actors: - if isinstance(actor, dict): - name = actor.get("name", "") - role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" - guest_attr = ' guest="yes"' if actor.get("guest") else "" - program_xml.append(f" {html.escape(name)}") - else: - program_xml.append(f" {html.escape(actor)}") - else: - program_xml.append(f" {html.escape(actors)}") - - program_xml.append(" ") - - if "date" in custom_data: - program_xml.append(f' {html.escape(custom_data["date"])}') - - if "country" in custom_data: - program_xml.append(f' {html.escape(custom_data["country"])}') - - if "icon" in custom_data: - program_xml.append(f' ') - elif "sd_icon" in custom_data: - program_xml.append(f' ') - - # Add special flags as proper tags with enhanced handling - if custom_data.get("previously_shown", False): - prev_shown_details = custom_data.get("previously_shown_details", {}) - attrs = [] - if "start" in prev_shown_details: - attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') - if "channel" in prev_shown_details: - attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f" ") - - if custom_data.get("premiere", False): - premiere_text = custom_data.get("premiere_text", "") - if premiere_text: - program_xml.append(f" {html.escape(premiere_text)}") - else: - program_xml.append(" ") - - if custom_data.get("last_chance", False): - last_chance_text = custom_data.get("last_chance_text", "") - if last_chance_text: - program_xml.append(f" {html.escape(last_chance_text)}") - else: - program_xml.append(" ") - - if custom_data.get("new", False): - program_xml.append(" ") - - if custom_data.get('live', False): - program_xml.append(' ') - - program_xml.append(" ") - - xml_text = '\n'.join(program_xml) - pending.append((prog['start_time'], prog['id'], xml_text)) - - del program_chunk - - yield from flush_pending() - - if program_batch: - yield '\n'.join(program_batch) + '\n' - - del real_epg_map - - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source, - export_lookback=lookback_cutoff, - export_cutoff=cutoff_date, - ) - if not dummy_programs: - continue - dummy_batch = [] - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - lines = [ - f' ', - f" {html.escape(program['title'])}", - ] - if program.get('sub_title'): - lines.append(f" {html.escape(program['sub_title'])}") - lines.append(f" {html.escape(program['description'])}") - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - lines.append(f" {html.escape(cat)}") - if 'date' in custom_data: - lines.append(f" {html.escape(custom_data['date'])}") - if custom_data.get('live', False): - lines.append(" ") - if custom_data.get('new', False): - lines.append(" ") - if 'icon' in custom_data: - lines.append(f' ') - lines.append(" ") - dummy_batch.append('\n'.join(lines)) - if len(dummy_batch) >= batch_size: - yield '\n'.join(dummy_batch) + '\n' - dummy_batch = [] - del dummy_programs - if dummy_batch: - yield '\n'.join(dummy_batch) + '\n' - - del dummy_program_list - - yield "\n" - - client_id, client_ip, user_agent = get_client_identifier(request) - event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - - def _log_epg_download(): - from django.core.cache import cache as event_cache - - if not event_cache.get(event_cache_key): - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=channel_count, - client_ip=client_ip, - user_agent=user_agent, - ) - event_cache.set(event_cache_key, True, 2) - - try: - from core.utils import _is_gevent_monkey_patched - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_log_epg_download) - else: - _log_epg_download() - except Exception: - _log_epg_download() - - def build_epg_stream(): - try: - yield from epg_generator() - finally: - _epg_export_teardown() - - return stream_epg_response(content_cache_key, build_epg_stream) - - def xc_get_user(request): username = request.GET.get("username") password = request.GET.get("password") From 107a891359fc0319bae9389e6fa608b0e3d556b4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 13:20:17 -0500 Subject: [PATCH 57/81] enhancement(epg): optimize XMLTV escaping and channel prefetch for improved performance --- CHANGELOG.md | 1 + apps/output/epg.py | 10 +++++----- apps/output/streaming_chunk_cache.py | 7 ++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840cb349..825a06fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. diff --git a/apps/output/epg.py b/apps/output/epg.py index 354d6e5b..4e11fbe8 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1188,7 +1188,7 @@ def generate_epg(request, profile_name=None, user=None): # once per channel (was ~13s of the request on large guides). .defer("epg_data__epg_source__programme_index") .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) ) channel_count = len(channels) @@ -1386,6 +1386,7 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None + escaped_primary_cid = None pending = [] program_batch = [] chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE @@ -1399,8 +1400,7 @@ def generate_epg(request, profile_name=None, user=None): return pending.sort(key=lambda row: (row[0], row[1])) escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None + escaped_primary_cid if len(channel_ids_for_epg) > 1 else None ) for _, _, xml_text in pending: program_batch.append(xml_text) @@ -1435,8 +1435,8 @@ def generate_epg(request, profile_name=None, user=None): yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] + escaped_primary_cid = html.escape(channel_ids_for_epg[0]) - primary_cid = channel_ids_for_epg[0] # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format # directly instead of strftime("%Y%m%d%H%M%S %z"), which is # ~10x slower and dominates XML build over 750k rows. @@ -1445,7 +1445,7 @@ def generate_epg(request, profile_name=None, user=None): start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - program_xml = [f' '] + program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') if prog['sub_title']: diff --git a/apps/output/streaming_chunk_cache.py b/apps/output/streaming_chunk_cache.py index f86bb060..4373f5ce 100644 --- a/apps/output/streaming_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -114,9 +114,14 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + refresh_interval = max(1, lock_ttl // 4) + last_refresh = 0.0 for chunk in source(): redis.rpush(chunks_key, _encode_chunk(chunk)) - _refresh_build_ttl(redis, base_key, lock_ttl) + now = time.monotonic() + if now - last_refresh >= refresh_interval: + _refresh_build_ttl(redis, base_key, lock_ttl) + last_refresh = now yield chunk redis.set(status_key, STATUS_READY) redis.set(_ready_key(base_key), "1") From 0dc8898e8b413764012ef14e80e0a0329c8dd7d1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:10:27 -0500 Subject: [PATCH 58/81] refactor(epg): separate programme index into dedicated table for optimized data handling - Moved the `programme_index` from the `EPGSource` model to a new `EPGSourceIndex` table, ensuring that the large JSON blob is only loaded when explicitly accessed, thus improving query performance and memory efficiency. - Updated related queries and API views to utilize the new structure, including adjustments to EPG generation and import logic to prevent unnecessary data loading. - Enhanced memory management in the EPG grid endpoint to reduce worker RSS during response handling. --- CHANGELOG.md | 7 ++- apps/epg/api_views.py | 25 ++++++----- apps/epg/migrations/0026_epgsourceindex.py | 52 ++++++++++++++++++++++ apps/epg/models.py | 37 ++++++++++++--- apps/epg/tasks.py | 17 ++++--- apps/epg/tests/test_epg_import_api.py | 7 ++- apps/epg/tests/test_programme_index.py | 1 - apps/output/epg.py | 38 +++++----------- apps/output/tests.py | 12 ++++- core/tasks.py | 8 +++- core/tests.py | 6 +-- core/utils.py | 24 ++++++++++ 12 files changed, 169 insertions(+), 65 deletions(-) create mode 100644 apps/epg/migrations/0026_epgsourceindex.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 825a06fc..0ad1ef3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed +- **`programme_index` moved off `EPGSource` into a dedicated `EPGSourceIndex` table.** The multi-MB byte-offset index was repeatedly pulled into web and Celery workers by ordinary `EPGSource` queries and `select_related` JOINs (which ignore manager-level `defer()`). It now lives in a one-to-one `EPGSourceIndex` row, read only when explicitly accessed through the `EPGSource.programme_index` property, so no list, detail, or JOIN query can load it by accident. Migration `0026` copies existing indices across. No API or index-build behavior change. + - **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. @@ -29,7 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. -- **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. +- **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index fbc419c5..1e0bfc41 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -64,8 +64,6 @@ class EPGSourceViewSet(viewsets.ModelViewSet): from apps.channels.models import Channel return EPGSource.objects.select_related( "refresh_task__crontab", "refresh_task__interval" - ).defer( - 'programme_index' ).annotate( has_channels=Exists( Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk')) @@ -1082,13 +1080,19 @@ class EPGGridAPIView(APIView): f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" ) - # Combine regular and dummy programs - all_programs = list(serialized_programs) + dummy_programs + # Combine regular and dummy programs in place to avoid copying the large list + serialized_programs.extend(dummy_programs) logger.debug( - f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)." + f"EPGGridAPIView: Returning {len(serialized_programs)} total programs (including {len(dummy_programs)} dummy programs)." ) - return Response({"data": all_programs}, status=status.HTTP_200_OK) + # The grid materializes tens of thousands of program dicts plus the + # rendered JSON; trim once the response is sent so worker RSS does not + # ratchet up per request. + from core.utils import spawn_memory_trim + response = Response({"data": serialized_programs}, status=status.HTTP_200_OK) + response._resource_closers.append(spawn_memory_trim) + return response # ───────────────────────────── @@ -1119,7 +1123,7 @@ class EPGImportAPIView(APIView): epg_id = request.data.get("id", None) force = bool(request.data.get("force", False)) - # Reject dummy sources without loading programme_index (multi-MB JSON). + # Reject dummy sources with a narrow existence query, no full row load. if epg_id is not None: from .models import EPGSource @@ -1236,10 +1240,9 @@ class CurrentProgramsAPIView(APIView): # Limit to 50 IDs per request epg_data_ids = epg_data_ids[:50] - # Defer the multi-MB programme_index the JOIN would pull once per row. The lookup reads it via a targeted refresh_from_db - epg_data_entries = EPGData.objects.select_related('epg_source').defer( - 'epg_source__programme_index' - ).filter(id__in=epg_data_ids) + epg_data_entries = EPGData.objects.select_related('epg_source').filter( + id__in=epg_data_ids + ) # Batch-fetch current programs for all requested EPG entries in one query db_programs = ProgramData.objects.filter( diff --git a/apps/epg/migrations/0026_epgsourceindex.py b/apps/epg/migrations/0026_epgsourceindex.py new file mode 100644 index 00000000..75d9d4eb --- /dev/null +++ b/apps/epg/migrations/0026_epgsourceindex.py @@ -0,0 +1,52 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def copy_index_forward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + rows = list( + EPGSource.objects.exclude(programme_index__isnull=True).values_list( + 'id', 'programme_index' + ) + ) + for source_id, data in rows: + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': data} + ) + + +def copy_index_backward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + for source_id, data in EPGSourceIndex.objects.values_list('source_id', 'data'): + EPGSource.objects.filter(id=source_id).update(programme_index=data) + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0025_programdata_epg_id_index'), + ] + + operations = [ + migrations.CreateModel( + name='EPGSourceIndex', + fields=[ + ('source', models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name='index_record', + serialize=False, + to='epg.epgsource', + )), + ('data', models.JSONField(blank=True, default=None, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.RunPython(copy_index_forward, copy_index_backward), + migrations.RemoveField( + model_name='epgsource', + name='programme_index', + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 04c6072c..b85453af 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -62,12 +62,6 @@ class EPGSource(models.Model): blank=True, help_text="Last status message, including success results or error information" ) - programme_index = models.JSONField( - null=True, - blank=True, - default=None, - help_text="Byte-offset index mapping tvg_id to file positions, built after each EPG refresh" - ) created_at = models.DateTimeField( auto_now_add=True, help_text="Time when this source was created" @@ -124,6 +118,37 @@ class EPGSource(models.Model): kwargs['update_fields'].remove('updated_at') super().save(*args, **kwargs) + @property + def programme_index(self): + """Byte-offset index for this source, read on demand from the separate + EPGSourceIndex table so the multi-MB blob is never pulled into EPGSource + queries or select_related JOINs. Returns the stored dict or None.""" + return ( + EPGSourceIndex.objects.filter(source_id=self.pk) + .values_list('data', flat=True) + .first() + ) + + +class EPGSourceIndex(models.Model): + """Byte-offset programme index for an EPGSource, stored in its own table. + + Kept out of EPGSource so the multi-MB JSON blob is only loaded when read + explicitly, never when querying or joining EPGSource rows. + """ + source = models.OneToOneField( + EPGSource, + on_delete=models.CASCADE, + related_name='index_record', + primary_key=True, + ) + data = models.JSONField(null=True, blank=True, default=None) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return f"Programme index for source {self.source_id}" + + class EPGData(models.Model): tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=512) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 9661c5b9..83aaa275 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -26,7 +26,7 @@ from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 +from .models import EPGSource, EPGSourceIndex, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 from core.utils import ( acquire_task_lock, is_task_lock_held, @@ -653,7 +653,9 @@ def _refresh_epg_data_impl(source_id, force=False): if source.source_type == 'xmltv': # Invalidate the byte-offset index before downloading the new file # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) + EPGSourceIndex.objects.update_or_create( + source_id=source.id, defaults={'data': None} + ) if not fetch_xmltv(source): logger.error(f"Failed to fetch XMLTV for source {source.name}") return @@ -4484,7 +4486,7 @@ def _programme_to_dict(elem, start_time, end_time): def build_programme_index(source_id): """ Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map. - Persists the result to EPGSource.programme_index. Most XMLTV files group programmes + Persists the result to the EPGSourceIndex table. Most XMLTV files group programmes by channel, but some split a channel across multiple non-contiguous blocks, so we record block starts up to _OFFSET_CAP and mark only channels that exceed the cap as interleaved. @@ -4573,7 +4575,9 @@ def build_programme_index(source_id): 'channels': index, 'interleaved_channels': sorted(interleaved_channels), } - EPGSource.objects.filter(id=source_id).update(programme_index=result) + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': result} + ) @shared_task @@ -4620,9 +4624,8 @@ def find_current_program_for_tvg_id(epg_or_id): return None now = timezone.now() - # Force a fresh read of the DB-backed index to avoid using stale related-object - # state when an EPG refresh invalidates/rebuilds the index concurrently. - source.refresh_from_db(fields=['programme_index']) + # The property reads the EPGSourceIndex table fresh on each access, so a + # concurrent refresh invalidating/rebuilding the index can't serve stale state. index = source.programme_index if index is not None: diff --git a/apps/epg/tests/test_epg_import_api.py b/apps/epg/tests/test_epg_import_api.py index c58f2a45..cc96c214 100644 --- a/apps/epg/tests/test_epg_import_api.py +++ b/apps/epg/tests/test_epg_import_api.py @@ -7,7 +7,7 @@ from django.test import TestCase from rest_framework import status from rest_framework.test import APIClient -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex User = get_user_model() @@ -47,7 +47,10 @@ class EPGImportAPITests(TestCase): name="Large Index XMLTV", source_type="xmltv", url="http://example.com/epg.xml", - programme_index={ + ) + EPGSourceIndex.objects.create( + source=source, + data={ "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)}, "interleaved_channels": [], }, diff --git a/apps/epg/tests/test_programme_index.py b/apps/epg/tests/test_programme_index.py index 5ece0ae1..829afc87 100644 --- a/apps/epg/tests/test_programme_index.py +++ b/apps/epg/tests/test_programme_index.py @@ -519,7 +519,6 @@ class FindCurrentProgramTests(TestCase): name="No Index", source_type="xmltv", file_path=FIXTURE_XML, - programme_index=None, ) epg = EPGData.objects.create( tvg_id="channel.current", diff --git a/apps/output/epg.py b/apps/output/epg.py index 4e11fbe8..626b21aa 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -40,34 +40,20 @@ def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cut def _ceil_to_half_hour(dt): """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) + original = dt.replace(microsecond=0) + aligned = dt.replace(second=0, microsecond=0) + remainder = aligned.minute % 30 + if remainder != 0: + aligned += timedelta(minutes=30 - remainder) + if aligned < original: + aligned += timedelta(minutes=30) + return aligned def _epg_export_teardown(): - from django.db import close_old_connections + from core.utils import spawn_memory_trim - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() + spawn_memory_trim(close_connections=True) def _ordered_channel_streams(channel): @@ -1183,10 +1169,6 @@ def generate_epg(request, profile_name=None, user=None): with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) diff --git a/apps/output/tests.py b/apps/output/tests.py index 5f4fd5f3..870ff765 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -355,4 +355,14 @@ class OutputEPGHelperTest(SimpleTestCase): aligned = _ceil_to_half_hour(dt) self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) - self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + def test_ceil_to_half_hour_past_boundary_second(self): + from django.utils import timezone + from apps.output.epg import _ceil_to_half_hour + + dt = timezone.now().replace(minute=0, second=52, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) diff --git a/core/tasks.py b/core/tasks.py index b562f335..a64469b4 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -398,12 +398,16 @@ def scan_and_process_files(): def _rebuild_programme_indices(): """Queue index builds for active EPG sources that are missing their DB index.""" try: + from django.db.models import Q from apps.epg.tasks import build_programme_index_task sources = EPGSource.objects.filter( is_active=True, - programme_index__isnull=True, - ).exclude(source_type__in=('dummy', 'schedules_direct')) + ).exclude( + source_type__in=('dummy', 'schedules_direct') + ).filter( + Q(index_record__isnull=True) | Q(index_record__data__isnull=True) + ) count = 0 for source in sources: diff --git a/core/tests.py b/core/tests.py index 33475ae7..2af52f07 100644 --- a/core/tests.py +++ b/core/tests.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock from django.test import TestCase, SimpleTestCase -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -37,14 +37,10 @@ class DispatcharrUserAgentTests(TestCase): class ProgrammeIndexRebuildTests(TestCase): def test_startup_rebuild_does_not_lock_out_queued_build_task(self): - EPGSource.objects.update( - programme_index={"channels": {}, "interleaved_channels": []} - ) source = EPGSource.objects.create( name="Missing Index", source_type="xmltv", is_active=True, - programme_index=None, ) class FakeRedis: diff --git a/core/utils.py b/core/utils.py index 89228088..44504b6a 100644 --- a/core/utils.py +++ b/core/utils.py @@ -608,6 +608,30 @@ def cleanup_memory(log_usage=False, force_collection=True): pass logger.trace("Memory cleanup complete for django") + +def spawn_memory_trim(close_connections=False): + """Reclaim a request's heap pages: GC, then return freed C pages to the OS. + + On gevent uWSGI workers the trim runs in a spawned greenlet so it never + blocks the caller; Celery prefork workers (no gevent hub) run it inline. + Set close_connections=True when called from a streaming generator's teardown + so the pooled DB connection is released first. + """ + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if close_connections: + from django.db import close_old_connections + close_old_connections() + + if _is_gevent_monkey_patched(): + import gevent + gevent.spawn(_run) + else: + _run() + + def safe_upload_path(filename: str, base_dir) -> str: """Return a safe absolute path for an uploaded file within base_dir. From e984b234e34fd5ad4d5804c2fef8348c8cff3a65 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:27:15 -0500 Subject: [PATCH 59/81] enhancement(epg): add export lookback and cutoff parameters to EPG generation - Updated the `generate_dummy_programs` function to include `export_lookback` and `export_cutoff` parameters, allowing for more precise control over the EPG data generation window. - Added a new test case to verify that future events are correctly represented in the grid window, ensuring upcoming fillers are displayed as expected. --- apps/epg/api_views.py | 4 +++- apps/output/tests.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 1e0bfc41..cdb39a47 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -984,7 +984,9 @@ class EPGGridAPIView(APIView): channel_name=name_to_parse, num_days=1, program_length_hours=4, - epg_source=epg_source + epg_source=epg_source, + export_lookback=one_hour_ago, + export_cutoff=twenty_four_hours_later, ) # Custom dummy should always return data (either from patterns or fallback) diff --git a/apps/output/tests.py b/apps/output/tests.py index 870ff765..e713df21 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -338,6 +338,55 @@ class OutputEPGCustomDummyTest(TestCase): ) self.assertGreaterEqual(programs[0]['start_time'], lookback) + def test_custom_dummy_future_event_fills_grid_window_with_upcoming(self): + """Grid-style window: future event should show upcoming filler, not empty.""" + from django.utils import timezone + from apps.output.epg import _programme_overlaps_export_window, generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy Future", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + now = timezone.now() + grid_start = now - timedelta(hours=1) + grid_end = now + timedelta(hours=24) + future = now + timedelta(days=3) + channel_name = ( + f"NHL 01: Washington Capitals vs Philadelphia Flyers @ " + f"{future.strftime('%B')} {future.day} 07:30 PM ET" + ) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=1, + epg_source=epg_source, + export_lookback=grid_start, + export_cutoff=grid_end, + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all( + _programme_overlaps_export_window( + p["start_time"], p["end_time"], grid_start, grid_end + ) + for p in programs + ), + "All programmes should overlap the grid query window", + ) + self.assertTrue( + any("Upcoming" in p.get("description", "") for p in programs), + "Future events outside the window should show upcoming filler", + ) + class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): From 7b6adf62d7f84e69f2d121b6d24fa2e17bea499d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:15:29 -0500 Subject: [PATCH 60/81] enhancement(vod): optimize VOD stream retrieval and performance - Improved `xc_get_vod_streams` and `xc_get_series` functions to utilize a new helper method for fetching distinct relations based on account priority, significantly reducing memory usage and response times for large libraries. - Updated the handling of VOD streams to avoid unnecessary ORM instantiation, enhancing performance for endpoints dealing with extensive movie and series data. - Added comprehensive tests to ensure the correctness of the new logic and verify that the highest priority relations are correctly selected. --- CHANGELOG.md | 1 + apps/output/tests.py | 483 ++++++++++++++++++++++++++++++++++++++++++- apps/output/views.py | 189 ++++++++++------- 3 files changed, 595 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad1ef3b..6bc8054f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. diff --git a/apps/output/tests.py b/apps/output/tests.py index e713df21..30b22a18 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,9 +1,23 @@ -from django.test import TestCase, Client, SimpleTestCase +from django.test import TestCase, Client, SimpleTestCase, RequestFactory from django.urls import reverse +from unittest import skipUnless from unittest.mock import patch from uuid import uuid4 +from django.db import connection +from django.test.utils import CaptureQueriesContext from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource +from apps.accounts.models import User +from apps.m3u.models import M3UAccount +from apps.output.views import xc_get_series, xc_get_vod_streams +from apps.vod.models import ( + M3UMovieRelation, + M3USeriesRelation, + Movie, + Series, + VODCategory, + VODLogo, +) import xml.etree.ElementTree as ET from datetime import timedelta @@ -415,3 +429,470 @@ class OutputEPGHelperTest(SimpleTestCase): self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + +class XcVodSeriesDistinctTests(TestCase): + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0, is_active=True): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + is_active=is_active, + ) + + def test_vod_streams_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + movie = Movie.objects.create(name="Shared Movie", year=2020) + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + stream_id="low-stream", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + stream_id="high-stream", + container_extension="mp4", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Shared Movie") + self.assertEqual(streams[0]["container_extension"], "mp4") + + def test_vod_streams_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}", priority=1) + inactive = self._account( + f"inactive-{uuid4().hex[:6]}", priority=99, is_active=False + ) + active_movie = Movie.objects.create(name="Active Movie") + inactive_movie = Movie.objects.create(name="Inactive Only Movie") + M3UMovieRelation.objects.create( + m3u_account=active, + movie=active_movie, + stream_id="active-1", + ) + M3UMovieRelation.objects.create( + m3u_account=inactive, + movie=inactive_movie, + stream_id="inactive-1", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + names = {s["name"] for s in streams} + self.assertEqual(names, {"Active Movie"}) + + def test_vod_streams_category_filter(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + action_movie = Movie.objects.create(name="Action Movie") + comedy_movie = Movie.objects.create(name="Comedy Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=action_movie, + category=action, + stream_id="action-1", + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=comedy_movie, + category=comedy, + stream_id="comedy-1", + ) + + streams = xc_get_vod_streams(self.request, self.user, category_id=action.id) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Action Movie") + self.assertEqual(streams[0]["category_id"], str(action.id)) + + def test_vod_streams_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + zebra = Movie.objects.create(name="Zebra Film") + apple = Movie.objects.create(name="Apple Film") + M3UMovieRelation.objects.create( + m3u_account=account, movie=zebra, stream_id="z-1" + ) + M3UMovieRelation.objects.create( + m3u_account=account, movie=apple, stream_id="a-1" + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual([s["name"] for s in streams], ["Apple Film", "Zebra Film"]) + + def test_vod_streams_includes_metadata_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create( + name="Rich Movie", + description="A plot", + genre="Drama", + year=2021, + rating="8", + custom_properties={ + "director": "Dir", + "actors": "Cast", + "release_date": "2021-01-01", + "youtube_trailer": "yt123", + }, + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="rich-1", + container_extension="avi", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["plot"], "A plot") + self.assertEqual(stream["genre"], "Drama") + self.assertEqual(stream["year"], 2021) + self.assertEqual(stream["director"], "Dir") + self.assertEqual(stream["cast"], "Cast") + self.assertEqual(stream["release_date"], "2021-01-01") + self.assertEqual(stream["trailer"], "yt123") + self.assertEqual(stream["container_extension"], "avi") + + def test_vod_streams_stream_icon_uses_logo_id_without_logo_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Poster", url="http://example.com/poster.png") + movie = Movie.objects.create(name="Logo Movie", logo=logo) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="logo-1", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIn(f"/{logo.id}/", stream["stream_icon"]) + + def test_series_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + series = Series.objects.create(name="Shared Series", year=2019) + M3USeriesRelation.objects.create( + m3u_account=low, + series=series, + external_series_id="low-series", + ) + high_rel = M3USeriesRelation.objects.create( + m3u_account=high, + series=series, + external_series_id="high-series", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["name"], "Shared Series") + self.assertEqual(results[0]["series_id"], high_rel.id) + + def test_series_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}") + inactive = self._account(f"inactive-{uuid4().hex[:6]}", is_active=False) + active_series = Series.objects.create(name="Active Series") + inactive_series = Series.objects.create(name="Inactive Only Series") + M3USeriesRelation.objects.create( + m3u_account=active, + series=active_series, + external_series_id="active-s", + ) + M3USeriesRelation.objects.create( + m3u_account=inactive, + series=inactive_series, + external_series_id="inactive-s", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual({r["name"] for r in results}, {"Active Series"}) + + def test_series_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + z = Series.objects.create(name="Zulu Show") + a = Series.objects.create(name="Alpha Show") + M3USeriesRelation.objects.create( + m3u_account=account, series=z, external_series_id="z" + ) + M3USeriesRelation.objects.create( + m3u_account=account, series=a, external_series_id="a" + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual([r["name"] for r in results], ["Alpha Show", "Zulu Show"]) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_vod_streams_dedupe_query_avoids_movie_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Query Shape Movie") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="qs-1" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_vod_streams(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_movie"', distinct_queries[0]["sql"]) + self.assertNotIn('"vod_vodlogo"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_movie"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_series_dedupe_query_avoids_series_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Query Shape Series") + M3USeriesRelation.objects.create( + m3u_account=account, series=series, external_series_id="qs-s" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_series(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_series"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_series"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + +XC_VOD_STREAM_KEYS = frozenset({ + "num", "name", "stream_type", "stream_id", "stream_icon", "rating", + "rating_5based", "added", "is_adult", "tmdb_id", "imdb_id", "trailer", + "plot", "genre", "year", "director", "cast", "release_date", "category_id", + "category_ids", "container_extension", "custom_sid", "direct_source", +}) + +XC_SERIES_KEYS = frozenset({ + "num", "name", "series_id", "cover", "plot", "cast", "director", "genre", + "release_date", "releaseDate", "last_modified", "rating", "rating_5based", + "backdrop_path", "youtube_trailer", "episode_run_time", "category_id", + "category_ids", "tmdb_id", "imdb_id", +}) + + +class XcVodSeriesRegressionTests(TestCase): + """Full output-shape and edge-case regressions for XC list endpoints.""" + + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-reg-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + ) + + def test_vod_streams_empty_library(self): + self.assertEqual(xc_get_vod_streams(self.request, self.user), []) + + def test_series_empty_library(self): + self.assertEqual(xc_get_series(self.request, self.user), []) + + def test_vod_streams_response_keys(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Schema Movie", rating="10") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="schema-1" + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(set(stream.keys()), XC_VOD_STREAM_KEYS) + self.assertEqual(stream["stream_type"], "movie") + self.assertEqual(stream["stream_id"], movie.id) + self.assertEqual(stream["rating_5based"], 5.0) + self.assertEqual(stream["custom_sid"], None) + self.assertEqual(stream["direct_source"], "") + + def test_vod_streams_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Sparse Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="sparse-1", + container_extension=None, + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIsNone(stream["stream_icon"]) + self.assertEqual(stream["category_id"], "0") + self.assertEqual(stream["category_ids"], []) + self.assertEqual(stream["container_extension"], "mp4") + self.assertEqual(stream["plot"], "") + self.assertEqual(stream["trailer"], "") + self.assertEqual(stream["tmdb_id"], "") + self.assertEqual(stream["imdb_id"], "") + + def test_vod_streams_category_from_winning_relation(self): + """Category must come from the highest-priority relation, not any relation.""" + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + movie = Movie.objects.create(name="Dual Category Movie") + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + category=action, + stream_id="low-cat", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + category=comedy, + stream_id="high-cat", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["category_id"], str(comedy.id)) + self.assertEqual(stream["category_ids"], [comedy.id]) + + def test_series_response_keys_and_metadata(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Cover", url="http://example.com/cover.png") + category = VODCategory.objects.create(name="Drama", category_type="series") + series = Series.objects.create( + name="Schema Series", + description="Series plot", + genre="Sci-Fi", + year=2022, + rating="8", + tmdb_id="tm123", + imdb_id="tt123", + logo=logo, + custom_properties={ + "cast": "Actor A", + "director": "Director B", + "release_date": "2022-06-01", + "backdrop_path": ["/img1.jpg"], + "youtube_trailer": "yt-series", + "episode_run_time": "45", + }, + ) + relation = M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + category=category, + external_series_id="schema-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(set(row.keys()), XC_SERIES_KEYS) + self.assertEqual(row["series_id"], relation.id) + self.assertIn(f"/{logo.id}/", row["cover"]) + self.assertEqual(row["plot"], "Series plot") + self.assertEqual(row["cast"], "Actor A") + self.assertEqual(row["director"], "Director B") + self.assertEqual(row["genre"], "Sci-Fi") + self.assertEqual(row["release_date"], "2022-06-01") + self.assertEqual(row["releaseDate"], "2022-06-01") + self.assertEqual(row["backdrop_path"], ["/img1.jpg"]) + self.assertEqual(row["youtube_trailer"], "yt-series") + self.assertEqual(row["episode_run_time"], "45") + self.assertEqual(row["tmdb_id"], "tm123") + self.assertEqual(row["imdb_id"], "tt123") + self.assertEqual(row["category_id"], str(category.id)) + self.assertEqual(row["category_ids"], [category.id]) + self.assertEqual(row["last_modified"], str(int(relation.updated_at.timestamp()))) + + def test_series_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Sparse Series") + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="sparse-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertIsNone(row["cover"]) + self.assertEqual(row["category_id"], "0") + self.assertEqual(row["category_ids"], []) + self.assertEqual(row["release_date"], "") + self.assertEqual(row["releaseDate"], "") + self.assertEqual(row["backdrop_path"], []) + self.assertEqual(row["youtube_trailer"], "") + self.assertEqual(row["episode_run_time"], "") + + def test_series_release_date_falls_back_to_year(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Year Only", year=2018) + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="year-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(row["release_date"], "2018") + self.assertEqual(row["releaseDate"], "2018") + + def test_priority_tiebreaker_uses_lower_relation_id(self): + """Same priority: DISTINCT ON tie-breaks on relation id ascending.""" + a1 = self._account(f"a1-{uuid4().hex[:6]}", priority=5) + a2 = self._account(f"a2-{uuid4().hex[:6]}", priority=5) + movie = Movie.objects.create(name="Tie Movie") + first = M3UMovieRelation.objects.create( + m3u_account=a1, + movie=movie, + stream_id="first", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=a2, + movie=movie, + stream_id="second", + container_extension="mp4", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["container_extension"], first.container_extension) diff --git a/apps/output/views.py b/apps/output/views.py index 4224fb35..699c9f95 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -918,6 +918,73 @@ def xc_get_epg(request, user, short=False): return output +XC_MOVIE_VALUE_FIELDS = ( + 'id', 'movie_id', 'category_id', 'container_extension', + 'movie__id', 'movie__name', 'movie__rating', 'movie__created_at', + 'movie__tmdb_id', 'movie__imdb_id', 'movie__description', 'movie__genre', + 'movie__year', 'movie__custom_properties', 'movie__logo_id', +) + +XC_SERIES_VALUE_FIELDS = ( + 'id', 'series_id', 'category_id', 'updated_at', + 'series__id', 'series__name', 'series__description', 'series__genre', + 'series__year', 'series__rating', 'series__custom_properties', 'series__logo_id', + 'series__tmdb_id', 'series__imdb_id', +) + + +def _xc_fetch_priority_distinct_relations( + *, + manager, + rel_filters, + distinct_field, + value_fields, + order_by_name_field, +): + """ + Return one row dict per distinct content ID (highest account priority wins). + + On PostgreSQL, dedupe on narrow relation rows first, then fetch display + columns via values() (no ORM model instantiation). That avoids sorting + wide joined rows during DISTINCT ON and reduces parallel worker /dev/shm + pressure in Docker. + """ + from django.db import connection, transaction + + narrow_qs = manager.filter(**rel_filters) + + def _fetch_by_ids(ids): + return list( + manager.filter(pk__in=ids) + .values(*value_fields) + .order_by(Lower(order_by_name_field)) + ) + + if connection.vendor == 'postgresql': + winning_ids_qs = ( + narrow_qs + .order_by(distinct_field, '-m3u_account__priority', 'id') + .distinct(distinct_field) + .values('pk') + ) + with transaction.atomic(): + #with connection.cursor() as cursor: + #cursor.execute("SET LOCAL max_parallel_workers_per_gather = 0") + winning_ids = list(winning_ids_qs.values_list('pk', flat=True)) + if not winning_ids: + return [] + return _fetch_by_ids(winning_ids) + + seen = {} + for row in narrow_qs.values(*value_fields).order_by('-m3u_account__priority', 'id'): + key = row[distinct_field] + if key not in seen: + seen[key] = row + rows = list(seen.values()) + rows.sort(key=lambda r: (r[order_by_name_field] or '').lower()) + return rows + + def xc_get_vod_categories(user): """Get VOD categories for XtreamCodes API""" from apps.vod.models import VODCategory, M3UMovieRelation @@ -943,33 +1010,19 @@ def xc_get_vod_categories(user): def xc_get_vod_streams(request, user, category_id=None): """Get VOD streams (movies) for XtreamCodes API""" from apps.vod.models import M3UMovieRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3UMovieRelation.objects - .filter(**rel_filters) - .select_related('movie', 'movie__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3UMovieRelation.objects, + rel_filters=rel_filters, + distinct_field='movie_id', + value_fields=XC_MOVIE_VALUE_FIELDS, + order_by_name_field='movie__name', ) - if connection.vendor == 'postgresql': - # DISTINCT ON returns one row per movie (highest-priority active relation) - # in a single query. ORDER BY must lead with the DISTINCT field. - relations = list( - base_qs.order_by('movie_id', '-m3u_account__priority', 'id') - .distinct('movie_id') - ) - else: - # SQLite fallback: fetch all matching relations, deduplicate in Python. - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.movie_id not in seen: - seen[rel.movie_id] = rel - relations = list(seen.values()) - # Precompute logo URL prefix/suffix once (mirrors _xc_live_streams_setup) # so each row only needs a string concat instead of reverse() + URI build. _base_url = build_absolute_uri_with_port(request, "") @@ -978,44 +1031,40 @@ def xc_get_vod_streams(request, user, category_id=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - # Sort by name (DISTINCT ON forces ORDER BY movie_id; SQLite path is unsorted). - relations.sort(key=lambda r: (r.movie.name or "").lower()) - streams = [] append = streams.append - for num, relation in enumerate(relations, 1): - movie = relation.movie - custom_props = movie.custom_properties or {} - category = relation.category - category_id_str = str(category.id) if category else "0" - category_id_list = [category.id] if category else [] - rating = movie.rating - logo = movie.logo + for num, row in enumerate(relations, 1): + custom_props = row['movie__custom_properties'] or {} + category_id = row['category_id'] + category_id_str = str(category_id) if category_id else "0" + category_id_list = [category_id] if category_id else [] + rating = row['movie__rating'] + logo_id = row['movie__logo_id'] append({ "num": num, - "name": movie.name, + "name": row['movie__name'], "stream_type": "movie", - "stream_id": movie.id, + "stream_id": row['movie__id'], "stream_icon": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), "rating": rating or "0", "rating_5based": round(float(rating or 0) / 2, 2) if rating else 0, - "added": str(int(movie.created_at.timestamp())), + "added": str(int(row['movie__created_at'].timestamp())), "is_adult": 0, - "tmdb_id": movie.tmdb_id or "", - "imdb_id": movie.imdb_id or "", + "tmdb_id": row['movie__tmdb_id'] or "", + "imdb_id": row['movie__imdb_id'] or "", "trailer": custom_props.get('youtube_trailer') or "", - "plot": movie.description or "", - "genre": movie.genre or "", - "year": movie.year or "", + "plot": row['movie__description'] or "", + "genre": row['movie__genre'] or "", + "year": row['movie__year'] or "", "director": custom_props.get('director', ''), "cast": custom_props.get('actors', ''), "release_date": custom_props.get('release_date', ''), "category_id": category_id_str, "category_ids": category_id_list, - "container_extension": relation.container_extension or "mp4", + "container_extension": row['container_extension'] or "mp4", "custom_sid": None, "direct_source": "", }) @@ -1048,72 +1097,58 @@ def xc_get_series_categories(user): def xc_get_series(request, user, category_id=None): """Get series list for XtreamCodes API""" from apps.vod.models import M3USeriesRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3USeriesRelation.objects - .filter(**rel_filters) - .select_related('series', 'series__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3USeriesRelation.objects, + rel_filters=rel_filters, + distinct_field='series_id', + value_fields=XC_SERIES_VALUE_FIELDS, + order_by_name_field='series__name', ) - if connection.vendor == 'postgresql': - relations = list( - base_qs.order_by('series_id', '-m3u_account__priority', 'id') - .distinct('series_id') - ) - else: - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.series_id not in seen: - seen[rel.series_id] = rel - relations = list(seen.values()) - _base_url = build_absolute_uri_with_port(request, "") _sample_logo_path = reverse("api:vod:vodlogo-cache", args=[0]) _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - relations.sort(key=lambda r: (r.series.name or "").lower()) - series_list = [] append = series_list.append - for num, relation in enumerate(relations, 1): - series = relation.series - custom_props = series.custom_properties or {} - category = relation.category - rating = series.rating - logo = series.logo - year_str = str(series.year) if series.year else "" + for num, row in enumerate(relations, 1): + custom_props = row['series__custom_properties'] or {} + category_id = row['category_id'] + rating = row['series__rating'] + logo_id = row['series__logo_id'] + year_str = str(row['series__year']) if row['series__year'] else "" release_date = custom_props.get('release_date', year_str) append({ "num": num, - "name": series.name, - "series_id": relation.id, + "name": row['series__name'], + "series_id": row['id'], "cover": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), - "plot": series.description or "", + "plot": row['series__description'] or "", "cast": custom_props.get('cast', ''), "director": custom_props.get('director', ''), - "genre": series.genre or "", + "genre": row['series__genre'] or "", "release_date": release_date, "releaseDate": release_date, - "last_modified": str(int(relation.updated_at.timestamp())), + "last_modified": str(int(row['updated_at'].timestamp())), "rating": str(rating or "0"), "rating_5based": str(round(float(rating or 0) / 2, 2)) if rating else "0", "backdrop_path": custom_props.get('backdrop_path', []), "youtube_trailer": custom_props.get('youtube_trailer', ''), "episode_run_time": custom_props.get('episode_run_time', ''), - "category_id": str(category.id) if category else "0", - "category_ids": [category.id] if category else [], - "tmdb_id": series.tmdb_id or "", - "imdb_id": series.imdb_id or "", + "category_id": str(category_id) if category_id else "0", + "category_ids": [category_id] if category_id else [], + "tmdb_id": row['series__tmdb_id'] or "", + "imdb_id": row['series__imdb_id'] or "", }) return series_list From 53fa1e42a0a10ae8fcc078d9bb978e464e0190e2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:54:12 -0500 Subject: [PATCH 61/81] enhancement(tests): introduce isolated backend test settings and improve test documentation - Added `dispatcharr.settings_test` for isolated testing, automatically creating an empty PostgreSQL test database and ensuring transaction isolation during tests. - Updated `manage.py` to switch to the test settings when running tests. - Enhanced the CONTRIBUTING.md file with detailed instructions on running the backend test suite and handling Celery tasks in tests. - Refactored test cases to use static methods for `worker_id` in `test_process_label.py` and adjusted assertions in `test_user_preferences.py` for better clarity and correctness. --- CHANGELOG.md | 4 +++ CONTRIBUTING.md | 14 +++++++- dispatcharr/settings_test.py | 58 ++++++++++++++++++++++++++++++++++ manage.py | 7 +++- tests/test_process_label.py | 4 +-- tests/test_user_preferences.py | 4 +-- 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 dispatcharr/settings_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc8054f..dfdeeda5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. + ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 725e21e8..6da68a61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,7 +144,19 @@ Untested code is significantly less likely to be merged. - Use Django's `TestCase` for unit/integration tests. - Test files live at `apps//tests/`. -- Run the test suite with: `uv run python manage.py test` +- Run the backend test suite with: + + ```bash + python manage.py test + ``` + + `manage.py` automatically uses `dispatcharr.settings_test`, which creates an empty PostgreSQL database `test_` (same engine as production), runs migrations, and rolls back each test in a transaction. Your live VOD/channels data is not used. + + Optional: `TEST_USE_SQLITE=1` for machines without Postgres (some PostgreSQL-only tests skip automatically). + + Tests that exercise Celery task bodies should use `@override_settings(CELERY_TASK_ALWAYS_EAGER=True)` locally. Global eager mode is off because `post_save` signals on M3U/EPG models call `.delay()` and would break `TestCase` transaction isolation. + +- Do **not** override with `--settings=dispatcharr.settings` on a live instance. ### Frontend diff --git a/dispatcharr/settings_test.py b/dispatcharr/settings_test.py new file mode 100644 index 00000000..fe4225fa --- /dev/null +++ b/dispatcharr/settings_test.py @@ -0,0 +1,58 @@ +""" +Django settings for running the backend test suite in isolation. + +Always use this module instead of dispatcharr.settings when running tests: + + python manage.py test + +`manage.py` selects this module automatically for the ``test`` command. + +Django creates a separate empty database (``test_``) and runs +migrations — your live data under /data/db is not used. + +Why not dispatcharr.settings? +- Production/AIO points at the live ``dispatcharr`` database. +- django-db-geventpool breaks TestCase transaction isolation on pooled connections. + +SQLite (``TEST_USE_SQLITE=1``) is an optional fallback for machines without +Postgres; production and CI should use the default Postgres test database. +""" +import os + +from dispatcharr.settings import * # noqa: F401,F403 + +# Fast password hashing for tests. +PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +# Do NOT run Celery tasks inline during tests. post_save signals on M3UAccount and +# EPGSource call .delay(); eager mode runs them inside TestCase transactions and +# closes/poisons the DB connection for subsequent queries in the same test. +CELERY_TASK_ALWAYS_EAGER = False +CELERY_TASK_EAGER_PROPAGATES = False + +_use_sqlite = os.environ.get("TEST_USE_SQLITE", "").lower() in ("1", "true", "yes") + +if _use_sqlite: + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } + } +else: + # Default: PostgreSQL with Django-managed test_dispatcharr (matches production). + # Uses the standard backend (not geventpool) so TestCase transactions isolate. + _pg_name = os.environ.get("POSTGRES_DB", "dispatcharr") + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": _pg_name, + "USER": os.environ.get("POSTGRES_USER", "dispatch"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), + "HOST": os.environ.get("POSTGRES_HOST", "localhost"), + "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), + "TEST": { + "NAME": "test_" + _pg_name, + }, + } + } diff --git a/manage.py b/manage.py index f3c22dfd..f0a85a39 100644 --- a/manage.py +++ b/manage.py @@ -5,7 +5,12 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') + # Use isolated test DB settings for `manage.py test` (empty test_). + # Override with --settings=... on the command line if needed. + if len(sys.argv) > 1 and sys.argv[1] == "test": + os.environ["DJANGO_SETTINGS_MODULE"] = "dispatcharr.settings_test" + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dispatcharr.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/tests/test_process_label.py b/tests/test_process_label.py index bb4c8429..e4525a85 100644 --- a/tests/test_process_label.py +++ b/tests/test_process_label.py @@ -17,13 +17,13 @@ class ProcessLabelTests(SimpleTestCase): self.assertEqual(role, "uwsgi") def test_uwsgi_labeled_when_worker_module_present(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 2})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 2)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "uwsgi") def test_uwsgi_master_not_labeled_as_uwsgi(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 0})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 0)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "django") diff --git a/tests/test_user_preferences.py b/tests/test_user_preferences.py index 5749011a..80dcc992 100644 --- a/tests/test_user_preferences.py +++ b/tests/test_user_preferences.py @@ -128,13 +128,13 @@ class UserPreferencesAPITests(TestCase): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) def test_patch_me_cannot_escalate_privileges(self): - """Test PATCH /me/ rejects attempts to change user_level or is_staff""" + """PATCH /me/ ignores privilege fields; they are stripped before save.""" original_level = self.user.user_level data = {"user_level": 99, "is_staff": True, "is_superuser": True} response = self.client.patch(self.me_url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.status_code, status.HTTP_200_OK) self.user.refresh_from_db() self.assertEqual(self.user.user_level, original_level) From 1b7840b71541ca95a6fcb533d48fa7f145c9fdd3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 07:52:13 -0500 Subject: [PATCH 62/81] changelog: Update for pr 1331 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdeeda5..0ea59832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. +- **Auto-sync range conflict warning no longer flags a group's own channels when using a channel-group override.** The M3U group settings "Range conflict" check compared occupant channel groups against the source group being configured. Auto-sync with `group_override` creates channels in the override target group, so those channels were misclassified as conflicts. The frontend now resolves the effective target group (override target when set, otherwise the source group) for classification. Genuine conflicts—manual channels, other accounts, different groups, or user-pinned numbers—still surface. (Fixes #1331) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.27.0] - 2026-06-16 From 578c50ea962d12f7d9f956c910accca431fc3e4f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 08:01:13 -0500 Subject: [PATCH 63/81] changelog: corrected --- CHANGELOG.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea59832..ce3d0b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance -- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. @@ -28,9 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. -### Performance - -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. ### Fixed From 971065b8a874f5837c818fe28267b2da554324bd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:14:00 -0500 Subject: [PATCH 64/81] chore(dependencies): update Django, requests, gevent, torch, sentence-transformers, and lxml to latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a003bc5c..8d4fad78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,18 +6,18 @@ license = "AGPL-3.0-only" requires-python = ">=3.13" dynamic = ["version"] dependencies = [ - "Django==6.0.5", + "Django==6.0.6", "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", - "requests==2.33.1", + "requests==2.34.2", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", "streamlink", "python-vlc", "yt-dlp", - "gevent==26.4.0", + "gevent==26.5.0", "django-db-geventpool", "daphne", "uwsgi", @@ -28,14 +28,14 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.11.0+cpu", - "sentence-transformers==5.4.1", + "torch==2.12.1+cpu", + "sentence-transformers==5.6.0", "channels", "channels-redis==4.3.0", "django-filter", "django-redis", "django-celery-beat>=2.9.0", - "lxml==6.1.0", + "lxml==6.1.1", "packaging", ] From 107246ff0b43d676e4cffc0219b4a047ca1c77d4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:17:38 -0500 Subject: [PATCH 65/81] changelog: Update for dependency updates. --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3d0b06..b2645f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- Updated `Django` 6.0.5 → 6.0.6, resolving the following CVEs: + - **CVE-2026-6873**: Signed cookie salt namespace collision in `HttpRequest.get_signed_cookie()`. + - **CVE-2026-7666**: Potential unencrypted email transmission via STARTTLS in the SMTP backend. + - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. + - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. + - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. + ### Added - **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. @@ -29,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. +- Dependency updates: + - `Django` 6.0.5 → 6.0.6 (security patch; see Security section) + - `requests` 2.33.1 → 2.34.2 + - `gevent` 26.4.0 → 26.5.0 + - `torch` 2.11.0+cpu → 2.12.1+cpu + - `sentence-transformers` 5.4.1 → 5.6.0 + - `lxml` 6.1.0 → 6.1.1 + ### Fixed From e483fc203b2645b5c5f9cd7b5530ade64a36f3c6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:14:00 -0500 Subject: [PATCH 66/81] chore(dependencies): update Django, requests, gevent, torch, sentence-transformers, and lxml to latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a003bc5c..8d4fad78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,18 +6,18 @@ license = "AGPL-3.0-only" requires-python = ">=3.13" dynamic = ["version"] dependencies = [ - "Django==6.0.5", + "Django==6.0.6", "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", - "requests==2.33.1", + "requests==2.34.2", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", "streamlink", "python-vlc", "yt-dlp", - "gevent==26.4.0", + "gevent==26.5.0", "django-db-geventpool", "daphne", "uwsgi", @@ -28,14 +28,14 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.11.0+cpu", - "sentence-transformers==5.4.1", + "torch==2.12.1+cpu", + "sentence-transformers==5.6.0", "channels", "channels-redis==4.3.0", "django-filter", "django-redis", "django-celery-beat>=2.9.0", - "lxml==6.1.0", + "lxml==6.1.1", "packaging", ] From 6b6eb11cc03ad1bb65b321c2b27c92b869fbe24b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:31:15 -0500 Subject: [PATCH 67/81] chore(dependencies): update Vite, esbuild and js-yaml to latest versions in package.json --- CHANGELOG.md | 4 + frontend/package-lock.json | 240 +++++++++++++++++++------------------ frontend/package.json | 7 +- 3 files changed, 133 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2645f45..f7bedbf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. +- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (1 low, 2 moderate, 1 high): + - Updated `vite` 7.3.2 → 7.3.5, resolving **moderate** NTLMv2 hash disclosure via UNC path handling on Windows ([GHSA-v6wh-96g9-6wx3](https://github.com/advisories/GHSA-v6wh-96g9-6wx3)) and **high** `server.fs.deny` bypass on Windows alternate paths ([GHSA-fx2h-pf6j-xcff](https://github.com/advisories/GHSA-fx2h-pf6j-xcff)) + - Updated `js-yaml` 4.1.1 → 5.1.0, resolving **moderate** quadratic-complexity DoS in merge key handling via repeated aliases ([GHSA-h67p-54hq-rp68](https://github.com/advisories/GHSA-h67p-54hq-rp68)) + - Updated `esbuild` 0.27.3 → 0.28.1, resolving **low** arbitrary file read when running the development server on Windows ([GHSA-g7r4-m6w7-qqqr](https://github.com/advisories/GHSA-g7r4-m6w7-qqqr)) ### Added diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8ca15bc7..ad8ad268 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -57,7 +57,7 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" } }, @@ -595,9 +595,9 @@ "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", "cpu": [ "ppc64" ], @@ -612,9 +612,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", "cpu": [ "arm" ], @@ -629,9 +629,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", "cpu": [ "arm64" ], @@ -646,9 +646,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", "cpu": [ "x64" ], @@ -663,9 +663,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", "cpu": [ "arm64" ], @@ -680,9 +680,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", "cpu": [ "x64" ], @@ -697,9 +697,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", "cpu": [ "arm64" ], @@ -714,9 +714,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", "cpu": [ "x64" ], @@ -731,9 +731,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", "cpu": [ "arm" ], @@ -748,9 +748,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", "cpu": [ "arm64" ], @@ -765,9 +765,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", "cpu": [ "ia32" ], @@ -782,9 +782,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", "cpu": [ "loong64" ], @@ -799,9 +799,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", "cpu": [ "mips64el" ], @@ -816,9 +816,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", "cpu": [ "ppc64" ], @@ -833,9 +833,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", "cpu": [ "riscv64" ], @@ -850,9 +850,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", "cpu": [ "s390x" ], @@ -867,9 +867,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", "cpu": [ "x64" ], @@ -884,9 +884,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", "cpu": [ "arm64" ], @@ -901,9 +901,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", "cpu": [ "x64" ], @@ -918,9 +918,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", "cpu": [ "arm64" ], @@ -935,9 +935,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", "cpu": [ "x64" ], @@ -952,9 +952,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", "cpu": [ "arm64" ], @@ -969,9 +969,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", "cpu": [ "x64" ], @@ -986,9 +986,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", "cpu": [ "arm64" ], @@ -1003,9 +1003,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", "cpu": [ "ia32" ], @@ -1020,9 +1020,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", "cpu": [ "x64" ], @@ -3163,9 +3163,9 @@ "license": "MIT" }, "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3176,32 +3176,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" } }, "node_modules/escape-string-regexp": { @@ -3814,16 +3814,26 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.1.0.tgz", + "integrity": "sha512-s8VA5jkR8f22S3NAXmhKPFqGUduqZGlsufabVOgN14iTdw/RXcym7bKkbwjxLK9Yw2lEvvmJjFp119+KPeo8Kg==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsdom": { @@ -5425,9 +5435,9 @@ } }, "node_modules/vite": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", - "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.5.tgz", + "integrity": "sha512-KuOaNhcnGFN2zIPGA7wRmzF+lJA1sea7rHq17aiJ++9lzY1WWG6Jpwqwe1KNbRVPIqHmr8GLYx7jbrQcN/7/ww==", "dev": true, "license": "MIT", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 2c1b78f5..8675351d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -61,16 +61,17 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" }, "resolutions": { - "vite": "7.1.7", + "vite": "7.3.5", "react": "19.1.0", "react-dom": "19.1.0" }, "overrides": { - "js-yaml": "^4.1.1", + "esbuild": "^0.28.1", + "js-yaml": "^5.1.0", "minimatch": "^10.2.1" } } From e2ceef521779467f1f4947f492dc1d040d5404cb Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:46:37 -0500 Subject: [PATCH 68/81] changelog: refactor xmltv changes and link issue. --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7bedbf8..059e0d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,11 +27,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. -- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. -- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. -- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **XMLTV EPG export is faster and no longer balloons worker memory.** `generate_epg()` was reworked end-to-end for large guides. (Fixes #1366) + - Streams incrementally: on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker); repeat requests within 300s stream chunks back from Redis. `malloc_trim` runs after cold builds. + - Channel streams are prefetched once (only `id`/`name`) instead of one query per custom-dummy channel; dummy `EPGData` programme existence is bulk-checked in a single query. + - The primary channel id is escaped once per `epg_id` group instead of once per programme (~750k fewer `html.escape` calls on a large guide). + - The channel query no longer JOINs multi-MB `programme_index` blobs per channel (~13s saved on a ~2000-channel guide; indices live in `EPGSourceIndex`). + - Programme export uses `(epg_id, id)` keyset pagination with a per-source `start_time` sort; a matching composite index on `ProgramData` (created `CONCURRENTLY` on PostgreSQL) lets each chunk use an ordered index range scan instead of re-sorting every chunk. - **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed From c5e50167285504e03487447d0bb3b2e0f8d12144 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:02:31 -0500 Subject: [PATCH 69/81] fix(dvr): Fix in-progress DVR playback to prevent jumping to live edge and update FloatingVideo component for improved error handling. Added tests to verify HLS configuration behavior. (Fixes #1329) --- CHANGELOG.md | 1 + frontend/src/components/FloatingVideo.jsx | 115 +++++++++--------- .../__tests__/FloatingVideo.test.jsx | 94 ++++++++++++++ 3 files changed, 154 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 059e0d23..5365e79a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **In-progress DVR playback no longer jumps to the live edge.** The floating video player still opens in-progress recordings at the start of the seekable range, but hls.js was configured with `liveMaxLatencyDurationCount: 10`, which forced the playhead forward to "now" shortly after playback began. Live-edge sync is now disabled for recording HLS so users can watch, pause, and scrub from the beginning while the recording continues. (Fixes #1329) - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 507e0caf..e49c3bb2 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -333,65 +333,68 @@ export default function FloatingVideo() { let hls = null; if (isHls && Hls.isSupported()) { - hls = new Hls({ - // Open at the very beginning of the recording rather than the live - // edge. Without this, an in-progress recording would start at "now" - // and hide everything already recorded. hls.js applies this AFTER - // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is - // also kept as a safety net for the Safari native-HLS path and for - // edge cases where this initial-position logic loses to the user's - // first interaction. - startPosition: 0, - // Allow seeking back to the start of the recording, regardless of - // current playhead position. Recordings can be hours long and the - // user may want to scrub anywhere; we explicitly disable buffer - // eviction by setting a very large back-buffer length. - backBufferLength: 90 * 60, // 90 minutes - maxBufferLength: 60, - maxMaxBufferLength: 600, - // For an in-progress recording, hls.js refreshes the playlist on - // its target-duration cadence; let it follow the live edge but keep - // the full DVR window seekable. - liveSyncDurationCount: 3, - liveMaxLatencyDurationCount: 10, - enableWorker: true, - lowLatencyMode: false, - // Inject the JWT into every playlist + segment XHR. Read the token - // from the auth store at request time rather than capturing the - // closure value at hls.js init, so a refreshed access token mid- - // playback is picked up on the next segment fetch. - xhrSetup: (xhr) => { - const token = useAuthStore.getState().accessToken; - if (token) { - xhr.setRequestHeader('Authorization', `Bearer ${token}`); - } - }, - }); - hls.on(Hls.Events.ERROR, (_evt, data) => { - if (data.fatal) { - // eslint-disable-next-line no-console - console.error('HLS fatal error:', data.type, data.details); - if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { - try { - hls.startLoad(); - } catch { - // ignore + try { + hls = new Hls({ + // Open at the very beginning of the recording rather than the live + // edge. Without this, an in-progress recording would start at "now" + // and hide everything already recorded. hls.js applies this AFTER + // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is + // also kept as a safety net for the Safari native-HLS path and for + // edge cases where this initial-position logic loses to the user's + // first interaction. + startPosition: 0, + // Allow seeking back to the start of the recording, regardless of + // current playhead position. Recordings can be hours long and the + // user may want to scrub anywhere; we explicitly disable buffer + // eviction by setting a very large back-buffer length. + backBufferLength: 90 * 60, // 90 minutes + maxBufferLength: 60, + maxMaxBufferLength: 600, + // Leave liveMaxLatencyDurationCount at the hls.js default (Infinity). + // A finite value forces the playhead to the live edge during playback. + enableWorker: true, + lowLatencyMode: false, + // Inject the JWT into every playlist + segment XHR. Read the token + // from the auth store at request time rather than capturing the + // closure value at hls.js init, so a refreshed access token mid- + // playback is picked up on the next segment fetch. + xhrSetup: (xhr) => { + const token = useAuthStore.getState().accessToken; + if (token) { + xhr.setRequestHeader('Authorization', `Bearer ${token}`); } - } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { - try { - hls.recoverMediaError(); - } catch { - // ignore + }, + }); + hls.on(Hls.Events.ERROR, (_evt, data) => { + if (data.fatal) { + // eslint-disable-next-line no-console + console.error('HLS fatal error:', data.type, data.details); + if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { + try { + hls.startLoad(); + } catch { + // ignore + } + } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { + try { + hls.recoverMediaError(); + } catch { + // ignore + } + } else { + setLoadError(`HLS playback error: ${data.details || data.type}`); } - } else { - setLoadError(`HLS playback error: ${data.details || data.type}`); } - } - }); - hls.attachMedia(video); - hls.on(Hls.Events.MEDIA_ATTACHED, () => { - hls.loadSource(streamUrl); - }); + }); + hls.attachMedia(video); + hls.on(Hls.Events.MEDIA_ATTACHED, () => { + hls.loadSource(streamUrl); + }); + } catch (error) { + setIsLoading(false); + setLoadError(`HLS initialization error: ${error.message}`); + return; + } } else if (isHls && video.canPlayType('application/vnd.apple.mpegurl')) { // Safari path: native HLS support, including seekable DVR windows. video.src = streamUrl; diff --git a/frontend/src/components/__tests__/FloatingVideo.test.jsx b/frontend/src/components/__tests__/FloatingVideo.test.jsx index f99a3dcc..9683fc59 100644 --- a/frontend/src/components/__tests__/FloatingVideo.test.jsx +++ b/frontend/src/components/__tests__/FloatingVideo.test.jsx @@ -20,8 +20,49 @@ vi.mock('mpegts.js', () => ({ }, })); +const mockHlsInstance = { + attachMedia: vi.fn(), + loadSource: vi.fn(), + destroy: vi.fn(), + on: vi.fn(), +}; + +let capturedHlsConfig = null; +let forceHlsInitError = false; + +vi.mock('hls.js', () => ({ + default: class MockHls { + static isSupported = vi.fn(() => true); + + static Events = { + ERROR: 'error', + MEDIA_ATTACHED: 'media_attached', + }; + + static ErrorTypes = { + NETWORK_ERROR: 'networkError', + MEDIA_ERROR: 'mediaError', + }; + + constructor(config) { + if (forceHlsInitError) { + throw new Error('Illegal hls.js config'); + } + capturedHlsConfig = config; + Object.assign(this, mockHlsInstance); + } + }, +})); + +vi.mock('../../store/auth', () => ({ + default: { + getState: vi.fn(() => ({ accessToken: 'test-token' })), + }, +})); + // Import the mocked module after mocking const mpegts = (await import('mpegts.js')).default; +const Hls = (await import('hls.js')).default; // Mock react-draggable vi.mock('react-draggable', () => ({ @@ -53,6 +94,8 @@ describe('FloatingVideo', () => { beforeEach(async () => { vi.clearAllMocks(); + capturedHlsConfig = null; + forceHlsInitError = false; // Mock HTMLVideoElement methods HTMLVideoElement.prototype.load = vi.fn(); @@ -239,6 +282,57 @@ describe('FloatingVideo', () => { expect(video.poster).toBe('http://example.com/poster.jpg'); }); + it('should disable live-edge sync for in-progress recording HLS', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + + render(); + + expect(capturedHlsConfig).toEqual( + expect.objectContaining({ + startPosition: 0, + }) + ); + expect(capturedHlsConfig).not.toHaveProperty( + 'liveMaxLatencyDurationCount' + ); + expect(capturedHlsConfig).not.toHaveProperty('liveSyncDurationCount'); + }); + + it('shows an in-player error when hls.js config is invalid', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + forceHlsInitError = true; + + render(); + + expect( + screen.getByText(/HLS initialization error: Illegal hls.js config/i) + ).toBeInTheDocument(); + }); + it('should show metadata overlay', () => { const { container } = render(); const video = container.querySelector('video'); From 562393b77e50d56ae51a2772b2d148d5a16aeca1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:32:48 -0500 Subject: [PATCH 70/81] fix(recording playback): Enhance completed DVR recording playback by allowing JWT token via query parameter for native
), Stack: ({ children }) =>
{children}
, + Collapse: ({ in: isOpen, children }) => + isOpen ?
{children}
: null, TextInput: ({ label, description, ...rest }) => (
@@ -353,11 +370,52 @@ describe('ProxySettingsForm', () => { // ── ProxySettingsOptions field routing ───────────────────────────────────── describe('ProxySettingsOptions field routing', () => { - it('calls getInputProps for each PROXY_SETTINGS_OPTIONS key', () => { + it('calls getInputProps for main settings on initial render', () => { render(); expect(formMock.getInputProps).toHaveBeenCalledWith('buffering_timeout'); expect(formMock.getInputProps).toHaveBeenCalledWith('buffering_speed'); expect(formMock.getInputProps).toHaveBeenCalledWith('redis_url'); + expect(formMock.getInputProps).not.toHaveBeenCalledWith( + 'channel_client_wait_period' + ); + expect(formMock.getInputProps).not.toHaveBeenCalledWith( + 'channel_init_grace_period' + ); + expect(formMock.getInputProps).not.toHaveBeenCalledWith('redis_chunk_ttl'); + }); + + it('hides advanced settings until expanded', () => { + render(); + expect( + screen.queryByTestId('number-input-Client Connect Grace Period') + ).not.toBeInTheDocument(); + expect( + screen.queryByTestId('number-input-Channel Initialization Timeout') + ).not.toBeInTheDocument(); + expect( + screen.queryByTestId('number-input-Buffer Chunk TTL') + ).not.toBeInTheDocument(); + expect(screen.getByText('Show Advanced Settings')).toBeInTheDocument(); + }); + + it('shows advanced settings when expanded', () => { + render(); + fireEvent.click(screen.getByText('Show Advanced Settings')); + expect(screen.getByTestId('collapse-open')).toBeInTheDocument(); + expect( + screen.getByTestId('number-input-Client Connect Grace Period') + ).toBeInTheDocument(); + expect( + screen.getByTestId('number-input-Channel Initialization Timeout') + ).toBeInTheDocument(); + expect(screen.getByTestId('number-input-Buffer Chunk TTL')).toBeInTheDocument(); + expect(formMock.getInputProps).toHaveBeenCalledWith( + 'channel_client_wait_period' + ); + expect(formMock.getInputProps).toHaveBeenCalledWith( + 'channel_init_grace_period' + ); + expect(formMock.getInputProps).toHaveBeenCalledWith('redis_chunk_ttl'); }); }); }); diff --git a/frontend/src/constants.js b/frontend/src/constants.js index fba0d7c6..ea6dec13 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -43,6 +43,7 @@ export const PROXY_SETTINGS_OPTIONS = { }, redis_chunk_ttl: { label: 'Buffer Chunk TTL', + advanced: true, description: 'Time-to-live for buffer chunks in seconds (how long stream data is cached)', }, @@ -53,13 +54,15 @@ export const PROXY_SETTINGS_OPTIONS = { }, channel_init_grace_period: { label: 'Channel Initialization Timeout', + advanced: true, description: 'Maximum seconds to wait for the initial buffer to fill while a channel is connecting. Channels that never receive enough buffered data are stopped after this limit.', }, channel_client_wait_period: { label: 'Client Connect Grace Period', + advanced: true, description: - 'Seconds to keep a ready channel alive when no client has connected yet (e.g. after an API warmup). Once a client connects the channel becomes active; if none connect within this window the channel stops.', + 'Seconds to keep a buffered channel alive when no viewer is connected yet. Rarely needed unless you start channels programmatically.', }, new_client_behind_seconds: { label: 'New Client Buffer (seconds)', From 97515c4d05acc8bc438f05b49057027d09e0ad1a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 30 Jun 2026 10:34:40 -0500 Subject: [PATCH 79/81] tests: fix proxy setting test --- .../__tests__/ProxySettingsForm.test.jsx | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/frontend/src/components/forms/settings/__tests__/ProxySettingsForm.test.jsx b/frontend/src/components/forms/settings/__tests__/ProxySettingsForm.test.jsx index b0fa3a56..2b0dc55f 100644 --- a/frontend/src/components/forms/settings/__tests__/ProxySettingsForm.test.jsx +++ b/frontend/src/components/forms/settings/__tests__/ProxySettingsForm.test.jsx @@ -370,18 +370,11 @@ describe('ProxySettingsForm', () => { // ── ProxySettingsOptions field routing ───────────────────────────────────── describe('ProxySettingsOptions field routing', () => { - it('calls getInputProps for main settings on initial render', () => { + it('binds main settings on initial render', () => { render(); expect(formMock.getInputProps).toHaveBeenCalledWith('buffering_timeout'); expect(formMock.getInputProps).toHaveBeenCalledWith('buffering_speed'); expect(formMock.getInputProps).toHaveBeenCalledWith('redis_url'); - expect(formMock.getInputProps).not.toHaveBeenCalledWith( - 'channel_client_wait_period' - ); - expect(formMock.getInputProps).not.toHaveBeenCalledWith( - 'channel_init_grace_period' - ); - expect(formMock.getInputProps).not.toHaveBeenCalledWith('redis_chunk_ttl'); }); it('hides advanced settings until expanded', () => { @@ -409,13 +402,6 @@ describe('ProxySettingsForm', () => { screen.getByTestId('number-input-Channel Initialization Timeout') ).toBeInTheDocument(); expect(screen.getByTestId('number-input-Buffer Chunk TTL')).toBeInTheDocument(); - expect(formMock.getInputProps).toHaveBeenCalledWith( - 'channel_client_wait_period' - ); - expect(formMock.getInputProps).toHaveBeenCalledWith( - 'channel_init_grace_period' - ); - expect(formMock.getInputProps).toHaveBeenCalledWith('redis_chunk_ttl'); }); }); }); From 22b957aee4934cc7546b57c836d96be1be3c10c4 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 30 Jun 2026 16:28:27 +0000 Subject: [PATCH 80/81] Release v0.27.2 --- CHANGELOG.md | 2 ++ version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 309ed5c1..a97f5158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.27.2] - 2026-06-30 + ### Added - **New proxy setting: Client Connect Grace Period (`channel_client_wait_period`, default 5s).** Adds a dedicated timeout for channels that have filled their buffer but still have no viewers (`waiting_for_clients`). Previously that window reused `channel_shutdown_delay` (default 0s), so those channels were torn down almost immediately. diff --git a/version.py b/version.py index db62d71f..25ccd35c 100644 --- a/version.py +++ b/version.py @@ -1,5 +1,5 @@ """ Dispatcharr version information. """ -__version__ = '0.27.1' # Follow semantic versioning (MAJOR.MINOR.PATCH) +__version__ = '0.27.2' # Follow semantic versioning (MAJOR.MINOR.PATCH) __timestamp__ = None # Set during CI/CD build process From a347e50e4806105e65def14795d8f7108a88f076 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 7 Jul 2026 20:51:49 +0000 Subject: [PATCH 81/81] test: Implement github workflow for backend tests. --- .github/workflows/backend-tests.yml | 161 ++++++++++++++++++++++++++++ scripts/ci_backend_test_labels.py | 104 ++++++++++++++++++ scripts/ci_bootstrap_backend.sh | 77 +++++++++++++ 3 files changed, 342 insertions(+) create mode 100644 .github/workflows/backend-tests.yml create mode 100644 scripts/ci_backend_test_labels.py create mode 100644 scripts/ci_bootstrap_backend.sh diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 00000000..036d6ece --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,161 @@ +name: Backend Tests + +on: + push: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + pull_request: + branches: [main, dev] + paths: + - 'apps/**' + - 'core/**' + - 'tests/**' + - 'dispatcharr/**' + - 'pyproject.toml' + - 'version.py' + - 'manage.py' + - 'scripts/ci_backend_test_labels.py' + - 'scripts/ci_bootstrap_backend.sh' + - '.github/workflows/backend-tests.yml' + workflow_dispatch: + inputs: + full_suite: + description: Run the full backend test suite + type: boolean + default: true + +permissions: + contents: read + packages: read + +concurrency: + group: backend-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + plan: + name: Plan test groups + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.resolve.outputs.labels }} + has_tests: ${{ steps.resolve.outputs.has_tests }} + base_image: ${{ steps.base_image.outputs.image }} + sync_python_deps: ${{ steps.base_image.outputs.sync_python_deps }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Collect changed paths + id: changed + shell: bash + run: | + set -euo pipefail + : > /tmp/changed_paths.txt + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ inputs.full_suite }}" = "true" ]; then + : > /tmp/changed_paths.txt + echo "mode=full" >> "$GITHUB_OUTPUT" + else + git diff --name-only HEAD~1 HEAD > /tmp/changed_paths.txt || true + echo "mode=diff" >> "$GITHUB_OUTPUT" + fi + elif [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin "${{ github.base_ref }}" + git diff --name-only "origin/${{ github.base_ref }}...HEAD" > /tmp/changed_paths.txt + echo "mode=pr" >> "$GITHUB_OUTPUT" + else + if [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then + git ls-files > /tmp/changed_paths.txt + echo "mode=initial-push" >> "$GITHUB_OUTPUT" + else + git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" > /tmp/changed_paths.txt + echo "mode=push" >> "$GITHUB_OUTPUT" + fi + fi + + - name: Select base image + id: base_image + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + TARGET_BRANCH="${{ github.base_ref }}" + else + TARGET_BRANCH="${{ github.ref_name }}" + fi + if [ "$TARGET_BRANCH" = "main" ]; then + TAG="base" + else + TAG="base-dev" + fi + REPO_OWNER="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + REPO_NAME="$(echo "${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]')" + echo "image=ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" >> "$GITHUB_OUTPUT" + if grep -qx 'pyproject.toml' /tmp/changed_paths.txt || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "sync_python_deps=true" >> "$GITHUB_OUTPUT" + else + echo "sync_python_deps=false" >> "$GITHUB_OUTPUT" + fi + echo "Using base image: ghcr.io/${REPO_OWNER}/${REPO_NAME}:${TAG}" + + - name: Resolve Django test labels + id: resolve + env: + FULL_SUITE: ${{ github.event_name == 'workflow_dispatch' && inputs.full_suite == true && 'true' || 'false' }} + run: | + set -euo pipefail + LABELS=$(python scripts/ci_backend_test_labels.py < /tmp/changed_paths.txt) + echo "labels=${LABELS}" >> "$GITHUB_OUTPUT" + if [ "${LABELS}" = "[]" ]; then + echo "has_tests=false" >> "$GITHUB_OUTPUT" + else + echo "has_tests=true" >> "$GITHUB_OUTPUT" + fi + echo "Selected labels: ${LABELS}" + + test: + name: ${{ matrix.label }} + needs: plan + if: needs.plan.outputs.has_tests == 'true' + runs-on: ubuntu-latest + container: + image: ${{ needs.plan.outputs.base_image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --entrypoint "" + strategy: + max-parallel: 6 + fail-fast: false + matrix: + label: ${{ fromJSON(needs.plan.outputs.labels) }} + env: + DISPATCHARR_ENV: aio + DJANGO_SECRET_KEY: ci-test-secret-key + POSTGRES_DB: dispatcharr + POSTGRES_USER: dispatch + POSTGRES_PASSWORD: secret + DISPATCHARR_LOG_LEVEL: WARNING + SYNC_PYTHON_DEPS: ${{ needs.plan.outputs.sync_python_deps }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Run tests in base image + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: bash scripts/ci_bootstrap_backend.sh "${{ matrix.label }}" -v2 diff --git a/scripts/ci_backend_test_labels.py b/scripts/ci_backend_test_labels.py new file mode 100644 index 00000000..e69356c4 --- /dev/null +++ b/scripts/ci_backend_test_labels.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Map changed repository paths to Django test package labels for CI.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import PurePosixPath + +# Keep in sync with dispatcharr.test_runner package discovery. +ALL_LABELS: tuple[str, ...] = ( + "apps.accounts.tests", + "apps.backups.tests", + "apps.channels.tests", + "apps.connect.tests", + "apps.dashboard.tests", + "apps.epg.tests", + "apps.m3u.tests", + "apps.output.tests", + "apps.plugins.tests", + "apps.timeshift.tests", + "apps.proxy.live_proxy.tests", + "apps.proxy.vod_proxy.tests", + "core.tests", + "tests", +) + +# Longest-prefix wins; order matters for overlapping rules. +_PATH_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("apps/channels/", ("apps.channels.tests",)), + ("apps/epg/", ("apps.epg.tests",)), + ("apps/m3u/", ("apps.m3u.tests",)), + ("apps/proxy/", ("apps.proxy.live_proxy.tests", "apps.proxy.vod_proxy.tests")), + ("apps/connect/", ("apps.connect.tests",)), + ("apps/output/", ("apps.output.tests",)), + ("apps/accounts/", ("apps.accounts.tests",)), + ("apps/backups/", ("apps.backups.tests",)), + ("apps/dashboard/", ("apps.dashboard.tests",)), + ("apps/plugins/", ("apps.plugins.tests",)), + ("apps/timeshift/", ("apps.timeshift.tests",)), + ("apps/vod/", ("apps.output.tests",)), + ("apps/hdhr/", ("apps.output.tests", "apps.channels.tests")), + ("apps/api/", ALL_LABELS), + ("core/", ("core.tests", "tests")), + ("tests/", ("tests",)), +) + +_SHARED_PREFIXES: tuple[str, ...] = ( + "dispatcharr/", + "pyproject.toml", + "manage.py", + "version.py", + "scripts/ci_backend_test_labels.py", + "scripts/ci_bootstrap_backend.sh", + ".github/workflows/backend-tests.yml", +) + + +def _normalize(path: str) -> str: + return PurePosixPath(path.strip().replace("\\", "/")).as_posix() + + +def labels_for_paths(paths: list[str], *, full_suite: bool = False) -> list[str]: + if full_suite: + return list(ALL_LABELS) + + labels: set[str] = set() + for raw in paths: + path = _normalize(raw) + if not path: + continue + if any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in _SHARED_PREFIXES): + return list(ALL_LABELS) + for prefix, group_labels in _PATH_RULES: + if path.startswith(prefix): + labels.update(group_labels) + break + + return sorted(labels) + + +def _read_paths(argv: list[str]) -> list[str]: + if argv: + return argv + if not sys.stdin.isatty(): + return [line for line in sys.stdin.read().splitlines() if line.strip()] + return [] + + +def main(argv: list[str] | None = None) -> int: + argv = argv if argv is not None else sys.argv[1:] + full_suite = os.environ.get("FULL_SUITE", "").lower() in {"1", "true", "yes"} + if full_suite: + print(json.dumps(list(ALL_LABELS))) + return 0 + paths = _read_paths(argv) + labels = labels_for_paths(paths, full_suite=False) + print(json.dumps(labels)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_bootstrap_backend.sh b/scripts/ci_bootstrap_backend.sh new file mode 100644 index 00000000..d13e0218 --- /dev/null +++ b/scripts/ci_bootstrap_backend.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Bootstrap internal Postgres/Redis (AIO-style) and run Django tests in the base image. +set -euo pipefail + +REPO_ROOT="${GITHUB_WORKSPACE:-$(pwd)}" +cd "${REPO_ROOT}" + +export DISPATCHARR_ENV="${DISPATCHARR_ENV:-aio}" +export PUID="${PUID:-1000}" +export PGID="${PGID:-1000}" +export POSTGRES_DB="${POSTGRES_DB:-dispatcharr}" +export POSTGRES_USER="${POSTGRES_USER:-dispatch}" +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-secret}" +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" +export POSTGRES_DIR="${POSTGRES_DIR:-/tmp/dispatcharr-ci/db}" +export REDIS_HOST="${REDIS_HOST:-localhost}" +export REDIS_PORT="${REDIS_PORT:-6379}" +export REDIS_DB="${REDIS_DB:-0}" +export DJANGO_SECRET_KEY="${DJANGO_SECRET_KEY:-ci-test-secret-key}" +export DISPATCHARR_LOG_LEVEL="${DISPATCHARR_LOG_LEVEL:-WARNING}" +export PATH="/dispatcharrpy/bin:${PATH}" +export PG_VERSION +PG_VERSION="$(ls /usr/lib/postgresql/ | sort -V | tail -n 1)" +export PG_BINDIR="/usr/lib/postgresql/${PG_VERSION}/bin" + +if [[ "$DISPATCHARR_ENV" == "aio" ]]; then + export POSTGRES_HOST="${POSTGRES_HOST:-/var/run/postgresql}" +else + export POSTGRES_HOST="${POSTGRES_HOST:-localhost}" +fi + +if [[ "${SYNC_PYTHON_DEPS:-}" == "true" ]]; then + echo "Syncing Python dependencies with uv..." + uv sync --python /dispatcharrpy/bin/python --no-install-project --no-dev +fi + +echo "Setting up CI user and PostgreSQL data directory..." +# shellcheck source=docker/init/01-user-setup.sh +. "${REPO_ROOT}/docker/init/01-user-setup.sh" +mkdir -p "${POSTGRES_DIR}" +chown "${PUID}:${PGID}" "${POSTGRES_DIR}" +chmod 700 "${POSTGRES_DIR}" +# shellcheck source=docker/init/02-postgres.sh +. "${REPO_ROOT}/docker/init/02-postgres.sh" + +echo "Starting internal PostgreSQL..." +prepare_pg_socket_dir +su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_ctl -D ${POSTGRES_DIR} start -w -t 120 -o '-c port=${POSTGRES_PORT}'" +until su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do + sleep 1 +done +set +e +promote_app_role +_promote_status=$? +ensure_app_database +_ensure_status=$? +set -e +if [ "$_promote_status" -ne 0 ] || [ "$_ensure_status" -ne 0 ]; then + echo "Failed to configure PostgreSQL role/database (promote=${_promote_status}, ensure=${_ensure_status})" + exit 1 +fi + +echo "Starting internal Redis..." +if redis-cli -p "${REDIS_PORT}" ping >/dev/null 2>&1; then + echo "Redis already listening on port ${REDIS_PORT}" +else + redis-server --daemonize yes --protected-mode no --bind 127.0.0.1 --port "${REDIS_PORT}" +fi +python "${REPO_ROOT}/scripts/wait_for_redis.py" + +cleanup() { + redis-cli -p "${REDIS_PORT}" shutdown nosave >/dev/null 2>&1 || true + su - "$POSTGRES_USER" -c "$PG_BINDIR/pg_ctl -D ${POSTGRES_DIR} stop -m fast" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +exec python manage.py test --keepdb "$@"