From 6b9d621cf778490fcb70f3b7b1d58d610609c68b Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 04:52:07 +0000 Subject: [PATCH 01/76] feat(epg): add Schedules Direct API integration - Add EPGSource username field and SDScheduleMD5 model for MD5 delta sync - Implement full Schedules Direct fetch with SHA1 auth, token caching, 20-day schedule window, and MD5-based delta sync to minimize API usage - Add lineup manager UI allowing users to search and manage SD lineups - Add stations-only fetch on source creation for immediate EPG matching - Enforce 2-hour minimum refresh interval to respect SD rate limits, with first-fetch bypass when no prior full refresh exists - Add real-time progress updates via synchronous Redis WebSocket delivery, bypassing gevent.spawn for reliable delivery from Celery prefork workers - Refactor EPGsTable status cell into standalone EPGStatusCell component with direct Zustand subscription for live progress bar updates - Guard XMLTV parse trigger for channels linked to SD EPG sources - Add 15 backend tests covering auth, credentials, signals, and serializers Migrations: 0023 (username field), 0024 (api_key help_text), 0025 (SD fields + SDScheduleMD5) --- apps/channels/signals.py | 9 + .../epg/migrations/0023_epgsource_username.py | 23 + .../0024_alter_epgsource_api_key.py | 21 + .../0025_sd_fields_and_schedule_md5.py | 65 ++ apps/epg/models.py | 56 +- apps/epg/serializers.py | 1 + apps/epg/signals.py | 13 +- apps/epg/tasks.py | 967 ++++++++++++++++-- apps/epg/tests/test_schedules_direct.py | 263 +++++ frontend/src/WebSocket.jsx | 20 +- frontend/src/api.js | 45 + frontend/src/components/forms/EPG.jsx | 469 ++++++++- frontend/src/components/tables/EPGsTable.jsx | 295 +++--- 13 files changed, 1991 insertions(+), 256 deletions(-) create mode 100644 apps/epg/migrations/0023_epgsource_username.py create mode 100644 apps/epg/migrations/0024_alter_epgsource_api_key.py create mode 100644 apps/epg/migrations/0025_sd_fields_and_schedule_md5.py create mode 100644 apps/epg/tests/test_schedules_direct.py diff --git a/apps/channels/signals.py b/apps/channels/signals.py index a1917b25..54dd30fc 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -201,10 +201,19 @@ def refresh_epg_programs(sender, instance, created, **kwargs): if not created and kwargs.get('update_fields') and 'epg_data' in kwargs['update_fields']: logger.info(f"Channel {instance.id} ({instance.name}) EPG data updated, refreshing program data") if instance.epg_data: + # Skip XMLTV program parser for SD sources — program data is handled + # by fetch_schedules_direct() directly. + if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'schedules_direct': + logger.info(f"Skipping XMLTV parse for SD source {instance.epg_data.tvg_id}") + return logger.info(f"Triggering EPG program refresh for {instance.epg_data.tvg_id}") parse_programs_for_tvg_id.delay(instance.epg_data.id) # For new channels with EPG data, also refresh elif created and instance.epg_data: + # Skip XMLTV program parser for SD sources + if instance.epg_data.epg_source and instance.epg_data.epg_source.source_type == 'schedules_direct': + logger.info(f"Skipping XMLTV parse for SD source {instance.epg_data.tvg_id}") + return logger.info(f"New channel {instance.id} ({instance.name}) created with EPG data, refreshing program data") parse_programs_for_tvg_id.delay(instance.epg_data.id) diff --git a/apps/epg/migrations/0023_epgsource_username.py b/apps/epg/migrations/0023_epgsource_username.py new file mode 100644 index 00000000..1a4c5ed3 --- /dev/null +++ b/apps/epg/migrations/0023_epgsource_username.py @@ -0,0 +1,23 @@ +# Generated migration — adds username field to EPGSource for Schedules Direct auth + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0022_alter_epgdata_name'), + ] + + operations = [ + migrations.AddField( + model_name='epgsource', + name='username', + field=models.CharField( + max_length=255, + blank=True, + null=True, + help_text='Username for Schedules Direct authentication', + ), + ), + ] diff --git a/apps/epg/migrations/0024_alter_epgsource_api_key.py b/apps/epg/migrations/0024_alter_epgsource_api_key.py new file mode 100644 index 00000000..e40a6baf --- /dev/null +++ b/apps/epg/migrations/0024_alter_epgsource_api_key.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0023_epgsource_username'), + ] + + operations = [ + migrations.AlterField( + model_name='epgsource', + name='api_key', + field=models.CharField( + max_length=255, + blank=True, + null=True, + help_text='Password for Schedules Direct authentication', + ), + ), + ] diff --git a/apps/epg/migrations/0025_sd_fields_and_schedule_md5.py b/apps/epg/migrations/0025_sd_fields_and_schedule_md5.py new file mode 100644 index 00000000..b02c94f2 --- /dev/null +++ b/apps/epg/migrations/0025_sd_fields_and_schedule_md5.py @@ -0,0 +1,65 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0024_alter_epgsource_api_key'), + ] + + operations = [ + # Add sd_changes_remaining to EPGSource + migrations.AddField( + model_name='epgsource', + name='sd_changes_remaining', + field=models.IntegerField( + blank=True, + null=True, + help_text='Number of Schedules Direct lineup additions remaining today (resets at midnight UTC)', + ), + ), + # Add sd_changes_reset_at to EPGSource + migrations.AddField( + model_name='epgsource', + name='sd_changes_reset_at', + field=models.DateTimeField( + blank=True, + null=True, + help_text='UTC datetime when the Schedules Direct daily lineup change counter resets', + ), + ), + # Add sd_program_md5 to ProgramData + migrations.AddField( + model_name='programdata', + name='sd_program_md5', + field=models.CharField( + blank=True, + max_length=22, + null=True, + help_text='MD5 hash from Schedules Direct for delta detection', + ), + ), + # Create SDScheduleMD5 model + migrations.CreateModel( + name='SDScheduleMD5', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('station_id', models.CharField(help_text='Schedules Direct stationID', max_length=20)), + ('date', models.DateField(help_text='Schedule date (UTC)')), + ('md5', models.CharField(help_text='MD5 hash of the schedule for this station/date from Schedules Direct', max_length=22)), + ('last_modified', models.DateTimeField(help_text='Last modified timestamp from Schedules Direct')), + ('epg_source', models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='sd_schedule_md5s', + to='epg.epgsource', + )), + ], + options={ + 'indexes': [ + models.Index(fields=['epg_source', 'station_id'], name='epg_sdsche_epg_sou_idx'), + ], + 'unique_together': {('epg_source', 'station_id', 'date')}, + }, + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index d5758ce2..0d4058fb 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -30,7 +30,10 @@ class EPGSource(models.Model): name = models.CharField(max_length=255, unique=True) source_type = models.CharField(max_length=20, choices=SOURCE_TYPE_CHOICES) url = models.URLField(max_length=1000, blank=True, null=True) # For XMLTV - api_key = models.CharField(max_length=255, blank=True, null=True) # For Schedules Direct + username = models.CharField(max_length=255, blank=True, null=True, + help_text='Username for Schedules Direct authentication') + api_key = models.CharField(max_length=255, blank=True, null=True, + help_text='Password for Schedules Direct authentication') # For Schedules Direct is_active = models.BooleanField(default=True) file_path = models.CharField(max_length=1024, blank=True, null=True) extracted_file_path = models.CharField(max_length=1024, blank=True, null=True, @@ -49,6 +52,16 @@ class EPGSource(models.Model): default=0, help_text="Priority for EPG matching (higher numbers = higher priority). Used when multiple EPG sources have matching entries for a channel." ) + sd_changes_remaining = models.IntegerField( + null=True, + blank=True, + help_text="Number of Schedules Direct lineup additions remaining today (resets at midnight UTC)" + ) + sd_changes_reset_at = models.DateTimeField( + null=True, + blank=True, + help_text="UTC datetime when the Schedules Direct daily lineup change counter resets" + ) status = models.CharField( max_length=20, choices=STATUS_CHOICES, @@ -163,6 +176,47 @@ class ProgramData(models.Model): description = models.TextField(blank=True, null=True) tvg_id = models.CharField(max_length=255, null=True, blank=True) custom_properties = models.JSONField(default=dict, blank=True, null=True) + sd_program_md5 = models.CharField( + max_length=22, + null=True, + blank=True, + help_text="MD5 hash from Schedules Direct for delta detection" + ) def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" + +class SDScheduleMD5(models.Model): + """ + Caches per-station per-date MD5 hashes from Schedules Direct. + Used to detect schedule changes and avoid unnecessary re-downloads, + minimizing API calls against SD's rate-limited endpoints. + """ + epg_source = models.ForeignKey( + EPGSource, + on_delete=models.CASCADE, + related_name="sd_schedule_md5s", + ) + station_id = models.CharField( + max_length=20, + help_text="Schedules Direct stationID" + ) + date = models.DateField( + help_text="Schedule date (UTC)" + ) + md5 = models.CharField( + max_length=22, + help_text="MD5 hash of the schedule for this station/date from Schedules Direct" + ) + last_modified = models.DateTimeField( + help_text="Last modified timestamp from Schedules Direct" + ) + + class Meta: + unique_together = ('epg_source', 'station_id', 'date') + indexes = [ + models.Index(fields=['epg_source', 'station_id']), + ] + + def __str__(self): + return f"SDScheduleMD5: {self.station_id} / {self.date} ({self.epg_source.name})" diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 4ab89a9b..049a9713 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -22,6 +22,7 @@ class EPGSourceSerializer(serializers.ModelSerializer): 'name', 'source_type', 'url', + 'username', 'api_key', 'is_active', 'file_path', diff --git a/apps/epg/signals.py b/apps/epg/signals.py index 97992ef3..7eaa8c0e 100644 --- a/apps/epg/signals.py +++ b/apps/epg/signals.py @@ -1,7 +1,7 @@ from django.db.models.signals import post_save, post_delete, pre_save from django.dispatch import receiver from .models import EPGSource, EPGData -from .tasks import refresh_epg_data, delete_epg_refresh_task_by_id +from .tasks import refresh_epg_data, delete_epg_refresh_task_by_id, fetch_schedules_direct_stations from core.scheduling import create_or_update_periodic_task, delete_periodic_task from core.utils import is_protected_path, send_websocket_update import json @@ -12,9 +12,16 @@ logger = logging.getLogger(__name__) @receiver(post_save, sender=EPGSource) def trigger_refresh_on_new_epg_source(sender, instance, created, **kwargs): - # Trigger refresh only if the source is newly created, active, and not a dummy EPG + # Trigger refresh only if the source is newly created, active, and not a dummy EPG. + # For Schedules Direct sources, run a stations-only fetch on creation so the user + # can run Auto-match EPG before committing to a full schedule/program fetch. + # A short countdown gives the frontend time to receive the API response and + # populate the store before WebSocket updates arrive. if created and instance.is_active and instance.source_type != 'dummy': - refresh_epg_data.delay(instance.id) + if instance.source_type == 'schedules_direct': + fetch_schedules_direct_stations.apply_async((instance.id,), countdown=3) + else: + refresh_epg_data.delay(instance.id) @receiver(post_save, sender=EPGSource) def create_dummy_epg_data(sender, instance, created, **kwargs): diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 8efe3618..479df9e2 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -29,6 +29,10 @@ from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, se logger = logging.getLogger(__name__) +SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' +SD_DAYS_TO_FETCH = 20 +SD_PROGRAM_BATCH_SIZE = 5000 + # DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named # entities so lxml/libxml2 can resolve references like é correctly # instead of silently dropping them in recovery mode. @@ -173,6 +177,64 @@ def send_epg_update(source_id, action, progress, **kwargs): gc.collect() +def _sd_send_ws_sync(source_id, action, progress, **kwargs): + """ + Sends a WebSocket progress update synchronously via Redis, bypassing gevent.spawn. + + In Celery prefork workers that inherit gevent monkey-patching from uWSGI, + gevent.spawn schedules coroutines that never execute because there is no + running gevent hub in the worker process. This function writes directly to + Redis using the channels_redis 4.x wire format, guaranteeing delivery + regardless of the execution context. + """ + try: + import msgpack + import redis as redis_lib + from django.conf import settings + + data = {"progress": progress, "type": "epg_refresh", "source": source_id, "action": action} + data.update(kwargs) + message = {"type": "update", "data": data} + + redis_url = getattr(settings, "CELERY_BROKER_URL", "redis://localhost:6379/0") + r = redis_lib.from_url(redis_url, decode_responses=False) + + prefix = "asgi" + group_name = "updates" + group_key = f"{prefix}:group:{group_name}" + group_expiry = 86400 + channel_expiry = 60 + rand_len = 12 + now = time.time() + + r.zremrangebyscore(group_key, 0, now - group_expiry) + raw = r.zrange(group_key, 0, -1) + if not raw: + return + + channels = [m.decode("utf-8") if isinstance(m, bytes) else m for m in raw] + nonlocal_map = {} + for ch in channels: + pos = ch.find("!") + nl = ch[:pos + 1] if pos >= 0 else ch + nonlocal_map.setdefault(nl, []).append(ch) + + pipe = r.pipeline(transaction=False) + for nl, chs in nonlocal_map.items(): + channel_key = prefix + nl + msg = dict(message) + msg["__asgi_channel__"] = chs + serialized = os.urandom(rand_len) + msgpack.packb(msg) + pipe.zadd(channel_key, {serialized: now}) + pipe.expire(channel_key, channel_expiry) + pipe.execute() + r.close() + except Exception as e: + logger.warning(f"SD WebSocket sync send failed: {e}") + # Fall back to standard path + send_epg_update(source_id, action, progress, **kwargs) + + def delete_epg_refresh_task_by_id(epg_id): """ Delete the periodic task associated with an EPG source ID. @@ -1255,6 +1317,17 @@ def parse_channels_only(source): @shared_task(time_limit=3600, soft_time_limit=3500) def parse_programs_for_tvg_id(epg_id): + # Skip XMLTV file parsing for Schedules Direct sources — program data is + # fetched and persisted directly by fetch_schedules_direct(). + try: + from apps.epg.models import EPGData + epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() + if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': + logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") + return "Skipped — Schedules Direct source" + except Exception as e: + logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") + if not acquire_task_lock('parse_epg_programs', epg_id): logger.info(f"Program parse for {epg_id} already in progress, skipping duplicate task") return "Task already running" @@ -1885,75 +1958,845 @@ def parse_programs_for_source(epg_source, tvg_id=None): logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") # Explicitly clear the process object to prevent potential memory leaks process = None -def fetch_schedules_direct(source): - logger.info(f"Fetching Schedules Direct data from source: {source.name}") +@shared_task(bind=True) +def fetch_schedules_direct_stations(self, source_id): + """ + Lightweight Celery task that runs a stations-only Schedules Direct fetch. + Called on initial source creation so EPGData entries exist for auto-matching + before the user commits to a full schedule/program fetch. + """ try: - # Get default user agent from settings - stream_settings = CoreSettings.get_stream_settings() - user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default - default_user_agent_id = stream_settings.get('default_user_agent') + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + logger.error(f"EPGSource {source_id} not found for SD stations fetch") + return + fetch_schedules_direct(source, stations_only=True) - if default_user_agent_id: - try: - user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() - if user_agent_obj and user_agent_obj.user_agent: - user_agent = user_agent_obj.user_agent - logger.debug(f"Using default user agent: {user_agent}") - except (ValueError, Exception) as e: - logger.warning(f"Error retrieving default user agent, using fallback: {e}") - api_url = '' - headers = { - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {source.api_key}', - 'User-Agent': user_agent - } - logger.debug(f"Requesting subscriptions from Schedules Direct using URL: {api_url}") - response = requests.get(api_url, headers=headers, timeout=30) - response.raise_for_status() - subscriptions = response.json() - logger.debug(f"Fetched subscriptions: {subscriptions}") +def fetch_schedules_direct(source, stations_only=False): + """ + Fetch EPG data from the Schedules Direct JSON API and persist it to the + EPGData / ProgramData models. - for sub in subscriptions: - tvg_id = sub.get('stationID') - logger.debug(f"Processing subscription for tvg_id: {tvg_id}") - schedules_url = f"/schedules/{tvg_id}" - logger.debug(f"Requesting schedules from URL: {schedules_url}") - sched_response = requests.get(schedules_url, headers=headers, timeout=30) - sched_response.raise_for_status() - schedules = sched_response.json() - logger.debug(f"Fetched schedules: {schedules}") + Authentication flow (as required by the SD API specification): + 1. POST credentials to the token endpoint — password must be SHA1-hashed + as required by the Schedules Direct API specification. + 2. Use the returned token for all subsequent requests via the 'token' header. + 3. Tokens are valid for 24 hours; SD returns the current valid token if one + already exists for the account. - epg_data, created = EPGData.objects.get_or_create( - tvg_id=tvg_id, - defaults={'name': tvg_id} - ) - if created: - logger.info(f"Created new EPGData for tvg_id '{tvg_id}'.") - else: - logger.debug(f"Found existing EPGData for tvg_id '{tvg_id}'.") + Data flow: + 1. Fetch subscribed lineups for the account. + 2. Fetch station metadata for each lineup. + 3. Persist station metadata to EPGData. + 4. If stations_only=True, stop here — used on initial source creation so + the user can run Auto-match EPG before the full program fetch. + 5. Fetch schedule grids in 14-day date-batched requests per station. + 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). + 7. Persist channels to EPGData and programs to ProgramData. - for sched in schedules.get('schedules', []): - title = sched.get('title', 'No Title') - desc = sched.get('description', '') - start_time = parse_schedules_direct_time(sched.get('startTime')) - end_time = parse_schedules_direct_time(sched.get('endTime')) - obj, created = ProgramData.objects.update_or_create( - epg=epg_data, - start_time=start_time, - title=title, - defaults={ - 'end_time': end_time, - 'description': desc, - 'sub_title': '' - } + Args: + source: EPGSource instance + stations_only: If True, only fetch and persist station metadata (no schedules/programs). + Used on initial source creation to populate EPGData for auto-matching + before channels are assigned. + """ + import hashlib + from datetime import date + + + logger.info(f"Fetching Schedules Direct data for source: {source.name}") + + # ------------------------------------------------------------------------- + # Validate credentials + # ------------------------------------------------------------------------- + username = (source.username or '').strip() + password = (source.api_key or '').strip() + + if not username or not password: + msg = "Schedules Direct source requires both a username and password." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Enforce 2-hour minimum interval between full fetches (not stations-only). + # Schedules Direct enforces rate limits of ~200 requests per 2-hour window. + # This prevents automated abuse regardless of how the refresh was triggered. + # + # Exception: if no SDScheduleMD5 records exist yet, this is the first full + # refresh after initial source creation (stations-only runs first and updates + # updated_at, which would otherwise incorrectly trigger this guard). Always + # allow the first full refresh through so guide data is immediately available. + # ------------------------------------------------------------------------- + if not stations_only and source.updated_at: + from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 + has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() + if has_prior_full_refresh: + elapsed = (timezone.now() - source.updated_at).total_seconds() + min_interval_seconds = 2 * 3600 # 2 hours + if elapsed < min_interval_seconds: + remaining_minutes = int((min_interval_seconds - elapsed) / 60) + msg = ( + f"Schedules Direct refresh skipped — minimum 2-hour interval not reached. " + f"Last refreshed {int(elapsed / 60)} minutes ago. " + f"Please wait {remaining_minutes} more minute(s)." ) - if created: - logger.info(f"Created ProgramData '{title}' for tvg_id '{tvg_id}'.") - else: - logger.info(f"Updated ProgramData '{title}' for tvg_id '{tvg_id}'.") - except Exception as e: - logger.error(f"Error fetching Schedules Direct data from {source.name}: {e}", exc_info=True) + logger.warning(f"SD source {source.id}: {msg}") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + else: + logger.info(f"SD source {source.id}: No prior full refresh detected — skipping 2-hour guard for first full fetch.") + + # ------------------------------------------------------------------------- + # Resolve user agent + # ------------------------------------------------------------------------- + stream_settings = CoreSettings.get_stream_settings() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" + default_user_agent_id = stream_settings.get('default_user_agent') + if default_user_agent_id: + try: + ua_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() + if ua_obj and ua_obj.user_agent: + user_agent = ua_obj.user_agent + except (ValueError, Exception) as e: + logger.warning(f"Could not resolve default user agent, using fallback: {e}") + + def _sd_headers(token=None): + h = { + 'Content-Type': 'application/json', + 'User-Agent': user_agent, + } + if token: + h['token'] = token + return h + + # ------------------------------------------------------------------------- + # Step 1 — Authenticate and obtain session token + # The SD API requires the password to be SHA1-hashed before transmission. + # This is a requirement of the Schedules Direct API specification, not an + # architectural choice. + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_FETCHING + source.last_message = "Authenticating with Schedules Direct..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") + + try: + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + token_response = requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers=_sd_headers(), + timeout=30, + ) + token_response.raise_for_status() + token_data = token_response.json() + + auth_code = token_data.get('code', 0) + if auth_code != 0: + if auth_code == 4007: + msg = "Schedules Direct: this application is not authorized. Please contact the Dispatcharr maintainers." + elif auth_code == 4004: + msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." + elif auth_code == 4009: + msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours — check for misconfiguration." + elif auth_code == 4001: + msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." + elif auth_code == 4008: + msg = "Schedules Direct: account is inactive. Please log in to schedulesdirect.org to reactivate." + else: + msg = f"Schedules Direct authentication failed (code {auth_code}): {token_data.get('message', 'Unknown error')}" + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + token = token_data.get('token') + if not token: + msg = "Schedules Direct returned no token." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info("Schedules Direct authentication successful.") + + except requests.exceptions.RequestException as e: + msg = f"Network error authenticating with Schedules Direct: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 2 — Check account status (respect OFFLINE system status) + # ------------------------------------------------------------------------- + try: + status_response = requests.get( + f"{SD_BASE_URL}/status", + headers=_sd_headers(token), + timeout=30, + ) + status_response.raise_for_status() + status_data = status_response.json() + system_status = status_data.get('systemStatus', [{}])[0].get('status', 'Online') + if system_status == 'Offline': + # Per SD API spec: if system is offline, disconnect and do not + # retry for 1 hour. We set idle status rather than error since + # this is a temporary SD-side condition. + msg = "Schedules Direct system is currently offline. Per SD guidelines, retrying in 1 hour." + logger.warning(msg) + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.debug(f"Schedules Direct system status: {system_status}") + except requests.exceptions.RequestException as e: + logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") + + # ------------------------------------------------------------------------- + # Step 3 — Fetch subscribed lineups and build station map + # ------------------------------------------------------------------------- + _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=_sd_headers(token), + timeout=30, + ) + # SD returns 400 with code 4102 when no lineups are configured. + # This is a valid account state — the user needs to add lineups via + # the Manage Lineups UI. Treat as idle rather than error. + if lineups_response.status_code == 400: + sd_data = lineups_response.json() + if sd_data.get('code') == 4102: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + lineups_response.raise_for_status() + lineups_data = lineups_response.json() + lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] + if not lineups: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no active lineups found.") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.info(f"Found {len(lineups)} lineup(s) in SD account.") + except requests.exceptions.RequestException as e: + msg = f"Failed to fetch Schedules Direct lineups: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # Build station metadata map: stationID -> {name, callsign, logo_url} + station_map = {} + _sd_send_ws_sync(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") + for lineup in lineups: + lineup_id = lineup.get('lineupID') or lineup.get('lineup') + if not lineup_id: + continue + try: + detail_response = requests.get( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=_sd_headers(token), + timeout=30, + ) + detail_response.raise_for_status() + detail_data = detail_response.json() + for station in detail_data.get('stations', []): + sid = station.get('stationID') + if not sid: + continue + logo_url = None + logos = station.get('stationLogos') or station.get('logo') or [] + if isinstance(logos, list) and logos: + logo_url = logos[0].get('URL') or logos[0].get('url') + elif isinstance(logos, dict): + logo_url = logos.get('URL') or logos.get('url') + station_map[sid] = { + 'name': station.get('name', sid), + 'callsign': station.get('callsign', ''), + 'logo_url': logo_url, + } + logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") + + if not station_map: + msg = "No stations found across all Schedules Direct lineups." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info(f"Built station map with {len(station_map)} stations.") + + # ------------------------------------------------------------------------- + # Step 4 — Persist station metadata to EPGData + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_PARSING + source.last_message = f"Syncing {len(station_map)} stations..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") + + existing_epg_map = { + epg.tvg_id: epg + for epg in EPGData.objects.filter(epg_source=source) + } + + epgs_to_create = [] + epgs_to_update = [] + icon_max_length = EPGData._meta.get_field('icon_url').max_length + name_max_length = EPGData._meta.get_field('name').max_length + + for sid, info in station_map.items(): + display_name = (info['name'] or sid)[:name_max_length] + logo = info['logo_url'] + if logo and len(logo) > icon_max_length: + logo = None + + if sid in existing_epg_map: + epg_obj = existing_epg_map[sid] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != logo: + epg_obj.icon_url = logo + needs_update = True + if needs_update: + epgs_to_update.append(epg_obj) + else: + epgs_to_create.append(EPGData( + tvg_id=sid, + name=display_name, + icon_url=logo, + epg_source=source, + )) + + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) + logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") + + gc.collect() + + # Rebuild map with fresh DB ids for all stations + epg_id_map = { + epg.tvg_id: epg.id + for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) + } + + # Station sync complete — send progress continuing into programs phase + # We deliberately do NOT send parsing_channels at 100 with status=success here + # because that would cause the frontend to mark the source as complete and + # stop rendering progress updates for the subsequent program fetch phases. + _sd_send_ws_sync(source.id, "parsing_programs", 30, + message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") + + # ------------------------------------------------------------------------- + # Stations-only mode — used on initial source creation. + # Stop here so the user can run Auto-match EPG before the full program fetch. + # ------------------------------------------------------------------------- + if stations_only: + success_msg = ( + f"{len(station_map)} stations loaded from Schedules Direct. " + f"Run Auto-match EPG to map your channels, then use the Refresh " + f"button to populate guide data." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_channels", 100, status="success", + message=success_msg, channels_count=len(station_map)) + logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") + return + + # ------------------------------------------------------------------------- + # Step 5 — MD5-delta schedule fetch + # First fetch MD5 hashes for all stations/dates. Compare against our + # locally cached hashes to determine which schedules have changed. + # Only download schedules that have actually changed — this minimises + # API calls against SD's rate-limited endpoints. + # ------------------------------------------------------------------------- + from apps.epg.models import SDScheduleMD5 + from django.utils.dateparse import parse_datetime + + _sd_send_ws_sync(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") + station_ids = list(station_map.keys()) + today = date.today() + date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] + + # Fetch MD5 hashes for all stations in batches of 5000 + STATION_BATCH_SIZE = 5000 + server_md5s = {} # (station_id, date) -> {md5, last_modified} + + logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") + + station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] + for batch in station_batches: + try: + md5_response = requests.post( + f"{SD_BASE_URL}/schedules/md5", + json=[{'stationID': sid, 'date': date_list} for sid in batch], + headers=_sd_headers(token), + timeout=120, + ) + md5_response.raise_for_status() + md5_data = md5_response.json() + for sid, dates in md5_data.items(): + for date_str, info in dates.items(): + if info.get('code', 0) == 0: + server_md5s[(sid, date_str)] = { + 'md5': info.get('md5', ''), + 'last_modified': info.get('lastModified', ''), + } + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule MD5s: {e}") + + # Load our cached MD5s from DB + cached_md5s = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) + } + + # Determine which station/date combinations need downloading + changed_by_station = {} # station_id -> [date_str, ...] + for (sid, date_str), server_info in server_md5s.items(): + if date_str not in date_list: + continue + cached = cached_md5s.get((sid, date_str)) + if cached != server_info['md5']: + changed_by_station.setdefault(sid, []).append(date_str) + + total_changed = sum(len(v) for v in changed_by_station.items()) + total_possible = len(station_ids) * len(date_list) + logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") + _sd_send_ws_sync(source.id, "parsing_programs", 38, + message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") + + # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} + schedules_by_station = {sid: [] for sid in station_ids} + program_ids_needed = set() + + # Existing cached program MD5s for delta detection in step 6 + existing_program_md5s = { + p.tvg_id + '|' + p.title: p.sd_program_md5 + for p in ProgramData.objects.filter( + epg__epg_source=source, + sd_program_md5__isnull=False, + ).only('tvg_id', 'title', 'sd_program_md5') + } + + if not changed_by_station: + logger.info("No schedule changes detected — skipping schedule and program downloads.") + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", + message="No schedule changes detected since last refresh. Guide data is up to date.") + source.status = EPGSource.STATUS_SUCCESS + source.last_message = "No schedule changes detected. Guide data is up to date." + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + return + + # Download only changed schedules, batched by 7-day windows per station + SCHEDULE_BATCH_DAYS = 7 + changed_station_ids = list(changed_by_station.keys()) + date_batches = [date_list[i:i + SCHEDULE_BATCH_DAYS] for i in range(0, len(date_list), SCHEDULE_BATCH_DAYS)] + new_md5_records = [] + updated_md5_records = [] + existing_md5_map = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=changed_station_ids) + } + + for batch_idx, date_batch in enumerate(date_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 38 + int((batch_idx / len(date_batches)) * 22) + logger.info(f"Fetching schedule batch {batch_idx + 1} of {len(date_batches)}...") + _sd_send_ws_sync(source.id, "parsing_programs", min(59, pre_progress), + message=f"Fetching schedules: batch {batch_idx + 1} of {len(date_batches)}...") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + # Only include stations that have changes in this date batch + request_body = [ + {'stationID': sid, 'date': [d for d in date_batch if d in changed_by_station.get(sid, [])]} + for sid in changed_station_ids + if any(d in changed_by_station.get(sid, []) for d in date_batch) + ] + if not request_body: + continue + try: + sched_response = requests.post( + f"{SD_BASE_URL}/schedules", + json=request_body, + headers=_sd_headers(token), + timeout=120, + ) + sched_response.raise_for_status() + sched_data = sched_response.json() + + for station_sched in sched_data: + sid = station_sched.get('stationID') + if not sid: + continue + programs = station_sched.get('programs', []) + schedules_by_station.setdefault(sid, []).extend(programs) + for prog in programs: + pid = prog.get('programID') + if pid: + program_ids_needed.add(pid) + + # Update MD5 cache for this station/date + meta = station_sched.get('metadata', {}) + start_date = meta.get('startDate') + md5_val = meta.get('md5', '') + last_mod_str = meta.get('modified', '') + if start_date and md5_val: + key = (sid, start_date) + last_mod = parse_datetime(last_mod_str) if last_mod_str else timezone.now() + if key in existing_md5_map: + rec = existing_md5_map[key] + rec.md5 = md5_val + rec.last_modified = last_mod + updated_md5_records.append(rec) + else: + import datetime as dt_module + try: + date_obj = dt_module.date.fromisoformat(start_date) + new_md5_records.append(SDScheduleMD5( + epg_source=source, + station_id=sid, + date=date_obj, + md5=md5_val, + last_modified=last_mod, + )) + except ValueError: + pass + + progress = 38 + int(((batch_idx + 1) / len(date_batches)) * 22) + _sd_send_ws_sync(source.id, "parsing_programs", min(60, progress), + message=f"Fetching changed schedules: batch {batch_idx + 1}/{len(date_batches)} ({len(program_ids_needed):,} programs found)") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule batch {batch_idx + 1}: {e}") + + # Persist updated MD5 cache + if new_md5_records: + SDScheduleMD5.objects.bulk_create(new_md5_records, ignore_conflicts=True) + logger.info(f"Cached {len(new_md5_records)} new schedule MD5s.") + if updated_md5_records: + SDScheduleMD5.objects.bulk_update(updated_md5_records, ['md5', 'last_modified']) + logger.info(f"Updated {len(updated_md5_records)} existing schedule MD5s.") + + if not program_ids_needed: + msg = "No schedule data returned from Schedules Direct." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 6 — MD5-delta program metadata fetch + # The schedule response includes an MD5 hash per program airing. + # Compare against our cached program MD5s to only download programs + # whose metadata has changed since our last fetch. + # ------------------------------------------------------------------------- + + # Build map of programID -> md5 from schedule data + schedule_program_md5s = {} # programID -> md5 from schedule + for sid, airings in schedules_by_station.items(): + for airing in airings: + pid = airing.get('programID') + md5 = airing.get('md5') + if pid and md5: + schedule_program_md5s[pid] = md5 + + # Load cached program MD5s from DB keyed by programID via tvg_id + cached_prog_md5s = { + p.tvg_id: p.sd_program_md5 + for p in ProgramData.objects.filter( + epg__epg_source=source, + sd_program_md5__isnull=False, + ).only('tvg_id', 'sd_program_md5').distinct('tvg_id') + } + + # Only fetch programs where MD5 differs from our cache + programs_to_fetch = [ + pid for pid in program_ids_needed + if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) + ] + + logger.info( + f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " + f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") + + program_metadata = {} + program_id_list = programs_to_fetch + total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) + + if program_id_list: + logger.info(f"Fetching metadata for {len(program_id_list)} programs in {total_batches} batch(es).") + for batch_idx in range(total_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 60 + int((batch_idx / total_batches) * 20) + logger.info(f"Fetching program metadata batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)...") + _sd_send_ws_sync(source.id, "parsing_programs", min(79, pre_progress), + message=f"Fetching program data: batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + batch = program_id_list[batch_idx * SD_PROGRAM_BATCH_SIZE:(batch_idx + 1) * SD_PROGRAM_BATCH_SIZE] + try: + prog_response = requests.post( + f"{SD_BASE_URL}/programs", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + prog_response.raise_for_status() + prog_data = prog_response.json() + for prog in prog_data: + pid = prog.get('programID') + if pid: + program_metadata[pid] = prog + + progress = 60 + int(((batch_idx + 1) / total_batches) * 20) + _sd_send_ws_sync(source.id, "parsing_programs", min(80, progress), + message=f"Fetching program details: batch {batch_idx + 1}/{total_batches} ({len(program_metadata):,} programs loaded)") + logger.debug(f"Fetched program metadata batch {batch_idx + 1}/{total_batches}") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch program metadata batch {batch_idx + 1}: {e}") + else: + logger.info("All program metadata unchanged - skipping program download.") + _sd_send_ws_sync(source.id, "parsing_programs", 80, message="Program metadata unchanged - using cached data.") + + gc.collect() + + # ------------------------------------------------------------------------- + # Step 7 — Build ProgramData records and persist atomically + # ------------------------------------------------------------------------- + logger.info("Building program records...") + _sd_send_ws_sync(source.id, "parsing_programs", 80) + + # Only process stations that are mapped to channels to match the existing + # XMLTV flow (parse_programs_for_source only processes mapped channels). + from apps.channels.models import Channel as ChannelModel + mapped_epg_ids = set( + ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) + ) + mapped_tvg_ids = set( + EPGData.objects.filter( + id__in=mapped_epg_ids, + epg_source=source, + ).values_list('tvg_id', flat=True) + ) + + all_programs_to_create = [] + total_programs = 0 + skipped_unmapped = 0 + + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + skipped_unmapped += len(airings) + continue + + epg_db_id = epg_id_map.get(sid) + if not epg_db_id: + continue + + for airing in airings: + pid = airing.get('programID') + air_time = airing.get('airDateTime') + duration_secs = airing.get('duration', 0) + + if not pid or not air_time or not duration_secs: + continue + + try: + start_dt = parse_schedules_direct_time(air_time) + end_dt = start_dt + timedelta(seconds=int(duration_secs)) + except Exception as e: + logger.debug(f"Could not parse air time '{air_time}': {e}") + continue + + meta = program_metadata.get(pid, {}) + titles = meta.get('titles', [{}]) + title = titles[0].get('title120', '') if titles else '' + if not title: + title = meta.get('episodeTitle150', '') or 'No Title' + title = title[:255] + + descriptions = meta.get('descriptions', {}) + desc = '' + for key in ('description1000', 'description255', 'description100'): + candidates = descriptions.get(key, []) + if candidates: + desc = candidates[0].get('description', '') + if desc: + break + + episode_title = meta.get('episodeTitle150', '') + + # Build custom_properties following the same pattern as the XMLTV parser + custom_props = {} + metadata_block = meta.get('metadata', [{}]) + if metadata_block: + m = metadata_block[0].get('Gracenote', {}) + season = m.get('season') + episode = m.get('episode') + if season: + custom_props['season'] = int(season) + if episode: + custom_props['episode'] = int(episode) + + content_rating = meta.get('contentRating', []) + if content_rating: + custom_props['rating'] = content_rating[0].get('code', '') + custom_props['rating_system'] = content_rating[0].get('body', '') + + genres = meta.get('genres', []) + if genres: + custom_props['categories'] = genres + + cast = meta.get('cast', []) + crew = meta.get('crew', []) + credits = {} + if cast: + credits['actor'] = [ + {'name': p.get('name', ''), 'role': p.get('role', '')} + for p in cast if p.get('name') + ] + if crew: + for member in crew: + role = member.get('role', '').lower() + name = member.get('name', '') + if not name: + continue + if 'director' in role: + credits.setdefault('director', []).append(name) + elif 'writer' in role or 'screenwriter' in role: + credits.setdefault('writer', []).append(name) + elif 'producer' in role: + credits.setdefault('producer', []).append(name) + if credits: + custom_props['credits'] = credits + + if airing.get('isLive'): + custom_props['live'] = True + if airing.get('isNew'): + custom_props['new'] = True + if airing.get('isPremiere'): + custom_props['premiere'] = True + + year = meta.get('movie', {}).get('year') or meta.get('originalAirDate', '')[:4] + if year: + custom_props['date'] = str(year) + + all_programs_to_create.append(ProgramData( + epg_id=epg_db_id, + start_time=start_dt, + end_time=end_dt, + title=title, + sub_title=episode_title or None, + description=desc or None, + tvg_id=sid, + custom_properties=custom_props or None, + sd_program_md5=schedule_program_md5s.get(pid), + )) + total_programs += 1 + + logger.info(f"Built {total_programs} program records " + f"({skipped_unmapped} skipped for unmapped stations).") + + _sd_send_ws_sync(source.id, "parsing_programs", 88) + + # Atomic delete + bulk insert — same pattern as parse_programs_for_source + BATCH_SIZE = 1000 + try: + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] + logger.debug(f"Deleted {deleted_count} existing SD programs.") + for i in range(0, len(all_programs_to_create), BATCH_SIZE): + ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) + progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) + _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) + + logger.info(f"Committed {total_programs} Schedules Direct programs to database.") + except Exception as db_error: + msg = f"Database error persisting Schedules Direct programs: {db_error}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + finally: + all_programs_to_create = None + gc.collect() + + # ------------------------------------------------------------------------- + # Done + # ------------------------------------------------------------------------- + success_msg = ( + f"Successfully fetched {total_programs:,} programs for " + f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " + f"({skipped_unmapped:,} programs skipped for unmapped stations)." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message=success_msg) + log_system_event( + event_type='epg_refresh', + source_name=source.name, + programs=total_programs, + channels=len(mapped_tvg_ids), + skipped_programs=skipped_unmapped, + ) + logger.info(f"Schedules Direct fetch complete for source: {source.name}") # ------------------------------- diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py new file mode 100644 index 00000000..782750af --- /dev/null +++ b/apps/epg/tests/test_schedules_direct.py @@ -0,0 +1,263 @@ +""" +Tests for the Schedules Direct EPG integration. + +Covers: +- EPGSource model: username field presence and help text +- EPGSource serializer: username field included in output +- fetch_schedules_direct: credential validation +- fetch_schedules_direct: SHA1 password hashing and token exchange +- fetch_schedules_direct: graceful error handling on auth failure +- parse_schedules_direct_time: correct UTC parsing +- EPG signals: SD sources skip the XMLTV program parser +""" + +import hashlib +from datetime import datetime +from unittest.mock import MagicMock, patch + +from django.test import TestCase +from django.utils import timezone + +from apps.epg.models import EPGSource +from apps.epg.serializers import EPGSourceSerializer + + +# --------------------------------------------------------------------------- +# Model tests +# --------------------------------------------------------------------------- + +class EPGSourceUsernameFieldTests(TestCase): + """EPGSource.username must exist, be nullable, and carry help text.""" + + def test_username_field_exists(self): + source = EPGSource.objects.create( + name='SD Test', + source_type='schedules_direct', + username='testuser', + api_key='testpass', + ) + source.refresh_from_db() + self.assertEqual(source.username, 'testuser') + + def test_username_nullable(self): + source = EPGSource.objects.create( + name='SD Nullable', + source_type='schedules_direct', + ) + source.refresh_from_db() + self.assertIsNone(source.username) + + def test_username_help_text(self): + field = EPGSource._meta.get_field('username') + self.assertIn('Schedules Direct', field.help_text) + + +# --------------------------------------------------------------------------- +# Serializer tests +# --------------------------------------------------------------------------- + +class EPGSourceSerializerSDTests(TestCase): + """EPGSourceSerializer must include the username field.""" + + def test_username_in_serializer_fields(self): + source = EPGSource.objects.create( + name='SD Serializer Test', + source_type='schedules_direct', + username='sduser', + api_key='sdpass', + ) + data = EPGSourceSerializer(source).data + self.assertIn('username', data) + self.assertEqual(data['username'], 'sduser') + + def test_api_key_in_serializer_fields(self): + source = EPGSource.objects.create( + name='SD API Key Test', + source_type='schedules_direct', + api_key='secret', + ) + data = EPGSourceSerializer(source).data + self.assertIn('api_key', data) + + +# --------------------------------------------------------------------------- +# fetch_schedules_direct tests +# --------------------------------------------------------------------------- + +class FetchSchedulesDirectCredentialTests(TestCase): + """fetch_schedules_direct must reject sources missing credentials.""" + + def _make_source(self, username=None, api_key=None): + return EPGSource.objects.create( + name='SD Cred Test', + source_type='schedules_direct', + username=username, + api_key=api_key, + ) + + def test_missing_username_sets_error_status(self): + from apps.epg.tasks import fetch_schedules_direct + source = self._make_source(username=None, api_key='pass') + fetch_schedules_direct(source) + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_ERROR) + + def test_missing_password_sets_error_status(self): + from apps.epg.tasks import fetch_schedules_direct + source = self._make_source(username='user', api_key=None) + fetch_schedules_direct(source) + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_ERROR) + + def test_empty_username_sets_error_status(self): + from apps.epg.tasks import fetch_schedules_direct + source = self._make_source(username=' ', api_key='pass') + fetch_schedules_direct(source) + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_ERROR) + + +class FetchSchedulesDirectAuthTests(TestCase): + """fetch_schedules_direct must SHA1-hash the password before sending.""" + + @patch('apps.epg.tasks.requests.post') + @patch('apps.epg.tasks.requests.get') + def test_password_sha1_hashed_in_token_request(self, mock_get, mock_post): + """The token POST body must contain the SHA1 hash of the plaintext password.""" + plaintext = 'mysecretpassword' + expected_hash = hashlib.sha1(plaintext.encode('utf-8')).hexdigest() + + # Auth succeeds, status check returns empty data, lineups returns empty + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={'code': 0, 'token': 'tok123'}), + ) + mock_get.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={}), + ) + + from apps.epg.tasks import fetch_schedules_direct + source = EPGSource.objects.create( + name='SD Hash Test', + source_type='schedules_direct', + username='sduser', + api_key=plaintext, + ) + + with patch('apps.epg.tasks.send_epg_update'): + fetch_schedules_direct(source) + + # Verify the POST was called and the body contained the hash + self.assertTrue(mock_post.called) + call_kwargs = mock_post.call_args + posted_json = call_kwargs[1].get('json') or call_kwargs[0][1] + self.assertEqual(posted_json.get('password'), expected_hash) + self.assertEqual(posted_json.get('username'), 'sduser') + + @patch('apps.epg.tasks.requests.post') + def test_auth_failure_sets_error_status(self, mock_post): + """A non-zero SD response code must set STATUS_ERROR on the source.""" + mock_post.return_value = MagicMock( + status_code=200, + json=MagicMock(return_value={ + 'code': 3000, + 'message': 'Invalid credentials', + }), + ) + + from apps.epg.tasks import fetch_schedules_direct + source = EPGSource.objects.create( + name='SD Auth Fail', + source_type='schedules_direct', + username='baduser', + api_key='badpass', + ) + + with patch('apps.epg.tasks.send_epg_update'): + fetch_schedules_direct(source) + + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_ERROR) + + @patch('apps.epg.tasks.requests.post') + def test_network_error_sets_error_status(self, mock_post): + """A network-level exception must set STATUS_ERROR and not crash.""" + import requests as req_lib + mock_post.side_effect = req_lib.exceptions.ConnectionError('timeout') + + from apps.epg.tasks import fetch_schedules_direct + source = EPGSource.objects.create( + name='SD Network Error', + source_type='schedules_direct', + username='user', + api_key='pass', + ) + + with patch('apps.epg.tasks.send_epg_update'): + fetch_schedules_direct(source) # Must not raise + + source.refresh_from_db() + self.assertEqual(source.status, EPGSource.STATUS_ERROR) + + +# --------------------------------------------------------------------------- +# parse_schedules_direct_time tests +# --------------------------------------------------------------------------- + +class ParseSchedulesDirectTimeTests(TestCase): + """parse_schedules_direct_time must parse SD ISO timestamps to UTC-aware datetimes.""" + + def test_parses_valid_timestamp(self): + from apps.epg.tasks import parse_schedules_direct_time + result = parse_schedules_direct_time('2026-05-16T20:00:00Z') + self.assertEqual(result.year, 2026) + self.assertEqual(result.month, 5) + self.assertEqual(result.day, 16) + self.assertEqual(result.hour, 20) + self.assertIsNotNone(result.tzinfo) + + def test_result_is_utc_aware(self): + from apps.epg.tasks import parse_schedules_direct_time + result = parse_schedules_direct_time('2026-01-01T00:00:00Z') + # Should be timezone-aware + self.assertIsNotNone(result.tzinfo) + + def test_raises_on_invalid_format(self): + from apps.epg.tasks import parse_schedules_direct_time + with self.assertRaises(Exception): + parse_schedules_direct_time('not-a-timestamp') + + +# --------------------------------------------------------------------------- +# Signal tests +# --------------------------------------------------------------------------- + +class SDSourceSignalTests(TestCase): + """SD EPG sources must skip the XMLTV program parser signal.""" + + @patch('apps.channels.signals.parse_programs_for_tvg_id') + def test_sd_source_skips_xmltv_parse_on_channel_create(self, mock_parse): + """Creating a channel linked to an SD EPG source must not trigger + the XMLTV program parser — SD data is handled by fetch_schedules_direct.""" + from apps.epg.models import EPGData + from apps.channels.models import Channel + + sd_source = EPGSource.objects.create( + name='SD Signal Test', + source_type='schedules_direct', + username='u', + api_key='p', + ) + epg_data = EPGData.objects.create( + tvg_id='sd-test-station', + name='SD Test Station', + epg_source=sd_source, + ) + + Channel.objects.create( + name='SD Channel', + epg_data=epg_data, + ) + + mock_parse.delay.assert_not_called() diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index 31ad1fc5..47eb91ca 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -51,6 +51,7 @@ export const WebsocketProvider = ({ children }) => { const epgs = useEPGsStore((s) => s.epgs); const updateEPG = useEPGsStore((s) => s.updateEPG); const updateEPGProgress = useEPGsStore((s) => s.updateEPGProgress); + const fetchEPGsForProgress = useEPGsStore((s) => s.fetchEPGs); const updatePlaylist = usePlaylistsStore((s) => s.updatePlaylist); @@ -642,19 +643,18 @@ export const WebsocketProvider = ({ children }) => { parsedEvent.data.source || parsedEvent.data.account; const epg = epgs[sourceId]; - // Only update progress if the EPG still exists in the store - // This prevents crashes when receiving updates for deleted EPGs - if (epg) { - // Update the store with progress information - updateEPGProgress(parsedEvent.data); - } else { - // EPG was deleted, ignore this update - console.debug( - `Ignoring EPG refresh update for deleted EPG ${sourceId}` - ); + // If EPG not in store yet (e.g. newly created and refresh started + // before the store was updated), fetch the EPG list and update progress. + if (!epg) { + fetchEPGsForProgress().then(() => { + updateEPGProgress(parsedEvent.data); + }); break; } + // Update the store with progress information + updateEPGProgress(parsedEvent.data); + if (epg) { // Check for any indication of an error (either via status or error field) const hasError = diff --git a/frontend/src/api.js b/frontend/src/api.js index 2a357a81..42f010c6 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3817,4 +3817,49 @@ export default class API { errorNotification('Failed to fetch connect logs', e); } } + + static async getSDLineups(sourceId) { + try { + const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`); + return response; + } catch (e) { + errorNotification('Failed to retrieve Schedules Direct lineups', e); + } + } + + static async addSDLineup(sourceId, lineup) { + try { + const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`, { + method: 'POST', + body: { lineup }, + }); + return response; + } catch (e) { + errorNotification(`Failed to add lineup ${lineup}`, e); + } + } + + static async deleteSDLineup(sourceId, lineup) { + try { + const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`, { + method: 'DELETE', + body: { lineup }, + }); + return response; + } catch (e) { + errorNotification(`Failed to remove lineup ${lineup}`, e); + } + } + + static async searchSDLineups(sourceId, country, postalcode) { + try { + const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/search/`, { + method: 'POST', + body: { country, postalcode }, + }); + return response; + } catch (e) { + errorNotification('Failed to search Schedules Direct lineups', e); + } + } } diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 4641af6c..82ddffe9 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -1,5 +1,5 @@ // Modal.js -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import { TextInput, Button, @@ -12,15 +12,333 @@ import { Divider, Box, Text, + Alert, + Select, + Loader, + Badge, + ScrollArea, + Table, } from '@mantine/core'; +import { TriangleAlert, Trash2, Plus, Search } from 'lucide-react'; import { isNotEmpty, useForm } from '@mantine/form'; import ScheduleInput from './ScheduleInput'; import { addEPG, updateEPG } from '../../utils/forms/DummyEpgUtils.js'; import { showNotification } from '../../utils/notificationUtils.js'; +import API from '../../api.js'; + +// Countries are fetched dynamically from the SD API on component mount. +// Fallback list used if the API call fails. +const SD_COUNTRIES_FALLBACK = [ + { value: 'USA', label: 'United States' }, + { value: 'CAN', label: 'Canada' }, + { value: 'GBR', label: 'United Kingdom' }, + { value: 'AUS', label: 'Australia' }, + { value: 'AUT', label: 'Austria' }, + { value: 'BEL', label: 'Belgium' }, + { value: 'DNK', label: 'Denmark' }, + { value: 'FIN', label: 'Finland' }, + { value: 'FRA', label: 'France' }, + { value: 'DEU', label: 'Germany' }, + { value: 'IRL', label: 'Ireland' }, + { value: 'ITA', label: 'Italy' }, + { value: 'NLD', label: 'Netherlands' }, + { value: 'NZL', label: 'New Zealand' }, +]; + +const SDLineupManager = ({ sourceId }) => { + const [countries, setCountries] = useState(SD_COUNTRIES_FALLBACK); + const [activeLineups, setActiveLineups] = useState([]); + const [lineupNotice, setLineupNotice] = useState(null); + const [changesRemaining, setChangesRemaining] = useState(null); + const [changesResetAt, setChangesResetAt] = useState(null); + const [searchResults, setSearchResults] = useState([]); + const [country, setCountry] = useState('USA'); + const [postalCode, setPostalCode] = useState(''); + const [loadingLineups, setLoadingLineups] = useState(false); + const [searching, setSearching] = useState(false); + const [addingLineup, setAddingLineup] = useState(null); + const [removingLineup, setRemovingLineup] = useState(null); + const maxLineups = 4; + const SD_DOCS_URL = 'https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform'; + + const fetchActiveLineups = useCallback(async () => { + setLoadingLineups(true); + try { + const data = await API.getSDLineups(sourceId); + if (data) { + setActiveLineups(data.lineups || []); + setLineupNotice(data.notice || null); + // Always update changesRemaining from server — includes null (unknown) and 0 (locked) + if (data.changes_remaining !== undefined) { + setChangesRemaining(data.changes_remaining); + } + if (data.changes_reset_at !== undefined) { + setChangesResetAt(data.changes_reset_at); + } + } + } finally { + setLoadingLineups(false); + } + }, [sourceId]); + + useEffect(() => { + if (sourceId) { + fetchActiveLineups(); + // Fetch country list from SD API per their recommendation to not hardcode + fetch('https://json.schedulesdirect.org/20141201/available/countries') + .then(r => r.json()) + .then(data => { + const all = Object.values(data).flat(); + const mapped = all + .filter(c => c.shortName && c.fullName) + .map(c => ({ value: c.shortName, label: c.fullName })) + .sort((a, b) => a.label.localeCompare(b.label)); + if (mapped.length > 0) setCountries(mapped); + }) + .catch(() => {}); // fallback list remains if fetch fails + } + }, [sourceId, fetchActiveLineups]); + + const handleSearch = async () => { + if (!postalCode.trim()) return; + setSearching(true); + setSearchResults([]); + try { + const data = await API.searchSDLineups(sourceId, country, postalCode.trim()); + if (data) { + setSearchResults(data.lineups || []); + } + } finally { + setSearching(false); + } + }; + + const handleAdd = async (lineup) => { + setAddingLineup(lineup.lineup); + try { + const result = await API.addSDLineup(sourceId, lineup.lineup); + if (!result) return; + + // Update changesRemaining from response + if (result.changes_remaining !== undefined && result.changes_remaining !== null) { + setChangesRemaining(result.changes_remaining); + } + + if (result.error === 'daily_limit_reached') { + setChangesRemaining(0); + await fetchActiveLineups(); // reload to get reset_at from server + return; + } + + if (result.error === 'duplicate_lineup') { + showNotification({ title: 'Already added', message: result.message, color: 'yellow' }); + return; + } + + if (result.error === 'max_lineups_reached') { + showNotification({ title: 'Maximum lineups reached', message: result.message, color: 'orange' }); + return; + } + + if (result.error) { + showNotification({ title: 'Unable to add lineup', message: result.message, color: 'red' }); + return; + } + + if (result.code === 0) { + showNotification({ title: 'Lineup added', message: lineup.name, color: 'green' }); + await fetchActiveLineups(); + } + } finally { + setAddingLineup(null); + } + }; + + const handleRemove = async (lineup) => { + setRemovingLineup(lineup.lineup); + try { + const result = await API.deleteSDLineup(sourceId, lineup.lineup); + if (result && result.code === 0) { + showNotification({ title: 'Lineup removed', message: lineup.name, color: 'blue' }); + if (result.changes_remaining !== undefined && result.changes_remaining !== null) { + setChangesRemaining(result.changes_remaining); + } + await fetchActiveLineups(); + setSearchResults([]); + } + } finally { + setRemovingLineup(null); + } + }; + + const activeLineupIds = new Set(activeLineups.map((l) => l.lineup)); + const atMax = activeLineups.length >= maxLineups; + + return ( + + + + + Manage Lineups + + + {loadingLineups ? '...' : `${activeLineups.length} / ${maxLineups}`} + + + + {/* Active lineups */} + {loadingLineups ? ( + + + + ) : activeLineups.length === 0 ? ( + + {lineupNotice || 'No lineups configured. Search below to add one.'} + + ) : ( + + {activeLineups.map((lineup) => ( + + + {lineup.name} + + {lineup.transport} · {lineup.location} · {lineup.lineup} + + + + + ))} + + )} + + {/* Search section */} + + Add a lineup + + + {atMax && ( + }> + Maximum of {maxLineups} lineups reached. Remove one to add another. + + )} + + {changesRemaining === 0 && ( + }> + You have reached your daily Schedules Direct lineup addition limit. SD allows 6 adds per 24-hour period.{' '} + {changesResetAt && ( + Limit resets at {new Date(changesResetAt).toUTCString()}. + )} + {!changesResetAt && Limit resets 24 hours after the first add of the day. } + Learn more + + )} + + {changesRemaining === 1 && ( + }> + You have 1 lineup addition remaining today. Use it carefully — Schedules Direct limits adds to 6 per 24-hour period.{' '} + Learn more + + )} + + {changesRemaining === 2 && ( + }> + You have 2 lineup additions remaining today. Schedules Direct limits adds to 6 per 24-hour period.{' '} + Learn more + + )} + + + onChange?.(e.target.value)} + > + + {(data ?? []).map((opt) => { + const val = typeof opt === 'string' ? opt : opt.value; + const lbl = typeof opt === 'string' ? opt : opt.label; + return ; + })} + + ), + Loader: ({ size }) =>
, + Badge: ({ children, color }) => {children}, + ScrollArea: ({ children }) =>
{children}
, + Table: ({ children }) => {children}
, + Tooltip: ({ children }) =>
{children}
, })); // ── Imports after mocks ──────────────────────────────────────────────────────── From cc41ab613f40927bbdfb75e400cd985c02e44a57 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 05:52:17 +0000 Subject: [PATCH 03/76] fix(tests): update EPG test for SD username/password fields --- frontend/src/components/forms/__tests__/EPG.test.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index ce004431..e1d926b5 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -399,10 +399,11 @@ describe('EPG', () => { expect(screen.queryByTestId('input-api-key')).not.toBeInTheDocument(); }); - it('shows API key input when source type requires it', () => { + it('shows username and password inputs when source type is schedules_direct', () => { const epg = makeEPG({ source_type: 'schedules_direct' }); render(); - expect(screen.getByTestId('input-api-key')).toBeInTheDocument(); + expect(screen.getByTestId('input-username')).toBeInTheDocument(); + expect(screen.getByTestId('input-password')).toBeInTheDocument(); }); }); From b236c965ebe330fd4d719481877a2b7bd4cfcf42 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 06:02:04 +0000 Subject: [PATCH 04/76] fix(tests): mock API module in EPG test to prevent real HTTP requests --- frontend/src/components/forms/__tests__/EPG.test.jsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index e1d926b5..e7e85372 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -2,6 +2,15 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react'; import { describe, it, expect, vi, beforeEach } from 'vitest'; // ── Utility mocks ────────────────────────────────────────────────────────────── +vi.mock('../../api.js', () => ({ + default: { + getSDLineups: vi.fn().mockResolvedValue([]), + addSDLineup: vi.fn().mockResolvedValue({ success: true }), + deleteSDLineup: vi.fn().mockResolvedValue({ success: true }), + searchSDLineups: vi.fn().mockResolvedValue([]), + }, +})); + vi.mock('../../../utils/notificationUtils.js', () => ({ showNotification: vi.fn(), })); From aa27fcc3623fbafcb75ac034fdd7657b79a3fa8e Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 06:08:47 +0000 Subject: [PATCH 05/76] fix: only fetch SD countries and lineups when source type is schedules_direct --- frontend/src/components/forms/EPG.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 82ddffe9..ffb91c24 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -82,7 +82,7 @@ const SDLineupManager = ({ sourceId }) => { }, [sourceId]); useEffect(() => { - if (sourceId) { + if (sourceId && sourceType === 'schedules_direct') { fetchActiveLineups(); // Fetch country list from SD API per their recommendation to not hardcode fetch('https://json.schedulesdirect.org/20141201/available/countries') From 60a61e78e3b4bd57e4ddb4df96e0cf323f09cac2 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 06:17:19 +0000 Subject: [PATCH 06/76] fix(tests): mock global fetch and restore SDLineupManager useEffect guard --- frontend/src/components/forms/EPG.jsx | 2 +- frontend/src/components/forms/__tests__/EPG.test.jsx | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index ffb91c24..82ddffe9 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -82,7 +82,7 @@ const SDLineupManager = ({ sourceId }) => { }, [sourceId]); useEffect(() => { - if (sourceId && sourceType === 'schedules_direct') { + if (sourceId) { fetchActiveLineups(); // Fetch country list from SD API per their recommendation to not hardcode fetch('https://json.schedulesdirect.org/20141201/available/countries') diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index e7e85372..01671a45 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -2,6 +2,10 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react'; import { describe, it, expect, vi, beforeEach } from 'vitest'; // ── Utility mocks ────────────────────────────────────────────────────────────── +global.fetch = vi.fn().mockResolvedValue({ + json: () => Promise.resolve({}), +}); + vi.mock('../../api.js', () => ({ default: { getSDLineups: vi.fn().mockResolvedValue([]), From 96f501b0fc9de3f04288c67a44ecaa0786f2f17d Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 06:24:46 +0000 Subject: [PATCH 07/76] fix(tests): correct API mock path in EPG test --- frontend/src/components/forms/__tests__/EPG.test.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index 01671a45..bd8ad1cd 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -6,7 +6,7 @@ global.fetch = vi.fn().mockResolvedValue({ json: () => Promise.resolve({}), }); -vi.mock('../../api.js', () => ({ +vi.mock('../../../api.js', () => ({ default: { getSDLineups: vi.fn().mockResolvedValue([]), addSDLineup: vi.fn().mockResolvedValue({ success: true }), From ba5623c248e05dac2c8080d68e41e7ab935d2149 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 21:10:47 +0000 Subject: [PATCH 08/76] refactor(migrations): consolidate SD migrations into single 0023_schedules_direct --- .../epg/migrations/0023_epgsource_username.py | 23 ------------------ ...hedule_md5.py => 0023_schedules_direct.py} | 24 ++++++++++++++++++- .../0024_alter_epgsource_api_key.py | 21 ---------------- 3 files changed, 23 insertions(+), 45 deletions(-) delete mode 100644 apps/epg/migrations/0023_epgsource_username.py rename apps/epg/migrations/{0025_sd_fields_and_schedule_md5.py => 0023_schedules_direct.py} (76%) delete mode 100644 apps/epg/migrations/0024_alter_epgsource_api_key.py diff --git a/apps/epg/migrations/0023_epgsource_username.py b/apps/epg/migrations/0023_epgsource_username.py deleted file mode 100644 index 1a4c5ed3..00000000 --- a/apps/epg/migrations/0023_epgsource_username.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated migration — adds username field to EPGSource for Schedules Direct auth - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('epg', '0022_alter_epgdata_name'), - ] - - operations = [ - migrations.AddField( - model_name='epgsource', - name='username', - field=models.CharField( - max_length=255, - blank=True, - null=True, - help_text='Username for Schedules Direct authentication', - ), - ), - ] diff --git a/apps/epg/migrations/0025_sd_fields_and_schedule_md5.py b/apps/epg/migrations/0023_schedules_direct.py similarity index 76% rename from apps/epg/migrations/0025_sd_fields_and_schedule_md5.py rename to apps/epg/migrations/0023_schedules_direct.py index b02c94f2..6c4efe1c 100644 --- a/apps/epg/migrations/0025_sd_fields_and_schedule_md5.py +++ b/apps/epg/migrations/0023_schedules_direct.py @@ -5,10 +5,32 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('epg', '0024_alter_epgsource_api_key'), + ('epg', '0022_alter_epgdata_name'), ] operations = [ + # Add username field to EPGSource + migrations.AddField( + model_name='epgsource', + name='username', + field=models.CharField( + max_length=255, + blank=True, + null=True, + help_text='Username for Schedules Direct authentication', + ), + ), + # Update api_key help_text to reflect SD password usage + migrations.AlterField( + model_name='epgsource', + name='api_key', + field=models.CharField( + max_length=255, + blank=True, + null=True, + help_text='Password for Schedules Direct authentication', + ), + ), # Add sd_changes_remaining to EPGSource migrations.AddField( model_name='epgsource', diff --git a/apps/epg/migrations/0024_alter_epgsource_api_key.py b/apps/epg/migrations/0024_alter_epgsource_api_key.py deleted file mode 100644 index e40a6baf..00000000 --- a/apps/epg/migrations/0024_alter_epgsource_api_key.py +++ /dev/null @@ -1,21 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('epg', '0023_epgsource_username'), - ] - - operations = [ - migrations.AlterField( - model_name='epgsource', - name='api_key', - field=models.CharField( - max_length=255, - blank=True, - null=True, - help_text='Password for Schedules Direct authentication', - ), - ), - ] From d066c3748aec5c5b06de67ac2a460782072a9467 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 17 May 2026 21:31:23 +0000 Subject: [PATCH 09/76] fix: set Dispatcharr/version User-Agent for SD API requests per SergeantPanda approval --- apps/epg/tasks.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 479df9e2..c4b58fe4 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2055,23 +2055,17 @@ def fetch_schedules_direct(source, stations_only=False): logger.info(f"SD source {source.id}: No prior full refresh detected — skipping 2-hour guard for first full fetch.") # ------------------------------------------------------------------------- - # Resolve user agent + # Build SD-specific headers + # SD API spec requires the User-Agent to identify the application and version. + # SergeantPanda confirmed Dispatcharr should identify itself properly. # ------------------------------------------------------------------------- - stream_settings = CoreSettings.get_stream_settings() - user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" - default_user_agent_id = stream_settings.get('default_user_agent') - if default_user_agent_id: - try: - ua_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() - if ua_obj and ua_obj.user_agent: - user_agent = ua_obj.user_agent - except (ValueError, Exception) as e: - logger.warning(f"Could not resolve default user agent, using fallback: {e}") + from version import __version__ as dispatcharr_version + sd_user_agent = f"Dispatcharr/{dispatcharr_version}" def _sd_headers(token=None): h = { 'Content-Type': 'application/json', - 'User-Agent': user_agent, + 'User-Agent': sd_user_agent, } if token: h['token'] = token From 6f316977ad423b2c9204e13924a3f526235fca26 Mon Sep 17 00:00:00 2001 From: mwhit Date: Mon, 18 May 2026 22:46:56 +0000 Subject: [PATCH 10/76] fix(epg): address reviewer feedback on SD integration - Rename api_key to password (RemoveField + AddField) for clarity and reusability - Move sd_changes_remaining/sd_changes_reset_at to custom_properties JSONField instead of nullable columns on all EPGSource rows - Replace sd_program_md5 column on ProgramData with SDProgramMD5 relation table keyed by epg_source + program_id, fixing the key mismatch bug in program MD5 delta detection - Fix total_changed counter: .items() -> .values() so len() returns date count not tuple length - Add missing SD lineup management endpoints to api_views.py (sd_lineups GET/POST/DELETE, sd_lineups_search POST) - Regenerate migration 0023_schedules_direct using Django to ensure correct index names and schema --- apps/epg/api_views.py | 320 +++++++++++++++++++ apps/epg/migrations/0023_schedules_direct.py | 84 ++--- apps/epg/models.py | 78 +++-- apps/epg/serializers.py | 2 +- apps/epg/tasks.py | 52 +-- apps/epg/tests/test_schedules_direct.py | 28 +- frontend/src/components/forms/EPG.jsx | 12 +- frontend/src/components/tables/EPGsTable.jsx | 6 +- 8 files changed, 441 insertions(+), 141 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 296f91b8..f0d80c5f 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -108,6 +108,326 @@ class EPGSourceViewSet(viewsets.ModelViewSet): return super().partial_update(request, *args, **kwargs) + def _sd_authenticate(self, source): + """ + Authenticate with Schedules Direct using stored credentials. + Returns (token, None) on success or (None, Response) on failure. + """ + import hashlib + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + username = (source.username or '').strip() + password = (source.password or '').strip() + if not username or not password: + return None, Response( + {"error": "Username and password are required."}, + status=status.HTTP_400_BAD_REQUEST + ) + + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + try: + auth_response = http_requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers={'Content-Type': 'application/json', 'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, + timeout=15, + ) + auth_response.raise_for_status() + token = auth_response.json().get('token') + if not token: + return None, Response( + {"error": "Authentication failed. Check your credentials."}, + status=status.HTTP_401_UNAUTHORIZED + ) + return token, None + except http_requests.exceptions.RequestException as e: + return None, Response( + {"error": f"Authentication failed: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + def _get_sd_reset_at(self, source): + """Retrieve stored reset timestamp from EPGSource model field.""" + reset_at_str = (source.custom_properties or {}).get('sd_changes_reset_at') + return reset_at_str + + def _get_sd_changes_remaining(self, source): + """ + Retrieve stored changesRemaining from EPGSource model field. + If a reset timestamp exists and has passed (midnight UTC), clears the + lockout automatically so the user can make adds again. + """ + from django.utils import timezone + + cp = source.custom_properties or {} + changes_remaining = cp.get('sd_changes_remaining') + reset_at_str = cp.get('sd_changes_reset_at') + from django.utils.dateparse import parse_datetime + reset_at = parse_datetime(reset_at_str) if reset_at_str else None + + # If we have a reset timestamp and it has passed, clear the lockout + if changes_remaining == 0 and reset_at: + if timezone.now() >= reset_at: + cp = source.custom_properties or {} + cp.pop('sd_changes_remaining', None) + cp.pop('sd_changes_reset_at', None) + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + return None + + return changes_remaining + + def _save_sd_changes_remaining(self, source, changes_remaining): + """Persist changesRemaining to EPGSource model field.""" + cp = source.custom_properties or {} + cp['sd_changes_remaining'] = changes_remaining + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + + def _save_sd_lockout(self, source): + """ + Persist a hard lockout to EPGSource model fields when SD returns + 4100 MAX_LINEUP_CHANGES_REACHED. Lockout clears automatically at next + midnight UTC per SD's documented reset schedule. + """ + from django.utils import timezone + from datetime import datetime, timedelta, timezone as dt_timezone + + now = timezone.now() + # SD resets on a 24-hour rolling window from when the limit was hit. + # We use 24 hours from now rather than next midnight UTC to be safe. + reset_at = now + timedelta(hours=24) + + cp = source.custom_properties or {} + cp['sd_changes_remaining'] = 0 + cp['sd_changes_reset_at'] = reset_at.isoformat() + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + logger.warning( + f"SD source {source.id}: daily add limit reached (4100). " + f"Lockout set until {reset_at.isoformat()}." + ) + + @action(detail=True, methods=["get", "post", "delete"], url_path="sd-lineups") + def sd_lineups(self, request, pk=None): + """ + GET — list lineups currently on the SD account + POST — add a lineup (body: {"lineup": "USA-NJ29486-X"}) + DELETE — remove a lineup (body: {"lineup": "USA-NJ29486-X"}) + """ + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + source = self.get_object() + if source.source_type != 'schedules_direct': + return Response( + {"error": "This action is only available for Schedules Direct sources."}, + status=status.HTTP_400_BAD_REQUEST + ) + + token, error = self._sd_authenticate(source) + if error: + return error + + headers = { + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + 'token': token, + } + + if request.method == "GET": + try: + resp = http_requests.get( + f"{SD_BASE_URL}/lineups", + headers=headers, + timeout=15, + ) + if resp.status_code == 400: + sd_data = resp.json() + sd_code = sd_data.get('code') + if sd_code == 4102: + return Response({ + "lineups": [], + "max_lineups": 4, + "changes_remaining": self._get_sd_changes_remaining(source), + "changes_reset_at": self._get_sd_reset_at(source), + "notice": "No lineups are currently configured on this Schedules Direct account. Use the search below to add one.", + }) + resp.raise_for_status() + data = resp.json() + lineups = [l for l in data.get('lineups', []) if not l.get('isDeleted', False)] + return Response({ + "lineups": lineups, + "max_lineups": 4, + "changes_remaining": self._get_sd_changes_remaining(source), + "changes_reset_at": self._get_sd_reset_at(source), + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to fetch lineups: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + elif request.method == "POST": + lineup_id = request.data.get('lineup') + if not lineup_id: + return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) + try: + resp = http_requests.put( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=headers, + timeout=15, + ) + sd_data = resp.json() + sd_code = sd_data.get('code') + + if resp.status_code == 400 or resp.status_code == 403: + if sd_code == 4100: + self._save_sd_lockout(source) + return Response({ + "error": "daily_limit_reached", + "message": "You have reached your daily Schedules Direct lineup addition limit. SD allows 6 adds per 24-hour period. Resets at midnight UTC.", + "changes_remaining": 0, + "docs_url": "https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform", + }, status=status.HTTP_200_OK) + if sd_code == 4101: + return Response({ + "error": "max_lineups_reached", + "message": "Your Schedules Direct account has reached the maximum of 4 lineups. Remove one before adding another.", + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + if sd_code == 2100: + return Response({ + "error": "duplicate_lineup", + "message": "This lineup is already on your Schedules Direct account.", + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + return Response({ + "error": sd_data.get('message', 'Failed to add lineup.'), + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + + resp.raise_for_status() + + # Persist changesRemaining to custom_properties + changes_remaining = sd_data.get('changesRemaining') + if changes_remaining is not None: + self._save_sd_changes_remaining(source, changes_remaining) + + logger.info( + f"SD lineup added for source {source.id}: {lineup_id}. " + f"changesRemaining: {changes_remaining}" + ) + + # Re-fetch stations so the new lineup's stations are available for matching + from apps.epg.tasks import fetch_schedules_direct_stations + fetch_schedules_direct_stations.delay(source.id) + + return Response({ + **sd_data, + "changes_remaining": changes_remaining, + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to add lineup: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + elif request.method == "DELETE": + lineup_id = request.data.get('lineup') + if not lineup_id: + return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) + try: + resp = http_requests.delete( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=headers, + timeout=15, + ) + if resp.status_code == 400: + sd_data = resp.json() + sd_code = sd_data.get('code') + if sd_code == 2103: + return Response({ + "response": "OK", + "code": 0, + "message": "Lineup not found on account — already removed.", + "changes_remaining": self._get_sd_changes_remaining(source), + }) + resp.raise_for_status() + logger.info(f"SD lineup deleted for source {source.id}: {lineup_id}") + return Response({ + **resp.json(), + "changes_remaining": self._get_sd_changes_remaining(source), + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to remove lineup: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + @action(detail=True, methods=["post"], url_path="sd-lineups/search") + def sd_lineups_search(self, request, pk=None): + """ + Search available headends/lineups by country and postal code. + Body: {"country": "USA", "postalcode": "07030"} + Returns a flat list of lineups across all matching headends. + """ + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + + source = self.get_object() + if source.source_type != 'schedules_direct': + return Response( + {"error": "This action is only available for Schedules Direct sources."}, + status=status.HTTP_400_BAD_REQUEST + ) + + country = request.data.get('country', '').strip() + postalcode = request.data.get('postalcode', '').strip() + if not country or not postalcode: + return Response( + {"error": "country and postalcode are required."}, + status=status.HTTP_400_BAD_REQUEST + ) + + token, error = self._sd_authenticate(source) + if error: + return error + + headers = { + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + 'token': token, + } + + try: + resp = http_requests.get( + f"{SD_BASE_URL}/headends", + params={'country': country, 'postalcode': postalcode}, + headers=headers, + timeout=15, + ) + resp.raise_for_status() + headends = resp.json() + lineups = [] + for headend in headends: + for lineup in headend.get('lineups', []): + lineups.append({ + 'lineup': lineup.get('lineup'), + 'name': lineup.get('name'), + 'transport': headend.get('transport'), + 'location': headend.get('location'), + 'headend': headend.get('headend'), + }) + return Response({"lineups": lineups}) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to search headends: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) # ───────────────────────────── # 2) Program API (CRUD) # ───────────────────────────── diff --git a/apps/epg/migrations/0023_schedules_direct.py b/apps/epg/migrations/0023_schedules_direct.py index 6c4efe1c..458f11f9 100644 --- a/apps/epg/migrations/0023_schedules_direct.py +++ b/apps/epg/migrations/0023_schedules_direct.py @@ -1,5 +1,7 @@ -from django.db import migrations, models +# Generated by Django 6.0.4 on 2026-05-18 22:38 + import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): @@ -9,60 +11,38 @@ class Migration(migrations.Migration): ] operations = [ - # Add username field to EPGSource + migrations.RemoveField( + model_name='epgsource', + name='api_key', + ), + migrations.AddField( + model_name='epgsource', + name='password', + field=models.CharField(blank=True, help_text='Password for credential-based EPG sources (e.g. Schedules Direct)', max_length=255, null=True), + ), migrations.AddField( model_name='epgsource', name='username', - field=models.CharField( - max_length=255, - blank=True, - null=True, - help_text='Username for Schedules Direct authentication', - ), + field=models.CharField(blank=True, help_text='Username for credential-based EPG sources (e.g. Schedules Direct)', max_length=255, null=True), ), - # Update api_key help_text to reflect SD password usage migrations.AlterField( model_name='epgsource', - name='api_key', - field=models.CharField( - max_length=255, - blank=True, - null=True, - help_text='Password for Schedules Direct authentication', - ), + name='custom_properties', + field=models.JSONField(blank=True, default=dict, help_text='Custom properties for source-specific configuration', null=True), ), - # Add sd_changes_remaining to EPGSource - migrations.AddField( - model_name='epgsource', - name='sd_changes_remaining', - field=models.IntegerField( - blank=True, - null=True, - help_text='Number of Schedules Direct lineup additions remaining today (resets at midnight UTC)', - ), + migrations.CreateModel( + name='SDProgramMD5', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('program_id', models.CharField(help_text='Schedules Direct programID (e.g. EP123456789)', max_length=64)), + ('md5', models.CharField(help_text='MD5 hash of the program metadata from Schedules Direct', max_length=22)), + ('epg_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sd_program_md5s', to='epg.epgsource')), + ], + options={ + 'indexes': [models.Index(fields=['epg_source', 'program_id'], name='epg_sdprogr_epg_sou_222ce1_idx')], + 'unique_together': {('epg_source', 'program_id')}, + }, ), - # Add sd_changes_reset_at to EPGSource - migrations.AddField( - model_name='epgsource', - name='sd_changes_reset_at', - field=models.DateTimeField( - blank=True, - null=True, - help_text='UTC datetime when the Schedules Direct daily lineup change counter resets', - ), - ), - # Add sd_program_md5 to ProgramData - migrations.AddField( - model_name='programdata', - name='sd_program_md5', - field=models.CharField( - blank=True, - max_length=22, - null=True, - help_text='MD5 hash from Schedules Direct for delta detection', - ), - ), - # Create SDScheduleMD5 model migrations.CreateModel( name='SDScheduleMD5', fields=[ @@ -71,16 +51,10 @@ class Migration(migrations.Migration): ('date', models.DateField(help_text='Schedule date (UTC)')), ('md5', models.CharField(help_text='MD5 hash of the schedule for this station/date from Schedules Direct', max_length=22)), ('last_modified', models.DateTimeField(help_text='Last modified timestamp from Schedules Direct')), - ('epg_source', models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='sd_schedule_md5s', - to='epg.epgsource', - )), + ('epg_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sd_schedule_md5s', to='epg.epgsource')), ], options={ - 'indexes': [ - models.Index(fields=['epg_source', 'station_id'], name='epg_sdsche_epg_sou_idx'), - ], + 'indexes': [models.Index(fields=['epg_source', 'station_id'], name='epg_sdsched_epg_sou_0d700e_idx')], 'unique_together': {('epg_source', 'station_id', 'date')}, }, ), diff --git a/apps/epg/models.py b/apps/epg/models.py index 0d4058fb..177623a3 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -31,9 +31,9 @@ class EPGSource(models.Model): source_type = models.CharField(max_length=20, choices=SOURCE_TYPE_CHOICES) url = models.URLField(max_length=1000, blank=True, null=True) # For XMLTV username = models.CharField(max_length=255, blank=True, null=True, - help_text='Username for Schedules Direct authentication') - api_key = models.CharField(max_length=255, blank=True, null=True, - help_text='Password for Schedules Direct authentication') # For Schedules Direct + help_text='Username for credential-based EPG sources (e.g. Schedules Direct)') + password = models.CharField(max_length=255, blank=True, null=True, + help_text='Password for credential-based EPG sources (e.g. Schedules Direct)') is_active = models.BooleanField(default=True) file_path = models.CharField(max_length=1024, blank=True, null=True) extracted_file_path = models.CharField(max_length=1024, blank=True, null=True, @@ -46,22 +46,12 @@ class EPGSource(models.Model): default=dict, blank=True, null=True, - help_text="Custom properties for dummy EPG configuration (regex patterns, timezone, duration, etc.)" + help_text="Custom properties for source-specific configuration" ) priority = models.PositiveIntegerField( default=0, help_text="Priority for EPG matching (higher numbers = higher priority). Used when multiple EPG sources have matching entries for a channel." ) - sd_changes_remaining = models.IntegerField( - null=True, - blank=True, - help_text="Number of Schedules Direct lineup additions remaining today (resets at midnight UTC)" - ) - sd_changes_reset_at = models.DateTimeField( - null=True, - blank=True, - help_text="UTC datetime when the Schedules Direct daily lineup change counter resets" - ) status = models.CharField( max_length=20, choices=STATUS_CHOICES, @@ -87,18 +77,13 @@ class EPGSource(models.Model): def get_cache_file(self): import mimetypes - # Use a temporary extension for initial download - # The actual extension will be determined after content inspection file_ext = ".tmp" - # If file_path is already set and contains an extension, use that - # This handles cases where we've already detected the proper type if self.file_path and os.path.exists(self.file_path): _, existing_ext = os.path.splitext(self.file_path) if existing_ext: file_ext = existing_ext else: - # Try to detect the MIME type and map to extension mime_type, _ = mimetypes.guess_type(self.file_path) if mime_type: if mime_type == 'application/gzip' or mime_type == 'application/x-gzip': @@ -107,48 +92,33 @@ class EPGSource(models.Model): file_ext = '.zip' elif mime_type == 'application/xml' or mime_type == 'text/xml': file_ext = '.xml' - # For files without mime type detection, try peeking at content else: try: with open(self.file_path, 'rb') as f: header = f.read(4) - # Check for gzip magic number (1f 8b) if header[:2] == b'\x1f\x8b': file_ext = '.gz' - # Check for zip magic number (PK..) elif header[:2] == b'PK': file_ext = '.zip' - # Check for XML elif header[:5] == b'': file_ext = '.xml' - except Exception as e: - # If we can't read the file, just keep the default extension + except Exception: pass filename = f"{self.id}{file_ext}" - - # Build full path in MEDIA_ROOT/cached_epg cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") - - # Create directory if it doesn't exist os.makedirs(cache_dir, exist_ok=True) - cache = os.path.join(cache_dir, filename) - return cache def save(self, *args, **kwargs): - # Prevent auto_now behavior by handling updated_at manually if 'update_fields' in kwargs and 'updated_at' not in kwargs['update_fields']: - # Don't modify updated_at for regular updates kwargs.setdefault('update_fields', []) if 'updated_at' in kwargs['update_fields']: kwargs['update_fields'].remove('updated_at') super().save(*args, **kwargs) class EPGData(models.Model): - # Removed the Channel foreign key. We now just store the original tvg_id - # and a name (which might simply be the tvg_id if no real channel exists). tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=512) icon_url = models.URLField(max_length=500, null=True, blank=True) @@ -167,7 +137,6 @@ class EPGData(models.Model): return f"EPG Data for {self.name}" class ProgramData(models.Model): - # Each programme is associated with an EPGData record. epg = models.ForeignKey(EPGData, on_delete=models.CASCADE, related_name="programs") start_time = models.DateTimeField() end_time = models.DateTimeField() @@ -176,12 +145,6 @@ class ProgramData(models.Model): description = models.TextField(blank=True, null=True) tvg_id = models.CharField(max_length=255, null=True, blank=True) custom_properties = models.JSONField(default=dict, blank=True, null=True) - sd_program_md5 = models.CharField( - max_length=22, - null=True, - blank=True, - help_text="MD5 hash from Schedules Direct for delta detection" - ) def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" @@ -220,3 +183,34 @@ class SDScheduleMD5(models.Model): def __str__(self): return f"SDScheduleMD5: {self.station_id} / {self.date} ({self.epg_source.name})" + + +class SDProgramMD5(models.Model): + """ + Caches per-program MD5 hashes from Schedules Direct. + Keyed by epg_source + program_id (SD's programID e.g. EP123456789). + Used for program-level delta detection to avoid re-downloading unchanged + program metadata, minimizing API calls against SD's rate-limited endpoints. + """ + epg_source = models.ForeignKey( + EPGSource, + on_delete=models.CASCADE, + related_name="sd_program_md5s", + ) + program_id = models.CharField( + max_length=64, + help_text="Schedules Direct programID (e.g. EP123456789)" + ) + md5 = models.CharField( + max_length=22, + help_text="MD5 hash of the program metadata from Schedules Direct" + ) + + class Meta: + unique_together = ('epg_source', 'program_id') + indexes = [ + models.Index(fields=['epg_source', 'program_id']), + ] + + def __str__(self): + return f"SDProgramMD5: {self.program_id} ({self.epg_source.name})" diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 049a9713..9d4e80c0 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -23,7 +23,7 @@ class EPGSourceSerializer(serializers.ModelSerializer): 'source_type', 'url', 'username', - 'api_key', + 'password', 'is_active', 'file_path', 'refresh_interval', diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index c4b58fe4..0ba20864 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -24,7 +24,7 @@ from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from .models import EPGSource, EPGData, ProgramData +from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event logger = logging.getLogger(__name__) @@ -2011,7 +2011,7 @@ def fetch_schedules_direct(source, stations_only=False): # Validate credentials # ------------------------------------------------------------------------- username = (source.username or '').strip() - password = (source.api_key or '').strip() + password = (source.password or '').strip() if not username or not password: msg = "Schedules Direct source requires both a username and password." @@ -2392,7 +2392,7 @@ def fetch_schedules_direct(source, stations_only=False): if cached != server_info['md5']: changed_by_station.setdefault(sid, []).append(date_str) - total_changed = sum(len(v) for v in changed_by_station.items()) + total_changed = sum(len(v) for v in changed_by_station.values()) total_possible = len(station_ids) * len(date_list) logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") _sd_send_ws_sync(source.id, "parsing_programs", 38, @@ -2402,15 +2402,6 @@ def fetch_schedules_direct(source, stations_only=False): schedules_by_station = {sid: [] for sid in station_ids} program_ids_needed = set() - # Existing cached program MD5s for delta detection in step 6 - existing_program_md5s = { - p.tvg_id + '|' + p.title: p.sd_program_md5 - for p in ProgramData.objects.filter( - epg__epg_source=source, - sd_program_md5__isnull=False, - ).only('tvg_id', 'title', 'sd_program_md5') - } - if not changed_by_station: logger.info("No schedule changes detected — skipping schedule and program downloads.") _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", @@ -2539,16 +2530,16 @@ def fetch_schedules_direct(source, stations_only=False): if pid and md5: schedule_program_md5s[pid] = md5 - # Load cached program MD5s from DB keyed by programID via tvg_id + # Load cached program MD5s from SDProgramMD5 table, keyed by programID cached_prog_md5s = { - p.tvg_id: p.sd_program_md5 - for p in ProgramData.objects.filter( - epg__epg_source=source, - sd_program_md5__isnull=False, - ).only('tvg_id', 'sd_program_md5').distinct('tvg_id') + r.program_id: r.md5 + for r in SDProgramMD5.objects.filter( + epg_source=source, + program_id__in=program_ids_needed, + ).only('program_id', 'md5') } - # Only fetch programs where MD5 differs from our cache + # Only fetch programs where MD5 differs from our cached value programs_to_fetch = [ pid for pid in program_ids_needed if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) @@ -2735,7 +2726,6 @@ def fetch_schedules_direct(source, stations_only=False): description=desc or None, tvg_id=sid, custom_properties=custom_props or None, - sd_program_md5=schedule_program_md5s.get(pid), )) total_programs += 1 @@ -2758,6 +2748,28 @@ def fetch_schedules_direct(source, stations_only=False): _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) logger.info(f"Committed {total_programs} Schedules Direct programs to database.") + + # Upsert SDProgramMD5 records for programs we just downloaded + # This updates the cache so future fetches can skip unchanged programs + if schedule_program_md5s: + md5_records = [ + SDProgramMD5( + epg_source=source, + program_id=pid, + md5=md5, + ) + for pid, md5 in schedule_program_md5s.items() + if pid in programs_to_fetch # Only cache what we actually downloaded + ] + if md5_records: + SDProgramMD5.objects.bulk_create( + md5_records, + update_conflicts=True, + unique_fields=['epg_source', 'program_id'], + update_fields=['md5'], + ) + logger.info(f"Cached {len(md5_records)} program MD5s for future delta detection.") + except Exception as db_error: msg = f"Database error persisting Schedules Direct programs: {db_error}" logger.error(msg, exc_info=True) diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index 782750af..e73f5ad3 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -34,7 +34,7 @@ class EPGSourceUsernameFieldTests(TestCase): name='SD Test', source_type='schedules_direct', username='testuser', - api_key='testpass', + password='testpass', ) source.refresh_from_db() self.assertEqual(source.username, 'testuser') @@ -64,20 +64,20 @@ class EPGSourceSerializerSDTests(TestCase): name='SD Serializer Test', source_type='schedules_direct', username='sduser', - api_key='sdpass', + password='sdpass', ) data = EPGSourceSerializer(source).data self.assertIn('username', data) self.assertEqual(data['username'], 'sduser') - def test_api_key_in_serializer_fields(self): + def test_password_in_serializer_fields(self): source = EPGSource.objects.create( name='SD API Key Test', source_type='schedules_direct', - api_key='secret', + password='secret', ) data = EPGSourceSerializer(source).data - self.assertIn('api_key', data) + self.assertIn('password', data) # --------------------------------------------------------------------------- @@ -87,31 +87,31 @@ class EPGSourceSerializerSDTests(TestCase): class FetchSchedulesDirectCredentialTests(TestCase): """fetch_schedules_direct must reject sources missing credentials.""" - def _make_source(self, username=None, api_key=None): + def _make_source(self, username=None, password=None): return EPGSource.objects.create( name='SD Cred Test', source_type='schedules_direct', username=username, - api_key=api_key, + password=password, ) def test_missing_username_sets_error_status(self): from apps.epg.tasks import fetch_schedules_direct - source = self._make_source(username=None, api_key='pass') + source = self._make_source(username=None, password='pass') fetch_schedules_direct(source) source.refresh_from_db() self.assertEqual(source.status, EPGSource.STATUS_ERROR) def test_missing_password_sets_error_status(self): from apps.epg.tasks import fetch_schedules_direct - source = self._make_source(username='user', api_key=None) + source = self._make_source(username='user', password=None) fetch_schedules_direct(source) source.refresh_from_db() self.assertEqual(source.status, EPGSource.STATUS_ERROR) def test_empty_username_sets_error_status(self): from apps.epg.tasks import fetch_schedules_direct - source = self._make_source(username=' ', api_key='pass') + source = self._make_source(username=' ', password='pass') fetch_schedules_direct(source) source.refresh_from_db() self.assertEqual(source.status, EPGSource.STATUS_ERROR) @@ -142,7 +142,7 @@ class FetchSchedulesDirectAuthTests(TestCase): name='SD Hash Test', source_type='schedules_direct', username='sduser', - api_key=plaintext, + password=plaintext, ) with patch('apps.epg.tasks.send_epg_update'): @@ -171,7 +171,7 @@ class FetchSchedulesDirectAuthTests(TestCase): name='SD Auth Fail', source_type='schedules_direct', username='baduser', - api_key='badpass', + password='badpass', ) with patch('apps.epg.tasks.send_epg_update'): @@ -191,7 +191,7 @@ class FetchSchedulesDirectAuthTests(TestCase): name='SD Network Error', source_type='schedules_direct', username='user', - api_key='pass', + password='pass', ) with patch('apps.epg.tasks.send_epg_update'): @@ -247,7 +247,7 @@ class SDSourceSignalTests(TestCase): name='SD Signal Test', source_type='schedules_direct', username='u', - api_key='p', + password='p', ) epg_data = EPGData.objects.create( tvg_id='sd-test-station', diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 82ddffe9..369011f3 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -347,7 +347,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { source_type: 'xmltv', url: '', username: '', - api_key: '', + password: '', is_active: true, refresh_interval: 24, cron_expression: '', @@ -402,7 +402,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { source_type: epg.source_type, url: epg.url, username: epg.username || '', - api_key: epg.api_key, + password: epg.password || '', is_active: epg.is_active, refresh_interval: epg.refresh_interval, cron_expression: epg.cron_expression || '', @@ -565,13 +565,13 @@ const EPG = ({ epg = null, isOpen, onClose }) => { key={form.key('username')} /> )} diff --git a/frontend/src/components/tables/EPGsTable.jsx b/frontend/src/components/tables/EPGsTable.jsx index 01eefdbb..a18dcc7b 100644 --- a/frontend/src/components/tables/EPGsTable.jsx +++ b/frontend/src/components/tables/EPGsTable.jsx @@ -299,7 +299,7 @@ const EPGsTable = () => { } else { value = cell.getValue() || - row.original.api_key || + row.original.password || row.original.file_path || ''; tooltip = value; @@ -702,8 +702,8 @@ Source Type: ${epgToDelete.source_type} ${ epgToDelete.url ? `URL: ${epgToDelete.url}` - : epgToDelete.api_key - ? `API Key: ${epgToDelete.api_key}` + : epgToDelete.password + ? `API Key: ${epgToDelete.password}` : epgToDelete.file_path ? `File Path: ${epgToDelete.file_path}` : '' From d8c0850fd8472e53dceeb2608686c9d15375d01b Mon Sep 17 00:00:00 2001 From: mwhit Date: Tue, 19 May 2026 23:48:11 +0000 Subject: [PATCH 11/76] fix(epg): address second round of reviewer feedback - Fix NameError in sd_lineups_search: add dispatcharr_version import - Fix DELETE handler: persist changesRemaining from SD response to custom_properties - Fix lockout reset time: calculate next midnight UTC (00:00Z) instead of rolling now+24h, matching SD's documented reset behavior - Remove redundant explicit index from SDProgramMD5: unique_together already creates this index in Postgres --- apps/epg/api_views.py | 25 +++++++++++++------ ...024_remove_sdprogrammd5_redundant_index.py | 17 +++++++++++++ apps/epg/models.py | 3 --- 3 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index f0d80c5f..2cfa8f47 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -188,17 +188,22 @@ class EPGSourceViewSet(viewsets.ModelViewSet): def _save_sd_lockout(self, source): """ - Persist a hard lockout to EPGSource model fields when SD returns - 4100 MAX_LINEUP_CHANGES_REACHED. Lockout clears automatically at next - midnight UTC per SD's documented reset schedule. + Persist a hard lockout to EPGSource custom_properties when SD returns + 4100 MAX_LINEUP_CHANGES_REACHED. SD lineup change counters reset at + 00:00Z (midnight UTC) per SD's documented behavior — error 4100 states + "lineup changes for today" and all SD rate counters reset at midnight UTC. + Lockout clears automatically when the next midnight UTC passes. """ from django.utils import timezone from datetime import datetime, timedelta, timezone as dt_timezone now = timezone.now() - # SD resets on a 24-hour rolling window from when the limit was hit. - # We use 24 hours from now rather than next midnight UTC to be safe. - reset_at = now + timedelta(hours=24) + # Calculate next midnight UTC — SD resets at 00:00Z not on a rolling window + tomorrow = (now + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0, + tzinfo=dt_timezone.utc + ) + reset_at = tomorrow cp = source.custom_properties or {} cp['sd_changes_remaining'] = 0 @@ -357,9 +362,14 @@ class EPGSourceViewSet(viewsets.ModelViewSet): "changes_remaining": self._get_sd_changes_remaining(source), }) resp.raise_for_status() + sd_data = resp.json() + # SD returns changesRemaining on deletes — persist it + changes_remaining = sd_data.get('changesRemaining') + if changes_remaining is not None: + self._save_sd_changes_remaining(source, changes_remaining) logger.info(f"SD lineup deleted for source {source.id}: {lineup_id}") return Response({ - **resp.json(), + **sd_data, "changes_remaining": self._get_sd_changes_remaining(source), }) except http_requests.exceptions.RequestException as e: @@ -377,6 +387,7 @@ class EPGSourceViewSet(viewsets.ModelViewSet): """ import requests as http_requests from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version source = self.get_object() if source.source_type != 'schedules_direct': diff --git a/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py b/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py new file mode 100644 index 00000000..f1499657 --- /dev/null +++ b/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py @@ -0,0 +1,17 @@ +# Generated by Django 6.0.4 on 2026-05-19 23:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0023_schedules_direct'), + ] + + operations = [ + migrations.RemoveIndex( + model_name='sdprogrammd5', + name='epg_sdprogr_epg_sou_222ce1_idx', + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 177623a3..c9870d0b 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -208,9 +208,6 @@ class SDProgramMD5(models.Model): class Meta: unique_together = ('epg_source', 'program_id') - indexes = [ - models.Index(fields=['epg_source', 'program_id']), - ] def __str__(self): return f"SDProgramMD5: {self.program_id} ({self.epg_source.name})" From 4d5ffaf2231ace22b37a4431fd678deef36007b8 Mon Sep 17 00:00:00 2001 From: mwhit Date: Tue, 19 May 2026 23:56:56 +0000 Subject: [PATCH 12/76] fix(epg): address second round of reviewer feedback - Fix NameError in sd_lineups_search: add dispatcharr_version import - Fix DELETE handler: persist changesRemaining from SD response to custom_properties - Fix lockout reset time: calculate next midnight UTC (00:00Z) instead of rolling now+24h, matching SD's documented reset behavior - Remove redundant explicit index from SDProgramMD5: unique_together already creates this index in Postgres - Regenerate 0023_schedules_direct migration to incorporate all changes cleanly in a single migration --- apps/epg/migrations/0023_schedules_direct.py | 3 +-- .../0024_remove_sdprogrammd5_redundant_index.py | 17 ----------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py diff --git a/apps/epg/migrations/0023_schedules_direct.py b/apps/epg/migrations/0023_schedules_direct.py index 458f11f9..b0e835a9 100644 --- a/apps/epg/migrations/0023_schedules_direct.py +++ b/apps/epg/migrations/0023_schedules_direct.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.4 on 2026-05-18 22:38 +# Generated by Django 6.0.4 on 2026-05-19 23:55 import django.db.models.deletion from django.db import migrations, models @@ -39,7 +39,6 @@ class Migration(migrations.Migration): ('epg_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sd_program_md5s', to='epg.epgsource')), ], options={ - 'indexes': [models.Index(fields=['epg_source', 'program_id'], name='epg_sdprogr_epg_sou_222ce1_idx')], 'unique_together': {('epg_source', 'program_id')}, }, ), diff --git a/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py b/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py deleted file mode 100644 index f1499657..00000000 --- a/apps/epg/migrations/0024_remove_sdprogrammd5_redundant_index.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 6.0.4 on 2026-05-19 23:46 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('epg', '0023_schedules_direct'), - ] - - operations = [ - migrations.RemoveIndex( - model_name='sdprogrammd5', - name='epg_sdprogr_epg_sou_222ce1_idx', - ), - ] From d89f63421337e15bb31b58c027a62e0b2de275d7 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Wed, 20 May 2026 03:30:48 -0400 Subject: [PATCH 13/76] Enforce shared credential connection pools across M3U accounts (#1137) Group M3U/XC accounts and profiles that share the same provider login into auto-assigned ServerGroups keyed by credential fingerprint. Enforce combined Redis limits for live TV and VOD via apps/m3u/connection_pool.py. - Per-profile fingerprinting (XC transforms and STD stream URLs) - VOD profile selection tries alternates when default credential pool is full (fixes live then VOD failure) - Stats UI shows provider login from active stream URL - Tests: apps/m3u/tests/test_connection_pool.py (11 tests, all passing) Co-authored-by: Cursor --- apps/channels/models.py | 137 ++++---- apps/m3u/connection_pool.py | 299 ++++++++++++++++++ .../0020_servergroup_max_streams.py | 21 ++ ...0021_servergroup_credential_fingerprint.py | 102 ++++++ apps/m3u/models.py | 32 ++ apps/m3u/serializers.py | 2 +- apps/m3u/tests/test_connection_pool.py | 247 +++++++++++++++ apps/proxy/live_proxy/channel_status.py | 22 +- apps/proxy/live_proxy/url_utils.py | 79 +++-- .../multi_worker_connection_manager.py | 80 ++--- apps/proxy/vod_proxy/views.py | 34 +- .../components/cards/StreamConnectionCard.jsx | 22 +- .../components/cards/VodConnectionCard.jsx | 7 + 13 files changed, 917 insertions(+), 167 deletions(-) create mode 100644 apps/m3u/connection_pool.py create mode 100644 apps/m3u/migrations/0020_servergroup_max_streams.py create mode 100644 apps/m3u/migrations/0021_servergroup_credential_fingerprint.py create mode 100644 apps/m3u/tests/test_connection_pool.py diff --git a/apps/channels/models.py b/apps/channels/models.py index 59af3e00..59a2bc3d 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -4,7 +4,7 @@ from django.conf import settings from core.models import StreamProfile, CoreSettings from core.utils import RedisClient from apps.proxy.live_proxy.redis_keys import RedisKeys -from apps.proxy.live_proxy.constants import ChannelMetadataField +from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState import logging import uuid from django.utils import timezone @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) # If you have an M3UAccount model in apps.m3u, you can still import it: from apps.m3u.models import M3UAccount +from apps.m3u.connection_pool import reserve_profile_slot, release_profile_slot # Add fallback functions if Redis isn't available @@ -231,17 +232,8 @@ class Stream(models.Model): if profile.is_active == False: continue - # Atomic slot reservation: INCR first, check, rollback if over capacity - if profile.max_streams == 0: - reserved = True - else: - profile_connections_key = f"profile_connections:{profile.id}" - new_count = redis_client.incr(profile_connections_key) - if new_count <= profile.max_streams: - reserved = True - else: - redis_client.decr(profile_connections_key) - reserved = False + # Atomic slot reservation via shared connection pool helper + reserved, _count = reserve_profile_slot(profile, redis_client) if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) @@ -277,12 +269,7 @@ class Stream(models.Model): f"Stream {stream_id}: found profile_id={profile_id}" ) - profile_connections_key = f"profile_connections:{profile_id}" - - # Only decrement if the profile had a max_connections limit - current_count = int(redis_client.get(profile_connections_key) or 0) - if current_count > 0: - redis_client.decr(profile_connections_key) + release_profile_slot(profile_id, redis_client) return True @@ -599,37 +586,29 @@ class Channel(models.Model): return victim_id - def _check_and_reserve_profile_slot(self, profile, redis_client): - """ - Atomically check and reserve a connection slot for the given profile. + def _channel_proxy_is_active(self, redis_client) -> bool: + """True when live proxy metadata shows this channel is still running.""" + metadata_key = RedisKeys.channel_metadata(str(self.uuid)) + if not redis_client.exists(metadata_key): + return False + state = redis_client.hget(metadata_key, ChannelMetadataField.STATE) + if state is None: + return False + if isinstance(state, bytes): + state = state.decode() + return state in ( + ChannelState.ACTIVE, + ChannelState.WAITING_FOR_CLIENTS, + ChannelState.BUFFERING, + ChannelState.INITIALIZING, + ChannelState.CONNECTING, + ) - Uses an INCR-first-then-check pattern to eliminate the TOCTOU race - condition where separate GET + check + INCR operations could allow - concurrent requests to both pass the capacity check. - - For profiles with max_streams=0 (unlimited), no reservation is needed. - - Args: - profile: M3UAccountProfile instance - redis_client: Redis client instance - - Returns: - tuple: (reserved: bool, current_count: int) - """ - if profile.max_streams == 0: - return (True, 0) - - profile_connections_key = f"profile_connections:{profile.id}" - - # Atomically increment first — this is a single Redis command - new_count = redis_client.incr(profile_connections_key) - - if new_count <= profile.max_streams: - return (True, new_count) - - # Over capacity — roll back the increment - redis_client.decr(profile_connections_key) - return (False, new_count - 1) + def _clear_stream_assignment_keys(self, redis_client, stream_id=None) -> None: + """Remove channel/stream profile assignment keys from Redis.""" + redis_client.delete(f"channel_stream:{self.id}") + if stream_id is not None: + redis_client.delete(f"stream_profile:{stream_id}") def get_stream(self, requester=None): """ @@ -646,24 +625,40 @@ class Channel(models.Model): error_reason = "No streams assigned to channel" return None, None, error_reason - # Check if a stream is already active for this channel + # Reuse assignment only when this channel is still active in the proxy. + # Stale channel_stream keys after stop/disconnect caused every tune to + # reuse the default profile without re-running rotation or INCR. stream_id_bytes = redis_client.get(f"channel_stream:{self.id}") if stream_id_bytes: try: stream_id = int(stream_id_bytes) - profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") - if profile_id_bytes: - try: - profile_id = int(profile_id_bytes) - return stream_id, profile_id, None - except (ValueError, TypeError): - logger.debug( - f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" - ) except (ValueError, TypeError): logger.debug( f"Invalid stream ID retrieved from Redis: {stream_id_bytes}" ) + stream_id = None + + if stream_id is not None: + if self._channel_proxy_is_active(redis_client): + profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") + if profile_id_bytes: + try: + profile_id = int(profile_id_bytes) + logger.debug( + f"Channel {self.uuid}: reusing active assignment " + f"stream={stream_id} profile={profile_id}" + ) + return stream_id, profile_id, None + except (ValueError, TypeError): + logger.debug( + f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" + ) + else: + logger.info( + f"Channel {self.uuid}: clearing stale stream assignment " + f"(stream={stream_id}, proxy not active)" + ) + self._clear_stream_assignment_keys(redis_client, stream_id) # No existing active stream, attempt to assign a new one has_streams_but_maxed_out = False @@ -697,14 +692,16 @@ class Channel(models.Model): has_active_profiles = True # Atomically check and reserve a slot (INCR-first pattern) - reserved, current_count = self._check_and_reserve_profile_slot( - profile, redis_client - ) + reserved, current_count = reserve_profile_slot(profile, redis_client) if reserved: # Slot reserved — assign stream to this channel redis_client.set(f"channel_stream:{self.id}", stream.id) redis_client.set(f"stream_profile:{stream.id}", profile.id) + logger.info( + f"Channel {self.uuid}: assigned stream {stream.id} " + f"profile {profile.id} ({profile.name})" + ) return ( stream.id, @@ -782,12 +779,11 @@ class Channel(models.Model): ChannelMetadataField.M3U_PROFILE, ) - profile_connections_key = f"profile_connections:{profile_id}" - current_count = int( - redis_client.get(profile_connections_key) or 0 - ) - if current_count > 0: - redis_client.decr(profile_connections_key) + stream = Stream.objects.select_related("m3u_account").filter( + id=stream_id + ).first() + m3u_account = stream.m3u_account if stream else None + release_profile_slot(profile_id, redis_client) return True logger.debug( @@ -832,12 +828,7 @@ class Channel(models.Model): f"stream {stream_id}" ) - profile_connections_key = f"profile_connections:{profile_id}" - - # Only decrement if the profile had a max_connections limit - current_count = int(redis_client.get(profile_connections_key) or 0) - if current_count > 0: - redis_client.decr(profile_connections_key) + release_profile_slot(profile_id, redis_client) # Clear metadata fields so duplicate release_stream() calls # (e.g. from _clean_redis_keys or ChannelService.stop_channel) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py new file mode 100644 index 00000000..b06aff10 --- /dev/null +++ b/apps/m3u/connection_pool.py @@ -0,0 +1,299 @@ +""" +Shared connection pool enforcement for M3U accounts with identical credentials. + +All Redis INCR/DECR for profile and server-group limits should go through this +module so live TV and VOD stay consistent. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + +PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" +SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}" +EXCLUDE_FROM_POOL_KEY = "exclude_from_credential_pool" + +_XC_URL_CREDENTIALS_RE = re.compile( + r"/(?:live|movie|series)/([^/]+)/([^/]+)/", + re.IGNORECASE, +) + + +def profile_connections_key(profile_id: int) -> str: + return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) + + +def server_group_connections_key(group_id: int) -> str: + return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id) + + +def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: + """Return a stable hash for grouping accounts with the same IPTV login.""" + if not username or not password: + return None + normalized = f"{username.strip().lower()}\0{password.strip()}" + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def extract_credentials_from_stream_url(url: str) -> Tuple[Optional[str], Optional[str]]: + """Parse username/password embedded in an Xtream-style stream URL.""" + if not url: + return None, None + match = _XC_URL_CREDENTIALS_RE.search(url) + if not match: + return None, None + return match.group(1), match.group(2) + + +def _fingerprint_from_profile_stream_url(profile) -> Optional[str]: + """STD/M3U: fingerprint from a sample stream URL after profile rewrite.""" + from apps.channels.models import Stream + + sample_url = ( + Stream.objects.filter(m3u_account=profile.m3u_account) + .exclude(url="") + .values_list("url", flat=True) + .first() + ) + if not sample_url: + return None + + try: + from apps.proxy.live_proxy.url_utils import transform_url + + transformed = transform_url( + sample_url, + profile.search_pattern or "", + profile.replace_pattern or "", + ) + url_user, url_pass = extract_credentials_from_stream_url( + transformed or sample_url + ) + return compute_credential_fingerprint(url_user or "", url_pass or "") + except Exception as exc: + logger.debug( + "Could not derive profile %s fingerprint from stream URL: %s", + profile.pk, + exc, + ) + return None + + +def get_profile_credential_fingerprint(profile) -> Optional[str]: + """Fingerprint for credentials this profile uses at playback time.""" + m3u_account = profile.m3u_account + + if m3u_account.account_type == "XC": + try: + from apps.m3u.tasks import get_transformed_credentials + + _url, username, password = get_transformed_credentials(m3u_account, profile) + fingerprint = compute_credential_fingerprint(username or "", password or "") + if fingerprint: + return fingerprint + except Exception as exc: + logger.debug( + "Could not resolve transformed credentials for profile %s: %s", + profile.pk, + exc, + ) + + fingerprint = _fingerprint_from_profile_stream_url(profile) + if fingerprint: + return fingerprint + + return compute_credential_fingerprint( + m3u_account.username or "", + m3u_account.password or "", + ) + + +def account_excluded_from_pool(m3u_account) -> bool: + props = m3u_account.custom_properties or {} + return bool(props.get(EXCLUDE_FROM_POOL_KEY)) + + +def _get_pool_for_fingerprint(fingerprint: str): + """Return the auto credential pool ServerGroup for a fingerprint, if configured.""" + if not fingerprint: + return None + from apps.m3u.models import ServerGroup + + group = ServerGroup.objects.filter(credential_fingerprint=fingerprint).first() + if not group or group.max_streams == 0: + return None + return group + + +def get_enforced_server_group_for_profile(profile): + """Return the shared pool for this profile's effective provider login.""" + if account_excluded_from_pool(profile.m3u_account): + return None + + group = _get_pool_for_fingerprint(get_profile_credential_fingerprint(profile)) + if group: + return group + + manual = profile.m3u_account.server_group + if manual and not manual.credential_fingerprint and manual.max_streams > 0: + return manual + return None + + +def pool_has_capacity_for_profile(profile, redis_client) -> bool: + """Non-mutating check for the profile's credential pool.""" + group = get_enforced_server_group_for_profile(profile) + if not group: + return True + key = server_group_connections_key(group.id) + current = int(redis_client.get(key) or 0) + return current < group.max_streams + + +def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: + group = get_enforced_server_group_for_profile(profile) + if not group: + return True + key = server_group_connections_key(group.id) + group_count = redis_client.incr(key) + if group_count <= group.max_streams: + return True + redis_client.decr(key) + return False + + +def _release_server_group_slot_for_profile(profile, redis_client) -> None: + group = get_enforced_server_group_for_profile(profile) + if not group: + return + key = server_group_connections_key(group.id) + new_count = redis_client.decr(key) + if new_count < 0: + redis_client.set(key, 0) + + +def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: + """ + Atomically reserve profile + shared pool slots (INCR-first). + + Returns (reserved, profile_count_after_attempt). + """ + if profile.max_streams == 0: + if _reserve_server_group_slot_for_profile(profile, redis_client): + return True, 0 + return False, 0 + + profile_key = profile_connections_key(profile.id) + new_count = redis_client.incr(profile_key) + + if new_count <= profile.max_streams: + if _reserve_server_group_slot_for_profile(profile, redis_client): + return True, new_count + redis_client.decr(profile_key) + return False, new_count - 1 + + redis_client.decr(profile_key) + return False, new_count - 1 + + +def release_profile_slot(profile_id: int, redis_client) -> None: + """Release profile and shared pool slots after a stream ends.""" + from apps.m3u.models import M3UAccountProfile + + try: + profile = M3UAccountProfile.objects.get(id=profile_id) + except M3UAccountProfile.DoesNotExist: + profile = None + + profile_key = profile_connections_key(profile_id) + if profile is None or profile.max_streams > 0: + current = int(redis_client.get(profile_key) or 0) + if current > 0: + redis_client.decr(profile_key) + + if profile: + _release_server_group_slot_for_profile(profile, redis_client) + + +def recompute_pool_max_streams(server_group) -> None: + """Set pool max_streams to the minimum positive limit across members and profiles.""" + from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup + + if not server_group.credential_fingerprint: + return + + fp = server_group.credential_fingerprint + limits = list( + M3UAccount.objects.filter( + server_group=server_group, max_streams__gt=0 + ).values_list("max_streams", flat=True) + ) + + for profile in M3UAccountProfile.objects.filter(is_active=True).select_related( + "m3u_account" + ): + if account_excluded_from_pool(profile.m3u_account): + continue + if get_profile_credential_fingerprint(profile) != fp: + continue + if profile.max_streams > 0: + limits.append(profile.max_streams) + + new_max = min(limits) if limits else 0 + if server_group.max_streams != new_max: + ServerGroup.objects.filter(pk=server_group.pk).update(max_streams=new_max) + + +def _ensure_pool_for_fingerprint(fingerprint: str): + from apps.m3u.models import ServerGroup + + group, _created = ServerGroup.objects.get_or_create( + credential_fingerprint=fingerprint, + defaults={ + "name": f"credential-pool-{fingerprint[:16]}", + "max_streams": 0, + }, + ) + recompute_pool_max_streams(group) + return group + + +def sync_account_credential_pool(m3u_account) -> None: + """ + Ensure auto credential pools exist for each distinct login on this account. + + Sets M3UAccount.server_group only when every active profile shares one login. + """ + from apps.m3u.models import M3UAccount, M3UAccountProfile + + if account_excluded_from_pool(m3u_account): + return + + if m3u_account.server_group_id: + existing = m3u_account.server_group + if existing and not existing.credential_fingerprint: + return + + profile_fps = set() + for profile in M3UAccountProfile.objects.filter( + m3u_account=m3u_account, is_active=True + ): + fp = get_profile_credential_fingerprint(profile) + if not fp: + continue + profile_fps.add(fp) + _ensure_pool_for_fingerprint(fp) + + if len(profile_fps) == 1: + group = _get_pool_for_fingerprint(next(iter(profile_fps))) + if group and m3u_account.server_group_id != group.id: + M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=group) + elif m3u_account.server_group_id: + existing = m3u_account.server_group + if existing and existing.credential_fingerprint: + M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=None) diff --git a/apps/m3u/migrations/0020_servergroup_max_streams.py b/apps/m3u/migrations/0020_servergroup_max_streams.py new file mode 100644 index 00000000..ce17d830 --- /dev/null +++ b/apps/m3u/migrations/0020_servergroup_max_streams.py @@ -0,0 +1,21 @@ +# Generated by Django 6.0.3 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("m3u", "0019_m3uaccountprofile_exp_date"), + ] + + operations = [ + migrations.AddField( + model_name="servergroup", + name="max_streams", + field=models.PositiveIntegerField( + default=0, + help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + ), + ), + ] diff --git a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py new file mode 100644 index 00000000..8d958945 --- /dev/null +++ b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py @@ -0,0 +1,102 @@ +# Generated manually for shared credential connection pools (#1137) + +from django.db import migrations, models + +from apps.m3u.connection_pool import ( + compute_credential_fingerprint, + extract_credentials_from_stream_url, +) + + +def _historical_account_fingerprint(M3UAccount, M3UAccountProfile, Stream, account): + """Resolve fingerprint for migration using historical models only.""" + fingerprint = compute_credential_fingerprint( + account.username or "", + account.password or "", + ) + if fingerprint: + return fingerprint + + sample_url = ( + Stream.objects.filter(m3u_account_id=account.pk) + .exclude(url="") + .values_list("url", flat=True) + .first() + ) + if sample_url: + url_user, url_pass = extract_credentials_from_stream_url(sample_url) + return compute_credential_fingerprint(url_user or "", url_pass or "") + + return None + + +def backfill_credential_pools(apps, schema_editor): + from django.db.models import Min + + M3UAccount = apps.get_model("m3u", "M3UAccount") + ServerGroup = apps.get_model("m3u", "ServerGroup") + M3UAccountProfile = apps.get_model("m3u", "M3UAccountProfile") + Stream = apps.get_model("dispatcharr_channels", "Stream") + + seen = {} + for account in M3UAccount.objects.all(): + props = account.custom_properties or {} + if props.get("exclude_from_credential_pool"): + continue + if account.server_group_id: + group = ServerGroup.objects.filter(pk=account.server_group_id).first() + if group and not group.credential_fingerprint: + continue + + fingerprint = _historical_account_fingerprint( + M3UAccount, M3UAccountProfile, Stream, account + ) + if not fingerprint: + continue + + if fingerprint not in seen: + short = fingerprint[:16] + group, _ = ServerGroup.objects.get_or_create( + credential_fingerprint=fingerprint, + defaults={ + "name": f"credential-pool-{short}", + "max_streams": 0, + }, + ) + seen[fingerprint] = group + else: + group = seen[fingerprint] + + M3UAccount.objects.filter(pk=account.pk).update(server_group_id=group.pk) + + for group in seen.values(): + agg = M3UAccount.objects.filter( + server_group_id=group.pk, max_streams__gt=0 + ).aggregate(min_limit=Min("max_streams")) + new_max = agg["min_limit"] or 0 + if group.max_streams != new_max: + ServerGroup.objects.filter(pk=group.pk).update(max_streams=new_max) + + +class Migration(migrations.Migration): + + dependencies = [ + ("m3u", "0020_servergroup_max_streams"), + ("dispatcharr_channels", "0037_auto_sync_overhaul"), + ] + + operations = [ + migrations.AddField( + model_name="servergroup", + name="credential_fingerprint", + field=models.CharField( + blank=True, + db_index=True, + help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", + max_length=64, + null=True, + unique=True, + ), + ), + migrations.RunPython(backfill_credential_pools, migrations.RunPython.noop), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 6fc48bec..4b1325d5 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -199,6 +199,18 @@ class ServerGroup(models.Model): name = models.CharField( max_length=100, unique=True, help_text="Unique name for this server group." ) + max_streams = models.PositiveIntegerField( + default=0, + help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + ) + credential_fingerprint = models.CharField( + max_length=64, + null=True, + blank=True, + unique=True, + db_index=True, + help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", + ) def __str__(self): return self.name @@ -334,6 +346,26 @@ class M3UAccountProfile(models.Model): return None +@receiver(models.signals.post_save, sender=M3UAccount) +def assign_credential_pool_for_m3u_account(sender, instance, **kwargs): + """Link accounts with identical credentials into a shared connection pool.""" + if kwargs.get("raw"): + return + from apps.m3u.connection_pool import sync_account_credential_pool + + sync_account_credential_pool(instance) + + +@receiver(models.signals.post_save, sender=M3UAccountProfile) +def assign_credential_pool_for_m3u_profile(sender, instance, **kwargs): + """Re-sync pools when profile URL transforms change.""" + if kwargs.get("raw"): + return + from apps.m3u.connection_pool import sync_account_credential_pool + + sync_account_credential_pool(instance.m3u_account) + + @receiver(models.signals.post_save, sender=M3UAccount) def create_profile_for_m3u_account(sender, instance, created, **kwargs): """Automatically create an M3UAccountProfile when M3UAccount is created.""" diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 485a3687..29c4157f 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -408,4 +408,4 @@ class ServerGroupSerializer(serializers.ModelSerializer): class Meta: model = ServerGroup - fields = ["id", "name"] + fields = ["id", "name", "max_streams"] diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py new file mode 100644 index 00000000..79285484 --- /dev/null +++ b/apps/m3u/tests/test_connection_pool.py @@ -0,0 +1,247 @@ +"""Tests for shared credential connection pools (#1137).""" + +from django.test import TestCase +from unittest.mock import patch + +from apps.m3u.connection_pool import ( + compute_credential_fingerprint, + extract_credentials_from_stream_url, + get_enforced_server_group_for_profile, + get_profile_credential_fingerprint, + pool_has_capacity_for_profile, + release_profile_slot, + reserve_profile_slot, + server_group_connections_key, + sync_account_credential_pool, +) +from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup + + +class FakeRedis: + """Minimal in-memory Redis stand-in for counter tests.""" + + def __init__(self): + self._data = {} + + def get(self, key): + val = self._data.get(key) + if val is None: + return None + return str(val).encode() + + def set(self, key, value, ex=None): + self._data[key] = int(value) + + def incr(self, key): + self._data[key] = self._data.get(key, 0) + 1 + return self._data[key] + + def decr(self, key): + self._data[key] = self._data.get(key, 0) - 1 + return self._data[key] + + +class CredentialFingerprintTests(TestCase): + def test_same_credentials_same_fingerprint(self): + fp1 = compute_credential_fingerprint("User", "pass") + fp2 = compute_credential_fingerprint("user", "pass") + self.assertEqual(fp1, fp2) + self.assertIsNotNone(fp1) + + def test_different_password_different_fingerprint(self): + fp1 = compute_credential_fingerprint("user", "pass1") + fp2 = compute_credential_fingerprint("user", "pass2") + self.assertNotEqual(fp1, fp2) + + def test_empty_credentials_returns_none(self): + self.assertIsNone(compute_credential_fingerprint("", "pass")) + self.assertIsNone(compute_credential_fingerprint("user", "")) + + def test_extract_credentials_from_xc_style_url(self): + url = "http://example.com/live/alice/secret123/99999.ts" + user, password = extract_credentials_from_stream_url(url) + self.assertEqual(user, "alice") + self.assertEqual(password, "secret123") + + +class AutoAssignTests(TestCase): + def test_accounts_with_same_credentials_share_server_group(self): + account1 = M3UAccount.objects.create( + name="Provider A", + account_type="XC", + username="user1", + password="secret", + server_url="http://a.example.com", + max_streams=2, + ) + account2 = M3UAccount.objects.create( + name="Provider B", + account_type="XC", + username="user1", + password="secret", + server_url="http://b.example.com", + max_streams=3, + ) + + account1.refresh_from_db() + account2.refresh_from_db() + + self.assertIsNotNone(account1.server_group_id) + self.assertEqual(account1.server_group_id, account2.server_group_id) + self.assertTrue(account1.server_group.credential_fingerprint) + self.assertEqual(account1.server_group.max_streams, 2) + + def test_exclude_from_pool_opt_out(self): + account1 = M3UAccount.objects.create( + name="Pooled", + account_type="XC", + username="shared", + password="secret", + max_streams=1, + ) + account2 = M3UAccount.objects.create( + name="Opt-out", + account_type="XC", + username="shared", + password="secret", + custom_properties={"exclude_from_credential_pool": True}, + max_streams=1, + ) + + account1.refresh_from_db() + account2.refresh_from_db() + + self.assertIsNotNone(account1.server_group_id) + self.assertIsNone(account2.server_group_id) + + +class MultiProfilePoolTests(TestCase): + def test_profiles_with_different_credentials_get_separate_pools(self): + account = M3UAccount.objects.create( + name="Multi-login XC", + account_type="XC", + username="xc_user_a", + password="xc_pass_a", + server_url="http://xc.example.com", + max_streams=1, + ) + base = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + base.search_pattern = r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" + base.replace_pattern = r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" + base.save() + + p2 = M3UAccountProfile.objects.create( + m3u_account=account, + name="login_b", + is_default=False, + is_active=True, + max_streams=1, + search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", + replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + ) + + sync_account_credential_pool(account) + + fp1 = get_profile_credential_fingerprint(base) + fp2 = get_profile_credential_fingerprint(p2) + self.assertNotEqual(fp1, fp2) + + g1 = get_enforced_server_group_for_profile(base) + g2 = get_enforced_server_group_for_profile(p2) + self.assertIsNotNone(g1) + self.assertIsNotNone(g2) + self.assertNotEqual(g1.id, g2.id) + self.assertIsNone(account.server_group_id) + + +class PoolEnforcementTests(TestCase): + def setUp(self): + self.redis = FakeRedis() + self.group = ServerGroup.objects.create( + name="test-pool", + max_streams=1, + ) + self.account = M3UAccount.objects.create( + name="Test Account", + account_type="XC", + username="user", + password="pass", + server_group=self.group, + max_streams=5, + ) + self.profile = M3UAccountProfile.objects.get( + m3u_account=self.account, is_default=True + ) + + def test_reserve_and_release_pool_counter(self): + reserved, count = reserve_profile_slot(self.profile, self.redis) + self.assertTrue(reserved) + self.assertEqual(count, 1) + + group_key = server_group_connections_key(self.group.id) + self.assertEqual(self.redis._data[group_key], 1) + + release_profile_slot(self.profile.id, self.redis) + self.assertEqual(self.redis._data[group_key], 0) + + def test_reserve_fails_when_pool_at_capacity(self): + group_key = server_group_connections_key(self.group.id) + self.redis.set(group_key, 1) + + reserved, _count = reserve_profile_slot(self.profile, self.redis) + self.assertFalse(reserved) + self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) + + def test_live_slot_blocks_second_reserve_same_profile(self): + reserved1, _ = reserve_profile_slot(self.profile, self.redis) + self.assertTrue(reserved1) + + reserved2, _ = reserve_profile_slot(self.profile, self.redis) + self.assertFalse(reserved2) + + +class VodProfileSelectionTests(TestCase): + """VOD must try alternate profiles when the default pool is full (live TV).""" + + def test_get_m3u_profile_skips_default_when_pool_full(self): + from apps.proxy.vod_proxy.views import _get_m3u_profile + + account = M3UAccount.objects.create( + name="VOD multi-login", + account_type="XC", + username="xc_user_a", + password="xc_pass_a", + server_url="http://xc.example.com", + max_streams=1, + ) + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.search_pattern = ( + r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" + ) + default.replace_pattern = ( + r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" + ) + default.save() + + alt = M3UAccountProfile.objects.create( + m3u_account=account, + name="login_b", + is_default=False, + is_active=True, + max_streams=1, + search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", + replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + ) + + sync_account_credential_pool(account) + + redis = FakeRedis() + reserved, _ = reserve_profile_slot(default, redis) + self.assertTrue(reserved) + + with patch("core.utils.RedisClient.get_client", return_value=redis): + result = _get_m3u_profile(account, None, None) + + self.assertIsNotNone(result) + selected, _connections = result + self.assertEqual(selected.id, alt.id) diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index 3f75693e..6180f317 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -390,11 +390,12 @@ class ChannelStatus: created_at = float(init_time_bytes) uptime = time.time() - created_at if created_at > 0 else 0 + stream_url = metadata.get(ChannelMetadataField.URL, "") or "" # Simplified info info = { 'channel_id': channel_id, 'state': metadata.get(ChannelMetadataField.STATE), - 'url': metadata.get(ChannelMetadataField.URL, ""), + 'url': stream_url, 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""), 'owner': metadata.get(ChannelMetadataField.OWNER), 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, @@ -402,6 +403,15 @@ class ChannelStatus: 'uptime': uptime, 'started_at': created_at if created_at > 0 else None, } + if stream_url: + try: + from apps.m3u.connection_pool import extract_credentials_from_stream_url + + provider_user, _ = extract_credentials_from_stream_url(stream_url) + if provider_user: + info['provider_username'] = provider_user + except Exception as e: + logger.debug(f"Could not parse provider username from stream URL: {e}") channel_name = metadata.get(ChannelMetadataField.CHANNEL_NAME) if channel_name: @@ -498,7 +508,15 @@ class ChannelStatus: m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE) if m3u_profile_id: try: - info['m3u_profile_id'] = int(m3u_profile_id) + profile_id_int = int(m3u_profile_id) + info['m3u_profile_id'] = profile_id_int + try: + from apps.m3u.models import M3UAccountProfile + m3u_profile = M3UAccountProfile.objects.filter(id=profile_id_int).first() + if m3u_profile: + info['m3u_profile_name'] = m3u_profile.name + except Exception as e: + logger.warning(f"Failed to get M3U profile name for ID {profile_id_int}: {e}") except ValueError: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 5227a3fa..083b0fd1 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -8,6 +8,7 @@ from typing import Optional, Tuple, List from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.m3u.connection_pool import pool_has_capacity_for_profile from core.models import UserAgent, CoreSettings, StreamProfile from .utils import get_logger from uuid import UUID @@ -15,6 +16,35 @@ import requests logger = get_logger() + +def _resolve_live_stream_url(stream, m3u_account, m3u_profile): + """ + Build the upstream URL for live playback. + + XC accounts use current transformed credentials plus provider stream_id so + playback matches the account login (not a stale stream.url from an old sync). + STD/M3U accounts keep using the URL stored on the stream row. + """ + if ( + m3u_account.account_type == M3UAccount.Types.XC + and stream.stream_id + ): + from apps.m3u.tasks import get_transformed_credentials + + server_url, username, password = get_transformed_credentials( + m3u_account, m3u_profile + ) + if server_url and username and password: + base = server_url.rstrip("/") + return f"{base}/live/{username}/{password}/{stream.stream_id}.ts" + + return transform_url( + stream.url or "", + m3u_profile.search_pattern, + m3u_profile.replace_pattern, + ) + + def get_stream_object(id: str): try: logger.info(f"Fetching channel ID {id}") @@ -63,7 +93,7 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - stream_url = transform_url(stream.url, profile.search_pattern, profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, profile) stream_profile = stream.get_stream_profile() logger.debug(f"Using stream profile: {stream_profile.name}") @@ -106,9 +136,7 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - # Generate stream URL based on the selected profile - input_url = stream.url - stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, m3u_profile) # Check if transcoding is needed stream_profile = channel.get_stream_profile() @@ -204,35 +232,41 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: - - # Check connection availability if redis_client: + if not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"Profile {profile.id} credential pool at capacity for stream switch" + ) + continue + profile_connections_key = f"profile_connections:{profile.id}" current_connections = int(redis_client.get(profile_connections_key) or 0) - # Check if this channel is already using this profile channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: - # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id - existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") + existing_profile_id = redis_client.get( + f"stream_profile:{existing_stream_id}" + ) if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True - logger.debug(f"Channel {channel.id} already using profile {profile.id}") - # Calculate effective connections (subtract 1 if channel already using this profile) - effective_connections = current_connections - (1 if channel_using_profile else 0) + effective_connections = current_connections - ( + 1 if channel_using_profile else 0 + ) - # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: selected_profile = profile - logger.debug(f"Selected profile {profile.id} with {effective_connections}/{profile.max_streams} effective connections (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Selected profile {profile.id} with " + f"{effective_connections}/{profile.max_streams} effective connections" + ) break - else: - logger.debug(f"Profile {profile.id} at max connections: {effective_connections}/{profile.max_streams} (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Profile {profile.id} at max connections: " + f"{effective_connections}/{profile.max_streams}" + ) else: - # No Redis available, assume first active profile is okay selected_profile = profile break @@ -260,8 +294,7 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] # Get the user agent from the M3U account user_agent = m3u_account.get_user_agent().user_agent - # Generate URL using the transform function directly - stream_url = transform_url(stream.url, profile.search_pattern, profile.replace_pattern) + stream_url = _resolve_live_stream_url(stream, m3u_account, profile) # Get transcode info from the channel's stream profile stream_profile = channel.get_stream_profile() @@ -365,6 +398,12 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: + if not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"Credential pool at capacity for profile " + f"{profile.id} (account {m3u_account.id})" + ) + continue selected_profile = profile logger.debug(f"Found available profile {profile.id} for stream {stream.id}: {effective_connections}/{profile.max_streams} effective (current: {current_connections}, already using: {channel_using_profile})") break diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index 7d367531..77db3b1b 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -735,68 +735,33 @@ class MultiWorkerVODConnectionManager: """Get Redis key for tracking connections per profile - STANDARDIZED with TS proxy""" return f"profile_connections:{profile_id}" - def _check_profile_limits(self, m3u_profile) -> bool: - """Check if profile has available connection slots""" - if m3u_profile.max_streams == 0: # Unlimited - return True - - try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - current_connections = int(self.redis_client.get(profile_connections_key) or 0) - - logger.info(f"[PROFILE-CHECK] Profile {m3u_profile.id} has {current_connections}/{m3u_profile.max_streams} connections") - return current_connections < m3u_profile.max_streams - - except Exception as e: - logger.error(f"Error checking profile limits: {e}") - return False - def _check_and_reserve_profile_slot(self, m3u_profile) -> bool: """ Atomically check and reserve a connection slot for the given profile. - Uses an INCR-first-then-check pattern to eliminate the TOCTOU race - condition where separate GET > check > INCR operations could allow - concurrent requests to both pass the capacity check. - - For profiles with max_streams=0 (unlimited), no reservation is needed. - Returns: bool: True if slot was reserved (or unlimited), False if at capacity """ - if m3u_profile.max_streams == 0: # Unlimited - return True + from apps.m3u.connection_pool import reserve_profile_slot try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - - # Atomically increment first — single Redis command eliminates race window - new_count = self.redis_client.incr(profile_connections_key) - - if new_count <= m3u_profile.max_streams: - logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: {new_count}/{m3u_profile.max_streams}") - return True - - # Over capacity — roll back the increment - self.redis_client.decr(profile_connections_key) - logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: {new_count - 1}/{m3u_profile.max_streams}") - return False + reserved, new_count = reserve_profile_slot(m3u_profile, self.redis_client) + if reserved: + logger.info( + f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: " + f"{new_count}/{m3u_profile.max_streams}" + ) + else: + logger.info( + f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: " + f"{new_count}/{m3u_profile.max_streams}" + ) + return reserved except Exception as e: logger.error(f"Error reserving profile slot: {e}") return False - def _increment_profile_connections(self, m3u_profile): - """Increment profile connection count""" - try: - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) - new_count = self.redis_client.incr(profile_connections_key) - logger.info(f"[PROFILE-INCR] Profile {m3u_profile.id} connections: {new_count}") - return new_count - except Exception as e: - logger.error(f"Error incrementing profile connections: {e}") - return None - def _trigger_vod_stats_update(self): """Trigger a VOD stats WebSocket update in a background thread.""" threading.Thread(target=self._do_vod_stats_update, daemon=True).start() @@ -859,21 +824,14 @@ class MultiWorkerVODConnectionManager: logger.error(f"Failed to trigger VOD stats update: {e}") def _decrement_profile_connections(self, m3u_profile_id: int): - """Decrement profile connection count. + """Decrement profile and shared pool connection counters.""" + from apps.m3u.connection_pool import release_profile_slot - Uses a single atomic DECR (no GET-before-DECR) to avoid the race condition - where two concurrent decrements both pass a >0 guard and both fire, sending - the counter negative. If the counter would go below zero it is clamped to 0. - """ try: - profile_connections_key = self._get_profile_connections_key(m3u_profile_id) - new_count = self.redis_client.decr(profile_connections_key) - if new_count < 0: - self.redis_client.set(profile_connections_key, 0) - new_count = 0 - logger.warning(f"[PROFILE-DECR] Profile {m3u_profile_id} counter went negative, clamped to 0") - else: - logger.info(f"[PROFILE-DECR] Profile {m3u_profile_id} connections: {new_count}") + release_profile_slot(m3u_profile_id, self.redis_client) + profile_key = self._get_profile_connections_key(m3u_profile_id) + new_count = int(self.redis_client.get(profile_key) or 0) + logger.info(f"[PROFILE-DECR] Profile {m3u_profile_id} connections: {new_count}") return new_count except Exception as e: logger.error(f"Error decrementing profile connections: {e}") diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index f73f48b4..a7995107 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -178,6 +178,7 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): """ try: from core.utils import RedisClient + from apps.m3u.connection_pool import pool_has_capacity_for_profile redis_client = RedisClient.get_client() if not redis_client: @@ -214,7 +215,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): except Exception as e: logger.warning(f"[PROFILE-SELECTION] Error checking existing profile for session {session_id}: {e}") else: - logger.debug(f"[PROFILE-SELECTION] Session {session_id} exists but has no profile ID stored") # If specific profile requested, try to use it + logger.debug( + f"[PROFILE-SELECTION] Session {session_id} exists but has no profile ID stored" + ) + + # If specific profile requested, try to use it if profile_id: try: profile = M3UAccountProfile.objects.get( @@ -226,9 +231,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): profile_connections_key = f"profile_connections:{profile.id}" current_connections = int(redis_client.get(profile_connections_key) or 0) - if profile.max_streams == 0 or current_connections < profile.max_streams: + if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Using requested profile {profile.id}: {current_connections}/{profile.max_streams} connections") return (profile, current_connections) + elif not pool_has_capacity_for_profile(profile, redis_client): + logger.warning(f"[PROFILE-SELECTION] Shared credential pool at capacity for account {m3u_account.id}") else: logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") except M3UAccountProfile.DoesNotExist: @@ -253,9 +260,15 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): current_connections = int(redis_client.get(profile_connections_key) or 0) # Check if profile has available connection slots - if profile.max_streams == 0 or current_connections < profile.max_streams: + if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Selected profile {profile.id} ({profile.name}): {current_connections}/{profile.max_streams} connections") return (profile, current_connections) + elif not pool_has_capacity_for_profile(profile, redis_client): + logger.debug( + f"[PROFILE-SELECTION] Credential pool at capacity for profile " + f"{profile.id}, trying next profile" + ) + continue else: logger.debug(f"[PROFILE-SELECTION] Profile {profile.id} at capacity: {current_connections}/{profile.max_streams}") @@ -791,17 +804,30 @@ def build_vod_stats_data(redis_client): # Get M3U profile information m3u_profile_info = {} m3u_profile_id = combined_data.get('m3u_profile_id') + stream_url_for_creds = ( + combined_data.get('final_url') + or combined_data.get('stream_url') + or '' + ) if m3u_profile_id: try: from apps.m3u.models import M3UAccountProfile + from apps.m3u.connection_pool import extract_credentials_from_stream_url + profile = M3UAccountProfile.objects.select_related('m3u_account').get(id=m3u_profile_id) m3u_profile_info = { 'profile_name': profile.name, 'account_name': profile.m3u_account.name, 'account_id': profile.m3u_account.id, 'max_streams': profile.m3u_account.max_streams, - 'm3u_profile_id': int(m3u_profile_id) + 'm3u_profile_id': int(m3u_profile_id), } + if stream_url_for_creds: + provider_user, _ = extract_credentials_from_stream_url( + stream_url_for_creds + ) + if provider_user: + m3u_profile_info['provider_username'] = provider_user except Exception as e: logger.warning(f"Could not fetch M3U profile {m3u_profile_id}: {e}") diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 30509a86..0e6e221b 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -547,6 +547,7 @@ const StreamConnectionCard = ({ channel.m3u_profile?.name || channel.m3u_profile_name || 'Unknown M3U Profile'; + const providerUsername = channel.provider_username; // Create select options for available streams const streamOptions = getStreamOptions(availableStreams, m3uAccountsMap); @@ -648,12 +649,21 @@ const StreamConnectionCard = ({ {/* M3U Profile on right - absolutely positioned */} - - - - {m3uProfileName} - - + + + + + {m3uProfileName} + + + {providerUsername && ( + + + Login: {providerUsername} + + + )} + {/* Channel Name on left */} diff --git a/frontend/src/components/cards/VodConnectionCard.jsx b/frontend/src/components/cards/VodConnectionCard.jsx index 90b4c034..6539426a 100644 --- a/frontend/src/components/cards/VodConnectionCard.jsx +++ b/frontend/src/components/cards/VodConnectionCard.jsx @@ -349,6 +349,13 @@ const VodConnectionCard = ({ vodContent, stopVODClient }) => { {connection.m3u_profile.profile_name || 'Default Profile'} + {connection.m3u_profile.provider_username && ( + + + Login: {connection.m3u_profile.provider_username} + + + )} From bb9b95b4a4605674a6b8919997537bc08eb5ba0a Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Sat, 23 May 2026 20:18:45 -0400 Subject: [PATCH 14/76] Refactor connection pools to manual ServerGroups with profile rotation. Drop auto-fingerprint migration and restore per-profile selection for live/VOD. Enforce shared limits on reserve using login-scoped group counters, and add Server Groups UI for manual account assignment per maintainer feedback (#1137). Co-authored-by: Cursor --- apps/channels/models.py | 48 ++-- apps/m3u/connection_pool.py | 201 +++++--------- ...0021_servergroup_credential_fingerprint.py | 102 -------- apps/m3u/models.py | 28 -- apps/m3u/tests/test_connection_pool.py | 245 +++++++++--------- apps/proxy/live_proxy/url_utils.py | 65 ++--- apps/proxy/live_proxy/views.py | 17 +- apps/proxy/vod_proxy/views.py | 26 +- frontend/src/api.js | 54 ++++ frontend/src/components/forms/M3U.jsx | 24 ++ frontend/src/components/forms/ServerGroup.jsx | 90 +++++++ .../components/tables/ServerGroupsTable.jsx | 203 +++++++++++++++ frontend/src/pages/Settings.jsx | 16 ++ frontend/src/store/auth.jsx | 2 + frontend/src/store/serverGroups.jsx | 40 +++ frontend/src/utils/forms/M3uUtils.js | 4 + 16 files changed, 701 insertions(+), 464 deletions(-) delete mode 100644 apps/m3u/migrations/0021_servergroup_credential_fingerprint.py create mode 100644 frontend/src/components/forms/ServerGroup.jsx create mode 100644 frontend/src/components/tables/ServerGroupsTable.jsx create mode 100644 frontend/src/store/serverGroups.jsx diff --git a/apps/channels/models.py b/apps/channels/models.py index 59a2bc3d..38ebbbc7 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -210,13 +210,14 @@ class Stream(models.Model): Finds an available profile for this stream and reserves a connection slot. Returns: - Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason) + Tuple[Optional[int], Optional[int], Optional[str], bool]: + (stream_id, profile_id, error_reason, slot_reserved) """ redis_client = RedisClient.get_client() profile_id = redis_client.get(f"stream_profile:{self.id}") if profile_id: profile_id = int(profile_id) - return self.id, profile_id, None + return self.id, profile_id, None, False # Retrieve the M3U account associated with the stream. m3u_account = self.m3u_account @@ -238,9 +239,9 @@ class Stream(models.Model): if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) redis_client.set(f"stream_profile:{self.id}", profile.id) - return self.id, profile.id, None + return self.id, profile.id, None, True - return None, None, "All active M3U profiles have reached maximum connection limits" + return None, None, "All active M3U profiles have reached maximum connection limits", False def release_stream(self): """ @@ -615,7 +616,8 @@ class Channel(models.Model): Finds an available stream for the requested channel and returns the selected stream and profile. Returns: - Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason) + Tuple[Optional[int], Optional[int], Optional[str], bool]: + (stream_id, profile_id, error_reason, slot_reserved) """ redis_client = RedisClient.get_client() error_reason = None @@ -623,11 +625,11 @@ class Channel(models.Model): # Check if this channel has any streams if not self.streams.exists(): error_reason = "No streams assigned to channel" - return None, None, error_reason + return None, None, error_reason, False # Reuse assignment only when this channel is still active in the proxy. - # Stale channel_stream keys after stop/disconnect caused every tune to - # reuse the default profile without re-running rotation or INCR. + # Stale channel_stream keys after stop/disconnect skip INCR and break pool + # accounting, which lets a second stream reach the provider and fail validation. stream_id_bytes = redis_client.get(f"channel_stream:{self.id}") if stream_id_bytes: try: @@ -648,7 +650,7 @@ class Channel(models.Model): f"Channel {self.uuid}: reusing active assignment " f"stream={stream_id} profile={profile_id}" ) - return stream_id, profile_id, None + return stream_id, profile_id, None, False except (ValueError, TypeError): logger.debug( f"Invalid profile ID retrieved from Redis: {profile_id_bytes}" @@ -707,6 +709,7 @@ class Channel(models.Model): stream.id, profile.id, None, + True, ) # Return newly assigned stream and matched profile else: # At capacity: try to preempt a lower-impact channel on this profile @@ -735,7 +738,7 @@ class Channel(models.Model): else: error_reason = "No active profiles found for any assigned stream" - return None, None, error_reason + return None, None, error_reason, False def release_stream(self): """ @@ -779,10 +782,6 @@ class Channel(models.Model): ChannelMetadataField.M3U_PROFILE, ) - stream = Stream.objects.select_related("m3u_account").filter( - id=stream_id - ).first() - m3u_account = stream.m3u_account if stream else None release_profile_slot(profile_id, redis_client) return True @@ -874,10 +873,27 @@ class Channel(models.Model): if current_profile_id == new_profile_id: return True + from apps.m3u.models import M3UAccountProfile + from apps.m3u.connection_pool import ( + get_enforced_server_group_for_profile, + profile_connections_key, + ) + + new_profile = M3UAccountProfile.objects.get(id=new_profile_id) + if get_enforced_server_group_for_profile(new_profile): + # Shared group pool: one active stream uses one group slot regardless + # of which profile supplies the URL. + redis_client.set(f"stream_profile:{stream_id}", new_profile_id) + logger.info( + f"Updated stream {stream_id} profile from {current_profile_id} to " + f"{new_profile_id} (shared group pool unchanged)" + ) + return True + # Use pipeline for atomic profile switch to prevent counter drift # if an exception occurs between DECR and INCR - old_profile_connections_key = f"profile_connections:{current_profile_id}" - new_profile_connections_key = f"profile_connections:{new_profile_id}" + old_profile_connections_key = profile_connections_key(current_profile_id) + new_profile_connections_key = profile_connections_key(new_profile_id) old_count = int(redis_client.get(old_profile_connections_key) or 0) pipe = redis_client.pipeline() diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index b06aff10..9817d86c 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -1,8 +1,10 @@ """ -Shared connection pool enforcement for M3U accounts with identical credentials. +Shared connection pool enforcement for M3U accounts in the same ServerGroup. -All Redis INCR/DECR for profile and server-group limits should go through this -module so live TV and VOD stay consistent. +Profile selection rotates across M3UAccountProfile rows using each profile's own +Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup +with max_streams > 0, the group counter is scoped by provider login fingerprint +so profiles that rewrite to different IPTV credentials keep independent limits. """ from __future__ import annotations @@ -15,8 +17,7 @@ from typing import Optional, Tuple logger = logging.getLogger(__name__) PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" -SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}" -EXCLUDE_FROM_POOL_KEY = "exclude_from_credential_pool" +SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}:{fingerprint}" _XC_URL_CREDENTIALS_RE = re.compile( r"/(?:live|movie|series)/([^/]+)/([^/]+)/", @@ -28,8 +29,10 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) -def server_group_connections_key(group_id: int) -> str: - return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id) +def server_group_connections_key(group_id: int, fingerprint: Optional[str] = None) -> str: + """Redis key for a manual ServerGroup slot, scoped by provider login.""" + fp = (fingerprint or "unknown")[:16] + return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id, fingerprint=fp) def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: @@ -113,53 +116,58 @@ def get_profile_credential_fingerprint(profile) -> Optional[str]: ) -def account_excluded_from_pool(m3u_account) -> bool: - props = m3u_account.custom_properties or {} - return bool(props.get(EXCLUDE_FROM_POOL_KEY)) - - -def _get_pool_for_fingerprint(fingerprint: str): - """Return the auto credential pool ServerGroup for a fingerprint, if configured.""" - if not fingerprint: - return None - from apps.m3u.models import ServerGroup - - group = ServerGroup.objects.filter(credential_fingerprint=fingerprint).first() - if not group or group.max_streams == 0: - return None - return group - - def get_enforced_server_group_for_profile(profile): - """Return the shared pool for this profile's effective provider login.""" - if account_excluded_from_pool(profile.m3u_account): - return None - - group = _get_pool_for_fingerprint(get_profile_credential_fingerprint(profile)) - if group: + """Return the shared ServerGroup limit for this profile's account, if configured.""" + group = profile.m3u_account.server_group + if group and group.max_streams > 0: return group - - manual = profile.m3u_account.server_group - if manual and not manual.credential_fingerprint and manual.max_streams > 0: - return manual return None -def pool_has_capacity_for_profile(profile, redis_client) -> bool: - """Non-mutating check for the profile's credential pool.""" +def _group_counter_key(profile, group) -> str: + return server_group_connections_key( + group.id, + get_profile_credential_fingerprint(profile), + ) + + +def get_profile_connection_count(profile, redis_client) -> int: + return int(redis_client.get(profile_connections_key(profile.id)) or 0) + + +def get_group_connection_count(profile, redis_client) -> int: + group = get_enforced_server_group_for_profile(profile) + if not group: + return 0 + return int(redis_client.get(_group_counter_key(profile, group)) or 0) + + +def profile_has_capacity_for_selection(profile, redis_client) -> bool: + """Per-profile capacity check used when rotating across profiles on one account.""" + if profile.max_streams == 0: + return True + return get_profile_connection_count(profile, redis_client) < profile.max_streams + + +def group_has_capacity_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) if not group: return True - key = server_group_connections_key(group.id) - current = int(redis_client.get(key) or 0) - return current < group.max_streams + return get_group_connection_count(profile, redis_client) < group.max_streams + + +def pool_has_capacity_for_profile(profile, redis_client) -> bool: + """Non-mutating check before reserve: profile slot and group slot if applicable.""" + return profile_has_capacity_for_selection(profile, redis_client) and group_has_capacity_for_profile( + profile, redis_client + ) def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) if not group: return True - key = server_group_connections_key(group.id) + key = _group_counter_key(profile, group) group_count = redis_client.incr(key) if group_count <= group.max_streams: return True @@ -171,7 +179,10 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: group = get_enforced_server_group_for_profile(profile) if not group: return - key = server_group_connections_key(group.id) + key = _group_counter_key(profile, group) + current = int(redis_client.get(key) or 0) + if current <= 0: + return new_count = redis_client.decr(key) if new_count < 0: redis_client.set(key, 0) @@ -179,30 +190,29 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: """ - Atomically reserve profile + shared pool slots (INCR-first). + Atomically reserve profile + optional group slots (INCR-first). Returns (reserved, profile_count_after_attempt). """ - if profile.max_streams == 0: - if _reserve_server_group_slot_for_profile(profile, redis_client): - return True, 0 - return False, 0 - profile_key = profile_connections_key(profile.id) - new_count = redis_client.incr(profile_key) + profile_count = 0 - if new_count <= profile.max_streams: - if _reserve_server_group_slot_for_profile(profile, redis_client): - return True, new_count - redis_client.decr(profile_key) - return False, new_count - 1 + if profile.max_streams > 0: + profile_count = redis_client.incr(profile_key) + if profile_count > profile.max_streams: + redis_client.decr(profile_key) + return False, profile_count - 1 - redis_client.decr(profile_key) - return False, new_count - 1 + if not _reserve_server_group_slot_for_profile(profile, redis_client): + if profile.max_streams > 0: + redis_client.decr(profile_key) + return False, profile_count - 1 if profile.max_streams > 0 else 0 + + return True, profile_count def release_profile_slot(profile_id: int, redis_client) -> None: - """Release profile and shared pool slots after a stream ends.""" + """Release profile and shared group slots after a stream ends.""" from apps.m3u.models import M3UAccountProfile try: @@ -218,82 +228,3 @@ def release_profile_slot(profile_id: int, redis_client) -> None: if profile: _release_server_group_slot_for_profile(profile, redis_client) - - -def recompute_pool_max_streams(server_group) -> None: - """Set pool max_streams to the minimum positive limit across members and profiles.""" - from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup - - if not server_group.credential_fingerprint: - return - - fp = server_group.credential_fingerprint - limits = list( - M3UAccount.objects.filter( - server_group=server_group, max_streams__gt=0 - ).values_list("max_streams", flat=True) - ) - - for profile in M3UAccountProfile.objects.filter(is_active=True).select_related( - "m3u_account" - ): - if account_excluded_from_pool(profile.m3u_account): - continue - if get_profile_credential_fingerprint(profile) != fp: - continue - if profile.max_streams > 0: - limits.append(profile.max_streams) - - new_max = min(limits) if limits else 0 - if server_group.max_streams != new_max: - ServerGroup.objects.filter(pk=server_group.pk).update(max_streams=new_max) - - -def _ensure_pool_for_fingerprint(fingerprint: str): - from apps.m3u.models import ServerGroup - - group, _created = ServerGroup.objects.get_or_create( - credential_fingerprint=fingerprint, - defaults={ - "name": f"credential-pool-{fingerprint[:16]}", - "max_streams": 0, - }, - ) - recompute_pool_max_streams(group) - return group - - -def sync_account_credential_pool(m3u_account) -> None: - """ - Ensure auto credential pools exist for each distinct login on this account. - - Sets M3UAccount.server_group only when every active profile shares one login. - """ - from apps.m3u.models import M3UAccount, M3UAccountProfile - - if account_excluded_from_pool(m3u_account): - return - - if m3u_account.server_group_id: - existing = m3u_account.server_group - if existing and not existing.credential_fingerprint: - return - - profile_fps = set() - for profile in M3UAccountProfile.objects.filter( - m3u_account=m3u_account, is_active=True - ): - fp = get_profile_credential_fingerprint(profile) - if not fp: - continue - profile_fps.add(fp) - _ensure_pool_for_fingerprint(fp) - - if len(profile_fps) == 1: - group = _get_pool_for_fingerprint(next(iter(profile_fps))) - if group and m3u_account.server_group_id != group.id: - M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=group) - elif m3u_account.server_group_id: - existing = m3u_account.server_group - if existing and existing.credential_fingerprint: - M3UAccount.objects.filter(pk=m3u_account.pk).update(server_group=None) diff --git a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py b/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py deleted file mode 100644 index 8d958945..00000000 --- a/apps/m3u/migrations/0021_servergroup_credential_fingerprint.py +++ /dev/null @@ -1,102 +0,0 @@ -# Generated manually for shared credential connection pools (#1137) - -from django.db import migrations, models - -from apps.m3u.connection_pool import ( - compute_credential_fingerprint, - extract_credentials_from_stream_url, -) - - -def _historical_account_fingerprint(M3UAccount, M3UAccountProfile, Stream, account): - """Resolve fingerprint for migration using historical models only.""" - fingerprint = compute_credential_fingerprint( - account.username or "", - account.password or "", - ) - if fingerprint: - return fingerprint - - sample_url = ( - Stream.objects.filter(m3u_account_id=account.pk) - .exclude(url="") - .values_list("url", flat=True) - .first() - ) - if sample_url: - url_user, url_pass = extract_credentials_from_stream_url(sample_url) - return compute_credential_fingerprint(url_user or "", url_pass or "") - - return None - - -def backfill_credential_pools(apps, schema_editor): - from django.db.models import Min - - M3UAccount = apps.get_model("m3u", "M3UAccount") - ServerGroup = apps.get_model("m3u", "ServerGroup") - M3UAccountProfile = apps.get_model("m3u", "M3UAccountProfile") - Stream = apps.get_model("dispatcharr_channels", "Stream") - - seen = {} - for account in M3UAccount.objects.all(): - props = account.custom_properties or {} - if props.get("exclude_from_credential_pool"): - continue - if account.server_group_id: - group = ServerGroup.objects.filter(pk=account.server_group_id).first() - if group and not group.credential_fingerprint: - continue - - fingerprint = _historical_account_fingerprint( - M3UAccount, M3UAccountProfile, Stream, account - ) - if not fingerprint: - continue - - if fingerprint not in seen: - short = fingerprint[:16] - group, _ = ServerGroup.objects.get_or_create( - credential_fingerprint=fingerprint, - defaults={ - "name": f"credential-pool-{short}", - "max_streams": 0, - }, - ) - seen[fingerprint] = group - else: - group = seen[fingerprint] - - M3UAccount.objects.filter(pk=account.pk).update(server_group_id=group.pk) - - for group in seen.values(): - agg = M3UAccount.objects.filter( - server_group_id=group.pk, max_streams__gt=0 - ).aggregate(min_limit=Min("max_streams")) - new_max = agg["min_limit"] or 0 - if group.max_streams != new_max: - ServerGroup.objects.filter(pk=group.pk).update(max_streams=new_max) - - -class Migration(migrations.Migration): - - dependencies = [ - ("m3u", "0020_servergroup_max_streams"), - ("dispatcharr_channels", "0037_auto_sync_overhaul"), - ] - - operations = [ - migrations.AddField( - model_name="servergroup", - name="credential_fingerprint", - field=models.CharField( - blank=True, - db_index=True, - help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", - max_length=64, - null=True, - unique=True, - ), - ), - migrations.RunPython(backfill_credential_pools, migrations.RunPython.noop), - ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 4b1325d5..e6ccaadb 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -203,14 +203,6 @@ class ServerGroup(models.Model): default=0, help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", ) - credential_fingerprint = models.CharField( - max_length=64, - null=True, - blank=True, - unique=True, - db_index=True, - help_text="Auto-assigned hash for accounts sharing the same IPTV credentials", - ) def __str__(self): return self.name @@ -346,26 +338,6 @@ class M3UAccountProfile(models.Model): return None -@receiver(models.signals.post_save, sender=M3UAccount) -def assign_credential_pool_for_m3u_account(sender, instance, **kwargs): - """Link accounts with identical credentials into a shared connection pool.""" - if kwargs.get("raw"): - return - from apps.m3u.connection_pool import sync_account_credential_pool - - sync_account_credential_pool(instance) - - -@receiver(models.signals.post_save, sender=M3UAccountProfile) -def assign_credential_pool_for_m3u_profile(sender, instance, **kwargs): - """Re-sync pools when profile URL transforms change.""" - if kwargs.get("raw"): - return - from apps.m3u.connection_pool import sync_account_credential_pool - - sync_account_credential_pool(instance.m3u_account) - - @receiver(models.signals.post_save, sender=M3UAccount) def create_profile_for_m3u_account(sender, instance, created, **kwargs): """Automatically create an M3UAccountProfile when M3UAccount is created.""" diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 79285484..44281211 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -1,18 +1,21 @@ -"""Tests for shared credential connection pools (#1137).""" +"""Tests for shared ServerGroup connection pools (#1137).""" from django.test import TestCase from unittest.mock import patch from apps.m3u.connection_pool import ( - compute_credential_fingerprint, extract_credentials_from_stream_url, get_enforced_server_group_for_profile, + get_group_connection_count, + get_profile_connection_count, get_profile_credential_fingerprint, + group_has_capacity_for_profile, pool_has_capacity_for_profile, + profile_has_capacity_for_selection, + profile_connections_key, release_profile_slot, reserve_profile_slot, server_group_connections_key, - sync_account_credential_pool, ) from apps.m3u.models import M3UAccount, M3UAccountProfile, ServerGroup @@ -41,22 +44,7 @@ class FakeRedis: return self._data[key] -class CredentialFingerprintTests(TestCase): - def test_same_credentials_same_fingerprint(self): - fp1 = compute_credential_fingerprint("User", "pass") - fp2 = compute_credential_fingerprint("user", "pass") - self.assertEqual(fp1, fp2) - self.assertIsNotNone(fp1) - - def test_different_password_different_fingerprint(self): - fp1 = compute_credential_fingerprint("user", "pass1") - fp2 = compute_credential_fingerprint("user", "pass2") - self.assertNotEqual(fp1, fp2) - - def test_empty_credentials_returns_none(self): - self.assertIsNone(compute_credential_fingerprint("", "pass")) - self.assertIsNone(compute_credential_fingerprint("user", "")) - +class ExtractCredentialsTests(TestCase): def test_extract_credentials_from_xc_style_url(self): url = "http://example.com/live/alice/secret123/99999.ts" user, password = extract_credentials_from_stream_url(url) @@ -64,94 +52,76 @@ class CredentialFingerprintTests(TestCase): self.assertEqual(password, "secret123") -class AutoAssignTests(TestCase): - def test_accounts_with_same_credentials_share_server_group(self): - account1 = M3UAccount.objects.create( - name="Provider A", - account_type="XC", - username="user1", - password="secret", - server_url="http://a.example.com", - max_streams=2, - ) - account2 = M3UAccount.objects.create( - name="Provider B", - account_type="XC", - username="user1", - password="secret", - server_url="http://b.example.com", - max_streams=3, - ) - - account1.refresh_from_db() - account2.refresh_from_db() - - self.assertIsNotNone(account1.server_group_id) - self.assertEqual(account1.server_group_id, account2.server_group_id) - self.assertTrue(account1.server_group.credential_fingerprint) - self.assertEqual(account1.server_group.max_streams, 2) - - def test_exclude_from_pool_opt_out(self): - account1 = M3UAccount.objects.create( - name="Pooled", - account_type="XC", - username="shared", - password="secret", - max_streams=1, - ) - account2 = M3UAccount.objects.create( - name="Opt-out", - account_type="XC", - username="shared", - password="secret", - custom_properties={"exclude_from_credential_pool": True}, - max_streams=1, - ) - - account1.refresh_from_db() - account2.refresh_from_db() - - self.assertIsNotNone(account1.server_group_id) - self.assertIsNone(account2.server_group_id) - - -class MultiProfilePoolTests(TestCase): - def test_profiles_with_different_credentials_get_separate_pools(self): +class ManualServerGroupTests(TestCase): + def test_group_enforced_when_max_streams_set(self): + group = ServerGroup.objects.create(name="provider-a", max_streams=2) account = M3UAccount.objects.create( - name="Multi-login XC", + name="Account A", account_type="XC", - username="xc_user_a", - password="xc_pass_a", + username="user", + password="pass", + server_group=group, + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + + self.assertEqual(get_enforced_server_group_for_profile(profile), group) + + def test_accounts_in_same_group_share_counter(self): + group = ServerGroup.objects.create(name="shared", max_streams=1) + account1 = M3UAccount.objects.create( + name="XC Account", + account_type="XC", + username="user", + password="pass", server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + account2 = M3UAccount.objects.create( + name="M3U Account", + account_type="STD", + username="user", + password="pass", + server_group=group, + max_streams=5, + ) + profile1 = M3UAccountProfile.objects.get(m3u_account=account1, is_default=True) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + + redis = FakeRedis() + reserved1, _ = reserve_profile_slot(profile1, redis) + self.assertTrue(reserved1) + + reserved2, _ = reserve_profile_slot(profile2, redis) + self.assertFalse(reserved2) + self.assertFalse(group_has_capacity_for_profile(profile2, redis)) + + def test_profile_rotation_when_default_profile_full(self): + """Pre-pool behavior: try the next profile on the same account.""" + account = M3UAccount.objects.create( + name="Multi-profile", + account_type="XC", max_streams=1, ) - base = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) - base.search_pattern = r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" - base.replace_pattern = r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" - base.save() + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.max_streams = 1 + default.save() - p2 = M3UAccountProfile.objects.create( + alt = M3UAccountProfile.objects.create( m3u_account=account, - name="login_b", + name="alt_profile", is_default=False, is_active=True, max_streams=1, - search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", - replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + search_pattern="", + replace_pattern="", ) - sync_account_credential_pool(account) - - fp1 = get_profile_credential_fingerprint(base) - fp2 = get_profile_credential_fingerprint(p2) - self.assertNotEqual(fp1, fp2) - - g1 = get_enforced_server_group_for_profile(base) - g2 = get_enforced_server_group_for_profile(p2) - self.assertIsNotNone(g1) - self.assertIsNotNone(g2) - self.assertNotEqual(g1.id, g2.id) - self.assertIsNone(account.server_group_id) + redis = FakeRedis() + reserved, _ = reserve_profile_slot(default, redis) + self.assertTrue(reserved) + self.assertFalse(profile_has_capacity_for_selection(default, redis)) + self.assertTrue(profile_has_capacity_for_selection(alt, redis)) class PoolEnforcementTests(TestCase): @@ -172,42 +142,86 @@ class PoolEnforcementTests(TestCase): self.profile = M3UAccountProfile.objects.get( m3u_account=self.account, is_default=True ) + self.profile.max_streams = 5 + self.profile.save() - def test_reserve_and_release_pool_counter(self): + def test_reserve_and_release_both_counters(self): reserved, count = reserve_profile_slot(self.profile, self.redis) self.assertTrue(reserved) self.assertEqual(count, 1) - group_key = server_group_connections_key(self.group.id) + group_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) + profile_key = profile_connections_key(self.profile.id) self.assertEqual(self.redis._data[group_key], 1) + self.assertEqual(self.redis._data[profile_key], 1) release_profile_slot(self.profile.id, self.redis) self.assertEqual(self.redis._data[group_key], 0) + self.assertEqual(self.redis._data[profile_key], 0) - def test_reserve_fails_when_pool_at_capacity(self): - group_key = server_group_connections_key(self.group.id) + def test_reserve_fails_when_group_at_capacity(self): + group_key = server_group_connections_key( + self.group.id, + get_profile_credential_fingerprint(self.profile), + ) self.redis.set(group_key, 1) reserved, _count = reserve_profile_slot(self.profile, self.redis) self.assertFalse(reserved) self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) - def test_live_slot_blocks_second_reserve_same_profile(self): - reserved1, _ = reserve_profile_slot(self.profile, self.redis) - self.assertTrue(reserved1) + def test_different_logins_in_group_do_not_block_each_other(self): + """Profiles with different provider logins keep separate group counters.""" + account = M3UAccount.objects.create( + name="Grouped multi-login", + account_type="XC", + username="login_a", + password="pass_a", + server_group=self.group, + max_streams=5, + ) + default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + default.max_streams = 1 + default.save() - reserved2, _ = reserve_profile_slot(self.profile, self.redis) - self.assertFalse(reserved2) + alt = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt_login", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + fp_a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fp_b = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + side_effect=lambda profile: fp_a if profile.id == default.id else fp_b, + ): + reserved_default, _ = reserve_profile_slot(default, self.redis) + self.assertTrue(reserved_default) + + reserved_alt, _ = reserve_profile_slot(alt, self.redis) + self.assertTrue(reserved_alt) + + key_a = server_group_connections_key(self.group.id, fp_a) + key_b = server_group_connections_key(self.group.id, fp_b) + self.assertEqual(self.redis._data[key_a], 1) + self.assertEqual(self.redis._data[key_b], 1) class VodProfileSelectionTests(TestCase): - """VOD must try alternate profiles when the default pool is full (live TV).""" - - def test_get_m3u_profile_skips_default_when_pool_full(self): + def test_get_m3u_profile_skips_default_when_profile_full(self): from apps.proxy.vod_proxy.views import _get_m3u_profile account = M3UAccount.objects.create( - name="VOD multi-login", + name="VOD multi-profile", account_type="XC", username="xc_user_a", password="xc_pass_a", @@ -215,26 +229,19 @@ class VodProfileSelectionTests(TestCase): max_streams=1, ) default = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) - default.search_pattern = ( - r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$" - ) - default.replace_pattern = ( - r"http://xc.example.com/live/xc_user_a/xc_pass_a/\1" - ) + default.max_streams = 1 default.save() alt = M3UAccountProfile.objects.create( m3u_account=account, - name="login_b", + name="alt_profile", is_default=False, is_active=True, max_streams=1, - search_pattern=r"^http://xc\.example\.com/live/xc_user_a/xc_pass_a/(.*)$", - replace_pattern=r"http://xc.example.com/live/xc_user_b/xc_pass_b/\1", + search_pattern="", + replace_pattern="", ) - sync_account_credential_pool(account) - redis = FakeRedis() reserved, _ = reserve_profile_slot(default, redis) self.assertTrue(reserved) diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 083b0fd1..60d0e3d0 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -8,7 +8,7 @@ from typing import Optional, Tuple, List from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile -from apps.m3u.connection_pool import pool_has_capacity_for_profile +from apps.m3u.connection_pool import profile_connections_key from core.models import UserAgent, CoreSettings, StreamProfile from .utils import get_logger from uuid import UUID @@ -54,15 +54,12 @@ def get_stream_object(id: str): logger.info(f"Fetching stream hash {id}") return get_object_or_404(Stream, stream_hash=id) -def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]]: +def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int], bool]: """ Generate the appropriate stream URL for a channel or stream based on its profile settings. - Args: - channel_id: The UUID of the channel or stream hash - Returns: - Tuple[str, str, bool, Optional[int]]: (stream_url, user_agent, transcode_flag, profile_id) + Tuple: (stream_url, user_agent, transcode_flag, profile_id, slot_reserved) """ try: channel_or_stream = get_stream_object(channel_id) @@ -74,15 +71,12 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] if not stream.m3u_account: logger.error(f"Stream {stream.id} has no M3U account") - return None, None, False, None + return None, None, False, None, False - # Use get_stream() to atomically reserve a slot and write the - # channel_stream / stream_profile Redis keys, matching the channel - # path so stream_name and stream_stats work correctly. - stream_id, profile_id, error_reason = stream.get_stream() + stream_id, profile_id, error_reason, slot_reserved = stream.get_stream() if not stream_id or not profile_id: logger.error(f"No profile available for stream {stream.id}: {error_reason}") - return None, None, False, None + return None, None, False, None, False try: profile = M3UAccountProfile.objects.get(id=profile_id) @@ -101,11 +95,12 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] transcode = not stream_profile.is_proxy() stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved except Exception as e: logger.error(f"Error generating stream URL for stream {stream.id}: {e}") - stream.release_stream() - return None, None, False, None + if slot_reserved: + stream.release_stream() + return None, None, False, None, False # Handle channel preview (existing logic) @@ -113,11 +108,11 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] # Get stream and profile for this channel # Note: get_stream now returns 3 values (stream_id, profile_id, error_reason) - stream_id, profile_id, error_reason = channel.get_stream() + stream_id, profile_id, error_reason, slot_reserved = channel.get_stream() if not stream_id or not profile_id: logger.error(f"No stream available for channel {channel_id}: {error_reason}") - return None, None, False, None + return None, None, False, None, False # get_stream() allocated a connection slot - ensure it's released on any error try: @@ -147,15 +142,16 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] stream_profile_id = stream_profile.id - return stream_url, stream_user_agent, transcode, stream_profile_id + return stream_url, stream_user_agent, transcode, stream_profile_id, slot_reserved except Exception as e: logger.error(f"Error generating stream URL for channel {channel_id}: {e}") - if not channel.release_stream(): - logger.warning(f"Failed to release stream for channel {channel_id} after URL generation error") - return None, None, False, None + if slot_reserved: + if not channel.release_stream(): + logger.warning(f"Failed to release stream for channel {channel_id} after URL generation error") + return None, None, False, None, False except Exception as e: logger.error(f"Error generating stream URL: {e}") - return None, None, False, None + return None, None, False, None, False def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -233,14 +229,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: if redis_client: - if not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"Profile {profile.id} credential pool at capacity for stream switch" - ) - continue - - profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = int( + redis_client.get(profile_connections_key(profile.id)) or 0 + ) channel_using_profile = False existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") @@ -275,7 +266,7 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] m3u_profile_id = selected_profile.id else: - stream_id, m3u_profile_id, error_reason = channel.get_stream() + stream_id, m3u_profile_id, error_reason, _slot_reserved = channel.get_stream() if stream_id is None or m3u_profile_id is None: return {'error': error_reason or 'No stream assigned to channel'} @@ -398,14 +389,12 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No # Check if profile has available slots if profile.max_streams == 0 or effective_connections < profile.max_streams: - if not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"Credential pool at capacity for profile " - f"{profile.id} (account {m3u_account.id})" - ) - continue selected_profile = profile - logger.debug(f"Found available profile {profile.id} for stream {stream.id}: {effective_connections}/{profile.max_streams} effective (current: {current_connections}, already using: {channel_using_profile})") + logger.debug( + f"Found available profile {profile.id} for stream {stream.id}: " + f"{effective_connections}/{profile.max_streams} effective " + f"(current: {current_connections}, already using: {channel_using_profile})" + ) break else: logger.debug(f"Profile {profile.id} at max connections: {effective_connections}/{profile.max_streams} (current: {current_connections}, already using: {channel_using_profile})") diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 1369a364..e9e1b8ab 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -213,6 +213,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): stream_user_agent = None transcode = False profile_value = None + slot_reserved = False error_reason = None attempt = 0 should_retry = True @@ -220,7 +221,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # Try to get a stream with fixed interval retries while should_retry and time.time() - wait_start_time < retry_timeout: attempt += 1 - stream_url, stream_user_agent, transcode, profile_value = ( + stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( generate_stream_url(channel_id) ) @@ -232,7 +233,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # On first failure, check if the error is retryable if attempt == 1: - _, _, error_reason = channel.get_stream() + _, _, error_reason, _ = channel.get_stream() if error_reason and "maximum connection limits" not in error_reason: logger.warning( f"[{client_id}] Can't retry - error not related to connection limits: {error_reason}" @@ -265,7 +266,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.info( f"[{client_id}] Making final attempt {attempt} at timeout boundary" ) - stream_url, stream_user_agent, transcode, profile_value = ( + stream_url, stream_user_agent, transcode, profile_value, slot_reserved = ( generate_stream_url(channel_id) ) if stream_url is not None: @@ -274,9 +275,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ) if stream_url is None: - # Release any connection slot that may have been allocated - # by the error-checking get_stream() call during retries - if not channel.release_stream(): + if slot_reserved and not channel.release_stream(): logger.debug(f"[{client_id}] release_stream found no keys during failed init cleanup") # Get the specific error message if available @@ -295,7 +294,7 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): # generate_stream_url() called get_stream() which allocated a connection # slot (INCR'd profile_connections) - track this for cleanup on error - if needs_initialization: + if needs_initialization and slot_reserved: connection_allocated = True # Read stream assignment from Redis (already set by generate_stream_url → get_stream). @@ -378,8 +377,8 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): logger.warning( f"[{client_id}] Alternate stream #{alt['stream_id']} failed validation: {message}" ) - # Release stream lock before redirecting - if not channel.release_stream(): + # Release stream lock before redirecting only if we reserved a slot + if connection_allocated and not channel.release_stream(): logger.warning(f"[{client_id}] Failed to release stream before redirect") connection_allocated = False # Final decision based on validation results diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index a7995107..01272468 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -178,7 +178,10 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): """ try: from core.utils import RedisClient - from apps.m3u.connection_pool import pool_has_capacity_for_profile + from apps.m3u.connection_pool import ( + get_profile_connection_count, + profile_has_capacity_for_selection, + ) redis_client = RedisClient.get_client() if not redis_client: @@ -229,15 +232,12 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): ) # Check Redis-based current connections profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = get_profile_connection_count(profile, redis_client) - if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): + if profile_has_capacity_for_selection(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Using requested profile {profile.id}: {current_connections}/{profile.max_streams} connections") return (profile, current_connections) - elif not pool_has_capacity_for_profile(profile, redis_client): - logger.warning(f"[PROFILE-SELECTION] Shared credential pool at capacity for account {m3u_account.id}") - else: - logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") + logger.warning(f"[PROFILE-SELECTION] Requested profile {profile.id} is at capacity: {current_connections}/{profile.max_streams}") except M3UAccountProfile.DoesNotExist: logger.warning(f"[PROFILE-SELECTION] Requested profile {profile_id} not found") @@ -256,19 +256,11 @@ def _get_m3u_profile(m3u_account, profile_id, session_id=None): profiles = [default_profile] + list(m3u_profiles.filter(is_default=False)) for profile in profiles: - profile_connections_key = f"profile_connections:{profile.id}" - current_connections = int(redis_client.get(profile_connections_key) or 0) + current_connections = get_profile_connection_count(profile, redis_client) - # Check if profile has available connection slots - if (profile.max_streams == 0 or current_connections < profile.max_streams) and pool_has_capacity_for_profile(profile, redis_client): + if profile_has_capacity_for_selection(profile, redis_client): logger.info(f"[PROFILE-SELECTION] Selected profile {profile.id} ({profile.name}): {current_connections}/{profile.max_streams} connections") return (profile, current_connections) - elif not pool_has_capacity_for_profile(profile, redis_client): - logger.debug( - f"[PROFILE-SELECTION] Credential pool at capacity for profile " - f"{profile.id}, trying next profile" - ) - continue else: logger.debug(f"[PROFILE-SELECTION] Profile {profile.id} at capacity: {current_connections}/{profile.max_streams}") diff --git a/frontend/src/api.js b/frontend/src/api.js index 2a357a81..4f58527e 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3,6 +3,7 @@ import useAuthStore from './store/auth'; import useChannelsStore from './store/channels'; import useLogosStore from './store/logos'; import useUserAgentsStore from './store/userAgents'; +import useServerGroupsStore from './store/serverGroups'; import usePlaylistsStore from './store/playlists'; import useEPGsStore from './store/epgs'; import useStreamsStore from './store/streams'; @@ -1305,6 +1306,59 @@ export default class API { } } + static async getServerGroups() { + try { + const response = await request(`${host}/api/m3u/server-groups/`); + + return response; + } catch (e) { + errorNotification('Failed to retrieve server groups', e); + } + } + + static async addServerGroup(values) { + try { + const response = await request(`${host}/api/m3u/server-groups/`, { + method: 'POST', + body: values, + }); + + useServerGroupsStore.getState().addServerGroup(response); + + return response; + } catch (e) { + errorNotification('Failed to create server group', e); + } + } + + static async updateServerGroup(values) { + try { + const { id, ...payload } = values; + const response = await request(`${host}/api/m3u/server-groups/${id}/`, { + method: 'PUT', + body: payload, + }); + + useServerGroupsStore.getState().updateServerGroup(response); + + return response; + } catch (e) { + errorNotification('Failed to update server group', e); + } + } + + static async deleteServerGroup(id) { + try { + await request(`${host}/api/m3u/server-groups/${id}/`, { + method: 'DELETE', + }); + + useServerGroupsStore.getState().removeServerGroups([id]); + } catch (e) { + errorNotification('Failed to delete server group', e); + } + } + static async getPlaylist(id) { try { const response = await request(`${host}/api/m3u/accounts/${id}/`); diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index e27570db..3a4161cb 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -1,6 +1,7 @@ // Modal.js import React, { useEffect, useState } from 'react'; import useUserAgentsStore from '../../store/userAgents'; +import useServerGroupsStore from '../../store/serverGroups'; import M3UProfiles from './M3UProfiles'; import { Box, @@ -43,6 +44,7 @@ const M3U = ({ playlistCreated = false, }) => { const userAgents = useUserAgentsStore((s) => s.userAgents); + const serverGroups = useServerGroupsStore((s) => s.serverGroups); const fetchChannelGroups = useChannelsStore((s) => s.fetchChannelGroups); const fetchEPGs = useEPGsStore((s) => s.fetchEPGs); const fetchCategories = useVODStore((s) => s.fetchCategories); @@ -61,6 +63,7 @@ const M3U = ({ name: '', server_url: '', user_agent: '0', + server_group: '0', is_active: true, max_streams: 0, refresh_interval: 24, @@ -88,6 +91,9 @@ const M3U = ({ server_url: m3uAccount.server_url, max_streams: m3uAccount.max_streams, user_agent: m3uAccount.user_agent ? `${m3uAccount.user_agent}` : '0', + server_group: m3uAccount.server_group + ? `${m3uAccount.server_group}` + : '0', is_active: m3uAccount.is_active, refresh_interval: m3uAccount.refresh_interval, cron_expression: m3uAccount.cron_expression || '', @@ -355,6 +361,24 @@ const M3U = ({ key={form.key('max_streams')} /> + { + const defaultValues = useMemo( + () => ({ + name: serverGroup?.name || '', + max_streams: serverGroup?.max_streams ?? 0, + }), + [serverGroup] + ); + + const { + register, + handleSubmit, + formState: { errors, isSubmitting }, + reset, + setValue, + watch, + } = useForm({ + defaultValues, + resolver: yupResolver(schema), + }); + + const onSubmit = async (values) => { + const payload = { + ...values, + max_streams: Number(values.max_streams), + }; + + if (serverGroup?.id) { + await API.updateServerGroup({ id: serverGroup.id, ...payload }); + } else { + await API.addServerGroup(payload); + } + + reset(); + onClose(); + }; + + useEffect(() => { + reset(defaultValues); + }, [defaultValues, reset]); + + if (!isOpen) { + return null; + } + + const maxStreams = watch('max_streams'); + + return ( + +
+ + + setValue('max_streams', value ?? 0)} + error={errors.max_streams?.message} + /> + + + + + +
+ ); +}; + +export default ServerGroupForm; diff --git a/frontend/src/components/tables/ServerGroupsTable.jsx b/frontend/src/components/tables/ServerGroupsTable.jsx new file mode 100644 index 00000000..e4280037 --- /dev/null +++ b/frontend/src/components/tables/ServerGroupsTable.jsx @@ -0,0 +1,203 @@ +import { useEffect, useMemo, useState } from 'react'; +import API from '../../api'; +import useServerGroupsStore from '../../store/serverGroups'; +import ServerGroupForm from '../forms/ServerGroup'; +import { + ActionIcon, + Box, + Button, + Center, + Flex, + Paper, + Stack, + Text, + Tooltip, +} from '@mantine/core'; +import { SquareMinus, SquarePen, SquarePlus } from 'lucide-react'; +import { CustomTable, useTable } from './CustomTable'; +import useLocalStorage from '../../hooks/useLocalStorage'; + +const RowActions = ({ row, editServerGroup, deleteServerGroup }) => { + return ( + <> + editServerGroup(row.original)} + > + + + deleteServerGroup(row.original.id)} + > + + + + ); +}; + +const ServerGroupsTable = () => { + const [serverGroup, setServerGroup] = useState(null); + const [serverGroupModalOpen, setServerGroupModalOpen] = useState(false); + + const serverGroups = useServerGroupsStore((state) => state.serverGroups); + const fetchServerGroups = useServerGroupsStore( + (state) => state.fetchServerGroups + ); + const [tableSize] = useLocalStorage('table-size', 'default'); + + const columns = useMemo( + () => [ + { + header: 'Name', + accessorKey: 'name', + size: 175, + }, + { + header: 'Max Streams', + accessorKey: 'max_streams', + size: 100, + cell: ({ cell }) => { + const value = cell.getValue(); + return value === 0 ? 'Unlimited' : value; + }, + }, + { + id: 'actions', + header: 'Actions', + size: tableSize == 'compact' ? 50 : 75, + }, + ], + [tableSize] + ); + + const [isLoading, setIsLoading] = useState(true); + + const editServerGroup = (group = null) => { + setServerGroup(group); + setServerGroupModalOpen(true); + }; + + const deleteServerGroup = async (id) => { + await API.deleteServerGroup(id); + }; + + const closeServerGroupForm = () => { + setServerGroup(null); + setServerGroupModalOpen(false); + }; + + useEffect(() => { + fetchServerGroups().finally(() => setIsLoading(false)); + }, [fetchServerGroups]); + + const renderHeaderCell = (header) => { + return ( + + {header.column.columnDef.header} + + ); + }; + + const renderBodyCell = ({ row }) => { + return ( + + ); + }; + + const table = useTable({ + columns, + data: serverGroups, + allRowIds: serverGroups.map((group) => group.id), + bodyCellRenderFns: { + actions: renderBodyCell, + }, + headerCellRenderFns: { + name: renderHeaderCell, + max_streams: renderHeaderCell, + actions: renderHeaderCell, + }, + }); + + if (isLoading) { + return ( +
+ Loading server groups... +
+ ); + } + + return ( + + + + + + + + + + + + + +
+ +
+ + + + + + ); +}; + +export default ServerGroupsTable; diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index 29d46dd8..c3481984 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -14,6 +14,9 @@ import { const UserAgentsTable = React.lazy( () => import('../components/tables/UserAgentsTable.jsx') ); +const ServerGroupsTable = React.lazy( + () => import('../components/tables/ServerGroupsTable.jsx') +); const StreamProfilesTable = React.lazy( () => import('../components/tables/StreamProfilesTable.jsx') ); @@ -135,6 +138,19 @@ const SettingsPage = () => { + + Server Groups + + + }> + + + + + + User-Agents diff --git a/frontend/src/store/auth.jsx b/frontend/src/store/auth.jsx index cfea5d4f..8f362357 100644 --- a/frontend/src/store/auth.jsx +++ b/frontend/src/store/auth.jsx @@ -6,6 +6,7 @@ import useEPGsStore from './epgs'; import useStreamProfilesStore from './streamProfiles'; import useOutputProfilesStore from './outputProfiles'; import useUserAgentsStore from './userAgents'; +import useServerGroupsStore from './serverGroups'; import useUsersStore from './users'; import API from '../api'; import { USER_LEVELS } from '../constants'; @@ -124,6 +125,7 @@ const useAuthStore = create((set, get) => ({ useStreamProfilesStore.getState().fetchProfiles(), useOutputProfilesStore.getState().fetchProfiles(), useUserAgentsStore.getState().fetchUserAgents(), + useServerGroupsStore.getState().fetchServerGroups(), useChannelsStore.getState().fetchChannelIds(), ]); diff --git a/frontend/src/store/serverGroups.jsx b/frontend/src/store/serverGroups.jsx new file mode 100644 index 00000000..42406d70 --- /dev/null +++ b/frontend/src/store/serverGroups.jsx @@ -0,0 +1,40 @@ +import { create } from 'zustand'; +import api from '../api'; + +const useServerGroupsStore = create((set) => ({ + serverGroups: [], + isLoading: false, + error: null, + + fetchServerGroups: async () => { + set({ isLoading: true, error: null }); + try { + const serverGroups = await api.getServerGroups(); + set({ serverGroups: serverGroups || [], isLoading: false }); + } catch (error) { + console.error('Failed to fetch server groups:', error); + set({ error: 'Failed to load server groups.', isLoading: false }); + } + }, + + addServerGroup: (serverGroup) => + set((state) => ({ + serverGroups: [...state.serverGroups, serverGroup], + })), + + updateServerGroup: (serverGroup) => + set((state) => ({ + serverGroups: state.serverGroups.map((group) => + group.id === serverGroup.id ? serverGroup : group + ), + })), + + removeServerGroups: (serverGroupIds) => + set((state) => ({ + serverGroups: state.serverGroups.filter( + (group) => !serverGroupIds.includes(group.id) + ), + })), +})); + +export default useServerGroupsStore; diff --git a/frontend/src/utils/forms/M3uUtils.js b/frontend/src/utils/forms/M3uUtils.js index 7fc7e369..44692b78 100644 --- a/frontend/src/utils/forms/M3uUtils.js +++ b/frontend/src/utils/forms/M3uUtils.js @@ -50,5 +50,9 @@ export const prepareSubmitValues = (values, expDate) => { prepared.user_agent = null; } + if (prepared.server_group == '0') { + prepared.server_group = null; + } + return prepared; }; From ca6ab0cd023c2d315fc08d0963ed0497eb32e343 Mon Sep 17 00:00:00 2001 From: mwhit Date: Sun, 24 May 2026 16:49:12 +0000 Subject: [PATCH 15/76] fix(epg): address third round of reviewer feedback --- apps/epg/serializers.py | 1 + apps/epg/tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 9d4e80c0..a1354bf0 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -37,6 +37,7 @@ class EPGSourceSerializer(serializers.ModelSerializer): 'epg_data_count', 'has_channels', ] + extra_kwargs = {'password': {'write_only': True}} def get_epg_data_count(self, obj): """Return the count of EPG data entries instead of all IDs to prevent large payloads""" diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0ba20864..d5d99446 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2540,17 +2540,17 @@ def fetch_schedules_direct(source, stations_only=False): } # Only fetch programs where MD5 differs from our cached value - programs_to_fetch = [ + programs_to_fetch = { pid for pid in program_ids_needed if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) - ] + } logger.info( f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") program_metadata = {} - program_id_list = programs_to_fetch + program_id_list = list(programs_to_fetch) total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) if program_id_list: From 3e966cd1ddc9eb6dffcc18330f6b10c0702d879c Mon Sep 17 00:00:00 2001 From: Kanishk Sachdev Date: Mon, 25 May 2026 04:31:42 -0400 Subject: [PATCH 16/76] docker: split base image into builder and runtime stages Keeps compilers in the builder; runtime stage copies only the venv and NumPy wheel. --- docker/DispatcharrBase | 62 +++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index 3040e189..61c460ce 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -1,4 +1,10 @@ -FROM lscr.io/linuxserver/ffmpeg:latest +# ============================================================================== +# Stage 1: builder +# Installs the Python toolchain + compilers, creates the virtual environment and +# builds the legacy (CPU-baseline=none) NumPy wheel. None of these compilers end +# up in the final image — only the artifacts below are copied forward. +# ============================================================================== +FROM lscr.io/linuxserver/ffmpeg:latest AS builder ENV DEBIAN_FRONTEND=noninteractive ENV UV_PROJECT_ENVIRONMENT=/dispatcharrpy @@ -15,10 +21,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ && apt-get update \ && apt-get install --no-install-recommends -y \ python3.13 python3.13-dev python3.13-venv libpython3.13 \ - libpcre3 libpcre3-dev libpq-dev procps pciutils \ - nginx comskip \ - vlc-bin vlc-plugin-base \ - build-essential gcc g++ gfortran libopenblas-dev libopenblas0 ninja-build + libpcre3 libpcre3-dev libpq-dev \ + build-essential gcc g++ gfortran libopenblas-dev libopenblas0 ninja-build \ + && rm -rf /var/lib/apt/lists/* # --- Install UV --- COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ @@ -28,11 +33,12 @@ WORKDIR /tmp/build COPY pyproject.toml /tmp/build/ COPY version.py /tmp/build/ COPY README.md /tmp/build/ -RUN uv sync --python 3.13 --no-cache --no-install-project --no-dev && \ - rm -rf /tmp/build +RUN uv sync --python 3.13 --no-cache --no-install-project --no-dev WORKDIR / # --- Build legacy NumPy wheel for old hardware (store for runtime switching) --- +# build/pip are installed into the venv only long enough to produce the wheel, +# then uninstalled so they are not carried into the final image's venv. RUN uv pip install --python $UV_PROJECT_ENVIRONMENT/bin/python --no-cache build pip && \ cd /tmp && \ $UV_PROJECT_ENVIRONMENT/bin/python -m pip download --no-binary numpy --no-deps numpy && \ @@ -40,14 +46,44 @@ RUN uv pip install --python $UV_PROJECT_ENVIRONMENT/bin/python --no-cache build cd numpy-*/ && \ $UV_PROJECT_ENVIRONMENT/bin/python -m build --wheel -Csetup-args=-Dcpu-baseline="none" -Csetup-args=-Dcpu-dispatch="none" && \ mv dist/*.whl /opt/ && \ - cd / && rm -rf /tmp/numpy-* /tmp/*.tar.gz && \ + cd / && rm -rf /tmp/numpy-* /tmp/*.tar.gz /tmp/build && \ uv pip uninstall --python $UV_PROJECT_ENVIRONMENT/bin/python build pip -# --- Clean up build dependencies to reduce image size --- -RUN apt-get remove -y build-essential gcc g++ gfortran libopenblas-dev libpcre3-dev python3.13-dev ninja-build && \ - apt-get autoremove -y --purge && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /root/.cache /tmp/* +# ============================================================================== +# Stage 2: final runtime image +# Same ffmpeg base, but installs only the runtime system libraries (no compilers) +# and copies the prebuilt virtual environment + NumPy wheel from the builder. +# ============================================================================== +FROM lscr.io/linuxserver/ffmpeg:latest AS final + +ENV DEBIAN_FRONTEND=noninteractive +ENV UV_PROJECT_ENVIRONMENT=/dispatcharrpy +ENV VIRTUAL_ENV=/dispatcharrpy +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy + +# --- Install runtime system dependencies (no build toolchain) --- +# python3.13 + libpython3.13 back the copied venv; libpcre3 backs uWSGI; +# libopenblas0 backs the legacy NumPy wheel; ca-certificates/gnupg2/curl/wget +# and software-properties-common are kept for TLS and the AIO runtime apt step. +RUN apt-get update && apt-get install --no-install-recommends -y \ + ca-certificates software-properties-common gnupg2 curl wget \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + python3.13 python3.13-venv libpython3.13 \ + libpcre3 libpq-dev libopenblas0 procps pciutils \ + nginx comskip \ + vlc-bin vlc-plugin-base \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# --- Install UV (used at runtime to swap in the legacy NumPy wheel) --- +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +# --- Copy prebuilt virtual environment and legacy NumPy wheel from the builder --- +COPY --from=builder /dispatcharrpy /dispatcharrpy +COPY --from=builder /opt/numpy-*.whl /opt/ # --- Set up Redis 7.x --- RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \ From 7768d137c428d14db8d454e03d31151921de112a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 14:31:00 -0500 Subject: [PATCH 17/76] fix: remove unnecessary libpq-dev dependency from Dockerfile --- docker/DispatcharrBase | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index 61c460ce..bc2e9b4d 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -21,7 +21,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ && apt-get update \ && apt-get install --no-install-recommends -y \ python3.13 python3.13-dev python3.13-venv libpython3.13 \ - libpcre3 libpcre3-dev libpq-dev \ + libpcre3 libpcre3-dev \ build-essential gcc g++ gfortran libopenblas-dev libopenblas0 ninja-build \ && rm -rf /var/lib/apt/lists/* @@ -73,7 +73,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ && apt-get update \ && apt-get install --no-install-recommends -y \ python3.13 python3.13-venv libpython3.13 \ - libpcre3 libpq-dev libopenblas0 procps pciutils \ + libpcre3 libopenblas0 procps pciutils \ nginx comskip \ vlc-bin vlc-plugin-base \ && apt-get clean && rm -rf /var/lib/apt/lists/* From 3acb5ff0f50ad71a4e4fe26bdf3cf03a0b8b8ccd Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Thu, 28 May 2026 14:11:47 -0400 Subject: [PATCH 18/76] Introduce In-House Template & Compliance Actions --- .github/workflows/issue-template-check.yml | 31 +++++++++++ .github/workflows/pr-compliance-check.yml | 65 ++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 .github/workflows/issue-template-check.yml create mode 100644 .github/workflows/pr-compliance-check.yml diff --git a/.github/workflows/issue-template-check.yml b/.github/workflows/issue-template-check.yml new file mode 100644 index 00000000..676b37df --- /dev/null +++ b/.github/workflows/issue-template-check.yml @@ -0,0 +1,31 @@ +on: + issues: + types: [opened, reopened] + +jobs: + check: + runs-on: ubuntu-latest + steps: + + # Request a bot user token + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.BOT_APP_ID }} + private-key: ${{ secrets.BOT_PRIVATE_KEY }} + + # Do the actual check + - uses: Dispatcharr/repo-bot/actions/template-enforcer@v1 + with: + github-token: ${{ steps.app-token.outputs.token }} + event-type: issue + required-type: "Bug, Feature" + enforcement: close-and-lock + bypass-for-members: true + close-comment: | + ## Issue not opened from a template + + + This issue was closed because it was not opened using one of the available issue templates. + + Please [open a new issue]({new-issue-url}) and select the appropriate template. This helps us triage and address issues efficiently. diff --git a/.github/workflows/pr-compliance-check.yml b/.github/workflows/pr-compliance-check.yml new file mode 100644 index 00000000..135bd9c3 --- /dev/null +++ b/.github/workflows/pr-compliance-check.yml @@ -0,0 +1,65 @@ +on: + pull_request_target: + types: [opened, reopened, edited] + +jobs: + check: + runs-on: ubuntu-latest + steps: + + # Request a bot user token + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.BOT_APP_ID }} + private-key: ${{ secrets.BOT_PRIVATE_KEY }} + + # Delete old bot comments before posting fresh ones + - uses: Dispatcharr/repo-bot/actions/comment-collapse@v1 + with: + github-token: ${{ steps.app-token.outputs.token }} + mode: delete + + # Template + Agreement Check + - uses: Dispatcharr/repo-bot/actions/template-enforcer@v1 + with: + github-token: ${{ steps.app-token.outputs.token }} + event-type: pull_request + required-markers: "## How was it tested?, ## Checklist, - [x] I agree to the [Contributor License Agreement](../blob/dev/CONTRIBUTING.md#contributor-license-agreement)" + enforcement: comment-only + bypass-for-members: true + close-comment: | + ## PR requirements not met + + + Your PR description is missing one or more required sections. Please ensure all of the following are present and filled out exactly as they appear in the [PR template](../blob/dev/.github/pull_request_template.md).: + + - **How was it tested?** Heading (describe how you verified your changes) + - **Checklist** Heading (completed from the [pull request template](../blob/dev/.github/pull_request_template.md)) + - **Contributor License Agreement** Checklist Item (the following item must appear checked in your description): + + > - [x] I agree to the [Contributor License Agreement](../blob/dev/CONTRIBUTING.md#contributor-license-agreement) + + + Edit your PR description to add any missing items. See [CONTRIBUTING.md](../blob/dev/CONTRIBUTING.md) for full contribution guidelines. Pull requests that do not follow the template, or that are wholly AI-generated or CLI-created, will be closed. + + # Target Check + - uses: Dispatcharr/repo-bot/actions/branch-guard@v1 + with: + github-token: ${{ steps.app-token.outputs.token }} + allowed-targets: "dev" + enforcement: comment-only + bypass-for-members: true + comment: | + ## Wrong target branch + + + This PR targets `{target-branch}`, but all contributions must target the `dev` branch. + + > **To fix this:** + > 1. Open the PR and click **Edit** next to the title + > 2. Change the base branch from `{target-branch}` to `dev` + > 3. Save the change + + + Unsure about our contribution guidelines? See [CONTRIBUTING.md](../blob/dev/CONTRIBUTING.md). From 4a45f536bbb0b8dabb3d52608f22d73a197d5af8 Mon Sep 17 00:00:00 2001 From: Jonathan Caicedo Date: Mon, 25 May 2026 11:12:46 -0400 Subject: [PATCH 19/76] perf: replace per-request DB connections with geventpool --- dispatcharr/settings.py | 12 ++++++++---- pyproject.toml | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index e8818108..1b103ccf 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -111,7 +111,7 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries -DATABASE_CONN_MAX_AGE = 0 # Close after each request; gevent makes per-greenlet connections +DATABASE_CONN_MAX_AGE = None # Managed by django-db-geventpool; None = persistent pool connections # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -221,13 +221,17 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "django_db_geventpool.backends.postgresql", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), "HOST": os.environ.get("POSTGRES_HOST", "localhost"), "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), "CONN_MAX_AGE": DATABASE_CONN_MAX_AGE, + "OPTIONS": { + "MAX_CONNS": 20, # Per-worker pool size; 4 workers × 20 = 80 total < pg max_connections=100 + "REUSE_CONNS": 10, # Connections to keep warm between requests + }, } } @@ -238,9 +242,9 @@ else: ("POSTGRES_SSL_KEY", POSTGRES_SSL_KEY), ], "PostgreSQL") - DATABASES["default"]["OPTIONS"] = { + DATABASES["default"]["OPTIONS"].update({ "sslmode": POSTGRES_SSL_MODE, - } + }) if POSTGRES_SSL_CA_CERT: DATABASES["default"]["OPTIONS"]["sslrootcert"] = POSTGRES_SSL_CA_CERT if POSTGRES_SSL_CERT: diff --git a/pyproject.toml b/pyproject.toml index db7efc78..fe7112f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "yt-dlp", "gevent==26.4.0", "psycogreen", + "django-db-geventpool", "daphne", "uwsgi", "django-cors-headers", From 14dd28b7fbdfabec27be075ad22514c90b3b4183 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:27:11 -0500 Subject: [PATCH 20/76] Enhancement: Update psycopg from 2 to 3, remove psycogreen as it's no longer needed with psycopg3. --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe7112f3..a003bc5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.13" dynamic = ["version"] dependencies = [ "Django==6.0.5", - "psycopg2-binary==2.9.12", + "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", "requests==2.33.1", @@ -18,7 +18,6 @@ dependencies = [ "python-vlc", "yt-dlp", "gevent==26.4.0", - "psycogreen", "django-db-geventpool", "daphne", "uwsgi", From 38c89402f896a728573d3e18700ad09904cabaf0 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:27:52 -0500 Subject: [PATCH 21/76] Enhancement: Update psycopg2 to 3. --- core/management/commands/dropdb.py | 7 +++--- core/tests.py | 8 ++++--- dispatcharr/gevent_patch.py | 35 +++++++++--------------------- docker/uwsgi.debug.ini | 4 ++-- docker/uwsgi.dev.ini | 4 ++-- docker/uwsgi.ini | 4 ++-- docker/uwsgi.modular.ini | 4 ++-- 7 files changed, 26 insertions(+), 40 deletions(-) diff --git a/core/management/commands/dropdb.py b/core/management/commands/dropdb.py index d61b9891..09455366 100644 --- a/core/management/commands/dropdb.py +++ b/core/management/commands/dropdb.py @@ -1,6 +1,6 @@ import sys -import psycopg2 -from psycopg2 import sql +import psycopg +from psycopg import sql from django.core.management.base import BaseCommand from django.conf import settings from django.db import connection @@ -38,8 +38,7 @@ class Command(BaseCommand): maintenance_db = 'postgres' try: self.stdout.write("Connecting to maintenance database...") - conn = psycopg2.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, **ssl_kwargs) - conn.autocommit = True + conn = psycopg.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, autocommit=True, **ssl_kwargs) cur = conn.cursor() self.stdout.write(f"Dropping database '{db_name}'...") cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name))) diff --git a/core/tests.py b/core/tests.py index b4770f92..9c758e76 100644 --- a/core/tests.py +++ b/core/tests.py @@ -153,7 +153,7 @@ class EpgIgnoreListsTest(TestCase): class DropDBCommandTlsTest(TestCase): - """Verify dropdb management command passes TLS parameters to psycopg2.""" + """Verify dropdb management command passes TLS parameters to psycopg.""" databases = [] _DB_WITH_TLS = { @@ -184,7 +184,7 @@ class DropDBCommandTlsTest(TestCase): } } - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_passes_ssl_kwargs_when_tls_enabled(self, _inp, _conn, mock_connect): @@ -199,13 +199,14 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, sslmode='verify-full', sslrootcert='/certs/ca.crt', sslcert='/certs/client.crt', sslkey='/certs/client.key', ) - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_no_ssl_kwargs_when_tls_disabled(self, _inp, _conn, mock_connect): @@ -220,4 +221,5 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, ) diff --git a/dispatcharr/gevent_patch.py b/dispatcharr/gevent_patch.py index 86ac92ca..1c9f9727 100644 --- a/dispatcharr/gevent_patch.py +++ b/dispatcharr/gevent_patch.py @@ -1,24 +1,20 @@ """ Loaded via uWSGI's `import = dispatcharr.gevent_patch` directive. -Two things happen here: +gevent stdlib monkey-patching - replaces blocking socket/threading/os +primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` +in the ini should already have done this; calling it again is a safe no-op if +it did, and a necessary fallback if it didn't (e.g. older uWSGI build). -1. gevent stdlib monkey-patching - replaces blocking socket/threading/os - primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` - in the ini should already have done this; calling it again is a safe no-op if - it did, and a necessary fallback if it didn't (e.g. older uWSGI build). - -2. psycogreen - installs a wait callback on psycopg2 so libpq yields to the - gevent hub during I/O instead of blocking the OS thread. - -Without (1), `async_to_sync(channel_layer.group_send)` in send_websocket_update +Without this, `async_to_sync(channel_layer.group_send)` in send_websocket_update calls epoll_wait() directly, which blocks the OS thread and freezes all greenlets -on the worker until the call returns. With (1), select.epoll is replaced by -monkey-patching, which breaks asyncio event loop creation in threadpool threads. +on the worker until the call returns. With monkey-patching, select.epoll is +replaced, which breaks asyncio event loop creation in threadpool threads. send_websocket_update therefore uses a synchronous Redis path in gevent workers instead of asyncio - see _gevent_ws_send() in core/utils.py. -Without (2), psycopg2 network calls pin the worker during slow/stalled queries. +psycopg3 uses Python's socket layer for I/O, so monkey.patch_all() provides +gevent compatibility without any additional driver patching. Celery and Daphne run in separate daemon processes and do not load this module. @@ -40,15 +36,4 @@ if not monkey.is_module_patched("socket"): else: print("[gevent_patch] gevent stdlib monkey-patching already active.", flush=True) -try: - from psycogreen.gevent import patch_psycopg - patch_psycopg() - print("[gevent_patch] psycogreen: psycopg2 patched for gevent.", flush=True) -except ImportError: - print( - "[gevent_patch] WARNING: psycogreen not installed - " - "psycopg2 will block the gevent hub during DB I/O. " - "Run: uv pip install psycogreen", - file=sys.stderr, - flush=True, - ) + diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 9a65db66..d5f577b3 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -42,8 +42,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 852a5114..a481952d 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -45,8 +45,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 76aaebdf..8e8f08dc 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -48,8 +48,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.modular.ini b/docker/uwsgi.modular.ini index 290081b0..7124b10d 100644 --- a/docker/uwsgi.modular.ini +++ b/docker/uwsgi.modular.ini @@ -44,8 +44,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch From 77a3edf63aa84deacbc8a0e1addeb2f6d3eeeb98 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:31:50 -0500 Subject: [PATCH 22/76] perf/fix: migrate DB driver to psycopg3, fix pool settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch engine to postgresql_psycopg3 (PR had wrong engine name, and we're migrating from psycopg2 to psycopg3 as part of this work) - Set CONN_MAX_AGE=0 (required by geventpool; PR had None) - Reduce MAX_CONNS 20→8, REUSE_CONNS 10→3 to avoid exhausting pg max_connections across 4 uWSGI workers + Celery - Add pool=False to explicitly disable Django's native psycopg3 pool --- dispatcharr/settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 1b103ccf..6a6856db 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -111,7 +111,7 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries -DATABASE_CONN_MAX_AGE = None # Managed by django-db-geventpool; None = persistent pool connections +DATABASE_CONN_MAX_AGE = 0 # geventpool intercepts close(); pool handles reuse # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -221,7 +221,7 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django_db_geventpool.backends.postgresql", + "ENGINE": "django_db_geventpool.backends.postgresql_psycopg3", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), @@ -229,8 +229,9 @@ else: "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), "CONN_MAX_AGE": DATABASE_CONN_MAX_AGE, "OPTIONS": { - "MAX_CONNS": 20, # Per-worker pool size; 4 workers × 20 = 80 total < pg max_connections=100 - "REUSE_CONNS": 10, # Connections to keep warm between requests + "MAX_CONNS": 8, # Per-worker pool size; 4 workers × 8 = 32 total < pg max_connections=100 + "REUSE_CONNS": 3, # Connections to keep warm between requests + "pool": False, # Disable Django's native psycopg3 pool; geventpool manages connections }, } } From 6e07dce3f17532fc3519c3ef2b052467c06ead05 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Fri, 29 May 2026 17:05:59 -0400 Subject: [PATCH 23/76] Fix maintainer review items without breaking multi-login rotation. Cap credential-scoped group counters at profile.max_streams (not group.max_streams), skip credential counters when fingerprint resolution fails, and always swap profile counters on update_stream_profile. Restore per-profile-only selection for VOD/live switches so a second stream can use a different login without invalid HTTP errors. Co-authored-by: Cursor --- apps/channels/models.py | 44 +++--- apps/m3u/connection_pool.py | 94 ++++++++----- apps/m3u/tests/test_connection_pool.py | 178 +++++++++++++++++++++---- 3 files changed, 233 insertions(+), 83 deletions(-) diff --git a/apps/channels/models.py b/apps/channels/models.py index 38ebbbc7..154f6fa2 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -712,6 +712,11 @@ class Channel(models.Model): True, ) # Return newly assigned stream and matched profile else: + from apps.m3u.connection_pool import ( + group_has_capacity_for_profile, + profile_has_capacity_for_selection, + ) + # At capacity: try to preempt a lower-impact channel on this profile victim_channel_id = self._pick_channel_to_preempt( profile_id=profile.id, @@ -723,12 +728,21 @@ class Channel(models.Model): logger.info(f"Preempting channel {victim_channel_id} for new stream on profile {profile.id}") # return self.id, profile.id, victim_channel_id - # This profile is at max connections has_streams_but_maxed_out = True - logger.debug( - f"Profile {profile.id} at max connections: " - f"{current_count}/{profile.max_streams}" - ) + if not profile_has_capacity_for_selection(profile, redis_client): + logger.info( + f"Profile {profile.id} at max connections: " + f"{current_count}/{profile.max_streams}, trying next profile" + ) + elif not group_has_capacity_for_profile(profile, redis_client): + logger.info( + f"Profile {profile.id} login or server group pool full, trying next profile" + ) + else: + logger.debug( + f"Profile {profile.id} reservation failed: " + f"{current_count}/{profile.max_streams}" + ) # No available streams - determine specific reason if has_streams_but_maxed_out: @@ -873,25 +887,9 @@ class Channel(models.Model): if current_profile_id == new_profile_id: return True - from apps.m3u.models import M3UAccountProfile - from apps.m3u.connection_pool import ( - get_enforced_server_group_for_profile, - profile_connections_key, - ) + from apps.m3u.connection_pool import profile_connections_key - new_profile = M3UAccountProfile.objects.get(id=new_profile_id) - if get_enforced_server_group_for_profile(new_profile): - # Shared group pool: one active stream uses one group slot regardless - # of which profile supplies the URL. - redis_client.set(f"stream_profile:{stream_id}", new_profile_id) - logger.info( - f"Updated stream {stream_id} profile from {current_profile_id} to " - f"{new_profile_id} (shared group pool unchanged)" - ) - return True - - # Use pipeline for atomic profile switch to prevent counter drift - # if an exception occurs between DECR and INCR + # Profile counters always move on switch; group totals stay unchanged (one stream). old_profile_connections_key = profile_connections_key(current_profile_id) new_profile_connections_key = profile_connections_key(new_profile_id) old_count = int(redis_client.get(old_profile_connections_key) or 0) diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 9817d86c..35c35b0e 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -3,8 +3,9 @@ Shared connection pool enforcement for M3U accounts in the same ServerGroup. Profile selection rotates across M3UAccountProfile rows using each profile's own Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup -with max_streams > 0, the group counter is scoped by provider login fingerprint -so profiles that rewrite to different IPTV credentials keep independent limits. +with max_streams > 0, a credential-scoped counter is checked on reserve/release +so accounts sharing the same provider login share one limit without blocking +unrelated logins on the same group. """ from __future__ import annotations @@ -29,10 +30,12 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) -def server_group_connections_key(group_id: int, fingerprint: Optional[str] = None) -> str: - """Redis key for a manual ServerGroup slot, scoped by provider login.""" - fp = (fingerprint or "unknown")[:16] - return SERVER_GROUP_CONNECTIONS_KEY.format(group_id=group_id, fingerprint=fp) +def server_group_connections_key(group_id: int, fingerprint: str) -> str: + """Redis key for per-credential usage within a ServerGroup.""" + return SERVER_GROUP_CONNECTIONS_KEY.format( + group_id=group_id, + fingerprint=fingerprint[:16], + ) def compute_credential_fingerprint(username: str, password: str) -> Optional[str]: @@ -124,22 +127,30 @@ def get_enforced_server_group_for_profile(profile): return None -def _group_counter_key(profile, group) -> str: - return server_group_connections_key( - group.id, - get_profile_credential_fingerprint(profile), - ) +def _credential_counter_key(profile, group) -> Optional[str]: + fingerprint = get_profile_credential_fingerprint(profile) + if not fingerprint: + return None + return server_group_connections_key(group.id, fingerprint) def get_profile_connection_count(profile, redis_client) -> int: return int(redis_client.get(profile_connections_key(profile.id)) or 0) -def get_group_connection_count(profile, redis_client) -> int: +def get_credential_connection_count(profile, redis_client) -> int: group = get_enforced_server_group_for_profile(profile) if not group: return 0 - return int(redis_client.get(_group_counter_key(profile, group)) or 0) + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return 0 + return int(redis_client.get(cred_key) or 0) + + +def get_group_connection_count(profile, redis_client) -> int: + """Backwards-compatible alias for credential-scoped group usage.""" + return get_credential_connection_count(profile, redis_client) def profile_has_capacity_for_selection(profile, redis_client) -> bool: @@ -151,35 +162,22 @@ def profile_has_capacity_for_selection(profile, redis_client) -> bool: def group_has_capacity_for_profile(profile, redis_client) -> bool: group = get_enforced_server_group_for_profile(profile) - if not group: + if not group or profile.max_streams == 0: return True - return get_group_connection_count(profile, redis_client) < group.max_streams + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return True + return get_credential_connection_count(profile, redis_client) < profile.max_streams def pool_has_capacity_for_profile(profile, redis_client) -> bool: - """Non-mutating check before reserve: profile slot and group slot if applicable.""" + """Non-mutating check before reserve: profile slot and credential slot if applicable.""" return profile_has_capacity_for_selection(profile, redis_client) and group_has_capacity_for_profile( profile, redis_client ) -def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: - group = get_enforced_server_group_for_profile(profile) - if not group: - return True - key = _group_counter_key(profile, group) - group_count = redis_client.incr(key) - if group_count <= group.max_streams: - return True - redis_client.decr(key) - return False - - -def _release_server_group_slot_for_profile(profile, redis_client) -> None: - group = get_enforced_server_group_for_profile(profile) - if not group: - return - key = _group_counter_key(profile, group) +def _safe_decr(redis_client, key: str) -> None: current = int(redis_client.get(key) or 0) if current <= 0: return @@ -188,9 +186,35 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: redis_client.set(key, 0) +def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: + group = get_enforced_server_group_for_profile(profile) + if not group or profile.max_streams == 0: + return True + + cred_key = _credential_counter_key(profile, group) + if not cred_key: + return True + + cred_count = redis_client.incr(cred_key) + if cred_count <= profile.max_streams: + return True + + redis_client.decr(cred_key) + return False + + +def _release_server_group_slot_for_profile(profile, redis_client) -> None: + group = get_enforced_server_group_for_profile(profile) + if not group or profile.max_streams == 0: + return + cred_key = _credential_counter_key(profile, group) + if cred_key: + _safe_decr(redis_client, cred_key) + + def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: """ - Atomically reserve profile + optional group slots (INCR-first). + Atomically reserve profile + optional credential slots (INCR-first). Returns (reserved, profile_count_after_attempt). """ @@ -212,7 +236,7 @@ def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: def release_profile_slot(profile_id: int, redis_client) -> None: - """Release profile and shared group slots after a stream ends.""" + """Release profile and shared credential slots after a stream end.""" from apps.m3u.models import M3UAccountProfile try: diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 44281211..1dbfc2a9 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -5,6 +5,7 @@ from unittest.mock import patch from apps.m3u.connection_pool import ( extract_credentials_from_stream_url, + get_credential_connection_count, get_enforced_server_group_for_profile, get_group_connection_count, get_profile_connection_count, @@ -43,6 +44,37 @@ class FakeRedis: self._data[key] = self._data.get(key, 0) - 1 return self._data[key] + def pipeline(self): + return FakeRedisPipeline(self) + + +class FakeRedisPipeline: + def __init__(self, redis): + self.redis = redis + self._ops = [] + + def decr(self, key): + self._ops.append(("decr", key)) + return self + + def incr(self, key): + self._ops.append(("incr", key)) + return self + + def set(self, key, value): + self._ops.append(("set", key, value)) + return self + + def execute(self): + for op in self._ops: + if op[0] == "decr": + self.redis.decr(op[1]) + elif op[0] == "incr": + self.redis.incr(op[1]) + elif op[0] == "set": + self.redis.set(op[1], op[2]) + self._ops = [] + class ExtractCredentialsTests(TestCase): def test_extract_credentials_from_xc_style_url(self): @@ -66,8 +98,8 @@ class ManualServerGroupTests(TestCase): self.assertEqual(get_enforced_server_group_for_profile(profile), group) - def test_accounts_in_same_group_share_counter(self): - group = ServerGroup.objects.create(name="shared", max_streams=1) + def test_accounts_in_same_group_share_credential_counter(self): + group = ServerGroup.objects.create(name="shared", max_streams=2) account1 = M3UAccount.objects.create( name="XC Account", account_type="XC", @@ -87,6 +119,10 @@ class ManualServerGroupTests(TestCase): ) profile1 = M3UAccountProfile.objects.get(m3u_account=account1, is_default=True) profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile1.max_streams = 1 + profile1.save() + profile2.max_streams = 1 + profile2.save() redis = FakeRedis() reserved1, _ = reserve_profile_slot(profile1, redis) @@ -97,7 +133,6 @@ class ManualServerGroupTests(TestCase): self.assertFalse(group_has_capacity_for_profile(profile2, redis)) def test_profile_rotation_when_default_profile_full(self): - """Pre-pool behavior: try the next profile on the same account.""" account = M3UAccount.objects.create( name="Multi-profile", account_type="XC", @@ -129,20 +164,21 @@ class PoolEnforcementTests(TestCase): self.redis = FakeRedis() self.group = ServerGroup.objects.create( name="test-pool", - max_streams=1, + max_streams=2, ) self.account = M3UAccount.objects.create( name="Test Account", account_type="XC", username="user", password="pass", + server_url="http://xc.example.com", server_group=self.group, max_streams=5, ) self.profile = M3UAccountProfile.objects.get( m3u_account=self.account, is_default=True ) - self.profile.max_streams = 5 + self.profile.max_streams = 1 self.profile.save() def test_reserve_and_release_both_counters(self): @@ -150,31 +186,45 @@ class PoolEnforcementTests(TestCase): self.assertTrue(reserved) self.assertEqual(count, 1) - group_key = server_group_connections_key( + cred_key = server_group_connections_key( self.group.id, get_profile_credential_fingerprint(self.profile), ) profile_key = profile_connections_key(self.profile.id) - self.assertEqual(self.redis._data[group_key], 1) + self.assertEqual(self.redis._data[cred_key], 1) self.assertEqual(self.redis._data[profile_key], 1) release_profile_slot(self.profile.id, self.redis) - self.assertEqual(self.redis._data[group_key], 0) + self.assertEqual(self.redis._data[cred_key], 0) self.assertEqual(self.redis._data[profile_key], 0) - def test_reserve_fails_when_group_at_capacity(self): - group_key = server_group_connections_key( - self.group.id, - get_profile_credential_fingerprint(self.profile), + def test_same_credential_capped_at_profile_max_not_group_max(self): + """Maintainer example: group max=2 but each login only allows 1.""" + account2 = M3UAccount.objects.create( + name="Second Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=self.group, + max_streams=5, ) - self.redis.set(group_key, 1) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile2.max_streams = 1 + profile2.save() - reserved, _count = reserve_profile_slot(self.profile, self.redis) - self.assertFalse(reserved) - self.assertFalse(pool_has_capacity_for_profile(self.profile, self.redis)) + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + self.assertFalse(reserve_profile_slot(profile2, self.redis)[0]) + + fp = get_profile_credential_fingerprint(self.profile) + cred_key = server_group_connections_key(self.group.id, fp) + self.assertEqual(self.redis._data[cred_key], 1) + + def test_different_logins_both_stream_when_group_max_is_one(self): + """Regression: group max=1 must not block a second login on another profile.""" + self.group.max_streams = 1 + self.group.save() - def test_different_logins_in_group_do_not_block_each_other(self): - """Profiles with different provider logins keep separate group counters.""" account = M3UAccount.objects.create( name="Grouped multi-login", account_type="XC", @@ -204,17 +254,96 @@ class PoolEnforcementTests(TestCase): "apps.m3u.connection_pool.get_profile_credential_fingerprint", side_effect=lambda profile: fp_a if profile.id == default.id else fp_b, ): - reserved_default, _ = reserve_profile_slot(default, self.redis) - self.assertTrue(reserved_default) - - reserved_alt, _ = reserve_profile_slot(alt, self.redis) - self.assertTrue(reserved_alt) + self.assertTrue(reserve_profile_slot(default, self.redis)[0]) + self.assertTrue(reserve_profile_slot(alt, self.redis)[0]) key_a = server_group_connections_key(self.group.id, fp_a) key_b = server_group_connections_key(self.group.id, fp_b) self.assertEqual(self.redis._data[key_a], 1) self.assertEqual(self.redis._data[key_b], 1) + def test_no_fingerprint_skips_credential_counter(self): + account = M3UAccount.objects.create( + name="No creds", + account_type="STD", + server_group=self.group, + max_streams=5, + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile.max_streams = 1 + profile.save() + + with patch( + "apps.m3u.connection_pool.get_profile_credential_fingerprint", + return_value=None, + ): + self.assertTrue(reserve_profile_slot(profile, self.redis)[0]) + self.assertEqual(get_credential_connection_count(profile, self.redis), 0) + self.assertEqual(get_group_connection_count(profile, self.redis), 0) + + def test_release_when_profile_row_deleted(self): + profile_id = self.profile.id + fp = get_profile_credential_fingerprint(self.profile) + cred_key = server_group_connections_key(self.group.id, fp) + + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + self.profile.delete() + + release_profile_slot(profile_id, self.redis) + + self.assertEqual(self.redis._data[profile_connections_key(profile_id)], 0) + self.assertEqual(self.redis._data[cred_key], 1) + + +class UpdateStreamProfileTests(TestCase): + def test_switch_updates_profile_counters_when_group_assigned(self): + from apps.channels.models import Channel, Stream + + redis = FakeRedis() + group = ServerGroup.objects.create(name="switch-group", max_streams=2) + account = M3UAccount.objects.create( + name="Switch Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=group, + max_streams=5, + ) + profile_a = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + profile_a.max_streams = 1 + profile_a.save() + profile_b = M3UAccountProfile.objects.create( + m3u_account=account, + name="alt", + is_default=False, + is_active=True, + max_streams=1, + search_pattern="", + replace_pattern="", + ) + + stream = Stream.objects.create(name="Test Stream", m3u_account=account) + channel = Channel.objects.create(channel_number=501, name="Switch Channel") + channel.streams.add(stream) + + reserve_profile_slot(profile_a, redis) + redis.set(f"channel_stream:{channel.id}", stream.id) + redis.set(f"stream_profile:{stream.id}", profile_a.id) + + cred_key = server_group_connections_key( + group.id, get_profile_credential_fingerprint(profile_a) + ) + cred_before = redis._data[cred_key] + + with patch("core.utils.RedisClient.get_client", return_value=redis): + self.assertTrue(channel.update_stream_profile(profile_b.id)) + + self.assertEqual(int(redis.get(f"stream_profile:{stream.id}")), profile_b.id) + self.assertEqual(redis._data[profile_connections_key(profile_a.id)], 0) + self.assertEqual(redis._data[profile_connections_key(profile_b.id)], 1) + self.assertEqual(redis._data[cred_key], cred_before) + class VodProfileSelectionTests(TestCase): def test_get_m3u_profile_skips_default_when_profile_full(self): @@ -243,8 +372,7 @@ class VodProfileSelectionTests(TestCase): ) redis = FakeRedis() - reserved, _ = reserve_profile_slot(default, redis) - self.assertTrue(reserved) + self.assertTrue(reserve_profile_slot(default, redis)[0]) with patch("core.utils.RedisClient.get_client", return_value=redis): result = _get_m3u_profile(account, None, None) From 7d2f0cdab3b735c5dcf90c215d5edf1a08dec911 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 10:03:03 -0500 Subject: [PATCH 24/76] fix(epg): update permission handling for sd_lineups actions in EPGSourceViewSet --- apps/epg/api_views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 2cfa8f47..f368714f 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -22,6 +22,7 @@ from .tasks import refresh_epg_data from .query_utils import parse_text_query from apps.accounts.permissions import ( Authenticated, + IsAdmin, IsStandardUser, permission_classes_by_action, permission_classes_by_method, @@ -47,6 +48,10 @@ class EPGSourceViewSet(viewsets.ModelViewSet): try: return [perm() for perm in permission_classes_by_action[self.action]] except KeyError: + if self.action in ('sd_lineups', 'sd_lineups_search'): + if self.request.method == 'GET': + return [IsStandardUser()] + return [IsAdmin()] return [Authenticated()] def get_queryset(self): From d000e777f1e9464d26aebf3370bbd6bd35f4643a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 10:03:19 -0500 Subject: [PATCH 25/76] fix(epg): prevent updating password field with empty value in EPGSourceSerializer --- apps/epg/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index a1354bf0..608636a9 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -68,6 +68,8 @@ class EPGSourceSerializer(serializers.ModelSerializer): cron_expr = f'{ct.minute} {ct.hour} {ct.day_of_month} {ct.month_of_year} {ct.day_of_week}' instance._cron_expression = cron_expr for attr, value in validated_data.items(): + if attr == 'password' and not value: + continue setattr(instance, attr, value) instance.save() return instance From 51dbab211a71ad45e621044b8c9e1d2c7c370a24 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 10:03:42 -0500 Subject: [PATCH 26/76] fix(epg): update caching logic to use program metadata in fetch_schedules_direct --- apps/epg/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index d5d99446..f45c0751 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2759,7 +2759,7 @@ def fetch_schedules_direct(source, stations_only=False): md5=md5, ) for pid, md5 in schedule_program_md5s.items() - if pid in programs_to_fetch # Only cache what we actually downloaded + if pid in program_metadata # Only cache programs that were actually downloaded ] if md5_records: SDProgramMD5.objects.bulk_create( From 33c1db4c9f65d12654d23a4fb3cb9d48c04eb8c6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 10:04:07 -0500 Subject: [PATCH 27/76] fix(test): ensure password field is excluded from EPGSourceSerializer output --- apps/epg/tests/test_schedules_direct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index e73f5ad3..8b696899 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -70,14 +70,14 @@ class EPGSourceSerializerSDTests(TestCase): self.assertIn('username', data) self.assertEqual(data['username'], 'sduser') - def test_password_in_serializer_fields(self): + def test_password_not_in_serializer_output(self): source = EPGSource.objects.create( name='SD API Key Test', source_type='schedules_direct', password='secret', ) data = EPGSourceSerializer(source).data - self.assertIn('password', data) + self.assertNotIn('password', data) # --------------------------------------------------------------------------- From ae564074056d43f47aa24758744d04fcc1179104 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 10:49:49 -0500 Subject: [PATCH 28/76] fix(epg): enhance logo URL selection and add onscreen episode formatting as well as fix new, live and previously shown tags. --- apps/epg/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index f45c0751..2974cf74 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2226,9 +2226,11 @@ def fetch_schedules_direct(source, stations_only=False): if not sid: continue logo_url = None - logos = station.get('stationLogos') or station.get('logo') or [] + logos = station.get('stationLogo') or station.get('logo') or [] if isinstance(logos, list) and logos: - logo_url = logos[0].get('URL') or logos[0].get('url') + # Prefer 'dark' variant (best on dark backgrounds); fall back to first available + preferred = next((l for l in logos if l.get('category') == 'dark'), logos[0]) + logo_url = preferred.get('URL') or preferred.get('url') elif isinstance(logos, dict): logo_url = logos.get('URL') or logos.get('url') station_map[sid] = { @@ -2673,6 +2675,8 @@ def fetch_schedules_direct(source, stations_only=False): custom_props['season'] = int(season) if episode: custom_props['episode'] = int(episode) + if season and episode: + custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" content_rating = meta.get('contentRating', []) if content_rating: @@ -2706,11 +2710,13 @@ def fetch_schedules_direct(source, stations_only=False): if credits: custom_props['credits'] = credits - if airing.get('isLive'): + if airing.get('liveTapeDelay') == 'Live': custom_props['live'] = True - if airing.get('isNew'): + if airing.get('new'): custom_props['new'] = True - if airing.get('isPremiere'): + else: + custom_props['previously_shown'] = True + if airing.get('premiere'): custom_props['premiere'] = True year = meta.get('movie', {}).get('year') or meta.get('originalAirDate', '')[:4] From 5d2bc2606c74d866790bdf51988641478c9ee8a7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 11:22:04 -0500 Subject: [PATCH 29/76] fix(epg): add force refresh option for EPG data import and update related API methods --- apps/epg/api_views.py | 3 +- apps/epg/tasks.py | 10 +-- frontend/src/api.js | 41 ++++++----- frontend/src/components/tables/EPGsTable.jsx | 71 +++++++++++++++----- 4 files changed, 89 insertions(+), 36 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index f368714f..265532b0 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -999,6 +999,7 @@ class EPGImportAPIView(APIView): def post(self, request, format=None): logger.info("EPGImportAPIView: Received request to import EPG data.") epg_id = request.data.get("id", None) + force = bool(request.data.get("force", False)) # Check if this is a dummy EPG source try: @@ -1013,7 +1014,7 @@ class EPGImportAPIView(APIView): except EPGSource.DoesNotExist: pass # Let the task handle the missing source - refresh_epg_data.delay(epg_id) # Trigger Celery task + refresh_epg_data.delay(epg_id, force=force) # Trigger Celery task logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") return Response( {"success": True, "message": "EPG data refresh initiated."}, diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 2974cf74..d9f2315b 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -307,7 +307,7 @@ def refresh_all_epg_data(): @shared_task(time_limit=14400) -def refresh_epg_data(source_id): +def refresh_epg_data(source_id, force=False): if not acquire_task_lock('refresh_epg_data', source_id): logger.debug(f"EPG refresh for {source_id} already running") return @@ -378,7 +378,7 @@ def refresh_epg_data(source_id): parse_programs_for_source(source) elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source) + fetch_schedules_direct(source, force=force) source.save(update_fields=['updated_at']) # After successful EPG refresh, evaluate DVR series rules to schedule new episodes @@ -1973,7 +1973,7 @@ def fetch_schedules_direct_stations(self, source_id): fetch_schedules_direct(source, stations_only=True) -def fetch_schedules_direct(source, stations_only=False): +def fetch_schedules_direct(source, stations_only=False, force=False): """ Fetch EPG data from the Schedules Direct JSON API and persist it to the EPGData / ProgramData models. @@ -2032,7 +2032,7 @@ def fetch_schedules_direct(source, stations_only=False): # updated_at, which would otherwise incorrectly trigger this guard). Always # allow the first full refresh through so guide data is immediately available. # ------------------------------------------------------------------------- - if not stations_only and source.updated_at: + if not stations_only and not force and source.updated_at: from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() if has_prior_full_refresh: @@ -2053,6 +2053,8 @@ def fetch_schedules_direct(source, stations_only=False): return else: logger.info(f"SD source {source.id}: No prior full refresh detected — skipping 2-hour guard for first full fetch.") + elif force and not stations_only: + logger.info(f"SD source {source.id}: Force flag set — bypassing 2-hour refresh guard.") # ------------------------------------------------------------------------- # Build SD-specific headers diff --git a/frontend/src/api.js b/frontend/src/api.js index 42f010c6..e45fe1a9 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -1632,11 +1632,11 @@ export default class API { } } - static async refreshEPG(id) { + static async refreshEPG(id, force = false) { try { const response = await request(`${host}/api/epg/import/`, { method: 'POST', - body: { id }, + body: { id, force }, }); return response; @@ -3820,7 +3820,9 @@ export default class API { static async getSDLineups(sourceId) { try { - const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`); + const response = await request( + `${host}/api/epg/sources/${sourceId}/sd-lineups/` + ); return response; } catch (e) { errorNotification('Failed to retrieve Schedules Direct lineups', e); @@ -3829,10 +3831,13 @@ export default class API { static async addSDLineup(sourceId, lineup) { try { - const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`, { - method: 'POST', - body: { lineup }, - }); + const response = await request( + `${host}/api/epg/sources/${sourceId}/sd-lineups/`, + { + method: 'POST', + body: { lineup }, + } + ); return response; } catch (e) { errorNotification(`Failed to add lineup ${lineup}`, e); @@ -3841,10 +3846,13 @@ export default class API { static async deleteSDLineup(sourceId, lineup) { try { - const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/`, { - method: 'DELETE', - body: { lineup }, - }); + const response = await request( + `${host}/api/epg/sources/${sourceId}/sd-lineups/`, + { + method: 'DELETE', + body: { lineup }, + } + ); return response; } catch (e) { errorNotification(`Failed to remove lineup ${lineup}`, e); @@ -3853,10 +3861,13 @@ export default class API { static async searchSDLineups(sourceId, country, postalcode) { try { - const response = await request(`${host}/api/epg/sources/${sourceId}/sd-lineups/search/`, { - method: 'POST', - body: { country, postalcode }, - }); + const response = await request( + `${host}/api/epg/sources/${sourceId}/sd-lineups/search/`, + { + method: 'POST', + body: { country, postalcode }, + } + ); return response; } catch (e) { errorNotification('Failed to search Schedules Direct lineups', e); diff --git a/frontend/src/components/tables/EPGsTable.jsx b/frontend/src/components/tables/EPGsTable.jsx index a18dcc7b..8b464ee2 100644 --- a/frontend/src/components/tables/EPGsTable.jsx +++ b/frontend/src/components/tables/EPGsTable.jsx @@ -99,7 +99,6 @@ const RowActions = ({ tableSize, row, editEPG, deleteEPG, refreshEPG }) => { ); }; - const EPGStatusCell = ({ epg }) => { // Direct Zustand subscription scoped to this source only. // This component re-renders whenever its source's progress changes, @@ -138,9 +137,15 @@ const EPGStatusCell = ({ epg }) => { let additionalInfo = ''; if (progress.message) { additionalInfo = progress.message; - } else if (progress.processed !== undefined && progress.channels !== undefined) { + } else if ( + progress.processed !== undefined && + progress.channels !== undefined + ) { additionalInfo = `${progress.processed.toLocaleString()} programs for ${progress.channels} channels`; - } else if (progress.processed !== undefined && progress.total !== undefined) { + } else if ( + progress.processed !== undefined && + progress.total !== undefined + ) { additionalInfo = `${progress.processed.toLocaleString()} / ${progress.total.toLocaleString()}`; } @@ -186,7 +191,8 @@ const EPGStatusCell = ({ epg }) => { // Show success message if (epg.status === 'success') { - const successMessage = epg.last_message || 'EPG data refreshed successfully'; + const successMessage = + epg.last_message || 'EPG data refreshed successfully'; return ( { if (epg.status === 'idle' && epg.last_message) { return ( - + {epg.last_message} @@ -230,6 +231,8 @@ const EPGsTable = () => { const [epgToDelete, setEpgToDelete] = useState(null); const [data, setData] = useState([]); const [deleting, setDeleting] = useState(false); + const [confirmSDRefreshOpen, setConfirmSDRefreshOpen] = useState(false); + const [sdRefreshTarget, setSDRefreshTarget] = useState(null); const epgs = useEPGsStore((s) => s.epgs); @@ -274,9 +277,9 @@ const EPGsTable = () => { size: 130, cell: ({ cell }) => { const typeMap = { - 'xmltv': 'XMLTV', - 'schedules_direct': 'Schedules Direct', - 'dummy': 'Custom Dummy', + xmltv: 'XMLTV', + schedules_direct: 'Schedules Direct', + dummy: 'Custom Dummy', }; return typeMap[cell.getValue()] || cell.getValue(); }, @@ -439,13 +442,27 @@ const EPGsTable = () => { } }; - const refreshEPG = async (id) => { - await API.refreshEPG(id); + const refreshEPG = async (id, force = false) => { + await API.refreshEPG(id, force); notifications.show({ title: 'EPG refresh initiated', }); }; + const handleRefreshEPG = (id) => { + const epgObj = epgs[id]; + if ( + epgObj?.source_type === 'schedules_direct' && + epgObj?.updated_at && + Date.now() - new Date(epgObj.updated_at).getTime() < 2 * 60 * 60 * 1000 + ) { + setSDRefreshTarget(id); + setConfirmSDRefreshOpen(true); + return; + } + refreshEPG(id); + }; + const closeEPGForm = () => { setEPG(null); setEPGModalOpen(false); @@ -478,7 +495,7 @@ const EPGsTable = () => { row={row} editEPG={editEPG} deleteEPG={deleteEPG} - refreshEPG={refreshEPG} + refreshEPG={handleRefreshEPG} /> ); } @@ -686,6 +703,28 @@ const EPGsTable = () => { onClose={closeDummyEPGForm} /> + setConfirmSDRefreshOpen(false)} + onConfirm={() => { + setConfirmSDRefreshOpen(false); + refreshEPG(sdRefreshTarget, true); + }} + title="Refresh Schedules Direct Early?" + message={ +
+

This source was refreshed less than 2 hours ago.

+

+ Schedules Direct rate-limits requests per account. Refreshing too + frequently may cause your account to be temporarily blocked. +

+

Are you sure you want to force a refresh now?

+
+ } + confirmLabel="Refresh Anyway" + cancelLabel="Cancel" + /> + setConfirmDeleteOpen(false)} From 712fb08563ff6811a1c116e89b73228df8a4853f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 31 May 2026 12:02:58 -0500 Subject: [PATCH 30/76] fix(epg): prune expired SDScheduleMD5 and ProgramData records to optimize storage. Add program_id field to programdata table for tracking. --- ...ce_api_key_epgsource_password_and_more.py} | 7 +- apps/epg/models.py | 1 + apps/epg/tasks.py | 105 +++++++++++++----- 3 files changed, 87 insertions(+), 26 deletions(-) rename apps/epg/migrations/{0023_schedules_direct.py => 0023_remove_epgsource_api_key_epgsource_password_and_more.py} (90%) diff --git a/apps/epg/migrations/0023_schedules_direct.py b/apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py similarity index 90% rename from apps/epg/migrations/0023_schedules_direct.py rename to apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py index b0e835a9..55c9e58e 100644 --- a/apps/epg/migrations/0023_schedules_direct.py +++ b/apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.4 on 2026-05-19 23:55 +# Generated by Django 6.0.5 on 2026-05-31 16:48 import django.db.models.deletion from django.db import migrations, models @@ -25,6 +25,11 @@ class Migration(migrations.Migration): name='username', field=models.CharField(blank=True, help_text='Username for credential-based EPG sources (e.g. Schedules Direct)', max_length=255, null=True), ), + migrations.AddField( + model_name='programdata', + name='program_id', + field=models.CharField(blank=True, help_text='Schedules Direct programID (e.g. EP123456789). Null for XMLTV sources.', max_length=64, null=True), + ), migrations.AlterField( model_name='epgsource', name='custom_properties', diff --git a/apps/epg/models.py b/apps/epg/models.py index c9870d0b..c8bd60ec 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -144,6 +144,7 @@ class ProgramData(models.Model): sub_title = models.TextField(blank=True, null=True) description = models.TextField(blank=True, null=True) tvg_id = models.CharField(max_length=255, null=True, blank=True) + program_id = models.CharField(max_length=64, null=True, blank=True, help_text='Schedules Direct programID (e.g. EP123456789). Null for XMLTV sources.') custom_properties = models.JSONField(default=dict, blank=True, null=True) def __str__(self): diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index d9f2315b..9ebb6400 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -17,6 +17,7 @@ import zipfile from celery import shared_task from django.conf import settings from django.db import connection, transaction +from django.db.models import Q from django.utils import timezone from apps.channels.models import Channel from core.models import UserAgent, CoreSettings @@ -102,7 +103,7 @@ def _open_xmltv_file(file_path: str): Prepends a block that declares all 252 HTML 4 named entities so lxml/libxml2 resolves references like é correctly instead of silently dropping them in recovery mode. This involves zero - disk I/O — the DOCTYPE is streamed in-memory before the file content. + disk I/O (the DOCTYPE is streamed in-memory before the file content). If the file already contains a declaration the file is returned unchanged; a second DOCTYPE would be invalid XML. @@ -126,7 +127,7 @@ def _open_xmltv_file(file_path: str): f.seek(decl_end + 2) return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f) - # No XML declaration — insert DOCTYPE at the very start of the file. + # No XML declaration found; insert DOCTYPE at the very start of the file. f.seek(0) return _PrependStream(_HTML_ENTITY_DOCTYPE, f) @@ -1317,14 +1318,14 @@ def parse_channels_only(source): @shared_task(time_limit=3600, soft_time_limit=3500) def parse_programs_for_tvg_id(epg_id): - # Skip XMLTV file parsing for Schedules Direct sources — program data is + # Skip XMLTV file parsing for Schedules Direct sources. Program data is # fetched and persisted directly by fetch_schedules_direct(). try: from apps.epg.models import EPGData epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") - return "Skipped — Schedules Direct source" + return "Skipped (Schedules Direct source)" except Exception as e: logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") @@ -1979,7 +1980,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): EPGData / ProgramData models. Authentication flow (as required by the SD API specification): - 1. POST credentials to the token endpoint — password must be SHA1-hashed + 1. POST credentials to the token endpoint (password must be SHA1-hashed as required by the Schedules Direct API specification. 2. Use the returned token for all subsequent requests via the 'token' header. 3. Tokens are valid for 24 hours; SD returns the current valid token if one @@ -1989,7 +1990,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): 1. Fetch subscribed lineups for the account. 2. Fetch station metadata for each lineup. 3. Persist station metadata to EPGData. - 4. If stations_only=True, stop here — used on initial source creation so + 4. If stations_only=True, stop here. Used on initial source creation so the user can run Auto-match EPG before the full program fetch. 5. Fetch schedule grids in 14-day date-batched requests per station. 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). @@ -2041,7 +2042,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if elapsed < min_interval_seconds: remaining_minutes = int((min_interval_seconds - elapsed) / 60) msg = ( - f"Schedules Direct refresh skipped — minimum 2-hour interval not reached. " + f"Schedules Direct refresh skipped. Minimum 2-hour interval not reached. " f"Last refreshed {int(elapsed / 60)} minutes ago. " f"Please wait {remaining_minutes} more minute(s)." ) @@ -2052,9 +2053,9 @@ def fetch_schedules_direct(source, stations_only=False, force=False): _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) return else: - logger.info(f"SD source {source.id}: No prior full refresh detected — skipping 2-hour guard for first full fetch.") + logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") elif force and not stations_only: - logger.info(f"SD source {source.id}: Force flag set — bypassing 2-hour refresh guard.") + logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") # ------------------------------------------------------------------------- # Build SD-specific headers @@ -2074,7 +2075,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): return h # ------------------------------------------------------------------------- - # Step 1 — Authenticate and obtain session token + # Step 1: Authenticate and obtain session token # The SD API requires the password to be SHA1-hashed before transmission. # This is a requirement of the Schedules Direct API specification, not an # architectural choice. @@ -2102,7 +2103,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): elif auth_code == 4004: msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." elif auth_code == 4009: - msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours — check for misconfiguration." + msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours. Check for misconfiguration." elif auth_code == 4001: msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." elif auth_code == 4008: @@ -2138,7 +2139,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): return # ------------------------------------------------------------------------- - # Step 2 — Check account status (respect OFFLINE system status) + # Step 2: Check account status (respect OFFLINE system status) # ------------------------------------------------------------------------- try: status_response = requests.get( @@ -2165,7 +2166,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") # ------------------------------------------------------------------------- - # Step 3 — Fetch subscribed lineups and build station map + # Step 3: Fetch subscribed lineups and build station map # ------------------------------------------------------------------------- _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") try: @@ -2175,7 +2176,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): timeout=30, ) # SD returns 400 with code 4102 when no lineups are configured. - # This is a valid account state — the user needs to add lineups via + # This is a valid account state. The user needs to add lineups via # the Manage Lineups UI. Treat as idle rather than error. if lineups_response.status_code == 400: sd_data = lineups_response.json() @@ -2256,7 +2257,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): logger.info(f"Built station map with {len(station_map)} stations.") # ------------------------------------------------------------------------- - # Step 4 — Persist station metadata to EPGData + # Step 4: Persist station metadata to EPGData # ------------------------------------------------------------------------- source.status = EPGSource.STATUS_PARSING source.last_message = f"Syncing {len(station_map)} stations..." @@ -2313,7 +2314,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) } - # Station sync complete — send progress continuing into programs phase + # Station sync complete. Send progress update before continuing into programs phase. # We deliberately do NOT send parsing_channels at 100 with status=success here # because that would cause the frontend to mark the source as complete and # stop rendering progress updates for the subsequent program fetch phases. @@ -2321,7 +2322,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") # ------------------------------------------------------------------------- - # Stations-only mode — used on initial source creation. + # Stations-only mode. Used on initial source creation. # Stop here so the user can run Auto-match EPG before the full program fetch. # ------------------------------------------------------------------------- if stations_only: @@ -2340,10 +2341,10 @@ def fetch_schedules_direct(source, stations_only=False, force=False): return # ------------------------------------------------------------------------- - # Step 5 — MD5-delta schedule fetch + # Step 5: MD5-delta schedule fetch # First fetch MD5 hashes for all stations/dates. Compare against our # locally cached hashes to determine which schedules have changed. - # Only download schedules that have actually changed — this minimises + # Only download schedules that have actually changed; this minimises # API calls against SD's rate-limited endpoints. # ------------------------------------------------------------------------- from apps.epg.models import SDScheduleMD5 @@ -2354,6 +2355,12 @@ def fetch_schedules_direct(source, stations_only=False, force=False): today = date.today() date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] + # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. + # These accumulate one row per station per day and are never useful once past. + pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] + if pruned_sched_md5_count: + logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") + # Fetch MD5 hashes for all stations in batches of 5000 STATION_BATCH_SIZE = 5000 server_md5s = {} # (station_id, date) -> {md5, last_modified} @@ -2407,7 +2414,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): program_ids_needed = set() if not changed_by_station: - logger.info("No schedule changes detected — skipping schedule and program downloads.") + logger.info("No schedule changes detected, skipping schedule and program downloads.") _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message="No schedule changes detected since last refresh. Guide data is up to date.") source.status = EPGSource.STATUS_SUCCESS @@ -2519,7 +2526,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): return # ------------------------------------------------------------------------- - # Step 6 — MD5-delta program metadata fetch + # Step 6: MD5-delta program metadata fetch # The schedule response includes an MD5 hash per program airing. # Compare against our cached program MD5s to only download programs # whose metadata has changed since our last fetch. @@ -2599,7 +2606,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): gc.collect() # ------------------------------------------------------------------------- - # Step 7 — Build ProgramData records and persist atomically + # Step 7: Build ProgramData records and persist atomically # ------------------------------------------------------------------------- logger.info("Building program records...") _sd_send_ws_sync(source.id, "parsing_programs", 80) @@ -2733,6 +2740,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): sub_title=episode_title or None, description=desc or None, tvg_id=sid, + program_id=pid, custom_properties=custom_props or None, )) total_programs += 1 @@ -2742,14 +2750,37 @@ def fetch_schedules_direct(source, stations_only=False, force=False): _sd_send_ws_sync(source.id, "parsing_programs", 88) - # Atomic delete + bulk insert — same pattern as parse_programs_for_source + # Build a map of epg_db_id -> list of (day_start_utc, day_end_utc) for each changed date. + # Only programs that fall within changed station/date pairs will be deleted and replaced; + # programs for unchanged stations or unchanged dates are left intact. + import datetime as dt_module + epg_changed_date_ranges = {} + for sid, changed_date_strs in changed_by_station.items(): + epg_db_id = epg_id_map.get(sid) + if not epg_db_id or epg_db_id not in mapped_epg_ids: + continue + ranges = [] + for ds in changed_date_strs: + d = dt_module.date.fromisoformat(ds) + day_start = datetime(d.year, d.month, d.day, tzinfo=dt_timezone.utc) + ranges.append((day_start, day_start + timedelta(days=1))) + if ranges: + epg_changed_date_ranges[epg_db_id] = ranges + + # Atomic delete (surgical) + bulk insert BATCH_SIZE = 1000 try: with transaction.atomic(): with connection.cursor() as cursor: cursor.execute("SET LOCAL statement_timeout = '10min'") - deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] - logger.debug(f"Deleted {deleted_count} existing SD programs.") + total_deleted = 0 + for epg_db_id, day_ranges in epg_changed_date_ranges.items(): + q = Q() + for day_start, day_end in day_ranges: + q |= Q(start_time__gte=day_start, start_time__lt=day_end) + cnt = ProgramData.objects.filter(epg_id=epg_db_id).filter(q).delete()[0] + total_deleted += cnt + logger.debug(f"Deleted {total_deleted} changed SD programs across {len(epg_changed_date_ranges)} stations.") for i in range(0, len(all_programs_to_create), BATCH_SIZE): ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) @@ -2790,6 +2821,30 @@ def fetch_schedules_direct(source, stations_only=False, force=False): all_programs_to_create = None gc.collect() + # Prune ProgramData whose end_time has passed. With surgical per-date deletes, + # programs from dates that have rolled off the window are never explicitly removed. + today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) + try: + expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] + if expired_count: + logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") + except Exception as prune_err: + logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") + + # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. + try: + live_program_ids = set( + ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) + .values_list('program_id', flat=True) + ) + pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( + program_id__in=live_program_ids + ).delete()[0] + if pruned_prog_md5_count: + logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") + except Exception as prune_err: + logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + # ------------------------------------------------------------------------- # Done # ------------------------------------------------------------------------- From 37e11c8fc49478d11880d07fc62942318c85bb8d Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 3 Jun 2026 12:36:07 -0400 Subject: [PATCH 31/76] migrate official plugin repo URL to GitHub Pages --- .../0003_update_official_repo_url.py | 25 +++++++++++++++++++ apps/plugins/models.py | 4 +-- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 apps/plugins/migrations/0003_update_official_repo_url.py diff --git a/apps/plugins/migrations/0003_update_official_repo_url.py b/apps/plugins/migrations/0003_update_official_repo_url.py new file mode 100644 index 00000000..b2e87745 --- /dev/null +++ b/apps/plugins/migrations/0003_update_official_repo_url.py @@ -0,0 +1,25 @@ +from django.db import migrations + +def update_url(apps, schema_editor): + PluginRepo = apps.get_model("plugins", "PluginRepo") + PluginRepo.objects.filter(is_official=True).update( + url="https://dispatcharr.github.io/Plugins/manifest.json" + ) + + +def revert_url(apps, schema_editor): + PluginRepo = apps.get_model("plugins", "PluginRepo") + PluginRepo.objects.filter(is_official=True).update( + url="https://raw.githubusercontent.com/Dispatcharr/Plugins/releases/manifest.json" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("plugins", "0002_pluginrepo"), + ] + + operations = [ + migrations.RunPython(update_url, revert_url), + ] diff --git a/apps/plugins/models.py b/apps/plugins/models.py index f1960fd9..76772a96 100644 --- a/apps/plugins/models.py +++ b/apps/plugins/models.py @@ -36,9 +36,7 @@ class PluginConfig(models.Model): return f"{self.name} ({self.key})" -OFFICIAL_REPO_URL = ( - "https://raw.githubusercontent.com/Dispatcharr/Plugins/releases/manifest.json" -) +OFFICIAL_REPO_URL = "https://dispatcharr.github.io/Plugins/manifest.json" class PluginRepo(models.Model): From b6afcafc984c434deef97d1499ba4548bede2d15 Mon Sep 17 00:00:00 2001 From: mwhit Date: Thu, 4 Jun 2026 03:40:22 +0000 Subject: [PATCH 32/76] feat: SD data enrichment, artwork improvement, and bug fixes - Fix channel_group_id not updating on M3U stream refresh - Fix SD content rating region selection (country-aware filtering) - Fix SD season/episode metadata iteration - Add SD data enrichment: entityType, showType, contentAdvisory, originalAirDate, country, runtime, star ratings, event details, character names - Cap cast to top-billed, remove role noise - Switch artwork preference to Banner-L1 (branded key art) - Add content advisory, sports details, runtime to program detail modal - Logo cache-busting for browser cache invalidation - SD API compliance: rolling 24h lineup reset, tokenExpires caching, error caching --- ProgramDetailModal.jsx | 458 +++ api_views.py | 1285 ++++++++ apps/epg/tasks.py | 908 +++--- apps/m3u/tasks.py | 7049 +++++++++++++++++++--------------------- docker-compose.yml | 43 + serializers.py | 866 +++++ tasks.py | 3594 ++++++++++++++++++++ 7 files changed, 10067 insertions(+), 4136 deletions(-) create mode 100644 ProgramDetailModal.jsx create mode 100644 api_views.py create mode 100644 docker-compose.yml create mode 100644 serializers.py create mode 100644 tasks.py diff --git a/ProgramDetailModal.jsx b/ProgramDetailModal.jsx new file mode 100644 index 00000000..54289ba4 --- /dev/null +++ b/ProgramDetailModal.jsx @@ -0,0 +1,458 @@ +import React, { useState, useEffect, useCallback } from 'react'; +import { + Badge, + Button, + Divider, + Flex, + Group, + Image, + Modal, + Stack, + Text, + Title, +} from '@mantine/core'; +import { Calendar, Video } from 'lucide-react'; +import API from '../api'; +import useVideoStore from '../store/useVideoStore'; +import useSettingsStore from '../store/settings'; +import { getShowVideoUrl } from '../utils/cards/RecordingCardUtils'; +import { formatSeasonEpisode } from '../utils/guideUtils'; +import { + format, + initializeTime, + diff, + useDateTimeFormat, +} from '../utils/dateTimeUtils'; +import { imdbUrl, tmdbUrl } from '../utils/externalUrls'; + +const overlayProps = { color: '#000', backgroundOpacity: 0.55, blur: 0 }; + +function formatDurationMinutes(startTime, endTime) { + if (!startTime || !endTime) return null; + const start = initializeTime(startTime); + const end = initializeTime(endTime); + const minutes = diff(end, start, 'minute'); + if (minutes >= 60) { + const hours = Math.floor(minutes / 60); + const mins = minutes % 60; + return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`; + } + return `${minutes}m`; +} + +function resolveImageUrl(detail) { + if (detail?.tmdb_poster_url) return detail.tmdb_poster_url; + if (detail?.poster_url) return detail.poster_url; + if (detail?.images?.length > 0) return detail.images[0].url; + if (detail?.icon) return detail.icon; + return null; +} + +function formatCredits(actors) { + if (!actors?.length) return null; + return actors + .map((a) => (a.role ? `${a.name} (${a.role})` : a.name)) + .join(', '); +} + +export default function ProgramDetailModal({ + program, + channel, + opened, + onClose, + onRecord, +}) { + const [detailData, setDetailData] = useState(null); + + const showVideo = useVideoStore((s) => s.showVideo); + const env_mode = useSettingsStore((s) => s.environment.env_mode); + const { timeFormat } = useDateTimeFormat(); + + useEffect(() => { + if (!opened || !program) { + setDetailData(null); + return; + } + + // Dummy programs may use UUID-style IDs that aren't real DB PKs + const programId = program.id; + if (!programId || typeof programId === 'string') { + setDetailData(null); + return; + } + + let cancelled = false; + + API.getProgramDetail(programId) + .then((data) => { + if (!cancelled) setDetailData(data); + }) + .catch(() => { + if (!cancelled) setDetailData(null); + }); + + return () => { + cancelled = true; + }; + }, [opened, program?.id]); + + const handleWatchLive = useCallback(() => { + if (!channel) return; + showVideo(getShowVideoUrl(channel, env_mode), 'live', { + name: channel.name, + }); + onClose(); + }, [channel, env_mode, showVideo, onClose]); + + const handleRecord = useCallback(() => { + if (onRecord) onRecord(program); + }, [onRecord, program]); + + if (!program) return null; + + // Merge detail data with grid data (detail enriches, grid is baseline) + const d = detailData || {}; + const seasonEpisodeLabel = formatSeasonEpisode( + d.season ?? program.season, + d.episode ?? program.episode + ); + const hasBadges = + seasonEpisodeLabel || + program.is_live || + program.is_new || + d.is_previously_shown || + program.is_premiere || + program.is_finale || + d.video_quality || + d.rating; + + const categories = d.categories || []; + const credits = d.credits || {}; + const hasCredits = + credits.actors?.length > 0 || + credits.directors?.length > 0 || + credits.writers?.length > 0; + const starRatings = d.star_ratings || []; + const description = d.description || program.description; + const subtitle = d.sub_title ?? program.sub_title; + const posterUrl = + resolveImageUrl(d) || program?.custom_properties?.icon || null; + const duration = formatDurationMinutes(program.start_time, program.end_time); + const programStart = initializeTime(program.start_time || program.startMs); + const programEnd = initializeTime(program.end_time || program.endMs); + + return ( + + {channel.channel_number ? `${channel.channel_number} - ` : ''} + {channel.name} + + ) : null + } + size="lg" + centered + overlayProps={overlayProps} + zIndex={9999} + > + + + {posterUrl && ( + { + e.currentTarget.style.display = 'none'; + }} + /> + )} + + + + + {program.title} + + + {subtitle && ( + + {subtitle} + + )} + + {hasBadges && ( + + {program.is_live && ( + + LIVE + + )} + {program.is_new && ( + + NEW + + )} + {d.is_previously_shown && ( + + RERUN + + )} + {program.is_premiere && ( + + PREMIERE + + )} + {program.is_finale && ( + + FINALE + + )} + {seasonEpisodeLabel && ( + + {seasonEpisodeLabel} + + )} + {d.rating && ( + + {d.rating} + + )} + {d.video_quality && ( + + {d.video_quality} + + )} + + )} + + + + {format(programStart, timeFormat)} –{' '} + {format(programEnd, timeFormat)} + + {duration && ( + <> + + · + + + {duration} + + + )} + {categories.length > 0 && ( + <> + + · + + + {categories.join(', ')} + + + )} + + + + + {program.isLive && channel && ( + + )} + {!program.isPast && ( + + )} + + + + + {description && ( + <> + + + {description} + + + )} + + {d.content_advisory?.length > 0 && ( + + {d.content_advisory.join(', ')} + + )} + + {d.event_details && ( + <> + + + {d.event_details.venue100 && ( + + Venue: + {d.event_details.venue100} + + )} + {d.event_details.teams?.length > 0 && ( + + Teams: + {d.event_details.teams.map((t, i) => ( + + {i > 0 ? ' vs ' : ''} + {t.name}{t.isHome ? ' (Home)' : ''} + + ))} + + )} + + + )} + + {hasCredits && ( + <> + + + {credits.actors?.length > 0 && ( + + + Cast:{' '} + + {formatCredits(credits.actors)} + + )} + {credits.directors?.length > 0 && ( + + + Director:{' '} + + {credits.directors.join(', ')} + + )} + {credits.writers?.length > 0 && ( + + + Writer:{' '} + + {credits.writers.join(', ')} + + )} + + + )} + + {(d.country || + d.language || + d.original_air_date || + d.runtime || + (d.production_date && d.is_previously_shown) || + starRatings.length > 0) && ( + <> + + + {d.country && ( + + + Country:{' '} + + {d.country} + + )} + {d.language && ( + + + Language:{' '} + + {d.language} + + )} + {d.production_date && d.is_previously_shown && ( + + + First aired:{' '} + + {d.production_date} + + )} + {d.runtime && ( + + + Runtime:{' '} + + {d.runtime} {d.runtime_units || 'min'} + + )} + {d.original_air_date && ( + + + Original Air:{' '} + + {d.original_air_date} + + )} + {starRatings.map((sr, i) => ( + + ★ {sr.value} + {sr.system ? ` (${sr.system})` : ''} + + ))} + + + )} + + {(d.imdb_id || d.tmdb_id) && ( + + {d.imdb_id && ( + + IMDb ↗ + + )} + {d.tmdb_id && ( + + TMDB ↗ + + )} + + )} + + + ); +} diff --git a/api_views.py b/api_views.py new file mode 100644 index 00000000..b7fffb05 --- /dev/null +++ b/api_views.py @@ -0,0 +1,1285 @@ +import logging, os, re +from rest_framework import viewsets, status, serializers +from rest_framework.permissions import AllowAny +from rest_framework.pagination import PageNumberPagination +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.decorators import action +from drf_spectacular.utils import extend_schema, OpenApiParameter, inline_serializer +from drf_spectacular.types import OpenApiTypes +from django.db.models import Q +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from datetime import timedelta +from .models import EPGSource, ProgramData, EPGData +from .serializers import ( + ProgramDataSerializer, + ProgramDetailSerializer, + EPGSourceSerializer, + EPGDataSerializer, + ProgramSearchResultSerializer, +) +from .tasks import refresh_epg_data +from .query_utils import parse_text_query +from apps.accounts.permissions import ( + Authenticated, + IsAdmin, + IsStandardUser, + permission_classes_by_action, + permission_classes_by_method, +) + +logger = logging.getLogger(__name__) + + +# ───────────────────────────── +# 1) EPG Source API (CRUD) +# ───────────────────────────── +class EPGSourceViewSet(viewsets.ModelViewSet): + """ + API endpoint that allows EPG sources to be viewed or edited. + """ + + queryset = EPGSource.objects.select_related( + "refresh_task__crontab", "refresh_task__interval" + ).all() + serializer_class = EPGSourceSerializer + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + if self.action in ('sd_lineups', 'sd_lineups_search'): + if self.request.method == 'GET': + return [IsStandardUser()] + return [IsAdmin()] + return [Authenticated()] + + def get_queryset(self): + from django.db.models import Exists, OuterRef + from apps.channels.models import Channel + return EPGSource.objects.select_related( + "refresh_task__crontab", "refresh_task__interval" + ).annotate( + has_channels=Exists( + Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk')) + ) + ) + + def list(self, request, *args, **kwargs): + logger.debug("Listing all EPG sources.") + return super().list(request, *args, **kwargs) + + @action(detail=False, methods=["post"]) + def upload(self, request): + if "file" not in request.FILES: + return Response( + {"error": "No file uploaded"}, status=status.HTTP_400_BAD_REQUEST + ) + + file = request.FILES["file"] + file_name = file.name + file_path = os.path.join("/data/uploads/epgs", file_name) + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb+") as destination: + for chunk in file.chunks(): + destination.write(chunk) + + new_obj_data = request.data.copy() + new_obj_data["file_path"] = file_path + + serializer = self.get_serializer(data=new_obj_data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + + def partial_update(self, request, *args, **kwargs): + """Handle partial updates with special logic for is_active field""" + instance = self.get_object() + + # Check if we're toggling is_active + if ( + "is_active" in request.data + and instance.is_active != request.data["is_active"] + ): + # Set appropriate status based on new is_active value + if request.data["is_active"]: + request.data["status"] = "idle" + else: + request.data["status"] = "disabled" + + # Continue with regular partial update + return super().partial_update(request, *args, **kwargs) + + def create(self, request, *args, **kwargs): + """ + Override create to validate Schedules Direct credentials before saving. + If source_type is 'schedules_direct', authenticates against SD's /token + endpoint. Returns 400 if credentials are invalid — no source is created. + """ + source_type = request.data.get('source_type', '') + if source_type == 'schedules_direct': + import hashlib + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + username = (request.data.get('username') or '').strip() + password = (request.data.get('password') or '').strip() + + if not username or not password: + return Response( + {"error": "Username and password are required for Schedules Direct."}, + status=status.HTTP_400_BAD_REQUEST + ) + + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + try: + auth_response = http_requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers={ + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + }, + timeout=15, + ) + auth_data = auth_response.json() + if not auth_data.get('token'): + error_msg = auth_data.get('message', 'Authentication failed. Check your credentials.') + return Response( + {"error": error_msg}, + status=status.HTTP_400_BAD_REQUEST + ) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Could not reach Schedules Direct: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + return super().create(request, *args, **kwargs) + + + def _sd_authenticate(self, source): + """ + Authenticate with Schedules Direct using stored credentials. + Returns (token, None) on success or (None, Response) on failure. + """ + import hashlib + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + username = (source.username or '').strip() + password = (source.password or '').strip() + if not username or not password: + return None, Response( + {"error": "Username and password are required."}, + status=status.HTTP_400_BAD_REQUEST + ) + + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + try: + auth_response = http_requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers={'Content-Type': 'application/json', 'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, + timeout=15, + ) + auth_response.raise_for_status() + token = auth_response.json().get('token') + if not token: + return None, Response( + {"error": "Authentication failed. Check your credentials."}, + status=status.HTTP_401_UNAUTHORIZED + ) + return token, None + except http_requests.exceptions.RequestException as e: + return None, Response( + {"error": f"Authentication failed: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + def _get_sd_reset_at(self, source): + """Retrieve stored reset timestamp from EPGSource model field.""" + reset_at_str = (source.custom_properties or {}).get('sd_changes_reset_at') + return reset_at_str + + def _get_sd_changes_remaining(self, source): + """ + Retrieve stored changesRemaining from EPGSource model field. + If a reset timestamp exists and has passed (midnight UTC), clears the + lockout automatically so the user can make adds again. + """ + from django.utils import timezone + + cp = source.custom_properties or {} + changes_remaining = cp.get('sd_changes_remaining') + reset_at_str = cp.get('sd_changes_reset_at') + from django.utils.dateparse import parse_datetime + reset_at = parse_datetime(reset_at_str) if reset_at_str else None + + # If we have a reset timestamp and it has passed, clear the lockout + if changes_remaining == 0 and reset_at: + if timezone.now() >= reset_at: + cp = source.custom_properties or {} + cp.pop('sd_changes_remaining', None) + cp.pop('sd_changes_reset_at', None) + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + return None + + return changes_remaining + + def _save_sd_changes_remaining(self, source, changes_remaining): + """Persist changesRemaining to EPGSource model field.""" + cp = source.custom_properties or {} + cp['sd_changes_remaining'] = changes_remaining + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + + def _save_sd_lockout(self, source): + """ + Persist a hard lockout to EPGSource custom_properties when SD returns + 4100 MAX_LINEUP_CHANGES_REACHED. Per SD API documentation, lineup adds + are limited to 6 per rolling 24-hour period (not midnight UTC reset). + Lockout clears automatically after 24 hours from when the limit was hit. + """ + from django.utils import timezone + from datetime import timedelta + + now = timezone.now() + # Rolling 24-hour window per SD docs: "6 adds in a 24 hour period" + reset_at = now + timedelta(hours=24) + + cp = source.custom_properties or {} + cp['sd_changes_remaining'] = 0 + cp['sd_changes_reset_at'] = reset_at.isoformat() + source.custom_properties = cp + source.save(update_fields=['custom_properties']) + logger.warning( + f"SD source {source.id}: daily add limit reached (4100). " + f"Lockout set until {reset_at.isoformat()} (24h rolling window)." + ) + + @action(detail=True, methods=["get", "post", "delete"], url_path="sd-lineups") + def sd_lineups(self, request, pk=None): + """ + GET — list lineups currently on the SD account + POST — add a lineup (body: {"lineup": "USA-NJ29486-X"}) + DELETE — remove a lineup (body: {"lineup": "USA-NJ29486-X"}) + """ + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + source = self.get_object() + if source.source_type != 'schedules_direct': + return Response( + {"error": "This action is only available for Schedules Direct sources."}, + status=status.HTTP_400_BAD_REQUEST + ) + + token, error = self._sd_authenticate(source) + if error: + return error + + headers = { + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + 'token': token, + } + + if request.method == "GET": + try: + resp = http_requests.get( + f"{SD_BASE_URL}/lineups", + headers=headers, + timeout=15, + ) + if resp.status_code == 400: + sd_data = resp.json() + sd_code = sd_data.get('code') + if sd_code == 4102: + return Response({ + "lineups": [], + "max_lineups": 4, + "changes_remaining": self._get_sd_changes_remaining(source), + "changes_reset_at": self._get_sd_reset_at(source), + "notice": "No lineups are currently configured on this Schedules Direct account. Use the search below to add one.", + }) + resp.raise_for_status() + data = resp.json() + lineups = [l for l in data.get('lineups', []) if not l.get('isDeleted', False)] + return Response({ + "lineups": lineups, + "max_lineups": 4, + "changes_remaining": self._get_sd_changes_remaining(source), + "changes_reset_at": self._get_sd_reset_at(source), + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to fetch lineups: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + elif request.method == "POST": + lineup_id = request.data.get('lineup') + if not lineup_id: + return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) + try: + resp = http_requests.put( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=headers, + timeout=15, + ) + sd_data = resp.json() + sd_code = sd_data.get('code') + + if resp.status_code == 400 or resp.status_code == 403: + if sd_code == 4100: + self._save_sd_lockout(source) + return Response({ + "error": "daily_limit_reached", + "message": "You have reached your daily Schedules Direct lineup addition limit. SD allows 6 adds per 24-hour period. Resets at midnight UTC.", + "changes_remaining": 0, + "docs_url": "https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform", + }, status=status.HTTP_200_OK) + if sd_code == 4101: + return Response({ + "error": "max_lineups_reached", + "message": "Your Schedules Direct account has reached the maximum of 4 lineups. Remove one before adding another.", + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + if sd_code == 2100: + return Response({ + "error": "duplicate_lineup", + "message": "This lineup is already on your Schedules Direct account.", + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + return Response({ + "error": sd_data.get('message', 'Failed to add lineup.'), + "changes_remaining": self._get_sd_changes_remaining(source), + }, status=status.HTTP_200_OK) + + resp.raise_for_status() + + # Persist changesRemaining to custom_properties + changes_remaining = sd_data.get('changesRemaining') + if changes_remaining is not None: + self._save_sd_changes_remaining(source, changes_remaining) + + logger.info( + f"SD lineup added for source {source.id}: {lineup_id}. " + f"changesRemaining: {changes_remaining}" + ) + + # Re-fetch stations so the new lineup's stations are available for matching + from apps.epg.tasks import fetch_schedules_direct_stations + fetch_schedules_direct_stations.delay(source.id) + + return Response({ + **sd_data, + "changes_remaining": changes_remaining, + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to add lineup: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + elif request.method == "DELETE": + lineup_id = request.data.get('lineup') + if not lineup_id: + return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) + try: + resp = http_requests.delete( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=headers, + timeout=15, + ) + if resp.status_code == 400: + sd_data = resp.json() + sd_code = sd_data.get('code') + if sd_code == 2103: + return Response({ + "response": "OK", + "code": 0, + "message": "Lineup not found on account — already removed.", + "changes_remaining": self._get_sd_changes_remaining(source), + }) + resp.raise_for_status() + sd_data = resp.json() + # SD returns changesRemaining on deletes — persist it + changes_remaining = sd_data.get('changesRemaining') + if changes_remaining is not None: + self._save_sd_changes_remaining(source, changes_remaining) + logger.info(f"SD lineup deleted for source {source.id}: {lineup_id}") + return Response({ + **sd_data, + "changes_remaining": self._get_sd_changes_remaining(source), + }) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to remove lineup: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) + + @action(detail=True, methods=["post"], url_path="sd-lineups/search") + def sd_lineups_search(self, request, pk=None): + """ + Search available headends/lineups by country and postal code. + Body: {"country": "USA", "postalcode": "07030"} + Returns a flat list of lineups across all matching headends. + """ + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + from version import __version__ as dispatcharr_version + + source = self.get_object() + if source.source_type != 'schedules_direct': + return Response( + {"error": "This action is only available for Schedules Direct sources."}, + status=status.HTTP_400_BAD_REQUEST + ) + + country = request.data.get('country', '').strip() + postalcode = request.data.get('postalcode', '').strip() + if not country or not postalcode: + return Response( + {"error": "country and postalcode are required."}, + status=status.HTTP_400_BAD_REQUEST + ) + + token, error = self._sd_authenticate(source) + if error: + return error + + headers = { + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + 'token': token, + } + + try: + resp = http_requests.get( + f"{SD_BASE_URL}/headends", + params={'country': country, 'postalcode': postalcode}, + headers=headers, + timeout=15, + ) + resp.raise_for_status() + headends = resp.json() + lineups = [] + for headend in headends: + for lineup in headend.get('lineups', []): + lineups.append({ + 'lineup': lineup.get('lineup'), + 'name': lineup.get('name'), + 'transport': headend.get('transport'), + 'location': headend.get('location'), + 'headend': headend.get('headend'), + }) + return Response({"lineups": lineups}) + except http_requests.exceptions.RequestException as e: + return Response( + {"error": f"Failed to search headends: {str(e)}"}, + status=status.HTTP_502_BAD_GATEWAY + ) +# ───────────────────────────── +# 2) Program API (CRUD) +# ───────────────────────────── +class ProgramSearchPagination(PageNumberPagination): + page_size = 50 + page_size_query_param = 'page_size' + max_page_size = 500 + + +class ProgramViewSet(viewsets.ModelViewSet): + """Handles CRUD operations for EPG programs""" + + queryset = ProgramData.objects.select_related("epg").all() + serializer_class = ProgramDataSerializer + + def get_permissions(self): + if self.action == 'poster': + return [AllowAny()] + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + def get_serializer_class(self): + if self.action == 'retrieve': + return ProgramDetailSerializer + return ProgramDataSerializer + + def retrieve(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.get_serializer(instance) + return Response(serializer.data) + + # Cached SD token for poster proxy (module-level, shared across requests) + _sd_poster_token_cache = {} # {source_id: {'token': str, 'expires': float}} + # Cache SD errors to prevent hammering a blocked account + _sd_poster_error_cache = {} # {source_id: {'until': float, 'reason': str}} + + @action(detail=True, methods=['get'], url_path='poster', permission_classes=[AllowAny]) + def poster(self, request, pk=None): + """ + Proxy endpoint for SD program poster images. + Fetches from SD with auth on first request, caches to disk. + Subsequent requests served from disk cache. + """ + import time + import hashlib + import requests as http_requests + + program = self.get_object() + poster_url = (program.custom_properties or {}).get('poster_url') + if not poster_url: + return Response(status=status.HTTP_404_NOT_FOUND) + + # Extract filename hash for cache key + cache_hash = poster_url.rsplit('/', 1)[-1] + cache_dir = '/data/cache/posters' + os.makedirs(cache_dir, exist_ok=True) + cache_path = os.path.join(cache_dir, cache_hash) + + # Serve from disk cache if fresh (< 30 days) + CACHE_MAX_AGE_SECS = 30 * 24 * 3600 + if os.path.exists(cache_path): + file_age = time.time() - os.path.getmtime(cache_path) + if file_age < CACHE_MAX_AGE_SECS: + from django.http import HttpResponse + with open(cache_path, 'rb') as f: + response = HttpResponse(f.read(), content_type='image/jpeg') + response['Cache-Control'] = 'public, max-age=86400' + return response + + # Check if SD is in error state — don't hammer a blocked account + source = program.epg.epg_source if program.epg else None + if not source or source.source_type != 'schedules_direct': + return Response(status=status.HTTP_404_NOT_FOUND) + + error_cache = ProgramViewSet._sd_poster_error_cache.get(source.id) + if error_cache and time.time() < error_cache['until']: + return Response( + {'error': f"SD temporarily unavailable: {error_cache['reason']}"}, + status=status.HTTP_503_SERVICE_UNAVAILABLE + ) + + from apps.epg.tasks import SD_BASE_URL + + # Reuse cached token if still valid (using tokenExpires from SD response) + cached = ProgramViewSet._sd_poster_token_cache.get(source.id) + token = None + if cached and time.time() < cached['expires']: + token = cached['token'] + + if not token: + sha1_password = hashlib.sha1(source.password.encode('utf-8')).hexdigest() + try: + auth_resp = http_requests.post( + f"{SD_BASE_URL}/token", + json={'username': source.username, 'password': sha1_password}, + headers={'Content-Type': 'application/json'}, + timeout=10, + ) + auth_data = auth_resp.json() + token = auth_data.get('token') + if not token: + # Auth failed — cache error for 1 hour to prevent hammering + error_msg = auth_data.get('message', 'Authentication failed') + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 3600, + 'reason': error_msg, + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + + # Cache token using SD's tokenExpires (UNIX epoch) + token_expires = auth_data.get('tokenExpires', time.time() + 86400) + ProgramViewSet._sd_poster_token_cache[source.id] = { + 'token': token, + 'expires': token_expires, + } + except http_requests.exceptions.RequestException: + # Network error — cache for 5 minutes + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 300, + 'reason': 'Network error reaching Schedules Direct', + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + + # Fetch the poster image + try: + img_resp = http_requests.get( + poster_url, + headers={'token': token}, + timeout=15, + ) + if img_resp.status_code == 403 or img_resp.status_code == 401: + # Token expired or account blocked — invalidate token cache + ProgramViewSet._sd_poster_token_cache.pop(source.id, None) + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 3600, + 'reason': f'SD returned {img_resp.status_code} — possible rate limit or token expiry', + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + + if img_resp.status_code != 200: + return Response(status=status.HTTP_502_BAD_GATEWAY) + + # Save to disk cache + with open(cache_path, 'wb') as f: + f.write(img_resp.content) + + from django.http import HttpResponse + response = HttpResponse(img_resp.content, content_type='image/jpeg') + response['Cache-Control'] = 'public, max-age=86400' + return response + + except http_requests.exceptions.RequestException: + return Response(status=status.HTTP_502_BAD_GATEWAY) + + def list(self, request, *args, **kwargs): + logger.debug("Listing all EPG programs.") + return super().list(request, *args, **kwargs) + + @extend_schema( + summary="Search EPG programs", + description=""" +**Advanced EPG program search with multiple filter types and complex query support.** + +### Text Search Features + +**Title and Description Search**: +- Supports AND/OR logical operators (case-insensitive: `and`/`AND` both work) +- Wrap phrases in double quotes to match them literally: `"Law and Order"` +- Parenthetical grouping for complex queries: `(Newcastle OR NEW) AND (Villa OR AST)` +- Regex pattern matching with `title_regex=true` (evaluated by the database engine) +- Whole word matching with `title_whole_words=true` to avoid partial matches + +**Examples**: +- Simple: `title=football` +- AND operator: `title=premier AND league` +- OR operator: `title=Newcastle OR Villa` +- Quoted phrase: `title="Law and Order"` (matches the exact phrase; 'and' is literal) +- Mixed: `title="Law and Order" AND crime` +- Nested groups: `title=(Newcastle OR NEW) AND (Villa OR AST)` +- Regex: `title=^Premier&title_regex=true` (programs starting with "Premier") +- Whole words: `title=NEW&title_whole_words=true` (matches "NEW" but not "News") + +### Time Filtering + +**airing_at**: Find programs airing at a specific moment (start_time ≤ airing_at < end_time) + +**Time ranges**: Use combinations of start_after, start_before, end_after, end_before + +### Response Customization + +**fields**: Comma-separated list to include only specific fields in response +- Available: id, title, sub_title, description, start_time, end_time, tvg_id, custom_properties, epg_source, epg_name, epg_icon_url, channels, streams + +### Pagination + +- Default: 50 results per page +- Maximum: 500 results per page +- Use `page` and `page_size` parameters to navigate results + """, + parameters=[ + OpenApiParameter( + 'title', + OpenApiTypes.STR, + description='Title search query. Supports AND/OR operators (case-insensitive), quoted phrases, and parentheses. Double-quote a phrase to match it literally: `"Law and Order"`. Unquoted space-separated terms are matched as a phrase; use AND/OR to combine separate terms.', + ), + OpenApiParameter('title_regex', OpenApiTypes.BOOL, description='Enable regex matching for title (case-insensitive, default: false). e.g. `^The` matches titles starting with "The".'), + OpenApiParameter('title_whole_words', OpenApiTypes.BOOL, description='Match whole words only in title (default: false). e.g. `new` matches "Newcastle" normally but not with whole words enabled.'), + OpenApiParameter( + 'description', + OpenApiTypes.STR, + description='Description search query. Same syntax and features as title search.' + ), + OpenApiParameter('description_regex', OpenApiTypes.BOOL, description='Enable regex matching for description (case-insensitive, default: false).'), + OpenApiParameter('description_whole_words', OpenApiTypes.BOOL, description='Match whole words only in description (default: false). Same behaviour as title_whole_words.'), + OpenApiParameter('start_after', OpenApiTypes.DATETIME, description='Filter programs starting at or after this time. ISO 8601 format, e.g. `2026-02-14T18:00:00Z`.'), + OpenApiParameter('start_before', OpenApiTypes.DATETIME, description='Filter programs starting at or before this time. ISO 8601 format.'), + OpenApiParameter('end_after', OpenApiTypes.DATETIME, description='Filter programs ending at or after this time. ISO 8601 format.'), + OpenApiParameter('end_before', OpenApiTypes.DATETIME, description='Filter programs ending at or before this time. ISO 8601 format.'), + OpenApiParameter('airing_at', OpenApiTypes.DATETIME, description='Find programs airing at this exact moment (start_time ≤ airing_at < end_time). ISO 8601 format, e.g. `2026-02-14T20:00:00Z`.'), + OpenApiParameter('channel', OpenApiTypes.STR, description='Filter by channel name (case-insensitive substring match). e.g. `BBC One`, `Sky Sports`.'), + OpenApiParameter('channel_id', OpenApiTypes.INT, description='Filter by exact channel ID.'), + OpenApiParameter('tvg_id', OpenApiTypes.STR, description='Filter by EPG tvg_id (exact match). e.g. `bbcone.uk`.'), + OpenApiParameter('stream', OpenApiTypes.STR, description='Filter by stream name (case-insensitive substring match).'), + OpenApiParameter('group', OpenApiTypes.STR, description='Filter by channel group or stream group name (case-insensitive substring match). e.g. `Sports`, `UK Channels`.'), + OpenApiParameter('epg_source', OpenApiTypes.INT, description='Filter by EPG source ID.'), + OpenApiParameter('fields', OpenApiTypes.STR, description='Comma-separated list of fields to include. Omit to return all fields. e.g. `title,start_time,end_time`.'), + OpenApiParameter('page', OpenApiTypes.INT, description='Page number for pagination (default: 1).'), + OpenApiParameter('page_size', OpenApiTypes.INT, description='Results per page (default: 50, max: 500).'), + ], + responses={200: ProgramSearchResultSerializer(many=True)}, + tags=['EPG'], + ) + @action(detail=False, methods=['get'], url_path='search', permission_classes=[IsStandardUser]) + def search(self, request): + params = request.query_params + + # Build base queryset with prefetching + queryset = ProgramData.objects.select_related( + 'epg', 'epg__epg_source' + ).prefetch_related( + 'epg__channels', 'epg__channels__channel_group', + 'epg__channels__streams', 'epg__channels__streams__channel_group', + 'epg__channels__streams__m3u_account', + ) + + filters = Q() + + # Text filters + title = params.get('title') + if title: + title_regex = params.get('title_regex', '').lower() in ('true', '1', 'yes') + title_whole_words = params.get('title_whole_words', '').lower() in ('true', '1', 'yes') + filters &= parse_text_query('title', title, use_regex=title_regex, whole_words=title_whole_words) + + description = params.get('description') + if description: + desc_regex = params.get('description_regex', '').lower() in ('true', '1', 'yes') + desc_whole_words = params.get('description_whole_words', '').lower() in ('true', '1', 'yes') + filters &= parse_text_query('description', description, use_regex=desc_regex, whole_words=desc_whole_words) + + # Time filters with validation + start_after = params.get('start_after') + if start_after: + dt = parse_datetime(start_after) + if dt is None: + return Response( + {"error": f"Invalid datetime format for start_after: {start_after}. Use ISO 8601 format (e.g., 2026-02-14T18:00:00Z)"}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__gte=dt) + + start_before = params.get('start_before') + if start_before: + dt = parse_datetime(start_before) + if dt is None: + return Response( + {"error": f"Invalid datetime format for start_before: {start_before}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__lte=dt) + + end_after = params.get('end_after') + if end_after: + dt = parse_datetime(end_after) + if dt is None: + return Response( + {"error": f"Invalid datetime format for end_after: {end_after}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(end_time__gte=dt) + + end_before = params.get('end_before') + if end_before: + dt = parse_datetime(end_before) + if dt is None: + return Response( + {"error": f"Invalid datetime format for end_before: {end_before}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(end_time__lte=dt) + + airing_at = params.get('airing_at') + if airing_at: + dt = parse_datetime(airing_at) + if dt is None: + return Response( + {"error": f"Invalid datetime format for airing_at: {airing_at}. Use ISO 8601 format."}, + status=status.HTTP_400_BAD_REQUEST + ) + filters &= Q(start_time__lte=dt, end_time__gt=dt) + + # Channel/stream filters + channel = params.get('channel') + if channel: + filters &= Q(epg__channels__name__icontains=channel) + + channel_id = params.get('channel_id') + if channel_id: + try: + filters &= Q(epg__channels__id=int(channel_id)) + except (ValueError, TypeError): + pass + + tvg_id = params.get('tvg_id') + if tvg_id: + filters &= Q(epg__tvg_id=tvg_id) + + stream = params.get('stream') + if stream: + filters &= Q(epg__channels__streams__name__icontains=stream) + + group = params.get('group') + if group: + filters &= ( + Q(epg__channels__channel_group__name__icontains=group) + | Q(epg__channels__streams__channel_group__name__icontains=group) + ) + + epg_source = params.get('epg_source') + if epg_source: + try: + filters &= Q(epg__epg_source__id=int(epg_source)) + except (ValueError, TypeError): + pass + + queryset = queryset.filter(filters).distinct().order_by('start_time') + + # Restrict results to programs on channels the user can access + user = request.user + if user.user_level < 10: + access_filter = Q(epg__channels__user_level__lte=user.user_level) + custom_props = user.custom_properties or {} + if custom_props.get('hide_adult_content', False): + access_filter &= Q(epg__channels__is_adult=False) + queryset = queryset.filter(access_filter).distinct() + + # Resolve field selection before serialization so expensive methods can short-circuit + requested_fields = params.get('fields') + allowed = set(f.strip() for f in requested_fields.split(',')) if requested_fields else None + + # Paginate + paginator = ProgramSearchPagination() + page = paginator.paginate_queryset(queryset, request) + serializer = ProgramSearchResultSerializer(page, many=True, context={'fields': allowed, 'user': request.user}) + data = serializer.data + + if allowed: + data = [{k: v for k, v in item.items() if k in allowed} for item in data] + + return paginator.get_paginated_response(data) + + +# ───────────────────────────── +# 3) EPG Grid View +# ───────────────────────────── +class EPGGridAPIView(APIView): + """Returns all programs airing in the next 24 hours including currently running ones and recent ones""" + + def get_permissions(self): + try: + return [ + perm() for perm in permission_classes_by_method[self.request.method] + ] + except KeyError: + return [Authenticated()] + + @extend_schema( + description="Retrieve programs from the previous hour, currently running and upcoming for the next 24 hours", + responses={200: ProgramDataSerializer(many=True)}, + ) + def get(self, request, format=None): + # Use current time instead of midnight + now = timezone.now() + one_hour_ago = now - timedelta(hours=1) + twenty_four_hours_later = now + timedelta(hours=24) + logger.debug( + f"EPGGridAPIView: Querying programs between {one_hour_ago} and {twenty_four_hours_later}." + ) + + programs = ProgramData.objects.filter( + end_time__gt=one_hour_ago, + start_time__lt=twenty_four_hours_later, + ) + + # Generate dummy programs for channels that have no EPG data OR dummy EPG sources + from apps.channels.models import Channel + from apps.epg.models import EPGSource + from django.db.models import Q + + # Get channels with no EPG data at all (standard dummy) + channels_without_epg = Channel.objects.filter(Q(epg_data__isnull=True)) + + # Get channels with custom dummy EPG sources (generate on-demand with patterns) + channels_with_custom_dummy = Channel.objects.filter( + epg_data__epg_source__source_type='dummy' + ).select_related('epg_data__epg_source').distinct() + + # Log what we found + without_count = channels_without_epg.count() + custom_count = channels_with_custom_dummy.count() + + if without_count > 0: + channel_names = [f"{ch.name} (ID: {ch.id})" for ch in channels_without_epg] + logger.debug( + f"EPGGridAPIView: Channels needing standard dummy EPG: {', '.join(channel_names)}" + ) + + if custom_count > 0: + channel_names = [f"{ch.name} (ID: {ch.id})" for ch in channels_with_custom_dummy] + logger.debug( + f"EPGGridAPIView: Channels needing custom dummy EPG: {', '.join(channel_names)}" + ) + + logger.debug( + f"EPGGridAPIView: Found {without_count} channels needing standard dummy, {custom_count} needing custom dummy EPG." + ) + + # Serialize the regular programs using .values() to bypass DRF overhead + programs_qs = programs.values( + 'id', 'start_time', 'end_time', 'title', 'sub_title', + 'description', 'tvg_id', 'custom_properties', + ) + serialized_programs = [] + for p in programs_qs: + cp = p['custom_properties'] or {} + premiere_text = cp.get('premiere_text', '') + serialized_programs.append({ + 'id': p['id'], + 'start_time': p['start_time'], + 'end_time': p['end_time'], + 'title': p['title'], + 'sub_title': p['sub_title'], + 'description': p['description'], + 'tvg_id': p['tvg_id'], + 'season': cp.get('season'), + 'episode': cp.get('episode'), + 'is_new': bool(cp.get('new')), + 'is_live': bool(cp.get('live')), + 'is_premiere': bool(cp.get('premiere')), + 'is_finale': bool(premiere_text and 'finale' in premiere_text.lower()), + }) + logger.debug( + f"EPGGridAPIView: Found {len(serialized_programs)} program(s), including recently ended, currently running, and upcoming shows." + ) + + # Humorous program descriptions based on time of day - same as in output/views.py + time_descriptions = { + (0, 4): [ + "Late Night with {channel} - Where insomniacs unite!", + "The 'Why Am I Still Awake?' Show on {channel}", + "Counting Sheep - A {channel} production for the sleepless", + ], + (4, 8): [ + "Dawn Patrol - Rise and shine with {channel}!", + "Early Bird Special - Coffee not included", + "Morning Zombies - Before coffee viewing on {channel}", + ], + (8, 12): [ + "Mid-Morning Meetings - Pretend you're paying attention while watching {channel}", + "The 'I Should Be Working' Hour on {channel}", + "Productivity Killer - {channel}'s daytime programming", + ], + (12, 16): [ + "Lunchtime Laziness with {channel}", + "The Afternoon Slump - Brought to you by {channel}", + "Post-Lunch Food Coma Theater on {channel}", + ], + (16, 20): [ + "Rush Hour - {channel}'s alternative to traffic", + "The 'What's For Dinner?' Debate on {channel}", + "Evening Escapism - {channel}'s remedy for reality", + ], + (20, 24): [ + "Prime Time Placeholder - {channel}'s finest not-programming", + "The 'Netflix Was Too Complicated' Show on {channel}", + "Family Argument Avoider - Courtesy of {channel}", + ], + } + + # Generate and append dummy programs + dummy_programs = [] + + # Import the function from output.views + from apps.output.views import generate_dummy_programs as gen_dummy_progs + + # Handle channels with CUSTOM dummy EPG sources (with patterns) + for channel in channels_with_custom_dummy: + # For dummy EPGs, ALWAYS use channel UUID to ensure unique programs per channel + # This prevents multiple channels assigned to the same dummy EPG from showing identical data + # Each channel gets its own unique program data even if they share the same EPG source + dummy_tvg_id = str(channel.uuid) + + try: + # Get the custom dummy EPG source + epg_source = channel.epg_data.epg_source if channel.epg_data else None + + logger.debug(f"Generating custom dummy programs for channel: {channel.name} (ID: {channel.id})") + + # Determine which name to parse based on custom properties + name_to_parse = channel.name + if epg_source and epg_source.custom_properties: + custom_props = epg_source.custom_properties + name_source = custom_props.get('name_source') + + if name_source == 'stream': + # Get the stream index (1-based from user, convert to 0-based) + stream_index = custom_props.get('stream_index', 1) - 1 + + # Get streams ordered by channelstream order + channel_streams = channel.streams.all().order_by('channelstream__order') + + if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): + stream = list(channel_streams)[stream_index] + name_to_parse = stream.name + logger.debug(f"Using stream name for parsing: {name_to_parse} (stream index: {stream_index})") + else: + logger.warning(f"Stream index {stream_index} not found for channel {channel.name}, falling back to channel name") + elif name_source == 'channel': + logger.debug(f"Using channel name for parsing: {name_to_parse}") + + # Generate programs using custom patterns from the dummy EPG source + # Use the same tvg_id that will be set in the program data + generated = gen_dummy_progs( + channel_id=dummy_tvg_id, + channel_name=name_to_parse, + num_days=1, + program_length_hours=4, + epg_source=epg_source + ) + + # Custom dummy should always return data (either from patterns or fallback) + if generated: + logger.debug(f"Generated {len(generated)} custom dummy programs for {channel.name}") + # Convert generated programs to API format + for program in generated: + prog_custom = program.get('custom_properties') or {} + dummy_program = { + "id": f"dummy-custom-{channel.id}-{program['start_time'].hour}", + "epg": {"tvg_id": dummy_tvg_id, "name": channel.name}, + "start_time": program['start_time'].isoformat(), + "end_time": program['end_time'].isoformat(), + "title": program['title'], + "description": program['description'], + "tvg_id": dummy_tvg_id, + "sub_title": program.get('sub_title'), + "custom_properties": prog_custom if prog_custom else None, + "season": None, + "episode": None, + "is_new": prog_custom.get('new', False), + "is_live": bool(prog_custom.get('live')), + "is_premiere": False, + "is_finale": False, + } + dummy_programs.append(dummy_program) + else: + logger.warning(f"No programs generated for custom dummy EPG channel: {channel.name}") + + except Exception as e: + logger.error( + f"Error creating custom dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" + ) + + # Handle channels with NO EPG data (standard dummy with humorous descriptions) + for channel in channels_without_epg: + # For channels with no EPG, use UUID to ensure uniqueness (matches frontend logic) + # The frontend uses: tvgRecord?.tvg_id ?? channel.uuid + # Since there's no EPG data, it will fall back to UUID + dummy_tvg_id = str(channel.uuid) + + try: + logger.debug(f"Generating standard dummy programs for channel: {channel.name} (ID: {channel.id})") + + # Create programs every 4 hours for the next 24 hours with humorous descriptions + for hour_offset in range(0, 24, 4): + # Use timedelta for time arithmetic instead of replace() to avoid hour overflow + start_time = now + timedelta(hours=hour_offset) + # Set minutes/seconds to zero for clean time blocks + start_time = start_time.replace(minute=0, second=0, microsecond=0) + end_time = start_time + timedelta(hours=4) + + # Get the hour for selecting a description + hour = start_time.hour + day = 0 # Use 0 as we're only doing 1 day + + # Find the appropriate time slot for description + for time_range, descriptions in time_descriptions.items(): + start_range, end_range = time_range + if start_range <= hour < end_range: + # Pick a description using the sum of the hour and day as seed + # This makes it somewhat random but consistent for the same timeslot + description = descriptions[ + (hour + day) % len(descriptions) + ].format(channel=channel.name) + break + else: + # Fallback description if somehow no range matches + description = f"Placeholder program for {channel.name} - EPG data went on vacation" + + # Create a dummy program in the same format as regular programs + dummy_program = { + "id": f"dummy-standard-{channel.id}-{hour_offset}", + "epg": {"tvg_id": dummy_tvg_id, "name": channel.name}, + "start_time": start_time.isoformat(), + "end_time": end_time.isoformat(), + "title": f"{channel.name}", + "description": description, + "tvg_id": dummy_tvg_id, + "sub_title": None, + "custom_properties": None, + "season": None, + "episode": None, + "is_new": False, + "is_live": False, + "is_premiere": False, + "is_finale": False, + } + dummy_programs.append(dummy_program) + + except Exception as e: + logger.error( + f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" + ) + + # Combine regular and dummy programs + all_programs = list(serialized_programs) + dummy_programs + logger.debug( + f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)." + ) + + return Response({"data": all_programs}, status=status.HTTP_200_OK) + + +# ───────────────────────────── +# 4) EPG Import View +# ───────────────────────────── +class EPGImportAPIView(APIView): + """Triggers an EPG data refresh""" + + def get_permissions(self): + try: + return [ + perm() for perm in permission_classes_by_method[self.request.method] + ] + except KeyError: + return [Authenticated()] + + @extend_schema( + description="Triggers an EPG data refresh for the given source.", + request=inline_serializer( + name="EPGImportRequest", + fields={ + "id": serializers.IntegerField(help_text="ID of the EPG source to refresh."), + }, + ), + ) + def post(self, request, format=None): + logger.info("EPGImportAPIView: Received request to import EPG data.") + epg_id = request.data.get("id", None) + force = bool(request.data.get("force", False)) + + # Check if this is a dummy EPG source + try: + from .models import EPGSource + epg_source = EPGSource.objects.get(id=epg_id) + if epg_source.source_type == 'dummy': + logger.info(f"EPGImportAPIView: Skipping refresh for dummy EPG source {epg_id}") + return Response( + {"success": False, "message": "Dummy EPG sources do not require refreshing."}, + status=status.HTTP_400_BAD_REQUEST, + ) + except EPGSource.DoesNotExist: + pass # Let the task handle the missing source + + refresh_epg_data.delay(epg_id, force=force) # Trigger Celery task + logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") + return Response( + {"success": True, "message": "EPG data refresh initiated."}, + status=status.HTTP_202_ACCEPTED, + ) + + +# ───────────────────────────── +# 5) EPG Data View +# ───────────────────────────── +class EPGDataViewSet(viewsets.ReadOnlyModelViewSet): + """ + API endpoint that allows EPGData objects to be viewed. + """ + + queryset = EPGData.objects.all() + serializer_class = EPGDataSerializer + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + +# ───────────────────────────── +# 6) Current Programs API +# ───────────────────────────── +class CurrentProgramsAPIView(APIView): + """ + Lightweight endpoint that returns currently playing programs for specified channel IDs. + Accepts POST with JSON body containing channel_ids array, or null/empty to fetch all channels. + """ + + def get_permissions(self): + try: + return [ + perm() for perm in permission_classes_by_method[self.request.method] + ] + except KeyError: + return [Authenticated()] + + @extend_schema( + description="Get currently playing programs for specified channels or all channels", + request=inline_serializer( + name="CurrentProgramsRequest", + fields={ + "channel_uuids": serializers.ListField( + child=serializers.CharField(), + required=False, + allow_null=True, + help_text="Array of channel UUIDs. If null or omitted, returns all channels with current programs.", + ), + }, + ), + responses={200: ProgramDataSerializer(many=True)}, + ) + def post(self, request, format=None): + # Import Channel model + from apps.channels.models import Channel + + # Build query for channels with EPG data + query = Channel.objects.filter(epg_data__isnull=False) + + channel_uuids = request.data.get('channel_uuids', None) + + if channel_uuids is not None: + if not isinstance(channel_uuids, list): + return Response( + {"error": "channel_uuids must be an array of strings or null"}, + status=status.HTTP_400_BAD_REQUEST + ) + query = query.filter(uuid__in=channel_uuids) + + # Get channels with EPG data + channels = query.select_related('epg_data') + + # Get current time + now = timezone.now() + + # Build list of current programs + current_programs = [] + + for channel in channels: + # Query for current program + program = ProgramData.objects.select_related("epg").filter( + epg=channel.epg_data, + start_time__lte=now, + end_time__gt=now + ).first() + + if program: + program_data = ProgramDataSerializer(program).data + program_data['channel_uuid'] = str(channel.uuid) + current_programs.append(program_data) + + + return Response(current_programs, status=status.HTTP_200_OK) + diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 73b1a456..fc43c36d 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -4,7 +4,6 @@ import logging import gzip import html.entities import os -import re import uuid import requests import time # Add import for tracking download progress @@ -57,12 +56,6 @@ def _build_html_entity_doctype() -> bytes: _HTML_ENTITY_DOCTYPE = _build_html_entity_doctype() -def _parse_programme_element(element_bytes): - """Parse a single element, prepending the HTML-entity DOCTYPE so references like é in the text resolve instead of failing.""" - parser = etree.XMLParser(resolve_entities=True, load_dtd=True, no_network=True) - return etree.fromstring(_HTML_ENTITY_DOCTYPE + element_bytes, parser) - - class _PrependStream: """Wraps an open binary file and prepends a bytes prefix to its content. @@ -365,10 +358,6 @@ def refresh_epg_data(source_id, force=False): # Continue with the normal processing... logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") if source.source_type == 'xmltv': - # Invalidate the byte-offset index before downloading the new file - # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) - fetch_success = fetch_xmltv(source) if not fetch_success: logger.error(f"Failed to fetch XMLTV for source {source.name}") @@ -387,9 +376,6 @@ def refresh_epg_data(source_id, force=False): gc.collect() return - # Build byte-offset index for preview lookups in the background so refresh isn't blocked by it - build_programme_index_task.delay(source.id) - parse_programs_for_source(source) elif source.source_type == 'schedules_direct': @@ -1331,8 +1317,7 @@ def parse_channels_only(source): @shared_task(time_limit=3600, soft_time_limit=3500) - -def parse_programs_for_tvg_id(epg_id, force=False): +def parse_programs_for_tvg_id(epg_id): # Skip XMLTV file parsing for Schedules Direct sources. Program data is # fetched and persisted directly by fetch_schedules_direct(). try: @@ -1384,7 +1369,7 @@ def parse_programs_for_tvg_id(epg_id, force=False): release_task_lock('parse_epg_programs', epg_id) return - if not force and not Channel.objects.filter(epg_data=epg).exists(): + if not Channel.objects.filter(epg_data=epg).exists(): logger.info(f"No channels matched to EPG {epg.tvg_id}") lock_renewer.stop() release_task_lock('parse_epg_programs', epg_id) @@ -2215,6 +2200,15 @@ def fetch_schedules_direct(source, stations_only=False, force=False): _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) return logger.info(f"Found {len(lineups)} lineup(s) in SD account.") + + # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) + sd_lineup_country = None + for l in lineups: + lid = l.get('lineupID') or l.get('lineup') or '' + if '-' in lid: + sd_lineup_country = lid.split('-')[0] + break + logger.debug(f"SD lineup country: {sd_lineup_country}") except requests.exceptions.RequestException as e: msg = f"Failed to fetch Schedules Direct lineups: {e}" logger.error(msg, exc_info=True) @@ -2246,8 +2240,9 @@ def fetch_schedules_direct(source, stations_only=False, force=False): logo_url = None logos = station.get('stationLogo') or station.get('logo') or [] if isinstance(logos, list) and logos: - # Prefer 'dark' variant (best on dark backgrounds); fall back to first available - preferred = next((l for l in logos if l.get('category') == 'dark'), logos[0]) + # Read preferred logo style from source settings; default to 'dark' + logo_style = (source.custom_properties or {}).get('logo_style', 'dark') + preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) logo_url = preferred.get('URL') or preferred.get('url') elif isinstance(logos, dict): logo_url = logos.get('URL') or logos.get('url') @@ -2642,6 +2637,35 @@ def fetch_schedules_direct(source, stations_only=False, force=False): ).values_list('tvg_id', flat=True) ) + # Cache existing program data for unchanged programs BEFORE surgical delete. + # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only + # programs with changed program MD5s get metadata re-downloaded. The surgical delete + # wipes ALL ProgramData for changed dates, so unchanged programs lose their titles. + # This cache preserves their data for rebuilding. + unchanged_pids = set() + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + continue + for airing in airings: + pid = airing.get('programID') + if pid and pid not in program_metadata: + unchanged_pids.add(pid) + + existing_program_cache = {} + if unchanged_pids: + for pd in ProgramData.objects.filter( + epg__epg_source=source, + program_id__in=unchanged_pids, + ).only('program_id', 'title', 'description', 'sub_title', 'custom_properties'): + if pd.program_id not in existing_program_cache: + existing_program_cache[pd.program_id] = { + 'title': pd.title, + 'description': pd.description, + 'sub_title': pd.sub_title, + 'custom_properties': pd.custom_properties, + } + logger.info(f"Cached {len(existing_program_cache)} existing program records for unchanged programs.") + all_programs_to_create = [] total_programs = 0 skipped_unmapped = 0 @@ -2671,30 +2695,51 @@ def fetch_schedules_direct(source, stations_only=False, force=False): continue meta = program_metadata.get(pid, {}) - titles = meta.get('titles', [{}]) - title = titles[0].get('title120', '') if titles else '' - if not title: - title = meta.get('episodeTitle150', '') or 'No Title' + cached_prog = existing_program_cache.get(pid) if not meta else None + + if cached_prog: + # Unchanged program — reuse cached data from before surgical delete + title = cached_prog['title'] or 'No Title' + desc = cached_prog['description'] or '' + episode_title = cached_prog['sub_title'] or '' + custom_props = cached_prog['custom_properties'] or {} + else: + titles = meta.get('titles', [{}]) + title = titles[0].get('title120', '') if titles else '' + if not title: + title = meta.get('episodeTitle150', '') or 'No Title' title = title[:255] - descriptions = meta.get('descriptions', {}) - desc = '' - for key in ('description1000', 'description255', 'description100'): - candidates = descriptions.get(key, []) - if candidates: - desc = candidates[0].get('description', '') - if desc: + if not cached_prog: + descriptions = meta.get('descriptions', {}) + desc = '' + for key in ('description1000', 'description255', 'description100'): + candidates = descriptions.get(key, []) + if candidates: + desc = candidates[0].get('description', '') + if desc: + break + + episode_title = meta.get('episodeTitle150', '') + + # Build custom_properties following the same pattern as the XMLTV parser + custom_props = {} + + # Season/Episode — search all metadata entries, not just [0] + metadata_block = meta.get('metadata', []) + gracenote_meta = {} + for md_entry in metadata_block: + if 'Gracenote' in md_entry: + gracenote_meta = md_entry['Gracenote'] break - - episode_title = meta.get('episodeTitle150', '') - - # Build custom_properties following the same pattern as the XMLTV parser - custom_props = {} - metadata_block = meta.get('metadata', [{}]) - if metadata_block: - m = metadata_block[0].get('Gracenote', {}) - season = m.get('season') - episode = m.get('episode') + if not gracenote_meta: + # Fall back to TVmaze if Gracenote is absent + for md_entry in metadata_block: + if 'TVmaze' in md_entry: + gracenote_meta = md_entry['TVmaze'] + break + season = gracenote_meta.get('season') + episode = gracenote_meta.get('episode') if season: custom_props['season'] = int(season) if episode: @@ -2702,50 +2747,129 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if season and episode: custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" - content_rating = meta.get('contentRating', []) - if content_rating: - custom_props['rating'] = content_rating[0].get('code', '') - custom_props['rating_system'] = content_rating[0].get('body', '') + # Content rating — store full array, pick display rating by lineup country + content_rating = meta.get('contentRating', []) + if content_rating: + custom_props['content_ratings'] = content_rating + selected = None + if sd_lineup_country: + for cr in content_rating: + if cr.get('country', '') == sd_lineup_country: + selected = cr + break + if not selected: + # Fall back to USA, then first available + for cr in content_rating: + if cr.get('country', '') == 'USA': + selected = cr + break + if not selected: + selected = content_rating[0] + custom_props['rating'] = selected.get('code', '') + custom_props['rating_system'] = selected.get('body', '') - genres = meta.get('genres', []) - if genres: - custom_props['categories'] = genres + # Content advisory — content warnings + content_advisory = meta.get('contentAdvisory', []) + if content_advisory: + custom_props['content_advisory'] = content_advisory - cast = meta.get('cast', []) - crew = meta.get('crew', []) - credits = {} - if cast: - credits['actor'] = [ - {'name': p.get('name', ''), 'role': p.get('role', '')} - for p in cast if p.get('name') - ] - if crew: - for member in crew: - role = member.get('role', '').lower() - name = member.get('name', '') - if not name: - continue - if 'director' in role: - credits.setdefault('director', []).append(name) - elif 'writer' in role or 'screenwriter' in role: - credits.setdefault('writer', []).append(name) - elif 'producer' in role: - credits.setdefault('producer', []).append(name) - if credits: - custom_props['credits'] = credits + # Categories — combine entityType, showType, and genres + categories = [] + entity_type = meta.get('entityType', '') + show_type = meta.get('showType', '') + if entity_type: + categories.append(entity_type) + if show_type and show_type != entity_type: + categories.append(show_type) + genres = meta.get('genres', []) + categories.extend(genres) + if categories: + custom_props['categories'] = categories - if airing.get('liveTapeDelay') == 'Live': - custom_props['live'] = True - if airing.get('new'): - custom_props['new'] = True - else: - custom_props['previously_shown'] = True - if airing.get('premiere'): - custom_props['premiere'] = True + # Cast — top-billed only, store characterName, drop role noise + cast = meta.get('cast', []) + crew = meta.get('crew', []) + credits = {} + if cast: + # Sort by billingOrder and cap at top-billed actors + sorted_cast = sorted( + [p for p in cast if p.get('name')], + key=lambda p: int(p.get('billingOrder', '999')) + ) + # Separate main cast from guest stars + main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] + # Store top-billed main cast (matching XMLTV parity) + credits['actor'] = [ + { + 'name': p.get('name', ''), + **(({'character': p['characterName']} ) if p.get('characterName') else {}), + } + for p in (main_cast[:6] if main_cast else sorted_cast[:6]) + ] + if crew: + for member in crew: + role = member.get('role', '').lower() + name = member.get('name', '') + if not name: + continue + if 'director' in role: + credits.setdefault('director', []).append(name) + elif 'writer' in role or 'screenwriter' in role: + credits.setdefault('writer', []).append(name) + elif 'producer' in role: + credits.setdefault('producer', []).append(name) + if credits: + custom_props['credits'] = credits - year = meta.get('movie', {}).get('year') or meta.get('originalAirDate', '')[:4] - if year: - custom_props['date'] = str(year) + # Airing flags + if airing.get('liveTapeDelay') == 'Live': + custom_props['live'] = True + if airing.get('new'): + custom_props['new'] = True + else: + custom_props['previously_shown'] = True + if airing.get('premiere'): + custom_props['premiere'] = True + + # Original air date — full date, not just year + original_air_date = meta.get('originalAirDate', '') + movie_year = meta.get('movie', {}).get('year', '') + if original_air_date: + custom_props['date'] = original_air_date + elif movie_year: + custom_props['date'] = str(movie_year) + + # Country of production + country = meta.get('country', []) + if country: + custom_props['country'] = country[0] if len(country) == 1 else ', '.join(country) + + # Runtime — program duration without commercials (seconds → store for display) + runtime_secs = meta.get('duration') or meta.get('movie', {}).get('duration') + if runtime_secs: + runtime_mins = int(runtime_secs) // 60 + custom_props['length'] = {'value': str(runtime_mins), 'units': 'minutes'} + + # Movie quality ratings → star_ratings (matches XMLTV key) + movie_data = meta.get('movie', {}) + quality_ratings = movie_data.get('qualityRating', []) + if quality_ratings: + star_ratings = [] + for qr in quality_ratings: + rating_str = qr.get('rating', '') + max_rating = qr.get('maxRating', '') + if rating_str and max_rating: + star_ratings.append({ + 'value': f"{rating_str}/{max_rating}", + 'system': qr.get('ratingsBody', ''), + }) + if star_ratings: + custom_props['star_ratings'] = star_ratings + + # Sports event details + event_details = meta.get('eventDetails', {}) + if event_details: + custom_props['event_details'] = event_details all_programs_to_create.append(ProgramData( epg_id=epg_db_id, @@ -2836,6 +2960,177 @@ def fetch_schedules_direct(source, stations_only=False, force=False): all_programs_to_create = None gc.collect() + # ------------------------------------------------------------------------- + # Step 8: Fetch program artwork (posters) if enabled + # ------------------------------------------------------------------------- + fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) + if fetch_posters and program_metadata: + logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") + _sd_send_ws_sync(source.id, "parsing_programs", 98, + message="Fetching program artwork...") + + try: + # Build a set of unique artwork lookup IDs. + # For episodes (EP...), use the series root (SH...0000) so we get + # series-level artwork — one poster per series instead of per-episode. + artwork_lookup_ids = set() + pid_to_artwork_key = {} # maps original programID -> the key we looked up + + for pid in program_metadata: + if pid.startswith('EP'): + sh_root = 'SH' + pid[2:10] + '0000' + artwork_lookup_ids.add(sh_root) + pid_to_artwork_key[pid] = sh_root + else: + artwork_lookup_ids.add(pid) + pid_to_artwork_key[pid] = pid + + artwork_map = {} # artwork_key -> poster_url + artwork_list = list(artwork_lookup_ids) + SD_ARTWORK_BATCH_SIZE = 500 + + total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) + logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " + f"in {total_art_batches} batch(es).") + + for batch_idx in range(total_art_batches): + batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] + try: + art_response = requests.post( + f"{SD_BASE_URL}/metadata/programs/", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + art_response.raise_for_status() + art_data = art_response.json() + + for entry in art_data: + if not isinstance(entry, dict): + continue + entry_pid = entry.get('programID') + images = entry.get('data') or [] + if not entry_pid or not images: + continue + + # Filter to only dict entries — SD sometimes returns bare strings + images = [img for img in images if isinstance(img, dict)] + if not images: + continue + + # Pick the best poster image: + # Prefer portrait orientation (2x3, 3x4) in larger sizes + # SD categories include: Iconic, Banner-L1, Banner-L2, Logo + # SD uses width/height instead of a "size" field + poster_url = None + + # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic + for min_width in [240, 135, 120]: + for pref_cat in ['Banner-L1', 'Iconic']: + match = next((img for img in images + if img.get('aspect') in ('2x3', '3x4') + and img.get('category') == pref_cat + and (img.get('width', 0) or 0) >= min_width), None) + if match: + poster_url = match.get('uri') + break + if poster_url: + break + + # Fallback: any portrait image at any size + if not poster_url: + portrait = next((img for img in images + if img.get('aspect') in ('2x3', '3x4')), None) + if portrait: + poster_url = portrait.get('uri') + + if poster_url: + # Complete the URL if it's relative + if not poster_url.startswith('http'): + poster_url = f"{SD_BASE_URL}/image/{poster_url}" + artwork_map[entry_pid] = poster_url + + logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " + f"{len(artwork_map)} posters found so far.") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") + + # Bulk-update ProgramData records with poster URLs + if artwork_map: + programs_to_update = [] + for prog in ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).only('id', 'program_id', 'custom_properties'): + art_key = pid_to_artwork_key.get(prog.program_id) + poster = artwork_map.get(art_key) if art_key else None + if poster: + cp = prog.custom_properties or {} + cp['poster_url'] = poster + prog.custom_properties = cp + programs_to_update.append(prog) + + if programs_to_update: + ProgramData.objects.bulk_update( + programs_to_update, ['custom_properties'], batch_size=1000 + ) + logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") + else: + logger.info("No poster artwork matched committed programs.") + else: + logger.info("No poster artwork found from Schedules Direct.") + + except Exception as art_error: + logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) + + elif fetch_posters: + logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") + + # ------------------------------------------------------------------------- + # Step 9: Apply SD station logos to matched channels if enabled + # ------------------------------------------------------------------------- + use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) + if use_sd_logos: + try: + from apps.channels.models import Channel as ChannelModel, Logo + + channels_to_update = [] + logos_created = 0 + + for channel in ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).select_related('epg_data', 'logo'): + icon_url = (channel.epg_data.icon_url or '').strip() + if not icon_url: + continue + + # Skip if channel already has this logo URL + if channel.logo and channel.logo.url == icon_url: + continue + + # Find or create a Logo object for this URL + try: + logo = Logo.objects.get(url=icon_url) + except Logo.DoesNotExist: + logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" + logo = Logo.objects.create(name=logo_name, url=icon_url) + logos_created += 1 + + channel.logo = logo + channels_to_update.append(channel) + + if channels_to_update: + ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) + logger.info(f"Applied SD logos to {len(channels_to_update)} channels " + f"({logos_created} new logos created).") + else: + logger.info("All matched channels already have current SD logos.") + + except Exception as logo_error: + logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) + # Prune ProgramData whose end_time has passed. With surgical per-date deletes, # programs from dates that have rolled off the window are never explicitly removed. today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) @@ -2860,6 +3155,37 @@ def fetch_schedules_direct(source, stations_only=False, force=False): except Exception as prune_err: logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + # ------------------------------------------------------------------------- + # Prune stale poster cache files (>30 days old or orphaned) + # ------------------------------------------------------------------------- + try: + cache_dir = '/data/cache/posters' + if os.path.exists(cache_dir): + import time as time_module + # Collect all poster hashes currently referenced by ProgramData + active_hashes = set() + for url in ProgramData.objects.filter( + epg__epg_source=source, + custom_properties__has_key='poster_url', + ).values_list('custom_properties__poster_url', flat=True): + if url: + active_hashes.add(url.rsplit('/', 1)[-1]) + + pruned_posters = 0 + for fname in os.listdir(cache_dir): + fpath = os.path.join(cache_dir, fname) + if not os.path.isfile(fpath): + continue + file_age = time_module.time() - os.path.getmtime(fpath) + # Remove if older than 30 days OR not referenced by any current program + if file_age > 30 * 24 * 3600 or fname not in active_hashes: + os.remove(fpath) + pruned_posters += 1 + if pruned_posters: + logger.info(f"Pruned {pruned_posters} stale poster cache files.") + except Exception as poster_prune_err: + logger.warning(f"Failed to prune poster cache: {poster_prune_err}") + # ------------------------------------------------------------------------- # Done # ------------------------------------------------------------------------- @@ -3266,425 +3592,3 @@ def generate_dummy_epg(source): logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " f"Dummy EPG programs are now generated on-demand.") return True - - -# EPG program byte-offset index for channel preview lookups - - -def _resolve_source_file(epg_source): - """Resolve the XML file path for an EPG source.""" - file_path = epg_source.extracted_file_path or epg_source.file_path - if not file_path: - file_path = epg_source.get_cache_file() - return file_path - - -_CHANNEL_ATTR_RE = re.compile(rb"""channel\s*=\s*(?:"([^"]+)"|'([^']+)')""") -_PROGRAMME_TAG = b'/' -_MAX_START_TAG = 4096 # generous upper bound for a start tag with namespaces/extra attrs -_OFFSET_CAP = 10 # max block-starts recorded per channel; exceeding this flags the channel as interleaved - - -def _decode_channel_id(raw): - """Match how EPGData.tvg_id is stored: resolve XML entities and strip, so byte-level index keys equal the lxml-parsed channel ids.""" - s = raw.decode('utf-8', errors='replace') - if '&' in s: - s = html.unescape(s) - return s.strip() - - -def _find_programme_tag(buf, start): - """ - Find the next ' - follow = idx + _PROGRAMME_TAG_LEN - if follow >= len(buf): - return idx, -1 # need more data - if buf[follow: follow + 1] not in _TAG_FOLLOW: - pos = follow # false match (e.g. ' that closes the opening tag (scan up to _MAX_START_TAG bytes) - tag_end = buf.find(b'>', follow, idx + _MAX_START_TAG) - if tag_end == -1: - if len(buf) >= idx + _MAX_START_TAG: - logger.warning( - f'[_find_programme_tag] start tag exceeds {_MAX_START_TAG} bytes at offset {idx}, skipping' - ) - return -1, -1 - return idx, -1 # need more data - return idx, tag_end - - -def _programme_to_dict(elem, start_time, end_time): - """Convert a lxml element to a serializable dict.""" - title_el = elem.find('title') - desc_el = elem.find('desc') - sub_el = elem.find('sub-title') - return { - 'title': title_el.text if title_el is not None and title_el.text else '', - 'description': desc_el.text if desc_el is not None and desc_el.text else '', - 'sub_title': sub_el.text if sub_el is not None and sub_el.text else '', - 'start_time': start_time.isoformat(), - 'end_time': end_time.isoformat(), - } - - -def build_programme_index(source_id): - """ - Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map. - Persists the result to EPGSource.programme_index. Most XMLTV files group programmes - by channel, but some split a channel across multiple non-contiguous blocks, so we - record block starts up to _OFFSET_CAP and mark only channels that exceed the cap - as interleaved. - """ - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - logger.error(f'[build_programme_index] EPGSource {source_id} not found') - return - - file_path = _resolve_source_file(source) - if not file_path or not os.path.exists(file_path): - logger.warning( - f'[build_programme_index] File not found for source {source_id}: {file_path}' - ) - return - - logger.debug( - f'[build_programme_index] Building byte-offset index for source {source_id} from {file_path}' - ) - start = time.monotonic() - index = {} - prev_channel = None - interleaved_channels = set() - - CHUNK = 8 * 1024 * 1024 # 8MB - - with open(file_path, 'rb') as f: - buf = bytearray() - buf_offset = 0 # absolute file offset of buf[0] - - while True: - chunk = f.read(CHUNK) - if not chunk and not buf: - break - buf.extend(chunk) - search_from = 0 - - while True: - idx, tag_end = _find_programme_tag(buf, search_from) - if idx == -1: - break - if tag_end == -1 and chunk: - break # incomplete tag at buffer edge, need more data - - abs_pos = buf_offset + idx - m = _CHANNEL_ATTR_RE.search( - buf, idx, tag_end + 1 if tag_end != -1 else idx + _MAX_START_TAG - ) - if m: - channel_id = _decode_channel_id(m.group(1) or m.group(2)) - if channel_id not in index: - index[channel_id] = [abs_pos] - elif channel_id != prev_channel: - if len(index[channel_id]) < _OFFSET_CAP: - index[channel_id].append(abs_pos) - else: - interleaved_channels.add(channel_id) - prev_channel = channel_id - - search_from = ( - (tag_end + 1) if tag_end != -1 else (idx + _PROGRAMME_TAG_LEN) - ) - - if not chunk: - break - - # Keep unprocessed tail for next iteration - keep_from = ( - max(search_from, len(buf) - _MAX_START_TAG) if chunk else len(buf) - ) - del buf[:keep_from] - buf_offset += keep_from - - elapsed = time.monotonic() - start - logger.info( - f'[build_programme_index] Indexed {len(index)} channels in {elapsed:.1f}s for source {source_id}' - + ( - f' ({len(interleaved_channels)} interleaved)' - if interleaved_channels - else '' - ) - ) - - result = { - 'channels': index, - 'interleaved_channels': sorted(interleaved_channels), - } - EPGSource.objects.filter(id=source_id).update(programme_index=result) - - -@shared_task -def build_programme_index_task(source_id): - """Celery wrapper. Locks so refresh and preview don't both build the same source. Releases on finish rather than waiting out the TTL.""" - from core.utils import RedisClient - - redis_client = RedisClient.get_client() - lock_key = f'building_programme_index_{source_id}' - if not redis_client.set(lock_key, '1', nx=True, ex=300): - return - try: - build_programme_index(source_id) - finally: - redis_client.delete(lock_key) - - -def find_current_program_for_tvg_id(epg_or_id): - """ - Look up the currently-airing program for an EPGData instance (or id) using - the byte-offset index. If no index exists yet, queue an async build and let - the caller retry rather than doing a blocking scan. - - Returns dict, None, or "timeout". - """ - if isinstance(epg_or_id, EPGData): - epg = epg_or_id - else: - try: - epg = EPGData.objects.select_related('epg_source').get(id=epg_or_id) - except EPGData.DoesNotExist: - return None - - source = epg.epg_source - if not source or source.source_type in ('dummy', 'schedules_direct'): - return None - - tvg_id = epg.tvg_id - if not tvg_id: - return None - - file_path = _resolve_source_file(source) - if not file_path or not os.path.exists(file_path): - return None - - now = timezone.now() - # Force a fresh read of the DB-backed index to avoid using stale related-object - # state when an EPG refresh invalidates/rebuilds the index concurrently. - source.refresh_from_db(fields=['programme_index']) - index = source.programme_index - - if index is not None: - channels = index.get('channels', {}) - if tvg_id not in channels: - # Channel has no programmes in the file - return None - offsets = channels[tvg_id] - if tvg_id in (index.get('interleaved_channels') or ()): - # Check all stored offsets first (cheap: one seek + one element parse each) - result = _read_programs_at_offsets(file_path, tvg_id, offsets, now) - if result is not None: - return result - # Current programme is beyond the stored offsets; scan forward from the - # last known position to avoid re-reading the already-checked portion - result = _scan_from_offset_for_tvg_id(file_path, tvg_id, offsets[-1], now) - if result == 'timeout': - logger.warning( - f'[find_current_program_for_tvg_id] Interleaved scan timed out for ' - f'tvg_id={tvg_id} source={source.id}; index has {len(offsets)} offsets' - ) - return None - return result - return _read_programs_at_offsets(file_path, tvg_id, offsets, now) - - # No index yet: dispatch a background build and let the frontend retry. - # A sync scan can block a worker for ~10s on SMB-hosted EPGs. - build_programme_index_task.delay(source.id) - return 'timeout' - - -def _read_programs_at_offsets(file_path, tvg_id, offsets, now): - """ - Seek to each offset, extract elements for *tvg_id*, return the - first one currently airing. Chunk-based so it works on minified XML. - """ - PROG_CLOSE = b'' - CLOSE_LEN = len(PROG_CLOSE) - READ_SIZE = 2 * 1024 * 1024 # 2MB per read - - with open(file_path, 'rb') as f: - for offset in offsets: - f.seek(offset) - buf = bytearray() - done = False - - while not done: - chunk = f.read(READ_SIZE) - if not chunk and not buf: - break - buf.extend(chunk) - search_from = 0 - - while True: - tag_start, tag_end = _find_programme_tag(buf, search_from) - if tag_start == -1: - break - if tag_end == -1 and chunk: - break # incomplete tag, need more data - - # Check channel before searching for close tag - m = _CHANNEL_ATTR_RE.search( - buf, - tag_start, - tag_end + 1 if tag_end != -1 else tag_start + _MAX_START_TAG, - ) - if not m: - search_from = ( - (tag_end + 1) - if tag_end != -1 - else (tag_start + _PROGRAMME_TAG_LEN) - ) - continue - - ch = _decode_channel_id(m.group(1) or m.group(2)) - if ch != tvg_id: - done = True # different channel, end of block - break - - # Find the closing tag - close_pos = buf.find( - PROG_CLOSE, tag_end + 1 if tag_end != -1 else m.end() - ) - if close_pos == -1: - if not chunk: - done = True # EOF with no close tag - break # need more data - close_end = close_pos + CLOSE_LEN - - element_bytes = bytes(buf[tag_start:close_end]) - search_from = close_end - - try: - prog = _parse_programme_element(element_bytes) - except etree.XMLSyntaxError: - continue - - start_str = prog.get('start') - stop_str = prog.get('stop') - if not start_str or not stop_str: - continue - start_time = parse_xmltv_time(start_str) - end_time = parse_xmltv_time(stop_str) - if start_time is None or end_time is None: - continue - if start_time <= now < end_time: - return _programme_to_dict(prog, start_time, end_time) - - # Trim processed bytes - if search_from > 0: - del buf[:search_from] - search_from = 0 - - if not chunk: - break - - return None - - -def _scan_from_offset_for_tvg_id(file_path, tvg_id, start_offset, now, timeout_sec=10): - """ - Scan forward from start_offset for tvg_id, skipping other channels rather than - stopping at a channel boundary. Used for interleaved/time-sorted XMLTV files where - a channel exceeded the stored offset cap. - Returns dict, None, or 'timeout'. - """ - PROG_CLOSE = b'' - CLOSE_LEN = len(PROG_CLOSE) - READ_SIZE = 2 * 1024 * 1024 - deadline = time.monotonic() + timeout_sec - - with open(file_path, 'rb') as f: - f.seek(start_offset) - buf = bytearray() - - while True: - if time.monotonic() > deadline: - return 'timeout' - - chunk = f.read(READ_SIZE) - if not chunk and not buf: - break - buf.extend(chunk) - search_from = 0 - - trim_to = 0 - - while True: - tag_start, tag_end = _find_programme_tag(buf, search_from) - if tag_start == -1: - trim_to = search_from - break - if tag_end == -1 and chunk: - trim_to = tag_start # keep incomplete tag for next read - break - - m = _CHANNEL_ATTR_RE.search( - buf, - tag_start, - tag_end + 1 if tag_end != -1 else tag_start + _MAX_START_TAG, - ) - if not m: - search_from = ( - tag_end + 1 if tag_end != -1 else tag_start + _PROGRAMME_TAG_LEN - ) - continue - - ch = _decode_channel_id(m.group(1) or m.group(2)) - if ch != tvg_id: - search_from = ( - tag_end + 1 if tag_end != -1 else tag_start + _PROGRAMME_TAG_LEN - ) - continue - - close_pos = buf.find( - PROG_CLOSE, tag_end + 1 if tag_end != -1 else m.end() - ) - if close_pos == -1: - trim_to = tag_start # keep incomplete element for next read - break - close_end = close_pos + CLOSE_LEN - - element_bytes = bytes(buf[tag_start:close_end]) - search_from = close_end - - try: - prog = _parse_programme_element(element_bytes) - except etree.XMLSyntaxError: - continue - - start_str = prog.get('start') - stop_str = prog.get('stop') - if not start_str or not stop_str: - continue - start_time = parse_xmltv_time(start_str) - end_time = parse_xmltv_time(stop_str) - if start_time is None or end_time is None: - continue - if start_time <= now < end_time: - return _programme_to_dict(prog, start_time, end_time) - - if trim_to > 0: - del buf[:trim_to] - - if not chunk: - break - - return None diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index db6b1b6e..fc43c36d 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1,1640 +1,256 @@ -# apps/m3u/tasks.py +# apps/epg/tasks.py + import logging -import re -import regex -import requests +import gzip +import html.entities import os -import gc -import gzip, zipfile -from concurrent.futures import ThreadPoolExecutor, as_completed -from celery.app.control import Inspect -from celery.result import AsyncResult -from celery import shared_task, current_app, group +import uuid +import requests +import time # Add import for tracking download progress +from datetime import datetime, timedelta, timezone as dt_timezone +import gc # Add garbage collection module +import json +from lxml import etree # Using lxml exclusively +import psutil # Add import for memory tracking +import zipfile + +from celery import shared_task from django.conf import settings -from django.core.cache import cache -from django.db import models, transaction -from .models import M3UAccount -from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount +from django.db import connection, transaction +from django.db.models import Q +from django.utils import timezone +from apps.channels.models import Channel +from core.models import UserAgent, CoreSettings + from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from django.utils import timezone -import time -import json -from core.utils import ( - RedisClient, - acquire_task_lock, - release_task_lock, - TaskLockRenewer, - natural_sort_key, - log_system_event, -) -from core.models import CoreSettings, UserAgent -from asgiref.sync import async_to_sync -from core.xtream_codes import Client as XCClient -from core.utils import send_websocket_update -from .utils import normalize_stream_url + +from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 +from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event logger = logging.getLogger(__name__) -BATCH_SIZE = 1500 # Optimized batch size for threading -m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") +SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' +SD_DAYS_TO_FETCH = 20 +SD_PROGRAM_BATCH_SIZE = 5000 -_EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') +# DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named +# entities so lxml/libxml2 can resolve references like é correctly +# instead of silently dropping them in recovery mode. +# The 5 XML-predefined entities (amp, lt, gt, quot, apos) are always +# recognised by the XML spec and must not be redeclared. +_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'}) -def fetch_m3u_lines(account, use_cache=False): - os.makedirs(m3u_dir, exist_ok=True) - file_path = os.path.join(m3u_dir, f"{account.id}.m3u") - - """Fetch M3U file lines efficiently.""" - if account.server_url: - if not use_cache or not os.path.exists(file_path): - try: - # Try to get account-specific user agent first - user_agent_obj = account.get_user_agent() - user_agent = ( - user_agent_obj.user_agent - if user_agent_obj - else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - ) - - logger.debug( - f"Using user agent: {user_agent} for M3U account: {account.name}" - ) - headers = {"User-Agent": user_agent} - logger.info(f"Fetching from URL {account.server_url}") - - # Set account status to FETCHING before starting download - account.status = M3UAccount.Status.FETCHING - account.last_message = "Starting download..." - account.save(update_fields=["status", "last_message"]) - - response = requests.get( - account.server_url, headers=headers, stream=True, - timeout=(30, 60), # 30s connect, 60s read between chunks - ) - - # Log the actual response details for debugging - logger.debug(f"HTTP Response: {response.status_code} from {account.server_url}") - logger.debug(f"Content-Type: {response.headers.get('content-type', 'Not specified')}") - logger.debug(f"Content-Length: {response.headers.get('content-length', 'Not specified')}") - logger.debug(f"Response headers: {dict(response.headers)}") - - # Check if we've been redirected to a different URL - if hasattr(response, 'url') and response.url != account.server_url: - logger.warning(f"Request was redirected from {account.server_url} to {response.url}") - - # Check for ANY non-success status code FIRST (before raise_for_status) - if response.status_code < 200 or response.status_code >= 300: - # For error responses, read the content immediately (not streaming) - try: - response_content = response.text[:1000] # Capture up to 1000 characters - logger.error(f"Error response content: {response_content!r}") - except Exception as e: - logger.error(f"Could not read error response content: {e}") - response_content = "Could not read error response content" - - # Provide specific messages for known non-standard codes - if response.status_code == 884: - error_msg = f"Server returned HTTP 884 (authentication/authorization failure) from URL: {account.server_url}. Server message: {response_content}" - elif response.status_code >= 800: - error_msg = f"Server returned non-standard HTTP status {response.status_code} from URL: {account.server_url}. Server message: {response_content}" - elif response.status_code == 404: - error_msg = f"M3U file not found (404) at URL: {account.server_url}. Server message: {response_content}" - elif response.status_code == 403: - error_msg = f"Access forbidden (403) to M3U file at URL: {account.server_url}. Server message: {response_content}" - elif response.status_code == 401: - error_msg = f"Authentication required (401) for M3U file at URL: {account.server_url}. Server message: {response_content}" - elif response.status_code == 500: - error_msg = f"Server error (500) while fetching M3U file from URL: {account.server_url}. Server message: {response_content}" - else: - error_msg = f"HTTP error ({response.status_code}) while fetching M3U file from URL: {account.server_url}. Server message: {response_content}" - - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account.id, - "downloading", - 100, - status="error", - error=error_msg, - ) - return [], False - - # Only call raise_for_status if we have a success code (this should not raise now) - response.raise_for_status() - - total_size = int(response.headers.get("Content-Length", 0)) - downloaded = 0 - start_time = time.time() - last_update_time = start_time - progress = 0 - has_content = False - - # Stream directly to a temp file to avoid holding the entire - # M3U in memory (large files can be 100MB+, which would use - # ~3x that in RAM in certain approaches). - temp_path = file_path + ".tmp" - try: - send_m3u_update(account.id, "downloading", 0) - with open(temp_path, "wb") as tmp_file: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - tmp_file.write(chunk) - has_content = True - - downloaded += len(chunk) - elapsed_time = time.time() - start_time - - # Calculate download speed in KB/s - speed = downloaded / elapsed_time / 1024 # in KB/s - - # Calculate progress percentage - if total_size and total_size > 0: - progress = (downloaded / total_size) * 100 - - # Time remaining (in seconds) - time_remaining = ( - (total_size - downloaded) / (speed * 1024) - if speed > 0 - else 0 - ) - - current_time = time.time() - if current_time - last_update_time >= 0.5: - last_update_time = current_time - if progress > 0: - # Update the account's last_message with detailed progress info - progress_msg = f"Downloading: {progress:.1f}% - {speed:.1f} KB/s - {time_remaining:.1f}s remaining" - account.last_message = progress_msg - account.save(update_fields=["last_message"]) - - send_m3u_update( - account.id, - "downloading", - progress, - speed=speed, - elapsed_time=elapsed_time, - time_remaining=time_remaining, - message=progress_msg, - ) - - # Check if we actually received any content - logger.info(f"Download completed. Has content: {has_content}, Content length: {downloaded} bytes") - if not has_content or downloaded == 0: - error_msg = f"Server responded successfully (HTTP {response.status_code}) but provided empty M3U file from URL: {account.server_url}" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account.id, - "downloading", - 100, - status="error", - error=error_msg, - ) - return [], False - - # Validate the file by reading only the first portion from - # disk — no need to load the entire file into memory just - # to check the header. - VALIDATION_READ_SIZE = 32768 # 32KB covers headers comfortably - try: - with open(temp_path, "rb") as vf: - head_bytes = vf.read(VALIDATION_READ_SIZE) - head_str = head_bytes.decode('utf-8', errors='ignore') - head_lines = head_str.strip().split('\n') - - # Count total lines efficiently without loading full file - with open(temp_path, "rb") as vf: - total_lines = sum(1 for _ in vf) - - # Log first few lines for debugging (be careful not to log too much) - preview_lines = head_lines[:5] - logger.info(f"Content preview (first 5 lines): {preview_lines}") - logger.info(f"Total lines in content: {total_lines}") - - # Check if it's a valid M3U file (should start with #EXTM3U or contain M3U-like content) - is_valid_m3u = False - - # First, check if this looks like an error response disguised as 200 OK - head_lower = head_str.lower() - if any(error_indicator in head_lower for error_indicator in [ - ' bytes: + """Build a DOCTYPE internal subset declaring all HTML 4 named entities.""" + lines = [b'\n'.encode('ascii')) + lines.append(b']>\n') + return b''.join(lines) -def get_case_insensitive_attr(attributes, key, default=""): - """Get attribute value using case-insensitive key lookup.""" - for attr_key, attr_value in attributes.items(): - if attr_key.lower() == key.lower(): - return attr_value - return default +_HTML_ENTITY_DOCTYPE = _build_html_entity_doctype() -def parse_is_adult(value): - try: - return int(value) == 1 - except (TypeError, ValueError): - return False +class _PrependStream: + """Wraps an open binary file and prepends a bytes prefix to its content. - -def parse_extinf_line(line: str) -> dict: + Used by _open_xmltv_file to inject a DOCTYPE entity block before the + file content reaches lxml's iterparse, with zero disk I/O. """ - Parse an EXTINF line from an M3U file. - This function removes the "#EXTINF:" prefix, then extracts all key="value" attributes, - and treats everything after the last attribute as the display name. - Returns a dictionary with: - - 'attributes': a dict of attribute key/value pairs (e.g. tvg-id, tvg-logo, group-title) - - 'display_name': the text after the attributes (the fallback display name) - - 'name': the value from tvg-name (if present) or the display name otherwise. + __slots__ = ('_prefix', '_prefix_pos', '_file') + + def __init__(self, prefix: bytes, file_obj): + self._prefix = prefix + self._prefix_pos = 0 + self._file = file_obj + + def read(self, size=-1): + prefix_len = len(self._prefix) + if self._prefix_pos >= prefix_len: + return self._file.read(size) + remaining = prefix_len - self._prefix_pos + if size < 0: + chunk = self._prefix[self._prefix_pos:] + self._file.read() + self._prefix_pos = prefix_len + return chunk + if size <= remaining: + chunk = self._prefix[self._prefix_pos:self._prefix_pos + size] + self._prefix_pos += size + return chunk + chunk = self._prefix[self._prefix_pos:] + self._prefix_pos = prefix_len + return chunk + self._file.read(size - remaining) + + def close(self): + self._file.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +def _open_xmltv_file(file_path: str): + """Open an XMLTV file for lxml iterparse, injecting an HTML entity DOCTYPE. + + Prepends a block that declares all 252 HTML 4 named + entities so lxml/libxml2 resolves references like é correctly + instead of silently dropping them in recovery mode. This involves zero + disk I/O (the DOCTYPE is streamed in-memory before the file content). + + If the file already contains a declaration the file is returned + unchanged; a second DOCTYPE would be invalid XML. + + The caller is responsible for closing the returned object. """ - if not line.startswith("#EXTINF:"): + f = open(file_path, 'rb') + start = f.read(512) + + # Do not inject if the file already declares a DOCTYPE. + if b'= 0: + decl_end = start.find(b'?>', xml_pos) + if decl_end >= 0: + xml_decl = start[:decl_end + 2] + f.seek(decl_end + 2) + return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f) + + # No XML declaration found; insert DOCTYPE at the very start of the file. + f.seek(0) + return _PrependStream(_HTML_ENTITY_DOCTYPE, f) + + +def validate_icon_url_fast(icon_url, max_length=None): + """ + Fast validation for icon URLs during parsing. + Returns None if URL is too long, original URL otherwise. + If max_length is None, gets it dynamically from the EPGData model field. + """ + if max_length is None: + # Get max_length dynamically from the model field + max_length = EPGData._meta.get_field('icon_url').max_length + + if icon_url and len(icon_url) > max_length: + logger.warning(f"Icon URL too long ({len(icon_url)} > {max_length}), skipping: {icon_url[:100]}...") return None - content = line[len("#EXTINF:") :].strip() - - # Single pass: extract all attributes AND track the last attribute position. - # Keys are normalised to lowercase so downstream code can use plain dict.get() - attrs = {} - last_attr_end = 0 - - for match in _EXTINF_ATTR_RE.finditer(content): - attrs[match.group(1).lower()] = match.group(3) - last_attr_end = match.end() - - # Everything after the last attribute (skipping leading comma and whitespace) is the display name - if last_attr_end > 0: - remaining = content[last_attr_end:].strip() - # Remove leading comma if present - if remaining.startswith(','): - remaining = remaining[1:].strip() - display_name = remaining - else: - # No attributes found, try the old comma-split method as fallback - parts = content.split(',', 1) - if len(parts) == 2: - display_name = parts[1].strip() - else: - display_name = content.strip() - - # Per the base #EXTINF spec, the comma text is the canonical human-readable title. - # Fall back to tvc-guide-title, then tvg-name (which some providers use as an EPG key, - # not a display label), and finally the raw content if everything else is empty. - name = display_name or attrs.get("tvc-guide-title") or attrs.get("tvg-name") or content.strip() - return {"attributes": attrs, "display_name": display_name, "name": name} + return icon_url -def iter_m3u_entries(lines): - """ - Generator that yields fully-assembled M3U stream entries from raw lines. - - Each yielded dict is guaranteed to contain a ``url`` key in addition to the - fields produced by :func:`parse_extinf_line` (``attributes``, ``display_name``, - ``name``). Recognised extended-tag lines that appear *between* an ``#EXTINF`` - and its URL are accumulated into the pending entry so they are available for - downstream processing: - - - ``#EXTGRP`` — sets ``attributes["group-title"]`` when no ``group-title`` - attribute was present on the ``#EXTINF`` line (explicit attribute wins). - - ``#EXTVLCOPT`` — stored as a list under the ``vlc_opts`` key. - - Unknown directives (``#KODIPROP``, etc.) and blank lines are - silently skipped while keeping the pending entry intact. A second ``#EXTINF`` - before a URL discards the first entry with a warning. A trailing ``#EXTINF`` - at end-of-file with no URL is also discarded. - - Adding support for a new directive requires only a new ``elif`` branch here; - no other code needs to change. - """ - pending = None - pending_line_num = None - for line_num, raw_line in enumerate(lines, 1): - line = raw_line.strip() - if not line: - continue - - if line.startswith("#EXTINF"): - if pending is not None: - logger.warning( - f"Line {pending_line_num}: #EXTINF had no URL (next #EXTINF at line {line_num}); " - f"discarding entry: {list(pending['attributes'].items())[:3]}" - ) - parsed = parse_extinf_line(line) - if parsed is None: - logger.warning(f"Line {line_num}: Failed to parse #EXTINF: {line[:200]}") - pending = parsed # None if malformed; URL branch guards on `pending is not None` - pending_line_num = line_num - - elif line.startswith("#EXTGRP:"): - # Only apply when group-title is absent — explicit attribute wins. - if pending is not None and "group-title" not in pending["attributes"]: - pending["attributes"]["group-title"] = line[len("#EXTGRP:"):].strip() - # else: #EXTGRP outside an entry, or group-title already set — silently skip - - elif line.startswith("#EXTVLCOPT:"): - if pending is not None: - pending.setdefault("vlc_opts", []).append(line[len("#EXTVLCOPT:"):]) - # else: #EXTVLCOPT outside an entry — silently skip - - elif pending is not None and line.startswith(("http", "rtsp", "rtp", "udp")): - pending["url"] = normalize_stream_url(line) if line.startswith("udp") else line - yield pending - pending = None - pending_line_num = None - - # else: unknown directive or bare content — skip, keeping pending intact - - if pending is not None: - logger.warning( - f"Line {pending_line_num}: #EXTINF at end of file had no URL; " - f"discarding entry: {list(pending['attributes'].items())[:3]}" - ) +MAX_EXTRACT_CHUNK_SIZE = 65536 # 64kb (base2) -@shared_task -def refresh_m3u_accounts(): - """Queue background parse for all active M3UAccounts.""" - active_accounts = M3UAccount.objects.filter(is_active=True) - count = 0 - for account in active_accounts: - refresh_single_m3u_account.delay(account.id) - count += 1 - - msg = f"Queued M3U refresh for {count} active account(s)." - logger.info(msg) - return msg - - -def check_field_lengths(streams_to_create): - for stream in streams_to_create: - for field, value in stream.__dict__.items(): - if isinstance(value, str) and len(value) > 255: - print(f"{field} --- {value}") - - print("") - print("") - - -@shared_task -def process_groups(account, groups, scan_start_time=None): - """Process groups and update their relationships with the M3U account. - - Args: - account: M3UAccount instance - groups: Dict of group names to custom properties - scan_start_time: Timestamp when the scan started (for consistent last_seen marking) - """ - # Use scan_start_time if provided, otherwise current time - # This ensures consistency with stream processing and cleanup logic - if scan_start_time is None: - scan_start_time = timezone.now() - - existing_groups = { - group.name: group - for group in ChannelGroup.objects.filter(name__in=groups.keys()) - } - logger.info(f"Currently {len(existing_groups)} existing groups") - - # Check if we should auto-enable new groups based on account settings - account_custom_props = account.custom_properties or {} - auto_enable_new_groups_live = account_custom_props.get("auto_enable_new_groups_live", True) - - # Separate existing groups from groups that need to be created - existing_group_objs = [] - groups_to_create = [] - - for group_name, custom_props in groups.items(): - if group_name in existing_groups: - existing_group_objs.append(existing_groups[group_name]) - else: - groups_to_create.append(ChannelGroup(name=group_name)) - - # Create new groups and fetch them back with IDs - newly_created_group_objs = [] - if groups_to_create: - logger.info(f"Creating {len(groups_to_create)} new groups for account {account.id}") - newly_created_group_objs = list(ChannelGroup.bulk_create_and_fetch(groups_to_create)) - logger.debug(f"Successfully created {len(newly_created_group_objs)} new groups") - - # Combine all groups - all_group_objs = existing_group_objs + newly_created_group_objs - - # Get existing relationships for this account - existing_relationships = { - rel.channel_group.name: rel - for rel in ChannelGroupM3UAccount.objects.filter( - m3u_account=account, - channel_group__name__in=groups.keys() - ).select_related('channel_group') +def send_epg_update(source_id, action, progress, **kwargs): + """Send WebSocket update about EPG download/parsing progress""" + # Start with the base data dictionary + data = { + "progress": progress, + "type": "epg_refresh", + "source": source_id, + "action": action, } - relations_to_create = [] - relations_to_update = [] + # Add the additional key-value pairs from kwargs + data.update(kwargs) - for group in all_group_objs: - custom_props = groups.get(group.name, {}) + # Use the standardized update function with garbage collection for program parsing + # This is a high-frequency operation that needs more aggressive memory management + collect_garbage = action == "parsing_programs" and progress % 10 == 0 + send_websocket_update('updates', 'update', data, collect_garbage=collect_garbage) - if group.name in existing_relationships: - # Update existing relationship if xc_id has changed (preserve other custom properties) - existing_rel = existing_relationships[group.name] + # Explicitly clear references + data = None - # Get existing custom properties (now JSONB, no need to parse) - existing_custom_props = existing_rel.custom_properties or {} - - # Get the new xc_id from groups data - new_xc_id = custom_props.get("xc_id") - existing_xc_id = existing_custom_props.get("xc_id") - - # Only update if xc_id has changed - if new_xc_id != existing_xc_id: - # Merge new xc_id with existing custom properties to preserve user settings - updated_custom_props = existing_custom_props.copy() - if new_xc_id is not None: - updated_custom_props["xc_id"] = new_xc_id - elif "xc_id" in updated_custom_props: - # Remove xc_id if it's no longer provided (e.g., converting from XC to standard) - del updated_custom_props["xc_id"] - - existing_rel.custom_properties = updated_custom_props - existing_rel.last_seen = scan_start_time - existing_rel.is_stale = False - relations_to_update.append(existing_rel) - logger.debug(f"Updated xc_id for group '{group.name}' from '{existing_xc_id}' to '{new_xc_id}' - account {account.id}") - else: - # Update last_seen even if xc_id hasn't changed - existing_rel.last_seen = scan_start_time - existing_rel.is_stale = False - relations_to_update.append(existing_rel) - logger.debug(f"xc_id unchanged for group '{group.name}' - account {account.id}") - else: - # Create new relationship - this group is new to this M3U account - # Use the auto_enable setting to determine if it should start enabled - if not auto_enable_new_groups_live: - logger.info(f"Group '{group.name}' is new to account {account.id} - creating relationship but DISABLED (auto_enable_new_groups_live=False)") - - relations_to_create.append( - ChannelGroupM3UAccount( - channel_group=group, - m3u_account=account, - custom_properties=custom_props, - enabled=auto_enable_new_groups_live, - last_seen=scan_start_time, - is_stale=False, - ) - ) - - # Bulk create new relationships - if relations_to_create: - ChannelGroupM3UAccount.objects.bulk_create(relations_to_create, ignore_conflicts=True) - logger.debug(f"Created {len(relations_to_create)} new group relationships for account {account.id}") - - # Bulk update existing relationships - if relations_to_update: - ChannelGroupM3UAccount.objects.bulk_update(relations_to_update, ['custom_properties', 'last_seen', 'is_stale']) - logger.info(f"Updated {len(relations_to_update)} existing group relationships for account {account.id}") + # For high-frequency parsing, occasionally force additional garbage collection + # to prevent memory buildup + if action == "parsing_programs" and progress % 50 == 0: + gc.collect() -def cleanup_stale_group_relationships(account, scan_start_time): +def _sd_send_ws_sync(source_id, action, progress, **kwargs): """ - Remove group relationships that haven't been seen since the stale retention period. - This follows the same logic as stream cleanup for consistency. + Sends a WebSocket progress update synchronously via Redis, bypassing gevent.spawn. + + In Celery prefork workers that inherit gevent monkey-patching from uWSGI, + gevent.spawn schedules coroutines that never execute because there is no + running gevent hub in the worker process. This function writes directly to + Redis using the channels_redis 4.x wire format, guaranteeing delivery + regardless of the execution context. """ - # Calculate cutoff date for stale group relationships - stale_cutoff = scan_start_time - timezone.timedelta(days=account.stale_stream_days) - logger.info( - f"Removing group relationships not seen since {stale_cutoff} for M3U account {account.id}" - ) - - # Find stale relationships - stale_relationships = ChannelGroupM3UAccount.objects.filter( - m3u_account=account, - last_seen__lt=stale_cutoff - ).select_related('channel_group') - - relations_to_delete = list(stale_relationships) - deleted_count = len(relations_to_delete) - - if deleted_count > 0: - logger.info( - f"Found {deleted_count} stale group relationships for account {account.id}: " - f"{[rel.channel_group.name for rel in relations_to_delete]}" - ) - - # Delete the stale relationships - stale_relationships.delete() - - # Check if any of the deleted relationships left groups with no remaining associations - orphaned_group_ids = [] - for rel in relations_to_delete: - group = rel.channel_group - - # Check if this group has any remaining M3U account relationships - remaining_m3u_relationships = ChannelGroupM3UAccount.objects.filter( - channel_group=group - ).exists() - - # Check if this group has any direct channels (not through M3U accounts) - has_direct_channels = group.related_channels().exists() - - # If no relationships and no direct channels, it's safe to delete - if not remaining_m3u_relationships and not has_direct_channels: - orphaned_group_ids.append(group.id) - logger.debug(f"Group '{group.name}' has no remaining associations and will be deleted") - - # Delete truly orphaned groups - if orphaned_group_ids: - deleted_groups = list(ChannelGroup.objects.filter(id__in=orphaned_group_ids).values_list('name', flat=True)) - ChannelGroup.objects.filter(id__in=orphaned_group_ids).delete() - logger.info(f"Deleted {len(orphaned_group_ids)} orphaned groups that had no remaining associations: {deleted_groups}") - else: - logger.debug(f"No stale group relationships found for account {account.id}") - - return deleted_count - - -def collect_xc_streams(account_id, enabled_groups): - """Collect all XC streams in a single API call and filter by enabled groups.""" - account = M3UAccount.objects.get(id=account_id) - all_streams = [] - - # Create a mapping from category_id to group info for filtering - enabled_category_ids = {} - for group_name, props in enabled_groups.items(): - if "xc_id" in props: - enabled_category_ids[str(props["xc_id"])] = { - "name": group_name, - "props": props - } - try: - with XCClient( - account.server_url, - account.username, - account.password, - account.get_user_agent(), - ) as xc_client: + import msgpack + import redis as redis_lib + from django.conf import settings - # Fetch ALL live streams in a single API call (much more efficient) - logger.info("Fetching ALL live streams from XC provider...") - all_xc_streams = xc_client.get_all_live_streams() # Get all streams without category filter + data = {"progress": progress, "type": "epg_refresh", "source": source_id, "action": action} + data.update(kwargs) + message = {"type": "update", "data": data} - if not all_xc_streams: - logger.warning("No live streams returned from XC provider") - return [] + redis_url = getattr(settings, "CELERY_BROKER_URL", "redis://localhost:6379/0") + r = redis_lib.from_url(redis_url, decode_responses=False) - logger.info(f"Retrieved {len(all_xc_streams)} total live streams from provider") + prefix = "asgi" + group_name = "updates" + group_key = f"{prefix}:group:{group_name}" + group_expiry = 86400 + channel_expiry = 60 + rand_len = 12 + now = time.time() - # Filter streams based on enabled categories - filtered_count = 0 - for stream in all_xc_streams: - # Fall back to a generated name if the provider returns null/empty - stream_name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" - if not stream.get("name"): - logger.warning( - f"XC stream has null/empty name; using generated name '{stream_name}' " - f"(stream_id={stream.get('stream_id', 'unknown')})" - ) + r.zremrangebyscore(group_key, 0, now - group_expiry) + raw = r.zrange(group_key, 0, -1) + if not raw: + return - # Get the category_id for this stream - category_id = str(stream.get("category_id", "")) - - # Only include streams from enabled categories - if category_id in enabled_category_ids: - group_info = enabled_category_ids[category_id] - - # Convert XC stream to our standard format with all properties preserved - stream_data = { - "name": stream_name, - "url": xc_client.get_stream_url(stream["stream_id"]), - "attributes": { - "tvg-id": stream.get("epg_channel_id", ""), - "tvg-logo": stream.get("stream_icon", ""), - "group-title": group_info["name"], - # Preserve all XC stream properties as custom attributes - "stream_id": str(stream.get("stream_id", "")), - "num": stream.get("num"), - "category_id": category_id, - "stream_type": stream.get("stream_type", ""), - "added": stream.get("added", ""), - "is_adult": str(stream.get("is_adult", "0")), - "custom_sid": stream.get("custom_sid", ""), - # Include any other properties that might be present - **{k: str(v) for k, v in stream.items() if k not in [ - "name", "stream_id", "epg_channel_id", "stream_icon", - "category_id", "stream_type", "added", "is_adult", "custom_sid", "num" - ] and v is not None} - } - } - all_streams.append(stream_data) - filtered_count += 1 + channels = [m.decode("utf-8") if isinstance(m, bytes) else m for m in raw] + nonlocal_map = {} + for ch in channels: + pos = ch.find("!") + nl = ch[:pos + 1] if pos >= 0 else ch + nonlocal_map.setdefault(nl, []).append(ch) + pipe = r.pipeline(transaction=False) + for nl, chs in nonlocal_map.items(): + channel_key = prefix + nl + msg = dict(message) + msg["__asgi_channel__"] = chs + serialized = os.urandom(rand_len) + msgpack.packb(msg) + pipe.zadd(channel_key, {serialized: now}) + pipe.expire(channel_key, channel_expiry) + pipe.execute() + r.close() except Exception as e: - logger.error(f"Failed to fetch XC streams: {str(e)}") - return [] + logger.warning(f"SD WebSocket sync send failed: {e}") + # Fall back to standard path + send_epg_update(source_id, action, progress, **kwargs) - logger.info(f"Filtered {filtered_count} streams from {len(enabled_category_ids)} enabled categories") - return all_streams -def process_xc_category_direct(account_id, batch, groups, hash_keys): - from django.db import connections - - # Ensure clean database connections for threading - connections.close_all() - - account = M3UAccount.objects.get(id=account_id) - - streams_to_create = [] - streams_to_update = [] - stream_hashes = {} - - try: - with XCClient( - account.server_url, - account.username, - account.password, - account.get_user_agent(), - ) as xc_client: - # Log the batch details to help with debugging - logger.debug(f"Processing XC batch: {batch}") - - for group_name, props in batch.items(): - # Check if we have a valid xc_id for this group - if "xc_id" not in props: - logger.error( - f"Missing xc_id for group {group_name} in batch {batch}" - ) - continue - - # Get actual group ID from the mapping - group_id = groups.get(group_name) - if not group_id: - logger.error(f"Group {group_name} not found in enabled groups") - continue - - try: - logger.debug( - f"Fetching streams for XC category: {group_name} (ID: {props['xc_id']})" - ) - streams = xc_client.get_live_category_streams(props["xc_id"]) - - if not streams: - logger.warning( - f"No streams found for XC category {group_name} (ID: {props['xc_id']})" - ) - continue - - logger.debug( - f"Found {len(streams)} streams for category {group_name}" - ) - - for stream in streams: - name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" - if not stream.get("name"): - logger.warning( - f"XC stream has null/empty name in category {group_name}; " - f"using generated name '{name}' (stream_id={stream.get('stream_id', 'unknown')})" - ) - raw_stream_id = stream.get("stream_id", "") - provider_stream_id = None - if raw_stream_id: - try: - provider_stream_id = int(raw_stream_id) - except (ValueError, TypeError): - pass - url = xc_client.get_stream_url(stream["stream_id"]) - tvg_id = stream.get("epg_channel_id", "") - tvg_logo = stream.get("stream_icon", "") - group_title = group_name - stream_chno = stream.get("num") - # Convert stream_chno to float if valid, otherwise None - if stream_chno is not None: - try: - stream_chno = float(stream_chno) - except (ValueError, TypeError): - stream_chno = None - - stream_hash = Stream.generate_hash_key( - name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, - account_type='XC', stream_id=provider_stream_id - ) - stream_props = { - "name": name, - "url": url, - "logo_url": tvg_logo, - "tvg_id": tvg_id, - "m3u_account": account, - "channel_group_id": int(group_id), - "stream_hash": stream_hash, - "custom_properties": stream, - "is_adult": parse_is_adult(stream.get("is_adult", 0)), - "is_stale": False, - "stream_id": provider_stream_id, - "stream_chno": stream_chno, - } - - if stream_hash not in stream_hashes: - stream_hashes[stream_hash] = stream_props - except Exception as e: - logger.error( - f"Error processing XC category {group_name} (ID: {props['xc_id']}): {str(e)}" - ) - continue - - # Process all found streams - existing_streams = { - s.stream_hash: s - for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( - 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno' - ) - } - - for stream_hash, stream_props in stream_hashes.items(): - if stream_hash in existing_streams: - obj = existing_streams[stream_hash] - # Optimized field comparison for XC streams - changed = ( - obj.name != stream_props["name"] or - obj.url != stream_props["url"] or - obj.logo_url != stream_props["logo_url"] or - obj.tvg_id != stream_props["tvg_id"] or - obj.custom_properties != stream_props["custom_properties"] or - obj.is_adult != stream_props["is_adult"] or - obj.stream_id != stream_props["stream_id"] or - obj.stream_chno != stream_props["stream_chno"] - ) - - if changed: - for key, value in stream_props.items(): - setattr(obj, key, value) - obj.last_seen = timezone.now() - obj.updated_at = timezone.now() # Update timestamp only for changed streams - obj.is_stale = False - streams_to_update.append(obj) - else: - # Always update last_seen, even if nothing else changed - obj.last_seen = timezone.now() - obj.is_stale = False - # Don't update updated_at for unchanged streams - streams_to_update.append(obj) - - # Remove from existing_streams since we've processed it - del existing_streams[stream_hash] - else: - stream_props["last_seen"] = timezone.now() - stream_props["updated_at"] = ( - timezone.now() - ) # Set initial updated_at for new streams - stream_props["is_stale"] = False - streams_to_create.append(Stream(**stream_props)) - - try: - with transaction.atomic(): - if streams_to_create: - Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) - - if streams_to_update: - # Simplified bulk update for better performance - Stream.objects.bulk_update( - streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], - batch_size=150 # Smaller batch size for XC processing - ) - - # Update last_seen for any remaining existing streams that weren't processed - if len(existing_streams.keys()) > 0: - Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) - except Exception as e: - logger.error(f"Bulk operation failed for XC streams: {str(e)}") - - retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." - - except Exception as e: - logger.error(f"XC category processing error: {str(e)}") - retval = f"Error processing XC batch: {str(e)}" - finally: - # Clean up database connections for threading - connections.close_all() - - # Aggressive garbage collection - del streams_to_create, streams_to_update, stream_hashes, existing_streams - gc.collect() - - return retval - - -def process_m3u_batch_direct(account_id, batch, groups, hash_keys): - """Processes a batch of M3U streams using bulk operations with thread-safe DB connections.""" - from django.db import connections - - # Ensure clean database connections for threading - connections.close_all() - - account = M3UAccount.objects.get(id=account_id) - - compiled_filters = [ - ( - re.compile( - f.regex_pattern, - ( - re.IGNORECASE - if (f.custom_properties or {}).get( - "case_sensitive", True - ) - == False - else 0 - ), - ), - f, - ) - for f in account.filters.order_by("order") - ] - - streams_to_create = [] - streams_to_update = [] - stream_hashes = {} - - name_max_length = Stream._meta.get_field('name').max_length - - logger.debug(f"Processing batch of {len(batch)} for M3U account {account_id}") - if compiled_filters: - logger.debug(f"Using compiled filters: {[f[1].regex_pattern for f in compiled_filters]}") - for stream_info in batch: - try: - name, url = stream_info["name"], stream_info["url"] - - # Validate URL length - maximum of 4096 characters - if url and len(url) > 4096: - logger.warning(f"Skipping stream '{name}': URL too long ({len(url)} characters, max 4096)") - continue - - # Truncate name if it exceeds the model field limit - if name and len(name) > name_max_length: - logger.warning(f"Stream name too long ({len(name)} > {name_max_length}), truncating: {name[:80]}...") - name = name[:name_max_length] - - tvg_id, tvg_logo = get_case_insensitive_attr( - stream_info["attributes"], "tvg-id", "" - ), get_case_insensitive_attr(stream_info["attributes"], "tvg-logo", "") - group_title = get_case_insensitive_attr( - stream_info["attributes"], "group-title", "Default Group" - ) - logger.debug(f"Processing stream: {name} - {url} in group {group_title}") - include = True - for pattern, filter in compiled_filters: - logger.trace(f"Checking filter pattern {pattern}") - target = name - if filter.filter_type == "url": - target = url - elif filter.filter_type == "group": - target = group_title - - if pattern.search(target or ""): - logger.debug( - f"Stream {name} - {url} matches filter pattern {filter.regex_pattern}" - ) - include = not filter.exclude - break - - if not include: - logger.debug(f"Stream excluded by filter, skipping.") - continue - - # Filter out disabled groups for this account - if group_title not in groups: - logger.debug( - f"Skipping stream in disabled or excluded group: {group_title}" - ) - continue - - # Determine provider-specific fields first - provider_stream_id = None - channel_num = None - account_type_for_hash = None - - if account.account_type == M3UAccount.Types.XC: - account_type_for_hash = 'XC' - raw_stream_id = stream_info["attributes"].get("stream_id", "") - if raw_stream_id: - try: - provider_stream_id = int(raw_stream_id) - except (ValueError, TypeError): - pass - raw_num = stream_info["attributes"].get("num") - if raw_num is not None: - try: - channel_num = float(raw_num) - except (ValueError, TypeError): - pass - else: - # For standard M3U accounts, check for tvg-chno or channel-number - tvg_chno = get_case_insensitive_attr(stream_info["attributes"], "tvg-chno", None) - if tvg_chno is None: - tvg_chno = get_case_insensitive_attr(stream_info["attributes"], "channel-number", None) - if tvg_chno is not None: - try: - channel_num = float(tvg_chno) - except (ValueError, TypeError): - pass - - # Generate hash once with all parameters - stream_hash = Stream.generate_hash_key( - name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, - account_type=account_type_for_hash, stream_id=provider_stream_id - ) - - stream_props = { - "name": name, - "url": url, - "logo_url": tvg_logo, - "tvg_id": tvg_id, - "m3u_account": account, - "channel_group_id": int(groups.get(group_title)), - "stream_hash": stream_hash, - "custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"], - "is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)), - "is_stale": False, - "stream_id": provider_stream_id, - "stream_chno": channel_num, - } - - if stream_hash not in stream_hashes: - stream_hashes[stream_hash] = stream_props - except Exception as e: - logger.error(f"Failed to process stream {name}: {e}") - logger.error(json.dumps(stream_info)) - - existing_streams = { - s.stream_hash: s - for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( - 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno' - ) - } - - for stream_hash, stream_props in stream_hashes.items(): - if stream_hash in existing_streams: - obj = existing_streams[stream_hash] - # Optimized field comparison - changed = ( - obj.name != stream_props["name"] or - obj.url != stream_props["url"] or - obj.logo_url != stream_props["logo_url"] or - obj.tvg_id != stream_props["tvg_id"] or - obj.custom_properties != stream_props["custom_properties"] or - obj.is_adult != stream_props["is_adult"] or - obj.stream_id != stream_props["stream_id"] or - obj.stream_chno != stream_props["stream_chno"] - ) - - # Always update last_seen - obj.last_seen = timezone.now() - - if changed: - # Only update fields that changed and set updated_at - obj.name = stream_props["name"] - obj.url = stream_props["url"] - obj.logo_url = stream_props["logo_url"] - obj.tvg_id = stream_props["tvg_id"] - obj.custom_properties = stream_props["custom_properties"] - obj.is_adult = stream_props["is_adult"] - obj.stream_id = stream_props["stream_id"] - obj.stream_chno = stream_props["stream_chno"] - obj.updated_at = timezone.now() - - # Always mark as not stale since we saw it in this refresh - obj.is_stale = False - - streams_to_update.append(obj) - else: - # New stream - stream_props["last_seen"] = timezone.now() - stream_props["updated_at"] = timezone.now() - stream_props["is_stale"] = False - streams_to_create.append(Stream(**stream_props)) - - try: - with transaction.atomic(): - if streams_to_create: - Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) - - if streams_to_update: - # Update all streams in a single bulk operation - Stream.objects.bulk_update( - streams_to_update, - ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno'], - batch_size=200 - ) - except Exception as e: - logger.error(f"Bulk operation failed: {str(e)}") - - retval = f"M3U account: {account_id}, Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." - - # Clean up database connections for threading - connections.close_all() - - # Free batch data structures (reference-counted deallocation) - del streams_to_create, streams_to_update, stream_hashes, existing_streams - - return retval - - -def cleanup_streams(account_id, scan_start_time=timezone.now): - account = M3UAccount.objects.get(id=account_id, is_active=True) - existing_groups = ChannelGroup.objects.filter( - m3u_accounts__m3u_account=account, - m3u_accounts__enabled=True, - ).values_list("id", flat=True) - logger.info( - f"Found {len(existing_groups)} active groups for M3U account {account_id}" - ) - - # Calculate cutoff date for stale streams - stale_cutoff = scan_start_time - timezone.timedelta(days=account.stale_stream_days) - logger.info( - f"Removing streams not seen since {stale_cutoff} for M3U account {account_id}" - ) - - # Delete streams that are not in active groups - streams_to_delete = Stream.objects.filter(m3u_account=account).exclude( - channel_group__in=existing_groups - ) - - # Also delete streams that haven't been seen for longer than stale_stream_days - stale_streams = Stream.objects.filter( - m3u_account=account, last_seen__lt=stale_cutoff - ) - - deleted_count = streams_to_delete.count() - stale_count = stale_streams.count() - - streams_to_delete.delete() - stale_streams.delete() - - total_deleted = deleted_count + stale_count - logger.info( - f"Cleanup for M3U account {account_id} complete: {deleted_count} streams removed due to group filter, {stale_count} removed as stale" - ) - - # Return the total count of deleted streams - return total_deleted - - -@shared_task -def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_start_time=None): - """Refresh M3U groups for an account. - - Args: - account_id: ID of the M3U account - use_cache: Whether to use cached M3U file - full_refresh: Whether this is part of a full refresh - scan_start_time: Timestamp when the scan started (for consistent last_seen marking) +def delete_epg_refresh_task_by_id(epg_id): """ - if not acquire_task_lock("refresh_m3u_account_groups", account_id): - return f"Task already running for account_id={account_id}.", None - - lock_renewer = TaskLockRenewer("refresh_m3u_account_groups", account_id) - lock_renewer.start() - - try: - account = M3UAccount.objects.get(id=account_id, is_active=True) - except M3UAccount.DoesNotExist: - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return f"M3UAccount with ID={account_id} not found or inactive.", None - - extinf_data = [] - groups = {"Default Group": {}} - - if account.account_type == M3UAccount.Types.XC: - # Log detailed information about the account - logger.info( - f"Processing XC account {account_id} with URL: {account.server_url}" - ) - logger.debug( - f"Username: {account.username}, Has password: {'Yes' if account.password else 'No'}" - ) - - # Validate required fields - if not account.server_url: - error_msg = "Missing server URL for Xtream Codes account" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "processing_groups", 100, status="error", error=error_msg - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - - if not account.username or not account.password: - error_msg = "Missing username or password for Xtream Codes account" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "processing_groups", 100, status="error", error=error_msg - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - - try: - # Ensure server URL is properly formatted - server_url = account.server_url.rstrip("/") - if not ( - server_url.startswith("http://") or server_url.startswith("https://") - ): - server_url = f"http://{server_url}" - - # User agent handling - completely rewritten - try: - # Debug the user agent issue - logger.debug(f"Getting user agent for account {account.id}") - - # Use a hardcoded user agent string to avoid any issues with object structure - user_agent_string = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - ) - - try: - # Try to get the user agent directly from the database - if account.user_agent_id: - ua_obj = UserAgent.objects.get(id=account.user_agent_id) - if ( - ua_obj - and hasattr(ua_obj, "user_agent") - and ua_obj.user_agent - ): - user_agent_string = ua_obj.user_agent - logger.debug( - f"Using user agent from account: {user_agent_string}" - ) - else: - # Get default user agent from CoreSettings - default_ua_id = CoreSettings.get_default_user_agent_id() - logger.debug( - f"Default user agent ID from settings: {default_ua_id}" - ) - if default_ua_id: - ua_obj = UserAgent.objects.get(id=default_ua_id) - if ( - ua_obj - and hasattr(ua_obj, "user_agent") - and ua_obj.user_agent - ): - user_agent_string = ua_obj.user_agent - logger.debug( - f"Using default user agent: {user_agent_string}" - ) - except Exception as e: - logger.warning( - f"Error getting user agent, using fallback: {str(e)}" - ) - - logger.debug(f"Final user agent string: {user_agent_string}") - except Exception as e: - user_agent_string = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - ) - logger.warning( - f"Exception in user agent handling, using fallback: {str(e)}" - ) - - logger.info( - f"Creating XCClient with URL: {account.server_url}, Username: {account.username}, User-Agent: {user_agent_string}" - ) - - # Create XCClient with explicit error handling - try: - with XCClient( - account.server_url, account.username, account.password, user_agent_string - ) as xc_client: - logger.info(f"XCClient instance created successfully") - - # Queue async profile refresh task to run in background - # This prevents any delay in the main refresh process - try: - logger.info(f"Queueing background profile refresh for account {account.name}") - refresh_account_profiles.delay(account.id) - except Exception as e: - logger.warning(f"Failed to queue profile refresh task: {str(e)}") - # Don't fail the main refresh if profile refresh can't be queued - - # Get categories with detailed error handling - try: - logger.info(f"Getting live categories from XC server") - xc_categories = xc_client.get_live_categories() - logger.info( - f"Found {len(xc_categories)} categories: {xc_categories}" - ) - - # Validate response - if not isinstance(xc_categories, list): - error_msg = ( - f"Unexpected response from XC server: {xc_categories}" - ) - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, - "processing_groups", - 100, - status="error", - error=error_msg, - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - - if len(xc_categories) == 0: - logger.warning("No categories found in XC server response") - - for category in xc_categories: - cat_name = category.get("category_name", "Unknown Category") - cat_id = category.get("category_id", "0") - logger.info(f"Adding category: {cat_name} (ID: {cat_id})") - groups[cat_name] = { - "xc_id": cat_id, - } - except Exception as e: - # Determine if this is an authentication error or category retrieval error - error_str = str(e).lower() - # Check for authentication-related keywords or HTTP status codes commonly used for auth failures - is_auth_error = any(keyword in error_str for keyword in [ - 'auth', 'credential', 'login', 'unauthorized', 'forbidden', - '401', '403', '512', '513' # HTTP status codes: 401 Unauthorized, 403 Forbidden, 512-513 (non-standard auth failure) - ]) - - if is_auth_error: - error_msg = f"Failed to authenticate with XC server: {str(e)}" - else: - error_msg = f"Failed to get categories from XC server: {str(e)}" - - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, - "processing_groups", - 100, - status="error", - error=error_msg, - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - - except Exception as e: - error_msg = f"Failed to create XC Client: {str(e)}" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, - "processing_groups", - 100, - status="error", - error=error_msg, - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - except Exception as e: - error_msg = f"Unexpected error occurred in XC Client: {str(e)}" - logger.error(error_msg) - account.status = M3UAccount.Status.ERROR - account.last_message = error_msg - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "processing_groups", 100, status="error", error=error_msg - ) - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return error_msg, None - else: - lines, success = fetch_m3u_lines(account, use_cache) - if not success: - # If fetch failed, don't continue processing - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - return f"Failed to fetch M3U data for account_id={account_id}.", None - - # Log basic file structure for debugging - logger.debug(f"Processing {len(lines)} lines from M3U file") - - valid_stream_count = 0 - - for entry in iter_m3u_entries(lines): - valid_stream_count += 1 - group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") - if group_title_attr and group_title_attr not in groups: - logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") - groups[group_title_attr] = {} - extinf_data.append(entry) - - if valid_stream_count % 1000 == 0: - logger.debug(f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}") - - logger.info(f"M3U parsing complete - Valid streams: {valid_stream_count}") - - # Log group statistics - logger.info( - f"Found {len(groups)} groups in M3U file: {', '.join(list(groups.keys())[:20])}" - + ("..." if len(groups) > 20 else "") - ) - - # Cache processed data - cache_path = os.path.join(m3u_dir, f"{account_id}.json") - with open(cache_path, "w", encoding="utf-8") as f: - json.dump( - { - "extinf_data": extinf_data, - "groups": groups, - }, - f, - ) - logger.debug(f"Cached parsed M3U data to {cache_path}") - - send_m3u_update(account_id, "processing_groups", 0) - - process_groups(account, groups, scan_start_time) - - lock_renewer.stop() - release_task_lock("refresh_m3u_account_groups", account_id) - - if not full_refresh: - # Use update() instead of save() to avoid triggering signals - M3UAccount.objects.filter(id=account_id).update( - status=M3UAccount.Status.PENDING_SETUP, - last_message="M3U groups loaded. Please select groups or refresh M3U to complete setup.", - ) - send_m3u_update( - account_id, - "processing_groups", - 100, - status="pending_setup", - message="M3U groups loaded. Please select groups or refresh M3U to complete setup.", - ) - - return extinf_data, groups - - -def delete_m3u_refresh_task_by_id(account_id): - """ - Delete the periodic task associated with an M3U account ID. + Delete the periodic task associated with an EPG source ID. Can be called directly or from the post_delete signal. Returns True if a task was found and deleted, False otherwise. """ try: task = None - task_name = f"m3u_account-refresh-{account_id}" + task_name = f"epg_source-refresh-{epg_id}" # Look for task by name try: from django_celery_beat.models import PeriodicTask, IntervalSchedule - task = PeriodicTask.objects.get(name=task_name) - logger.debug(f"Found task by name: {task.id} for M3UAccount {account_id}") + logger.info(f"Found task by name: {task.id} for EPGSource {epg_id}") except PeriodicTask.DoesNotExist: logger.warning(f"No PeriodicTask found with name {task_name}") return False @@ -1643,2171 +259,3336 @@ def delete_m3u_refresh_task_by_id(account_id): if task: # Store interval info before deleting the task interval_id = None - if hasattr(task, "interval") and task.interval: + if hasattr(task, 'interval') and task.interval: interval_id = task.interval.id # Count how many TOTAL tasks use this interval (including this one) - tasks_with_same_interval = PeriodicTask.objects.filter( - interval_id=interval_id - ).count() - logger.debug( - f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total" - ) + tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() + logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") # Delete the task first task_id = task.id task.delete() - logger.debug(f"Successfully deleted periodic task {task_id}") + logger.info(f"Successfully deleted periodic task {task_id}") # Now check if we should delete the interval # We only delete if it was the ONLY task using this interval if interval_id and tasks_with_same_interval == 1: try: interval = IntervalSchedule.objects.get(id=interval_id) - logger.debug( - f"Deleting interval schedule {interval_id} (not shared with other tasks)" - ) + logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") interval.delete() - logger.debug(f"Successfully deleted interval {interval_id}") + logger.info(f"Successfully deleted interval {interval_id}") except IntervalSchedule.DoesNotExist: logger.warning(f"Interval {interval_id} no longer exists") elif interval_id: - logger.debug( - f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks" - ) + logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") return True return False except Exception as e: - logger.error( - f"Error deleting periodic task for M3UAccount {account_id}: {str(e)}", - exc_info=True, - ) + logger.error(f"Error deleting periodic task for EPGSource {epg_id}: {str(e)}", exc_info=True) return False -def _next_available_number(used_numbers, start, end=None): - """ - Return the smallest integer >= start that is not present in `used_numbers`. - - When `end` is provided (inclusive upper bound for range-constrained groups), - returns None if the search exceeds that bound instead of running - indefinitely. The search is O(cluster size) per call against the set; - maintaining the cursor monotonically across calls keeps the - "next_available" numbering mode from becoming O(N^2) on large groups. - """ - n = start - while n in used_numbers: - n += 1 - if end is not None and n > end: - return None - if end is not None and n > end: - return None - return n - - -def _pick_target_number( - mode, - stream, - used_numbers, - fixed_cursor, - fallback_start, - end_number=None, - range_start=None, -): - """ - Return the channel number a given stream should claim under the group's - numbering mode, or None if the configured range is exhausted. - - Shared by the existing-channel renumber pass and the new-channel create - pass so both honor identical mode semantics: provider-supplied number - when available and free, otherwise fall back; or always next-available; - or fixed-cursor sequential. - - `range_start`, when provided, is the inclusive lower bound for the - group's configured numbering range. Provider-supplied numbers below - this bound fall back to the next-available picker so freshly-created - channels never land outside the configured range. - """ - if mode == "provider": - chno = stream.stream_chno - if ( - chno is not None - and chno not in used_numbers - and (range_start is None or chno >= range_start) - and (end_number is None or chno <= end_number) - ): - return chno - # No usable provider number: walk from fallback_start, bumped up - # to range_start when set so the fallback never lands below the - # configured range. - effective_start = ( - max(fallback_start, range_start) - if range_start is not None - else fallback_start - ) - return _next_available_number(used_numbers, effective_start, end=end_number) - if mode == "next_available": - return _next_available_number(used_numbers, 1, end=end_number) - return _next_available_number(used_numbers, fixed_cursor, end=end_number) - - -def _custom_properties_as_dict(value): - """ - Normalize a JSONField-backed custom_properties value into a dict. - - Historical data has rows where the field holds a JSON-encoded string - instead of a dict. Django's JSONField serializes whatever it gets, so - `.get()` on one of those rows raises AttributeError and aborts the - entire sync. Treat string values as JSON to parse, and fall back to an - empty dict for anything that isn't a dict after parsing. - """ - import json - - if isinstance(value, dict): - return value - if isinstance(value, str): - try: - parsed = json.loads(value) - except (ValueError, TypeError): - logger.warning( - "custom_properties stored as non-JSON string; ignoring: %r", - value[:100], - ) - return {} - return parsed if isinstance(parsed, dict) else {} - return {} - - -def _classify_sync_failure(exc): - """ - Map an exception raised during per-stream sync to a coarse typed - reason used by the completion notification's grouped failure list. - Keeps the bucket count small so the modal stays readable; the - underlying exception text is preserved verbatim in ``error``. - """ - from django.db import IntegrityError - - if isinstance(exc, IntegrityError): - return "INTEGRITY_ERROR" - return "OTHER" - - @shared_task -def sync_auto_channels(account_id, scan_start_time=None): - """ - Automatically create/update/delete channels to match streams in groups with auto_channel_sync enabled. - Preserves existing channel UUIDs to maintain M3U link integrity. - Called after M3U refresh completes successfully. - """ - from apps.channels.models import ( - Channel, - ChannelGroup, - ChannelGroupM3UAccount, - Stream, - ChannelStream, - ) - from apps.epg.models import EPGData - from django.utils import timezone +def refresh_all_epg_data(): + logger.info("Starting refresh_epg_data task.") + # Exclude dummy EPG sources from refresh - they don't need refreshing + active_sources = EPGSource.objects.filter(is_active=True).exclude(source_type='dummy') + logger.debug(f"Found {active_sources.count()} active EPGSource(s) (excluding dummy EPGs).") - try: - account = M3UAccount.objects.get(id=account_id) - logger.info(f"Starting auto channel sync for M3U account {account.name}") + for source in active_sources: + refresh_epg_data(source.id) + # Force garbage collection between sources + gc.collect() - # Always use scan_start_time as the cutoff for last_seen - if scan_start_time is not None: - if isinstance(scan_start_time, str): - scan_start_time = timezone.datetime.fromisoformat(scan_start_time) - else: - scan_start_time = timezone.now() + logger.info("Finished refresh_epg_data task.") + return "EPG data refreshed." - # Get groups with auto sync enabled for this account - auto_sync_groups = ChannelGroupM3UAccount.objects.filter( - m3u_account=account, enabled=True, auto_channel_sync=True - ).select_related("channel_group") - channels_created = 0 - channels_updated = 0 - channels_deleted = 0 - channels_failed = 0 - # Per-failure context for the completion notification. Each entry - # carries a typed ``reason`` so the modal can group counts by - # cause; the cap keeps the WebSocket payload bounded but is sized - # generously to cover realistic multi-provider failure sets. - # Full per-stream detail still goes to ``logger.warning`` for - # power-user diagnostics regardless of the cap. - failed_stream_details = [] - FAILURE_LOG_LIMIT = 1000 +@shared_task(time_limit=14400) +def refresh_epg_data(source_id, force=False): + if not acquire_task_lock('refresh_epg_data', source_id): + logger.debug(f"EPG refresh for {source_id} already running") + return - # Group range reservations (start+end) are advisory and NOT seeded - # here: two groups with overlapping ranges must cooperate, so only - # actually-occupied numbers constrain assignment. - # Hidden auto-created channels stay in the seed because the renumber - # loop iterates current provider streams (which excludes hidden - # ones); excluding them here would let sync reclaim their numbers. - used_numbers = set( - Channel.objects.exclude( - auto_created=True, - auto_created_by=account, - hidden_from_output=False, - ).values_list("channel_number", flat=True) - ) - # Override pins are global reservations: effective_channel_number - # uses the override, so the picker must treat those numbers as - # taken or sync can produce duplicate effective channel numbers. - from apps.channels.models import ChannelOverride - - used_numbers.update( - ChannelOverride.objects.filter( - channel_number__isnull=False - ).values_list("channel_number", flat=True) - ) - used_numbers.discard(None) - - for group_relation in auto_sync_groups: - channel_group = group_relation.channel_group - start_number = group_relation.auto_sync_channel_start or 1.0 - # Optional upper bound; _next_available_number returns None when - # exhausted, which the per-stream loop converts to a failure. - end_number = group_relation.auto_sync_channel_end - - # Get force_dummy_epg, group_override, and regex patterns from group custom_properties - group_custom_props = {} - force_dummy_epg = False # Backward compatibility: legacy option to disable EPG - override_group_id = None - name_regex_pattern = None - name_replace_pattern = None - name_match_regex = None - name_match_exclude_regex = None - channel_profile_ids = None - channel_sort_order = None - channel_sort_reverse = False - stream_profile_id = None - custom_logo_id = None - custom_epg_id = None # New option: select specific EPG source (takes priority over force_dummy_epg) - channel_numbering_mode = "fixed" # Default mode - channel_numbering_fallback = 1 # Default fallback for provider mode - group_custom_props = _custom_properties_as_dict( - group_relation.custom_properties - ) - if group_custom_props: - force_dummy_epg = group_custom_props.get("force_dummy_epg", False) - override_group_id = group_custom_props.get("group_override") - name_regex_pattern = group_custom_props.get("name_regex_pattern") - name_replace_pattern = group_custom_props.get( - "name_replace_pattern" - ) - name_match_regex = group_custom_props.get("name_match_regex") - name_match_exclude_regex = group_custom_props.get( - "name_match_exclude_regex" - ) - channel_profile_ids = group_custom_props.get("channel_profile_ids") - custom_epg_id = group_custom_props.get("custom_epg_id") - channel_sort_order = group_custom_props.get("channel_sort_order") - channel_sort_reverse = group_custom_props.get( - "channel_sort_reverse", False - ) - stream_profile_id = group_custom_props.get("stream_profile_id") - custom_logo_id = group_custom_props.get("custom_logo_id") - channel_numbering_mode = group_custom_props.get("channel_numbering_mode", "fixed") - channel_numbering_fallback = group_custom_props.get("channel_numbering_fallback", 1) - - # Determine which group to use for created channels - target_group = channel_group - if override_group_id: - try: - target_group = ChannelGroup.objects.get(id=override_group_id) - logger.info( - f"Using override group '{target_group.name}' instead of '{channel_group.name}' for auto-created channels" - ) - except ChannelGroup.DoesNotExist: - logger.warning( - f"Override group with ID {override_group_id} not found, using original group '{channel_group.name}'" - ) - - logger.info( - f"Processing auto sync for group: {channel_group.name} (mode: {channel_numbering_mode}, start: {start_number})" - ) - - # Get all current streams in this group for this M3U account, filter out stale streams - current_streams = Stream.objects.filter( - m3u_account=account, - channel_group=channel_group, - last_seen__gte=scan_start_time, - ) - - # Filter streams in Python using the same `regex` module as the - # preview API. This ensures auto-sync accepts the same patterns - # the user tested in the frontend (e.g. `(?)` as a no-op inline - # modifier), and avoids passing potentially incompatible syntax - # to PostgreSQL's regex engine. - streams_is_list = False - if name_match_regex: - try: - match_re = regex.compile(name_match_regex, regex.IGNORECASE) - current_streams = [s for s in current_streams if match_re.search(s.name)] - streams_is_list = True - except regex.error as e: - logger.warning( - f"Invalid name_match_regex '{name_match_regex}' for group '{channel_group.name}': {e}. Skipping name filter." - ) - - # Exclude regex runs after the include filter so the two - # compose: include narrows, exclude removes from what's left. - if name_match_exclude_regex: - try: - exclude_re = regex.compile(name_match_exclude_regex, regex.IGNORECASE) - current_streams = [s for s in current_streams if not exclude_re.search(s.name)] - streams_is_list = True - except regex.error as e: - logger.warning( - f"Invalid name_match_exclude_regex '{name_match_exclude_regex}' for group '{channel_group.name}': {e}. Skipping exclude filter." - ) - - # --- APPLY CHANNEL SORT ORDER --- - if channel_sort_order and channel_sort_order != "": - if channel_sort_order == "name": - if not streams_is_list: - current_streams = list(current_streams) - streams_is_list = True - current_streams.sort( - key=lambda stream: natural_sort_key(stream.name), - reverse=channel_sort_reverse, - ) - elif channel_sort_order == "tvg_id": - if streams_is_list: - current_streams.sort( - key=lambda s: (s.tvg_id or ""), - reverse=channel_sort_reverse, - ) - else: - order_prefix = "-" if channel_sort_reverse else "" - current_streams = current_streams.order_by(f"{order_prefix}tvg_id") - elif channel_sort_order == "updated_at": - if streams_is_list: - current_streams.sort( - key=lambda s: (s.updated_at or ""), - reverse=channel_sort_reverse, - ) - else: - order_prefix = "-" if channel_sort_reverse else "" - current_streams = current_streams.order_by( - f"{order_prefix}updated_at" - ) - else: - logger.warning( - f"Unknown channel_sort_order '{channel_sort_order}' for group '{channel_group.name}'. Using provider order." - ) - if streams_is_list: - current_streams.sort( - key=lambda s: s.id, - reverse=channel_sort_reverse, - ) - else: - order_prefix = "-" if channel_sort_reverse else "" - current_streams = current_streams.order_by(f"{order_prefix}id") - else: - # Provider order (default) - can still be reversed - if streams_is_list: - current_streams.sort( - key=lambda s: s.id, - reverse=channel_sort_reverse, - ) - else: - order_prefix = "-" if channel_sort_reverse else "" - current_streams = current_streams.order_by(f"{order_prefix}id") - - # Scoped to this group so the loop below runs in O(group size). - # Multi-stream channels are deduped by channel_id so every - # stream_id maps to the same in-memory Channel instance and - # post-loop bulk_update writes the merged state. - existing_channel_map = {} - existing_channels_by_id = {} - existing_channel_streams = ( - ChannelStream.objects.filter( - channel__auto_created=True, - channel__auto_created_by=account, - stream__m3u_account=account, - stream__channel_group=channel_group, - ) - .select_related("channel") - ) - for cs in existing_channel_streams: - if cs.stream_id and cs.channel_id: - canonical = existing_channels_by_id.setdefault( - cs.channel_id, cs.channel - ) - existing_channel_map[cs.stream_id] = canonical - - # Track which streams we've processed - processed_stream_ids = set() - - # Check if we have streams - handle both QuerySet and list cases - has_streams = ( - len(current_streams) > 0 - if streams_is_list - else current_streams.exists() - ) - - # Bulk pre-fetch collapses N+1 Logo/EPGData lookups into a - # pair of in_bulk() calls. - from apps.channels.models import Logo - from apps.epg.models import EPGSource - - # Resolve the group's custom EPG source once. - custom_epg_source = None - custom_dummy_epg_data = None - if custom_epg_id: - try: - custom_epg_source = EPGSource.objects.get(id=custom_epg_id) - if custom_epg_source.source_type == "dummy": - custom_dummy_epg_data = ( - EPGData.objects.filter( - epg_source=custom_epg_source - ).first() - ) - if not custom_dummy_epg_data: - logger.warning( - f"No EPGData found for dummy EPG source " - f"{custom_epg_source.name} (ID: {custom_epg_id})" - ) - except EPGSource.DoesNotExist: - logger.warning( - f"Custom EPG source with ID {custom_epg_id} not found " - f"for group '{channel_group.name}', falling back to " - f"auto-match" - ) - - # Resolve the group's custom logo once. - custom_logo = None - if custom_logo_id: - try: - custom_logo = Logo.objects.get(id=custom_logo_id) - except Logo.DoesNotExist: - logger.warning( - f"Custom logo with ID {custom_logo_id} not found for " - f"group '{channel_group.name}', falling back to stream " - f"logos" - ) - - logo_cache_by_url = {} - epg_cache_by_tvg_id = {} - if has_streams: - # Collect unique URLs / tvg_ids in one DB call each. - stream_iter = ( - current_streams - if streams_is_list - else list(current_streams.values("logo_url", "tvg_id")) - ) - unique_logo_urls = { - s.get("logo_url") if isinstance(s, dict) else getattr(s, "logo_url", None) - for s in stream_iter - } - unique_logo_urls.discard(None) - unique_logo_urls.discard("") - if unique_logo_urls: - logo_cache_by_url = { - lg.url: lg - for lg in Logo.objects.filter(url__in=unique_logo_urls) - } - - unique_tvg_ids = { - s.get("tvg_id") if isinstance(s, dict) else getattr(s, "tvg_id", None) - for s in stream_iter - } - unique_tvg_ids.discard(None) - unique_tvg_ids.discard("") - # Skip the EPG cache when force_dummy_epg with no - # custom source override; the resolver always returns None. - want_epg_cache = unique_tvg_ids and ( - not force_dummy_epg or custom_epg_id - ) - if want_epg_cache: - # Scope to the group's pinned source so foreign-source - # tvg_id matches do not leak in. - epg_q = EPGData.objects.filter(tvg_id__in=unique_tvg_ids) - if ( - custom_epg_source is not None - and custom_epg_source.source_type != "dummy" - ): - epg_q = epg_q.filter(epg_source=custom_epg_source) - epg_cache_by_tvg_id = {d.tvg_id: d for d in epg_q} - - def _resolve_logo_for_stream(stream): - """Return a Logo for stream.logo_url, creating it once if needed.""" - url = getattr(stream, "logo_url", None) - if not url: - return None - cached = logo_cache_by_url.get(url) - if cached is not None: - return cached - created, _ = Logo.objects.get_or_create( - url=url, - defaults={"name": stream.name or stream.tvg_id or "Unknown"}, - ) - logo_cache_by_url[url] = created - return created - - def _resolve_epg_for_stream(stream): - """Return the EPGData row that should be assigned to this - stream's channel. Encodes all four group-level EPG modes: - - 1. custom dummy source: single shared EPGData - 2. custom non-dummy source: cache lookup, scoped to - that source - 3. force_dummy_epg with no custom: None (clear EPG) - 4. default auto-match: cache lookup, any source - - The cache (epg_cache_by_tvg_id) is built once above with the - correct scope so the per-stream lookup is a dict access. - """ - if custom_epg_source is not None: - if custom_epg_source.source_type == "dummy": - return custom_dummy_epg_data - tvg_id = getattr(stream, "tvg_id", None) - if not tvg_id: - return None - return epg_cache_by_tvg_id.get(tvg_id) - if force_dummy_epg: - return None - tvg_id = getattr(stream, "tvg_id", None) - if not tvg_id: - return None - return epg_cache_by_tvg_id.get(tvg_id) - - if not has_streams: - logger.debug(f"No streams found in group {channel_group.name}") - # No streams left in the group: drop the visible auto - # channels. Hidden channels are preserved so the hide - # flag survives temporary provider drops (event/PPV). - channels_to_delete = [ - ch - for ch in existing_channel_map.values() - if not ch.hidden_from_output - ] - if channels_to_delete: - deleted_count = len(channels_to_delete) - Channel.objects.filter( - id__in=[ch.id for ch in channels_to_delete] - ).delete() - channels_deleted += deleted_count - logger.debug( - f"Deleted {deleted_count} auto channels (no streams remaining)" - ) - continue - - # Prepare profiles to assign to new channels - from apps.channels.models import ChannelProfile, ChannelProfileMembership - - if ( - channel_profile_ids - and isinstance(channel_profile_ids, list) - and len(channel_profile_ids) > 0 - ): - # Convert all to int (in case they're strings) - try: - profile_ids = [int(pid) for pid in channel_profile_ids] - except Exception: - profile_ids = [] - profiles_to_assign = list( - ChannelProfile.objects.filter(id__in=profile_ids) - ) - else: - profiles_to_assign = list(ChannelProfile.objects.all()) - - # Get stream profile to assign if specified - from core.models import StreamProfile - stream_profile_to_assign = None - if stream_profile_id: - try: - stream_profile_to_assign = StreamProfile.objects.get(id=int(stream_profile_id)) - logger.info( - f"Will assign stream profile '{stream_profile_to_assign.name}' to auto-synced streams in group '{channel_group.name}'" - ) - except (StreamProfile.DoesNotExist, ValueError, TypeError): - logger.warning( - f"Stream profile with ID {stream_profile_id} not found for group '{channel_group.name}', streams will use default profile" - ) - stream_profile_to_assign = None - - current_channel_number = start_number - - # Renumber existing channels to match sort order. Compact - # mode skips this; the end-of-iteration pack is the source - # of truth and would overwrite the renumber. - compact_mode = bool(group_custom_props.get("compact_numbering")) - channels_to_renumber = [] - temp_channel_number = start_number - - for stream in current_streams if not compact_mode else []: - if stream.id in existing_channel_map: - channel = existing_channel_map[stream.id] - - target_number = _pick_target_number( - channel_numbering_mode, - stream, - used_numbers, - temp_channel_number, - channel_numbering_fallback, - end_number=end_number, - range_start=start_number, - ) - - # Range exhausted: leave the channel at its existing - # number. The renumber pass is sort-optimization only; - # no failure record needed. - if target_number is None: - # Preserve the channel's current number in used_numbers - if channel.channel_number is not None: - used_numbers.add(channel.channel_number) - continue - - # Add this number to used_numbers so we don't reuse it in this batch - used_numbers.add(target_number) - - if channel.channel_number != target_number: - channel.channel_number = target_number - channels_to_renumber.append(channel) - logger.debug( - f"Will renumber channel '{channel.name}' to {target_number}" - ) - - # Only increment temp_channel_number in fixed mode - if channel_numbering_mode == "fixed": - temp_channel_number += 1.0 - if temp_channel_number % 1 != 0: # Has decimal - temp_channel_number = int(temp_channel_number) + 1.0 - - # Bulk update channel numbers if any need renumbering - if channels_to_renumber: - Channel.objects.bulk_update( - channels_to_renumber, ["channel_number"], batch_size=500 - ) - logger.info( - f"Renumbered {len(channels_to_renumber)} channels to maintain sort order" - ) - - # When the group's configured range is narrower than its existing - # channels span, any non-hidden auto-created channel whose number - # falls outside [start, end] gets deleted. The new-channel - # creation loop below picks up the freed stream and re-creates - # the channel at a slot inside the new range, so the net user - # outcome is a renumber, not a failure. Counted in - # channels_deleted; the replacement counts in channels_created. - # Hidden channels are preserved. Runs BEFORE new-channel creation - # so slots freed by the deletions are available to incoming - # streams. - if end_number is not None: - overflow_delete_ids = [] - for stream_id, ch in list(existing_channel_map.items()): - if ch.hidden_from_output: - continue - num = ch.channel_number - if num is None: - continue - if num < start_number or num > end_number: - overflow_delete_ids.append(ch.id) - existing_channel_map.pop(stream_id, None) - used_numbers.discard(num) - if overflow_delete_ids: - deleted = Channel.objects.filter( - id__in=overflow_delete_ids - ).delete() - removed_count = ( - deleted[1].get("dispatcharr_channels.Channel", 0) - if isinstance(deleted, tuple) and len(deleted) > 1 - else len(overflow_delete_ids) - ) - channels_deleted += removed_count - logger.info( - f"Deleted {removed_count} channels outside the " - f"range {int(start_number)}-{int(end_number)} for " - f"group '{channel_group.name}'" - ) - - # Reset channel number counter for processing new channels - current_channel_number = start_number - - # Per-channel changes are buffered and bulk_update'd once after the - # loop. update_fields is set explicitly so post_save signals only - # fire for receivers whose tracked field actually changed. - existing_dirty_channels = [] - existing_dirty_ids = set() - existing_dirty_field_set = set() - # Subset of channels whose epg_data actually changed in this - # pass. Used by the dispatcher below to fire the EPG parse - # task only for those, not for every channel in - # existing_dirty_channels. - epg_dirty_channel_ids = set() - - # New channels are buffered and bulk_create'd after the loop. - # bulk_create skips post_save, so the EPG parse task is dispatched - # once per unique epg_data_id below rather than per channel. - # Pairs are (Channel(), Stream) so the post-loop step can attach - # ChannelStream rows using the IDs Postgres returns. - new_channels_pending = [] - - for stream in current_streams: - processed_stream_ids.add(stream.id) - try: - # Parse custom properties for additional info - stream_custom_props = stream.custom_properties or {} - tvc_guide_stationid = stream_custom_props.get("tvc-guide-stationid") - - # --- REGEX FIND/REPLACE LOGIC --- - original_name = stream.name - new_name = original_name - if name_regex_pattern is not None: - # If replace is None, treat as empty string (remove match) - replace = ( - name_replace_pattern - if name_replace_pattern is not None - else "" - ) - try: - # Convert $1, $2, etc. to \1, \2, etc. for consistency with M3U profiles - safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace) - new_name = re.sub( - name_regex_pattern, safe_replace_pattern, original_name - ) - except re.error as e: - logger.warning( - f"Regex error for group '{channel_group.name}': {e}. Using original name." - ) - new_name = original_name - - # Check if we already have a channel for this stream - existing_channel = existing_channel_map.get(stream.id) - - if existing_channel: - # Track only the fields that actually changed, so the - # eventual UPDATE writes one column per change instead - # of every column on every channel. The dirty list is - # accumulated and bulk_update'd after the loop - - # which avoids issuing an UPDATE per channel and - # avoids firing the EPG post_save signal on saves - # that didn't touch epg_data. - dirty_fields = [] - - if existing_channel.name != new_name: - existing_channel.name = new_name - dirty_fields.append("name") - - if existing_channel.tvg_id != stream.tvg_id: - existing_channel.tvg_id = stream.tvg_id - dirty_fields.append("tvg_id") - - if existing_channel.tvc_guide_stationid != tvc_guide_stationid: - existing_channel.tvc_guide_stationid = tvc_guide_stationid - dirty_fields.append("tvc_guide_stationid") - - # The group override may direct sync to a different - # ChannelGroup than the one currently on the row. - if existing_channel.channel_group_id != target_group.id: - existing_channel.channel_group = target_group - dirty_fields.append("channel_group") - - # Logo: custom group setting wins; otherwise stream logo - current_logo = ( - custom_logo - if custom_logo_id and custom_logo is not None - else _resolve_logo_for_stream(stream) - ) - current_logo_id = current_logo.id if current_logo else None - if existing_channel.logo_id != current_logo_id: - existing_channel.logo = current_logo - dirty_fields.append("logo") - - # EPG: handled centrally by _resolve_epg_for_stream - current_epg_data = _resolve_epg_for_stream(stream) - current_epg_id = ( - current_epg_data.id if current_epg_data else None - ) - if existing_channel.epg_data_id != current_epg_id: - existing_channel.epg_data = current_epg_data - dirty_fields.append("epg_data") - if current_epg_id is not None: - epg_dirty_channel_ids.add(existing_channel.id) - - # Stream profile: only set if group has one configured - if ( - stream_profile_to_assign is not None - and existing_channel.stream_profile_id - != stream_profile_to_assign.id - ): - existing_channel.stream_profile = stream_profile_to_assign - dirty_fields.append("stream_profile") - - if dirty_fields: - # Multi-stream channels appear once per stream; - # dedupe by id so bulk_update does not double-fire - # and channels_updated does not double-count. - if existing_channel.id not in existing_dirty_ids: - existing_dirty_channels.append(existing_channel) - existing_dirty_ids.add(existing_channel.id) - channels_updated += 1 - existing_dirty_field_set.update(dirty_fields) - - else: - # Range exhaustion is surfaced to the user via the - # completion notification, not swallowed. - target_number = _pick_target_number( - channel_numbering_mode, - stream, - used_numbers, - current_channel_number, - channel_numbering_fallback, - end_number=end_number, - range_start=start_number, - ) - - if target_number is None: - channels_failed += 1 - if len(failed_stream_details) < FAILURE_LOG_LIMIT: - failed_stream_details.append({ - "stream_name": stream.name, - "stream_id": stream.id, - "group": channel_group.name, - "reason": "RANGE_EXHAUSTED", - "error": ( - f"Channel number range " - f"{int(start_number)}-{int(end_number)} is full" - ), - }) - processed_stream_ids.add(stream.id) - continue - - # Add this number to used_numbers - used_numbers.add(target_number) - - # Resolve every FK BEFORE the create call so the - # initial INSERT carries the complete row. - new_logo = ( - custom_logo - if custom_logo_id and custom_logo is not None - else _resolve_logo_for_stream(stream) - ) - new_epg_data = _resolve_epg_for_stream(stream) - - new_channels_pending.append( - ( - Channel( - channel_number=target_number, - name=new_name, - tvg_id=stream.tvg_id, - tvc_guide_stationid=tvc_guide_stationid, - channel_group=target_group, - user_level=0, - auto_created=True, - auto_created_by=account, - logo=new_logo, - epg_data=new_epg_data, - stream_profile=stream_profile_to_assign, - ), - stream, - ) - ) - - # Increment channel number for next iteration (only in fixed mode) - if channel_numbering_mode == "fixed": - current_channel_number += 1.0 - if current_channel_number % 1 != 0: # Has decimal - current_channel_number = int(current_channel_number) + 1.0 - - except Exception as e: - logger.error( - f"Error processing auto channel for stream {stream.name}: {str(e)}" - ) - channels_failed += 1 - if len(failed_stream_details) < FAILURE_LOG_LIMIT: - failed_stream_details.append({ - "stream_name": stream.name, - "stream_id": stream.id, - "group": channel_group.name, - "reason": _classify_sync_failure(e), - "error": str(e), - }) - continue - - # Bulk-create channels, then dependent rows using the IDs - # Postgres returns. bulk_create skips post_save, so the EPG - # parse task is dispatched explicitly per-epg_data_id below - # to avoid flooding Celery at scale. - if new_channels_pending: - channel_objs = [pair[0] for pair in new_channels_pending] - streams_for_new = [pair[1] for pair in new_channels_pending] - Channel.objects.bulk_create(channel_objs, batch_size=500) - - ChannelStream.objects.bulk_create( - [ - ChannelStream( - channel_id=channel_objs[i].id, - stream_id=streams_for_new[i].id, - order=0, - ) - for i in range(len(channel_objs)) - ], - batch_size=500, - ) - - if profiles_to_assign: - ChannelProfileMembership.objects.bulk_create( - [ - ChannelProfileMembership( - channel_id=ch.id, - channel_profile_id=profile.id, - enabled=True, - ) - for ch in channel_objs - for profile in profiles_to_assign - ], - ignore_conflicts=True, - batch_size=500, - ) - - channels_created += len(channel_objs) - - # One EPG parse task per unique EPGData replaces the - # per-channel post_save dispatch bypassed by bulk_create. - from apps.epg.tasks import parse_programs_for_tvg_id - - unique_epg_ids = { - ch.epg_data_id for ch in channel_objs if ch.epg_data_id - } - for epg_id in unique_epg_ids: - parse_programs_for_tvg_id.delay(epg_id) - - logger.debug( - f"Bulk created {len(channel_objs)} channels in group " - f"'{channel_group.name}'; dispatched " - f"{len(unique_epg_ids)} unique EPG parse task(s)" - ) - - # bulk_update writes only the columns named in `fields` and - # bypasses post_save, so the EPG refresh signal cannot fire here. - # Dispatch one parse task per unique EPGData id when epg_data was - # in the dirty set, mirroring the new-channel path above. - if existing_dirty_channels: - Channel.objects.bulk_update( - existing_dirty_channels, - fields=list(existing_dirty_field_set), - batch_size=500, - ) - if epg_dirty_channel_ids: - from apps.epg.tasks import parse_programs_for_tvg_id - - # Dispatch only for channels whose epg_data_id changed. - # Other dirty channels would queue redundant parses. - unique_epg_ids = { - ch.epg_data_id - for ch in existing_dirty_channels - if ch.id in epg_dirty_channel_ids and ch.epg_data_id - } - for epg_id in unique_epg_ids: - parse_programs_for_tvg_id.delay(epg_id) - logger.debug( - f"Bulk updated {len(existing_dirty_channels)} existing " - f"channels (fields: {sorted(existing_dirty_field_set)})" - ) - - # Reconcile ChannelProfileMembership in two writes: one - # bulk_update for enable-flips, one bulk_create for missing - # rows. Avoids a per-channel save loop. - existing_channel_ids = [ - c.id for c in existing_channel_map.values() - ] - target_profile_ids = {p.id for p in profiles_to_assign} - if existing_channel_ids: - membership_rows = list( - ChannelProfileMembership.objects.filter( - channel_id__in=existing_channel_ids - ).only("id", "channel_id", "channel_profile_id", "enabled") - ) - memberships_by_channel = {} - for m in membership_rows: - memberships_by_channel.setdefault(m.channel_id, []).append(m) - - rows_to_flip = [] - rows_to_create = [] - for ch_id in existing_channel_ids: - rows = memberships_by_channel.get(ch_id, []) - have_for_target = set() - for m in rows: - if m.channel_profile_id in target_profile_ids: - have_for_target.add(m.channel_profile_id) - if not m.enabled: - m.enabled = True - rows_to_flip.append(m) - else: - if m.enabled: - m.enabled = False - rows_to_flip.append(m) - missing = target_profile_ids - have_for_target - for pid in missing: - rows_to_create.append( - ChannelProfileMembership( - channel_id=ch_id, - channel_profile_id=pid, - enabled=True, - ) - ) - - if rows_to_flip: - ChannelProfileMembership.objects.bulk_update( - rows_to_flip, ["enabled"], batch_size=500 - ) - if rows_to_create: - ChannelProfileMembership.objects.bulk_create( - rows_to_create, ignore_conflicts=True, batch_size=500 - ) - if rows_to_flip or rows_to_create: - logger.debug( - f"Reconciled memberships for " - f"{len(existing_channel_ids)} channels " - f"({len(rows_to_flip)} flipped, " - f"{len(rows_to_create)} created)" - ) - - # Delete channels whose streams have all disappeared. - # Hidden channels are preserved so event/PPV holds across - # provider drops. - channel_streams_in_group = {} - for stream_id, channel in existing_channel_map.items(): - channel_streams_in_group.setdefault(channel.id, []).append( - (stream_id, channel) - ) - channels_to_delete = [] - for ch_id, pairs in channel_streams_in_group.items(): - channel = pairs[0][1] - if channel.hidden_from_output: - continue - stream_ids = {sid for sid, _ in pairs} - if not (stream_ids & processed_stream_ids): - channels_to_delete.append(channel) - - if channels_to_delete: - deleted_count = len(channels_to_delete) - Channel.objects.filter( - id__in=[ch.id for ch in channels_to_delete] - ).delete() - channels_deleted += deleted_count - logger.debug( - f"Deleted {deleted_count} auto channels for removed streams" - ) - - # Compact-mode pack: hidden channels release their number and - # visible channels pack contiguously into [start, end]. Runs - # after create/update/delete so the channel set is stable. - if compact_mode: - from apps.channels.compact_numbering import repack_group - - pack_result = repack_group(group_relation) - if pack_result["failed"]: - channels_failed += pack_result["failed"] - if ( - len(failed_stream_details) < FAILURE_LOG_LIMIT - ): - failed_stream_details.append( - { - "stream_name": None, - "stream_id": None, - "group": channel_group.name, - "reason": "RANGE_EXHAUSTED", - "error": ( - f"Compact pack: {pack_result['failed']} " - f"visible channel(s) could not fit in range " - f"{int(start_number)}" - + ( - f"-{int(end_number)}" - if end_number - else "+" - ) - ), - } - ) - logger.debug( - f"Compact pack for group '{channel_group.name}': " - f"{pack_result['assigned']} assigned, " - f"{pack_result['released']} released, " - f"{pack_result['failed']} failed" - ) - - # Cleanup mode read from account.custom_properties.orphan_channel_cleanup: - # "always" (default; key absent) removes every orphan auto channel; - # "preserve_customized" keeps those with a ChannelOverride row; - # "never" disables cleanup. Hidden channels are preserved across all - # modes so event/PPV channels that come and go are not silently lost. - cleanup_mode = (account.custom_properties or {}).get( - "orphan_channel_cleanup", "always" - ) - if cleanup_mode != "never": - orphaned_channels = Channel.objects.filter( - auto_created=True, - auto_created_by=account, - hidden_from_output=False, - ).exclude( - id__in=ChannelStream.objects.filter( - stream__m3u_account=account, - stream__isnull=False, - ).values_list("channel_id", flat=True) - ) - if cleanup_mode == "preserve_customized": - orphaned_channels = orphaned_channels.filter(override__isnull=True) - - _, per_model = orphaned_channels.delete() - deleted_channels = per_model.get("dispatcharr_channels.Channel", 0) - if deleted_channels: - channels_deleted += deleted_channels - logger.info( - f"Deleted {deleted_channels} orphaned auto channels with no valid streams (mode={cleanup_mode})" - ) - - logger.info( - f"Auto channel sync complete for account {account.name}: " - f"{channels_created} created, {channels_updated} updated, " - f"{channels_deleted} deleted, {channels_failed} failed" - ) - return { - "status": "ok", - "channels_created": channels_created, - "channels_updated": channels_updated, - "channels_deleted": channels_deleted, - "channels_failed": channels_failed, - "failed_stream_details": failed_stream_details, - } - - except Exception as e: - logger.error(f"Error in auto channel sync for account {account_id}: {str(e)}") - return { - "status": "error", - "error": str(e), - "channels_created": 0, - "channels_updated": 0, - "channels_deleted": 0, - "channels_failed": 0, - "failed_stream_details": [], - } - - -def get_transformed_credentials(account, profile=None): - """ - Get transformed credentials for XtreamCodes API calls. - - Args: - account: M3UAccount instance - profile: M3UAccountProfile instance (optional, if not provided will use primary profile) - - Returns: - tuple: (transformed_url, transformed_username, transformed_password) - """ - import re - import urllib.parse - - # If no profile is provided, find the primary active profile - if profile is None: - try: - from apps.m3u.models import M3UAccountProfile - profile = M3UAccountProfile.objects.filter( - m3u_account=account, - is_active=True - ).first() - if profile: - logger.debug(f"Using primary profile '{profile.name}' for URL transformation") - else: - logger.debug(f"No active profiles found for account {account.name}, using base credentials") - except Exception as e: - logger.warning(f"Could not get primary profile for account {account.name}: {e}") - profile = None - - base_url = account.server_url - base_username = account.username - base_password = account.password # Build a complete URL with credentials (similar to how IPTV URLs are structured) - # Format: http://server.com:port/live/username/password/1234.ts - if base_url and base_username and base_password: - # Remove trailing slash from server URL if present - clean_server_url = base_url.rstrip('/') - - # Build the complete URL with embedded credentials - complete_url = f"{clean_server_url}/live/{base_username}/{base_password}/1234.ts" - logger.debug(f"Built complete URL: {complete_url}") - - # Apply profile-specific transformations if profile is provided - if profile and profile.search_pattern and profile.replace_pattern: - try: - # Handle backreferences: convert JS-style $ -> \g, $1 -> \1 - # regex module accepts JS-style (?...) named groups natively - safe_replace_pattern = regex.sub(r'\$<([^>]+)>', r'\\g<\1>', profile.replace_pattern) - safe_replace_pattern = regex.sub(r'\$(\d+)', r'\\\1', safe_replace_pattern) - - # Apply transformation to the complete URL - transformed_complete_url = regex.sub(profile.search_pattern, safe_replace_pattern, complete_url) - logger.info(f"Transformed complete URL: {complete_url} -> {transformed_complete_url}") - - # Extract components from the transformed URL - # Pattern: http://server.com:port/live/username/password/1234.ts - parsed_url = urllib.parse.urlparse(transformed_complete_url) - path_parts = [part for part in parsed_url.path.split('/') if part] - - if len(path_parts) >= 4 and path_parts[-1] == '1234.ts': - # Extract username and password from the known structure: - # .../{live}/{username}/{password}/1234.ts - # Using negative indices so sub-paths in the server URL don't shift extraction - transformed_username = path_parts[-3] - transformed_password = path_parts[-2] - - # Rebuild server URL: preserve any sub-path that precedes - # /live/username/password/1234.ts (path_parts[:-4]). - base_path_parts = path_parts[:-4] - base_path = ('/' + '/'.join(base_path_parts)) if base_path_parts else '' - transformed_url = f"{parsed_url.scheme}://{parsed_url.netloc}{base_path}" - - logger.debug(f"Extracted transformed credentials:") - logger.debug(f" Server URL: {transformed_url}") - logger.debug(f" Username: {transformed_username}") - logger.debug(f" Password: {transformed_password}") - - return transformed_url, transformed_username, transformed_password - else: - logger.warning(f"Could not extract credentials from transformed URL: {transformed_complete_url}") - return base_url, base_username, base_password - - except Exception as e: - logger.error(f"Error transforming URL for profile {profile.name if profile else 'unknown'}: {e}") - return base_url, base_username, base_password - else: - # No profile or no transformation patterns - return base_url, base_username, base_password - else: - logger.warning(f"Missing credentials for account {account.name}") - return base_url, base_username, base_password - - -@shared_task -def refresh_account_profiles(account_id): - """Refresh account information for all active profiles of an XC account. - - This task runs asynchronously in the background after account refresh completes. - It includes rate limiting delays between profile authentications to prevent provider bans. - """ - from django.conf import settings - import time - - try: - account = M3UAccount.objects.get(id=account_id, is_active=True) - - if account.account_type != M3UAccount.Types.XC: - logger.debug(f"Account {account_id} is not XC type, skipping profile refresh") - return f"Account {account_id} is not an XtreamCodes account" - - from apps.m3u.models import M3UAccountProfile - - profiles = M3UAccountProfile.objects.filter( - m3u_account=account, - is_active=True - ) - - if not profiles.exists(): - logger.info(f"No active profiles found for account {account.name}") - return f"No active profiles for account {account_id}" - - # Get user agent for this account - try: - user_agent_string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - if account.user_agent_id: - from core.models import UserAgent - ua_obj = UserAgent.objects.get(id=account.user_agent_id) - if ua_obj and hasattr(ua_obj, "user_agent") and ua_obj.user_agent: - user_agent_string = ua_obj.user_agent - except Exception as e: - logger.warning(f"Error getting user agent, using fallback: {str(e)}") - logger.debug(f"Using user agent for profile refresh: {user_agent_string}") - # Get rate limiting delay from settings - profile_delay = getattr(settings, 'XC_PROFILE_REFRESH_DELAY', 2.5) - - profiles_updated = 0 - profiles_failed = 0 - - logger.info(f"Starting background refresh for {profiles.count()} profiles of account {account.name}") - - for idx, profile in enumerate(profiles): - try: - # Add delay between profiles to prevent rate limiting (except for first profile) - if idx > 0: - logger.info(f"Waiting {profile_delay}s before refreshing next profile to avoid rate limiting") - time.sleep(profile_delay) - - # Get transformed credentials for this specific profile - profile_url, profile_username, profile_password = get_transformed_credentials(account, profile) - - # Create a separate XC client for this profile's credentials - with XCClient( - profile_url, - profile_username, - profile_password, - user_agent_string - ) as profile_client: - # Authenticate with this profile's credentials - if profile_client.authenticate(): - # Get account information specific to this profile's credentials - profile_account_info = profile_client.get_account_info() - - # Merge with existing custom_properties if they exist - existing_props = profile.custom_properties or {} - existing_props.update(profile_account_info) - profile.custom_properties = existing_props - profile.save(update_fields=['custom_properties', 'exp_date']) - - profiles_updated += 1 - logger.info(f"Updated account information for profile '{profile.name}' ({profiles_updated}/{profiles.count()})") - else: - profiles_failed += 1 - logger.warning(f"Failed to authenticate profile '{profile.name}' with transformed credentials") - - except Exception as profile_error: - profiles_failed += 1 - logger.error(f"Failed to update account information for profile '{profile.name}': {str(profile_error)}") - # Continue with other profiles even if one fails - - result_msg = f"Profile refresh complete for account {account.name}: {profiles_updated} updated, {profiles_failed} failed" - logger.info(result_msg) - return result_msg - - except M3UAccount.DoesNotExist: - error_msg = f"Account {account_id} not found" - logger.error(error_msg) - return error_msg - except Exception as e: - error_msg = f"Error refreshing profiles for account {account_id}: {str(e)}" - logger.error(error_msg) - return error_msg - - -@shared_task -def refresh_account_info(profile_id): - """Refresh only the account information for a specific M3U profile.""" - if not acquire_task_lock("refresh_account_info", profile_id): - return f"Account info refresh task already running for profile_id={profile_id}." - - try: - from apps.m3u.models import M3UAccountProfile - import re - - profile = M3UAccountProfile.objects.get(id=profile_id) - account = profile.m3u_account - - if account.account_type != M3UAccount.Types.XC: - release_task_lock("refresh_account_info", profile_id) - return f"Profile {profile_id} belongs to account {account.id} which is not an XtreamCodes account." - - # Get transformed credentials using the helper function - transformed_url, transformed_username, transformed_password = get_transformed_credentials(account, profile) - - # Initialize XtreamCodes client with extracted/transformed credentials - client = XCClient( - transformed_url, - transformed_username, - transformed_password, - account.get_user_agent(), - ) # Authenticate and get account info - auth_result = client.authenticate() - if not auth_result: - error_msg = f"Authentication failed for profile {profile.name} ({profile_id})" - logger.error(error_msg) - - # Send error notification to frontend via websocket - send_websocket_update( - "updates", - "update", - { - "type": "account_info_refresh_error", - "profile_id": profile_id, - "profile_name": profile.name, - "error": "Authentication failed with the provided credentials", - "message": f"Failed to authenticate profile '{profile.name}'. Please check the credentials." - } - ) - - release_task_lock("refresh_account_info", profile_id) - return error_msg - - # Get account information - account_info = client.get_account_info() - - # Update only this specific profile with the new account info - if not profile.custom_properties: - profile.custom_properties = {} - profile.custom_properties.update(account_info) - profile.save() - - # Send success notification to frontend via websocket - send_websocket_update( - "updates", - "update", - { - "type": "account_info_refresh_success", - "profile_id": profile_id, - "profile_name": profile.name, - "message": f"Account information successfully refreshed for profile '{profile.name}'" - } - ) - - release_task_lock("refresh_account_info", profile_id) - return f"Account info refresh completed for profile {profile_id} ({profile.name})." - - except M3UAccountProfile.DoesNotExist: - error_msg = f"Profile {profile_id} not found" - logger.error(error_msg) - - send_websocket_update( - "updates", - "update", - { - "type": "account_refresh_error", - "profile_id": profile_id, - "error": "Profile not found", - "message": f"Profile {profile_id} not found" - } - ) - - release_task_lock("refresh_account_info", profile_id) - return error_msg - except Exception as e: - error_msg = f"Error refreshing account info for profile {profile_id}: {str(e)}" - logger.error(error_msg) - - send_websocket_update( - "updates", - "update", - { - "type": "account_refresh_error", - "profile_id": profile_id, - "error": str(e), - "message": f"Failed to refresh account info: {str(e)}" - } - ) - - release_task_lock("refresh_account_info", profile_id) - return error_msg -@shared_task(time_limit=3600, soft_time_limit=3500) -def refresh_single_m3u_account(account_id): - """Splits M3U processing into chunks and dispatches them as parallel tasks.""" - if not acquire_task_lock("refresh_single_m3u_account", account_id): - return f"Task already running for account_id={account_id}." - - # Keep the lock alive while this long-running task is working. - # Without renewal, the 300s lock TTL can expire during large - # downloads/parses, allowing duplicate tasks to start. - lock_renewer = TaskLockRenewer("refresh_single_m3u_account", account_id) + lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) lock_renewer.start() + source = None try: - return _refresh_single_m3u_account_impl(account_id) - finally: - # Guaranteed cleanup on all exit paths (success, exception, early return) - lock_renewer.stop() - release_task_lock("refresh_single_m3u_account", account_id) + # Try to get the EPG source + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + # The EPG source doesn't exist, so delete the periodic task if it exists + logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") + # Call the shared function to delete the task + if delete_epg_refresh_task_by_id(source_id): + logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") + else: + logger.info(f"No orphaned task found for EPG source {source_id}") -def _refresh_single_m3u_account_impl(account_id): - """Implementation of M3U account refresh with guaranteed memory cleanup.""" - # Record start time - refresh_start_timestamp = timezone.now() # For the cleanup function - start_time = time.time() # For tracking elapsed time as float - streams_created = 0 - streams_updated = 0 - streams_deleted = 0 + # Release the lock and exit + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return f"EPG source {source_id} does not exist, task cleaned up" - try: - account = M3UAccount.objects.get(id=account_id, is_active=True) - if not account.is_active: - logger.debug(f"Account {account_id} is not active, skipping.") + # The source exists but is not active, just skip processing + if not source.is_active: + logger.info(f"EPG source {source_id} is not active. Skipping.") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() return - # Set status to fetching - account.status = M3UAccount.Status.FETCHING - account.save(update_fields=['status']) + # Skip refresh for dummy EPG sources - they don't need refreshing + if source.source_type == 'dummy': + logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + gc.collect() + return - filters = list(account.filters.all()) + # Continue with the normal processing... + logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") + if source.source_type == 'xmltv': + fetch_success = fetch_xmltv(source) + if not fetch_success: + logger.error(f"Failed to fetch XMLTV for source {source.name}") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return - # Check if VOD is enabled for this account - vod_enabled = False - if account.custom_properties: - custom_props = account.custom_properties or {} - vod_enabled = custom_props.get('enable_vod', False) + parse_channels_success = parse_channels_only(source) + if not parse_channels_success: + logger.error(f"Failed to parse channels for source {source.name}") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return - except M3UAccount.DoesNotExist: - # The M3U account doesn't exist, so delete the periodic task if it exists - logger.warning( - f"M3U account with ID {account_id} not found, but task was triggered. Cleaning up orphaned task." - ) + parse_programs_for_source(source) - # Call the helper function to delete the task - if delete_m3u_refresh_task_by_id(account_id): - logger.info( - f"Successfully cleaned up orphaned task for M3U account {account_id}" - ) - else: - logger.debug(f"No orphaned task found for M3U account {account_id}") + elif source.source_type == 'schedules_direct': + fetch_schedules_direct(source, force=force) - return f"M3UAccount with ID={account_id} not found or inactive, task cleaned up" - - # Fetch M3U lines and handle potential issues - extinf_data = [] - groups = None - - cache_path = os.path.join(m3u_dir, f"{account_id}.json") - if os.path.exists(cache_path): + source.save(update_fields=['updated_at']) + # After successful EPG refresh, evaluate DVR series rules to schedule new episodes try: - with open(cache_path, "r") as file: - data = json.load(file) - - extinf_data = data["extinf_data"] - groups = data["groups"] - del data # Free top-level dict; extinf_data/groups retain their references - except json.JSONDecodeError as e: - # Handle corrupted JSON file - logger.error( - f"Error parsing cached M3U data for account {account_id}: {str(e)}" - ) - - # Backup the corrupted file for potential analysis - backup_path = f"{cache_path}.corrupted" - try: - os.rename(cache_path, backup_path) - logger.info(f"Renamed corrupted cache file to {backup_path}") - except OSError as rename_err: - logger.warning( - f"Failed to rename corrupted cache file: {str(rename_err)}" - ) - - # Reset the data to empty structures - extinf_data = [] - groups = None - except Exception as e: - logger.error(f"Unexpected error reading cached M3U data: {str(e)}") - extinf_data = [] - groups = None - - if not extinf_data: + from apps.channels.tasks import evaluate_series_rules + evaluate_series_rules.delay() + except Exception: + pass + except Exception as e: + logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) try: - logger.info(f"Calling refresh_m3u_groups for account {account_id}") - result = refresh_m3u_groups(account_id, full_refresh=True, scan_start_time=refresh_start_timestamp) - logger.trace(f"refresh_m3u_groups result: {result}") + if source: + source.status = 'error' + source.last_message = f"Error refreshing EPG data: {str(e)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) + except Exception as inner_e: + logger.error(f"Error updating source status: {inner_e}") + finally: + # Clear references to ensure proper garbage collection + source = None + # Force garbage collection before releasing the lock + gc.collect() + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) - # Check for completely empty result or missing groups - if not result or result[1] is None: - logger.error( - f"Failed to refresh M3U groups for account {account_id}: {result}" - ) - return "Failed to update m3u account - download failed or other error" - extinf_data, groups = result +def fetch_xmltv(source): + # Handle cases with local file but no URL + if not source.url and source.file_path and os.path.exists(source.file_path): + logger.info(f"Using existing local file for EPG source: {source.name} at {source.file_path}") - # XC accounts can have empty extinf_data but valid groups + # Check if the existing file is compressed and we need to extract it + if source.file_path.endswith(('.gz', '.zip')) and not source.file_path.endswith('.xml'): try: - account = M3UAccount.objects.get(id=account_id) - is_xc_account = account.account_type == M3UAccount.Types.XC - except M3UAccount.DoesNotExist: - is_xc_account = False + # Define the path for the extracted file in the cache directory + cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") + os.makedirs(cache_dir, exist_ok=True) + xml_path = os.path.join(cache_dir, f"{source.id}.xml") - # For XC accounts, empty extinf_data is normal at this stage - if not extinf_data and not is_xc_account: - logger.error(f"No streams found for non-XC account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No streams found in M3U source" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "parsing", 100, status="error", error="No streams found" + # Extract to the cache location keeping the original + extracted_path = extract_compressed_file(source.file_path, xml_path, delete_original=False) + + if extracted_path: + logger.info(f"Extracted mapped compressed file to: {extracted_path}") + # Update to use extracted_file_path instead of changing file_path + source.extracted_file_path = extracted_path + source.save(update_fields=['extracted_file_path']) + else: + logger.error(f"Failed to extract mapped compressed file. Using original file: {source.file_path}") + except Exception as e: + logger.error(f"Failed to extract existing compressed file: {e}") + # Continue with the original file if extraction fails + + # Set the status to success in the database + source.status = 'success' + source.save(update_fields=['status']) + + # Send a download complete notification + send_epg_update(source.id, "downloading", 100, status="success") + + # Return True to indicate successful fetch, processing will continue with parse_channels_only + return True + + # Handle cases where no URL is provided and no valid file path exists + if not source.url: + # Update source status for missing URL + source.status = 'error' + source.last_message = "No URL provided and no valid local file exists" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "downloading", 100, status="error", error="No URL provided and no valid local file exists") + return False + + logger.info(f"Fetching XMLTV data from source: {source.name}") + try: + # Get default user agent from settings + stream_settings = CoreSettings.get_stream_settings() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + default_user_agent_id = stream_settings.get('default_user_agent') + if default_user_agent_id: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + + headers = { + 'User-Agent': user_agent + } + + # Update status to fetching before starting download + source.status = 'fetching' + source.save(update_fields=['status']) + + # Send initial download notification + send_epg_update(source.id, "downloading", 0) + + # Use streaming response to track download progress + with requests.get(source.url, headers=headers, stream=True, timeout=60) as response: + # Handle 404 specifically + if response.status_code == 404: + logger.error(f"EPG URL not found (404): {source.url}") + # Update status to error in the database + source.status = 'error' + source.last_message = f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" + source.save(update_fields=['status', 'last_message']) + + # Notify users through the WebSocket about the EPG fetch failure + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": 404, + "message": f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" + } + } ) - except Exception as e: - logger.error(f"Exception in refresh_m3u_groups: {str(e)}", exc_info=True) - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error refreshing M3U groups: {str(e)}" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, - "parsing", - 100, - status="error", - error=f"Error refreshing M3U groups: {str(e)}", - ) - return "Failed to update m3u account" + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error="URL not found (404)") + return False - # Only proceed with parsing if we actually have data and no errors were encountered - # Get account type to handle XC accounts differently - try: - is_xc_account = account.account_type == M3UAccount.Types.XC - except Exception: - is_xc_account = False + # For all other error status codes + if response.status_code >= 400: + error_message = f"HTTP error {response.status_code}" + user_message = f"EPG source '{source.name}' encountered HTTP error {response.status_code}" - # Modified validation logic for different account types - if (not groups) or (not is_xc_account and not extinf_data): - logger.error(f"No data to process for account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No data available for processing" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, - "parsing", - 100, - status="error", - error="No data available for processing", - ) - return "Failed to update m3u account, no data available" + # Update status to error in the database + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) - hash_keys = CoreSettings.get_m3u_hash_key().split(",") + # Notify users through the WebSocket + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": response.status_code, + "message": user_message + } + } + ) + # Update download progress + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False - existing_groups = { - group.name: group.id - for group in ChannelGroup.objects.filter( - m3u_accounts__m3u_account=account, # Filter by the M3UAccount - m3u_accounts__enabled=True, # Filter by the enabled flag in the join table - ) - } + response.raise_for_status() + logger.debug("XMLTV data fetched successfully.") - try: - # Set status to parsing - account.status = M3UAccount.Status.PARSING - account.save(update_fields=["status"]) + # Define base paths for consistent file naming + cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") + os.makedirs(cache_dir, exist_ok=True) - # Commit any pending transactions before threading - from django.db import transaction - transaction.commit() + # Create temporary download file with .tmp extension + temp_download_path = os.path.join(cache_dir, f"{source.id}.tmp") - # Initialize stream counters - streams_created = 0 - streams_updated = 0 + # Check if we have content length for progress tracking + total_size = int(response.headers.get('content-length', 0)) + downloaded = 0 + start_time = time.time() + last_update_time = start_time + update_interval = 0.5 # Only update every 0.5 seconds - if account.account_type == M3UAccount.Types.STADNARD: - logger.debug( - f"Processing Standard account ({account_id}) with groups: {existing_groups}" - ) - # Break into batches and process with threading - use global batch size - batches = [ - extinf_data[i : i + BATCH_SIZE] - for i in range(0, len(extinf_data), BATCH_SIZE) - ] + # Download to temporary file + with open(temp_download_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=16384): + f.write(chunk) - logger.info(f"Processing {len(extinf_data)} streams in {len(batches)} thread batches") + downloaded += len(chunk) + elapsed_time = time.time() - start_time - # Use 2 threads for optimal database connection handling - max_workers = min(2, len(batches)) - logger.debug(f"Using {max_workers} threads for processing") + # Calculate download speed in KB/s + speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0 - with ThreadPoolExecutor(max_workers=max_workers) as executor: - # Submit batch processing tasks using direct functions (now thread-safe) - future_to_batch = { - executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i - for i, batch in enumerate(batches) - } + # Calculate progress percentage + if total_size and total_size > 0: + progress = min(100, int((downloaded / total_size) * 100)) + else: + # If no content length header, estimate progress + progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown - completed_batches = 0 - total_batches = len(batches) + # Time remaining (in seconds) + time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0 - # Process completed batches as they finish - for future in as_completed(future_to_batch): - batch_idx = future_to_batch[future] - try: - result = future.result() - completed_batches += 1 - - # Extract stream counts from result - if isinstance(result, str): - try: - created_match = re.search(r"(\d+) created", result) - updated_match = re.search(r"(\d+) updated", result) - if created_match and updated_match: - created_count = int(created_match.group(1)) - updated_count = int(updated_match.group(1)) - streams_created += created_count - streams_updated += updated_count - except (AttributeError, ValueError): - pass - - # Send progress update - progress = int((completed_batches / total_batches) * 100) - current_elapsed = time.time() - start_time - - if progress > 0: - estimated_total = (current_elapsed / progress) * 100 - time_remaining = max(0, estimated_total - current_elapsed) - else: - time_remaining = 0 - - send_m3u_update( - account_id, - "parsing", + # Only send updates at specified intervals to avoid flooding + current_time = time.time() + if current_time - last_update_time >= update_interval and progress > 0: + last_update_time = current_time + send_epg_update( + source.id, + "downloading", progress, - elapsed_time=current_elapsed, - time_remaining=time_remaining, - streams_processed=streams_created + streams_updated, + speed=round(speed, 2), + elapsed_time=round(elapsed_time, 1), + time_remaining=round(time_remaining, 1), + downloaded=f"{downloaded / (1024 * 1024):.2f} MB" ) - logger.debug(f"Thread batch {completed_batches}/{total_batches} completed") + # Explicitly delete the chunk to free memory immediately + del chunk - except Exception as e: - logger.error(f"Error in thread batch {batch_idx}: {str(e)}") - completed_batches += 1 # Still count it to avoid hanging + # Send completion notification + send_epg_update(source.id, "downloading", 100) - logger.info(f"Thread-based processing completed for account {account_id}") - else: - # For XC accounts, get the groups with their custom properties containing xc_id - logger.debug(f"Processing XC account with groups: {existing_groups}") + # Determine the appropriate file extension based on content detection + with open(temp_download_path, 'rb') as f: + content_sample = f.read(1024) # Just need the first 1KB to detect format - # Get the ChannelGroupM3UAccount entries with their custom_properties - channel_group_relationships = ChannelGroupM3UAccount.objects.filter( - m3u_account=account, enabled=True - ).select_related("channel_group") - - filtered_groups = {} - for rel in channel_group_relationships: - group_name = rel.channel_group.name - group_id = rel.channel_group.id - - # Load the custom properties with the xc_id - custom_props = rel.custom_properties or {} - if "xc_id" in custom_props: - filtered_groups[group_name] = { - "xc_id": custom_props["xc_id"], - "channel_group_id": group_id, - } - logger.debug( - f"Added group {group_name} with xc_id {custom_props['xc_id']}" - ) - else: - logger.warning( - f"No xc_id found in custom properties for group {group_name}" - ) - - logger.info( - f"Filtered {len(filtered_groups)} groups for processing: {filtered_groups}" + # Use our helper function to detect the format + format_type, is_compressed, file_extension = detect_file_format( + file_path=source.url, # Original URL as a hint + content=content_sample # Actual file content for detection ) - # Collect all XC streams in a single API call and filter by enabled categories - logger.info("Fetching all XC streams from provider and filtering by enabled categories...") - all_xc_streams = collect_xc_streams(account_id, filtered_groups) + logger.debug(f"File format detection results: type={format_type}, compressed={is_compressed}, extension={file_extension}") - if not all_xc_streams: - logger.warning("No streams collected from XC groups") + # Ensure consistent final paths + compressed_path = os.path.join(cache_dir, f"{source.id}{file_extension}" if is_compressed else f"{source.id}.compressed") + xml_path = os.path.join(cache_dir, f"{source.id}.xml") + + # Clean up old files before saving new ones + if os.path.exists(compressed_path): + try: + os.remove(compressed_path) + logger.debug(f"Removed old compressed file: {compressed_path}") + except OSError as e: + logger.warning(f"Failed to remove old compressed file: {e}") + + if os.path.exists(xml_path): + try: + os.remove(xml_path) + logger.debug(f"Removed old XML file: {xml_path}") + except OSError as e: + logger.warning(f"Failed to remove old XML file: {e}") + + # Rename the temp file to appropriate final path + if is_compressed: + try: + os.rename(temp_download_path, compressed_path) + logger.debug(f"Renamed temp file to compressed file: {compressed_path}") + current_file_path = compressed_path + except OSError as e: + logger.error(f"Failed to rename temp file to compressed file: {e}") + current_file_path = temp_download_path # Fall back to using temp file else: - # Now batch by stream count (like standard M3U processing) - batches = [ - all_xc_streams[i : i + BATCH_SIZE] - for i in range(0, len(all_xc_streams), BATCH_SIZE) - ] + try: + os.rename(temp_download_path, xml_path) + logger.debug(f"Renamed temp file to XML file: {xml_path}") + current_file_path = xml_path + except OSError as e: + logger.error(f"Failed to rename temp file to XML file: {e}") + current_file_path = temp_download_path # Fall back to using temp file - logger.info(f"Processing {len(all_xc_streams)} XC streams in {len(batches)} batches") + # Now extract the file if it's compressed + if is_compressed: + try: + logger.info(f"Extracting compressed file {current_file_path}") + send_epg_update(source.id, "extracting", 0, message="Extracting downloaded file") - # Free the original list; batches hold independent sliced copies - del all_xc_streams + # Always extract to the standard XML path - set delete_original to True to clean up + extracted = extract_compressed_file(current_file_path, xml_path, delete_original=True) - # Use threading for XC stream processing - now with consistent batch sizes - max_workers = min(4, len(batches)) - logger.debug(f"Using {max_workers} threads for XC stream processing") + if extracted: + logger.info(f"Successfully extracted to {xml_path}, compressed file deleted") + send_epg_update(source.id, "extracting", 100, message=f"File extracted successfully, temporary file removed") + # Update to store only the extracted file path since the compressed file is now gone + source.file_path = xml_path + source.extracted_file_path = None + else: + logger.error("Extraction failed, using compressed file") + send_epg_update(source.id, "extracting", 100, status="error", message="Extraction failed, using compressed file") + # Use the compressed file + source.file_path = current_file_path + source.extracted_file_path = None + except Exception as e: + logger.error(f"Error extracting file: {str(e)}", exc_info=True) + send_epg_update(source.id, "extracting", 100, status="error", message=f"Error during extraction: {str(e)}") + # Use the compressed file if extraction fails + source.file_path = current_file_path + source.extracted_file_path = None + else: + # It's already an XML file + source.file_path = current_file_path + source.extracted_file_path = None - with ThreadPoolExecutor(max_workers=max_workers) as executor: - # Submit stream batch processing tasks (reuse standard M3U processing) - future_to_batch = { - executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i - for i, batch in enumerate(batches) - } + # Update the source's file paths + source.save(update_fields=['file_path', 'status', 'extracted_file_path']) - completed_batches = 0 - total_batches = len(batches) + # Update status to parsing + source.status = 'parsing' + source.save(update_fields=['status']) - # Process completed batches as they finish - for future in as_completed(future_to_batch): - batch_idx = future_to_batch[future] - try: - result = future.result() - completed_batches += 1 + logger.info(f"Cached EPG file saved to {source.file_path}") - # Extract stream counts from result - if isinstance(result, str): - try: - created_match = re.search(r"(\d+) created", result) - updated_match = re.search(r"(\d+) updated", result) - if created_match and updated_match: - created_count = int(created_match.group(1)) - updated_count = int(updated_match.group(1)) - streams_created += created_count - streams_updated += updated_count - except (AttributeError, ValueError): - pass + return True - # Send progress update - progress = int((completed_batches / total_batches) * 100) - current_elapsed = time.time() - start_time + except requests.exceptions.HTTPError as e: + logger.error(f"HTTP Error fetching XMLTV from {source.name}: {e}", exc_info=True) - if progress > 0: - estimated_total = (current_elapsed / progress) * 100 - time_remaining = max(0, estimated_total - current_elapsed) - else: - time_remaining = 0 + # Get error details + status_code = e.response.status_code if hasattr(e, 'response') and e.response else 'unknown' + error_message = str(e) - send_m3u_update( - account_id, - "parsing", - progress, - elapsed_time=current_elapsed, - time_remaining=time_remaining, - streams_processed=streams_created + streams_updated, - ) + # Create a user-friendly message + user_message = f"EPG source '{source.name}' encountered HTTP error {status_code}" - logger.debug(f"XC thread batch {completed_batches}/{total_batches} completed") + # Add specific handling for common HTTP errors + if status_code == 404: + user_message = f"EPG source '{source.name}' URL not found (404) - will retry on next scheduled run" + elif status_code == 401 or status_code == 403: + user_message = f"EPG source '{source.name}' access denied (HTTP {status_code}) - check credentials" + elif status_code == 429: + user_message = f"EPG source '{source.name}' rate limited (429) - try again later" + elif status_code >= 500: + user_message = f"EPG source '{source.name}' server error (HTTP {status_code}) - will retry later" - except Exception as e: - logger.error(f"Error in XC thread batch {batch_idx}: {str(e)}") - completed_batches += 1 # Still count it to avoid hanging + # Update source status to error with the error message + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) - logger.info(f"XC thread-based processing completed for account {account_id}") - - # Ensure all database transactions are committed before cleanup - logger.info( - f"All thread processing completed, ensuring DB transactions are committed before cleanup" - ) - # Force a simple DB query to ensure connection sync - Stream.objects.filter( - id=-1 - ).exists() # This will never find anything but ensures DB sync - - # Mark streams that weren't seen in this refresh as stale (pending deletion) - stale_stream_count = Stream.objects.filter( - m3u_account=account, - last_seen__lt=refresh_start_timestamp - ).update(is_stale=True) - logger.info(f"Marked {stale_stream_count} streams as stale for account {account_id}") - - # Mark group relationships that weren't seen in this refresh as stale (pending deletion) - stale_group_count = ChannelGroupM3UAccount.objects.filter( - m3u_account=account, - last_seen__lt=refresh_start_timestamp - ).update(is_stale=True) - logger.info(f"Marked {stale_group_count} group relationships as stale for account {account_id}") - - # Now run cleanup - streams_deleted = cleanup_streams(account_id, refresh_start_timestamp) - - # Cleanup stale group relationships (follows same retention policy as streams) - cleanup_stale_group_relationships(account, refresh_start_timestamp) - - # Run auto channel sync after successful refresh - auto_sync_message = "" - auto_sync_result = {} - try: - auto_sync_result = sync_auto_channels( - account_id, scan_start_time=str(refresh_start_timestamp) - ) or {} - logger.info( - f"Auto channel sync result for account {account_id}: {auto_sync_result}" - ) - if auto_sync_result.get("status") == "ok": - created = auto_sync_result.get("channels_created", 0) - updated = auto_sync_result.get("channels_updated", 0) - deleted = auto_sync_result.get("channels_deleted", 0) - failed = auto_sync_result.get("channels_failed", 0) - if created or updated or deleted or failed: - parts = [] - if created: - parts.append(f"{created} channel(s) created") - if updated: - parts.append(f"{updated} updated") - if deleted: - parts.append(f"{deleted} deleted") - if failed: - parts.append(f"{failed} failed") - auto_sync_message = f" Auto-sync: {', '.join(parts)}." - elif auto_sync_result.get("status") == "error": - auto_sync_message = ( - f" Auto-sync error: {auto_sync_result.get('error', 'unknown')}." - ) - except Exception as e: - logger.error( - f"Error running auto channel sync for account {account_id}: {str(e)}" - ) - - # Calculate elapsed time - elapsed_time = time.time() - start_time - - # Calculate total streams processed - streams_processed = streams_created + streams_updated - - # Set status to success and update timestamp BEFORE sending the final update - account.status = M3UAccount.Status.SUCCESS - account.last_message = ( - f"Processing completed in {elapsed_time:.1f} seconds. " - f"Streams: {streams_created} created, {streams_updated} updated, {streams_deleted} removed. " - f"Total processed: {streams_processed}.{auto_sync_message}" - ) - account.updated_at = timezone.now() - account.save(update_fields=["status", "last_message", "updated_at"]) - - # Log system event for M3U refresh - log_system_event( - event_type='m3u_refresh', - account_name=account.name, - elapsed_time=round(elapsed_time, 2), - streams_created=streams_created, - streams_updated=streams_updated, - streams_deleted=streams_deleted, - total_processed=streams_processed, + # Notify users through the WebSocket about the EPG fetch failure + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": status_code, + "message": user_message, + "details": error_message + } + } ) - # Send final update with complete metrics and explicitly include success status - send_m3u_update( - account_id, - "parsing", - 100, - status="success", # Explicitly set status to success - elapsed_time=elapsed_time, - time_remaining=0, - streams_processed=streams_processed, - streams_created=streams_created, - streams_updated=streams_updated, - streams_deleted=streams_deleted, - # Structured auto-sync counts so the frontend can render a - # warning card when anything failed, without parsing the - # free-text last_message. - channels_created=auto_sync_result.get("channels_created", 0), - channels_updated=auto_sync_result.get("channels_updated", 0), - channels_deleted=auto_sync_result.get("channels_deleted", 0), - channels_failed=auto_sync_result.get("channels_failed", 0), - failed_stream_details=auto_sync_result.get("failed_stream_details", []), - message=account.last_message, - ) + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except requests.exceptions.ConnectionError as e: + # Handle connection errors separately + error_message = str(e) + user_message = f"Connection error: Unable to connect to EPG source '{source.name}'" + logger.error(f"Connection error fetching XMLTV from {source.name}: {e}", exc_info=True) - # Trigger VOD refresh if enabled and account is XtreamCodes type - if vod_enabled and account.account_type == M3UAccount.Types.XC: - logger.info(f"VOD is enabled for account {account_id}, triggering VOD refresh") + # Update source status + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Send notifications + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": "connection_error", + "message": user_message + } + } + ) + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except requests.exceptions.Timeout as e: + # Handle timeout errors specifically + error_message = str(e) + user_message = f"Timeout error: EPG source '{source.name}' took too long to respond" + logger.error(f"Timeout error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Update source status + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Send notifications + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except Exception as e: + error_message = str(e) + logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Update source status for general exceptions too + source.status = 'error' + source.last_message = f"Error: {error_message}" + source.save(update_fields=['status', 'last_message']) + + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error=f"Error: {error_message}") + return False + + +def extract_compressed_file(file_path, output_path=None, delete_original=False): + """ + Extracts a compressed file (.gz or .zip) to an XML file. + + Args: + file_path: Path to the compressed file + output_path: Specific path where the file should be extracted (optional) + delete_original: Whether to delete the original compressed file after successful extraction + + Returns: + Path to the extracted XML file, or None if extraction failed + """ + try: + if output_path is None: + base_path = os.path.splitext(file_path)[0] + extracted_path = f"{base_path}.xml" + else: + extracted_path = output_path + + # Make sure the output path doesn't already exist + if os.path.exists(extracted_path): try: - from apps.vod.tasks import refresh_vod_content - refresh_vod_content.delay(account_id) - logger.info(f"VOD refresh task queued for account {account_id}") + os.remove(extracted_path) + logger.info(f"Removed existing extracted file: {extracted_path}") except Exception as e: - logger.error(f"Failed to queue VOD refresh for account {account_id}: {str(e)}") + logger.warning(f"Failed to remove existing extracted file: {e}") + # If we can't delete the existing file and no specific output was requested, + # create a unique filename instead + if output_path is None: + base_path = os.path.splitext(file_path)[0] + extracted_path = f"{base_path}_{uuid.uuid4().hex[:8]}.xml" + + # Use our detection helper to determine the file format instead of relying on extension + with open(file_path, 'rb') as f: + content_sample = f.read(4096) # Read a larger sample to ensure accurate detection + + format_type, is_compressed, _ = detect_file_format(file_path=file_path, content=content_sample) + + if format_type == 'gzip': + logger.debug(f"Extracting gzip file: {file_path}") + try: + # First check if the content is XML by reading a sample + with gzip.open(file_path, 'rb') as gz_file: + content_sample = gz_file.read(4096) # Read first 4KB for detection + detected_format, _, _ = detect_file_format(content=content_sample) + + if detected_format != 'xml': + logger.warning(f"GZIP file does not appear to contain XML content: {file_path} (detected as: {detected_format})") + # Continue anyway since GZIP only contains one file + + # Reset file pointer and extract the content + gz_file.seek(0) + with open(extracted_path, 'wb') as out_file: + while True: + chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk or len(chunk) == 0: + break + out_file.write(chunk) + except Exception as e: + logger.error(f"Error extracting GZIP file: {e}", exc_info=True) + return None + + logger.info(f"Successfully extracted gzip file to: {extracted_path}") + + # Delete original compressed file if requested + if delete_original: + try: + os.remove(file_path) + logger.info(f"Deleted original compressed file: {file_path}") + except Exception as e: + logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + + return extracted_path + + elif format_type == 'zip': + logger.debug(f"Extracting zip file: {file_path}") + with zipfile.ZipFile(file_path, 'r') as zip_file: + # Find the first XML file in the ZIP archive + xml_files = [f for f in zip_file.namelist() if f.lower().endswith('.xml')] + + if not xml_files: + logger.info("No files with .xml extension found in ZIP archive, checking content of all files") + # Check content of each file to see if any are XML without proper extension + for filename in zip_file.namelist(): + if not filename.endswith('/'): # Skip directories + try: + # Read a sample of the file content + content_sample = zip_file.read(filename, 4096) # Read up to 4KB for detection + format_type, _, _ = detect_file_format(content=content_sample) + if format_type == 'xml': + logger.info(f"Found XML content in file without .xml extension: {filename}") + xml_files = [filename] + break + except Exception as e: + logger.warning(f"Error reading file {filename} from ZIP: {e}") + + if not xml_files: + logger.error("No XML file found in ZIP archive") + return None + + # Extract the first XML file + with open(extracted_path, 'wb') as out_file: + with zip_file.open(xml_files[0], "r") as xml_file: + while True: + chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk or len(chunk) == 0: + break + out_file.write(chunk) + + logger.info(f"Successfully extracted zip file to: {extracted_path}") + + # Delete original compressed file if requested + if delete_original: + try: + os.remove(file_path) + logger.info(f"Deleted original compressed file: {file_path}") + except Exception as e: + logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + + return extracted_path + + else: + logger.error(f"Unsupported or unrecognized compressed file format: {file_path} (detected as: {format_type})") + return None except Exception as e: - logger.error(f"Error processing M3U for account {account_id}: {str(e)}") - try: - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error processing M3U: {str(e)}" - account.save(update_fields=["status", "last_message"]) - except Exception: - logger.debug(f"Failed to update account {account_id} status during error handling") - raise # Re-raise the exception for Celery to handle - finally: - # Free large data structures regardless of success or failure - if 'existing_groups' in locals(): - del existing_groups - if 'extinf_data' in locals(): - del extinf_data - if 'groups' in locals(): - del groups - if 'batches' in locals(): - del batches - if 'all_xc_streams' in locals(): - del all_xc_streams - if 'data' in locals(): - del data - if 'filtered_groups' in locals(): - del filtered_groups - if 'channel_group_relationships' in locals(): - del channel_group_relationships + logger.error(f"Error extracting {file_path}: {str(e)}", exc_info=True) + return None - # Remove cache file after processing (success or failure) - cache_path = os.path.join(m3u_dir, f"{account_id}.json") + +def parse_channels_only(source): + # Use extracted file if available, otherwise use the original file path + file_path = source.extracted_file_path if source.extracted_file_path else source.file_path + if not file_path: + file_path = source.get_cache_file() + + # Send initial parsing notification + send_epg_update(source.id, "parsing_channels", 0) + + process = None + should_log_memory = False + + try: + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file does not exist at path: {file_path}") + + # Update the source's file_path to the default cache location + new_path = source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + source.file_path = new_path + source.save(update_fields=['file_path']) + + # If the source has a URL, fetch the data before continuing + if source.url: + logger.info(f"Fetching new EPG data from URL: {source.url}") + fetch_success = fetch_xmltv(source) # Store the result + + # Only proceed if fetch was successful AND file exists + if not fetch_success: + logger.error(f"Failed to fetch EPG data from URL: {source.url}") + # Update status to error + source.status = 'error' + source.last_message = f"Failed to fetch EPG data from URL" + source.save(update_fields=['status', 'last_message']) + # Send error notification + send_epg_update(source.id, "parsing_channels", 100, status="error", error="Failed to fetch EPG data") + return False + + # Verify the file was downloaded successfully + if not os.path.exists(source.file_path): + logger.error(f"Failed to fetch EPG data, file still missing at: {source.file_path}") + # Update status to error + source.status = 'error' + source.last_message = f"Failed to fetch EPG data, file missing after download" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found after download") + return False + + # Update file_path with the new location + file_path = source.file_path + else: + logger.error(f"No URL provided for EPG source {source.name}, cannot fetch new data") + # Update status to error + source.status = 'error' + source.last_message = f"No URL provided, cannot fetch EPG data" + source.save(update_fields=['updated_at']) + + # Initialize process variable for memory tracking only in debug mode try: - os.remove(cache_path) - except OSError: + process = None + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is DEBUG (10) or more verbose + # This is more future-proof than string comparisons + should_log_memory = current_log_level <= logging.DEBUG or settings.DEBUG + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_channels_only] Initial memory usage: {initial_memory:.2f} MB") + except (ImportError, NameError): + process = None + should_log_memory = False + logger.warning("psutil not available for memory tracking") + + # Replace full dictionary load with more efficient lookup set + existing_tvg_ids = set() + existing_epgs = {} + scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup + last_id = 0 + chunk_size = 5000 + + while True: + tvg_id_chunk = set(EPGData.objects.filter( + epg_source=source, + id__gt=last_id + ).order_by('id').values_list('tvg_id', flat=True)[:chunk_size]) + + if not tvg_id_chunk: + break + + existing_tvg_ids.update(tvg_id_chunk) + last_id = EPGData.objects.filter(tvg_id__in=tvg_id_chunk).order_by('-id')[0].id + # Update progress to show file read starting + send_epg_update(source.id, "parsing_channels", 10) + + # Stream parsing instead of loading entire file at once + # This can be simplified since we now always have XML files + epgs_to_create = [] + epgs_to_update = [] + total_channels = 0 + processed_channels = 0 + batch_size = 500 # Process in batches to limit memory usage + progress = 0 # Initialize progress variable here + icon_url_max_length = EPGData._meta.get_field('icon_url').max_length # Get max length for icon_url field + name_max_length = EPGData._meta.get_field('name').max_length # Get max length for name field + + # Track memory at key points + if process: + logger.debug(f"[parse_channels_only] Memory before opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + try: + # Attempt to count existing channels in the database + try: + total_channels = EPGData.objects.filter(epg_source=source).count() + logger.info(f"Found {total_channels} existing channels for this source") + except Exception as e: + logger.error(f"Error counting channels: {e}") + total_channels = 500 # Default estimate + if process: + logger.debug(f"[parse_channels_only] Memory after closing initial file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Update progress after counting + send_epg_update(source.id, "parsing_channels", 25, total_channels=total_channels) + + # Open the file - no need to check file type since it's always XML now + logger.debug(f"Opening file for channel parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + if process: + logger.debug(f"[parse_channels_only] Memory after opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Change iterparse to look for both channel and programme elements + logger.debug(f"Creating iterparse context for channels and programmes") + channel_parser = etree.iterparse(source_file, events=('end',), tag=('channel', 'programme'), remove_blank_text=True, recover=True) + if process: + logger.debug(f"[parse_channels_only] Memory after creating iterparse: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + channel_count = 0 + total_elements_processed = 0 # Track total elements processed, not just channels + for _, elem in channel_parser: + total_elements_processed += 1 + # Only process channel elements + if elem.tag == 'channel': + channel_count += 1 + tvg_id = elem.get('id', '').strip() + if tvg_id: + scanned_tvg_ids.add(tvg_id) + display_name = None + icon_url = None + for child in elem: + if display_name is None and child.tag == 'display-name' and child.text: + display_name = child.text.strip() + elif child.tag == 'icon': + raw_icon_url = child.get('src', '').strip() + icon_url = validate_icon_url_fast(raw_icon_url, icon_url_max_length) + if display_name and icon_url: + break # No need to continue if we have both + + if not display_name: + display_name = tvg_id + + if display_name and len(display_name) > name_max_length: + logger.warning(f"EPG display name too long ({len(display_name)} > {name_max_length}), truncating: {display_name[:80]}...") + display_name = display_name[:name_max_length] + + # Use lazy loading approach to reduce memory usage + if tvg_id in existing_tvg_ids: + # Only fetch the object if we need to update it and it hasn't been loaded yet + if tvg_id not in existing_epgs: + try: + # This loads the full EPG object from the database and caches it + existing_epgs[tvg_id] = EPGData.objects.get(tvg_id=tvg_id, epg_source=source) + except EPGData.DoesNotExist: + # Handle race condition where record was deleted + existing_tvg_ids.remove(tvg_id) + epgs_to_create.append(EPGData( + tvg_id=tvg_id, + name=display_name, + icon_url=icon_url, + epg_source=source, + )) + logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 1: {tvg_id} - {display_name}") + processed_channels += 1 + continue + + # We use the cached object to check if the name or icon_url has changed + epg_obj = existing_epgs[tvg_id] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != icon_url: + epg_obj.icon_url = icon_url + needs_update = True + + if needs_update: + epgs_to_update.append(epg_obj) + logger.debug(f"[parse_channels_only] Added channel to update to epgs_to_update: {tvg_id} - {display_name}") + else: + # No changes needed, just clear the element + logger.debug(f"[parse_channels_only] No changes needed for channel {tvg_id} - {display_name}") + else: + # This is a new channel that doesn't exist in our database + epgs_to_create.append(EPGData( + tvg_id=tvg_id, + name=display_name, + icon_url=icon_url, + epg_source=source, + )) + logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 2: {tvg_id} - {display_name}") + + processed_channels += 1 + + # Batch processing + if len(epgs_to_create) >= batch_size: + logger.info(f"[parse_channels_only] Bulk creating {len(epgs_to_create)} EPG entries") + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + if process: + logger.info(f"[parse_channels_only] Memory after bulk_create: {process.memory_info().rss / 1024 / 1024:.2f} MB") + del epgs_to_create # Explicit deletion + epgs_to_create = [] + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + logger.info(f"[parse_channels_only] Memory after gc.collect(): {process.memory_info().rss / 1024 / 1024:.2f} MB") + + if len(epgs_to_update) >= batch_size: + logger.info(f"[parse_channels_only] Bulk updating {len(epgs_to_update)} EPG entries") + if process: + logger.info(f"[parse_channels_only] Memory before bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) + if process: + logger.info(f"[parse_channels_only] Memory after bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") + epgs_to_update = [] + # Force garbage collection + cleanup_memory(log_usage=should_log_memory, force_collection=True) + + # Periodically clear the existing_epgs cache to prevent memory buildup + if processed_channels % 1000 == 0: + logger.info(f"[parse_channels_only] Clearing existing_epgs cache at {processed_channels} channels") + existing_epgs.clear() + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + logger.info(f"[parse_channels_only] Memory after clearing cache: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Send progress updates + if processed_channels % 100 == 0 or processed_channels == total_channels: + progress = 25 + int((processed_channels / total_channels) * 65) if total_channels > 0 else 90 + send_epg_update( + source.id, + "parsing_channels", + progress, + processed=processed_channels, + total=total_channels + ) + if processed_channels > total_channels: + logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels - total_channels} additional channels") + else: + logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels}/{total_channels}") + if process: + logger.debug(f"[parse_channels_only] Memory before elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + # Clear memory + try: + # First clear the element's content + clear_element(elem) + + except Exception as e: + # Just log the error and continue - don't let cleanup errors stop processing + logger.debug(f"[parse_channels_only] Non-critical error during XML element cleanup: {e}") + if process: + logger.debug(f"[parse_channels_only] Memory after elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + logger.debug(f"[parse_channels_only] Total elements processed: {total_elements_processed}") + + else: + logger.trace(f"[parse_channels_only] Skipping non-channel element: {elem.get('channel', 'unknown')} - {elem.get('start', 'unknown')} {elem.tag}") + clear_element(elem) + continue + + except (etree.XMLSyntaxError, Exception) as xml_error: + logger.error(f"[parse_channels_only] XML parsing failed: {xml_error}") + # Update status to error + source.status = 'error' + source.last_message = f"Error parsing XML file: {str(xml_error)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(xml_error)) + return False + if process: + logger.info(f"[parse_channels_only] Processed {processed_channels} channels current memory: {process.memory_info().rss / 1024 / 1024:.2f} MB") + else: + logger.info(f"[parse_channels_only] Processed {processed_channels} channels") + # Process any remaining items + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.debug(f"[parse_channels_only] Created final batch of {len(epgs_to_create)} EPG entries") + + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) + logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") + + # Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel. + # Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list. + potentially_stale = existing_tvg_ids - scanned_tvg_ids + if potentially_stale: + stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True) + deleted_count, _ = stale_qs.delete() + if deleted_count: + logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel") + + if process: + logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Update source status with channel count + source.status = 'success' + source.last_message = f"Successfully parsed {processed_channels} channels" + source.save(update_fields=['status', 'last_message']) + + # Send completion notification + send_epg_update( + source.id, + "parsing_channels", + 100, + status="success", + channels_count=processed_channels + ) + + send_websocket_update('updates', 'update', {"success": True, "type": "epg_channels"}) + + logger.info(f"Finished parsing channel info. Found {processed_channels} channels.") + + return True + + except FileNotFoundError: + logger.error(f"EPG file not found at: {file_path}") + # Update status to error + source.status = 'error' + source.last_message = f"EPG file not found: {file_path}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found") + return False + except Exception as e: + logger.error(f"Error reading EPG file {file_path}: {e}", exc_info=True) + # Update status to error + source.status = 'error' + source.last_message = f"Error parsing EPG file: {str(e)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(e)) + return False + finally: + # Cleanup memory and close file + if process: + logger.debug(f"[parse_channels_only] Memory before cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + try: + # Output any errors in the channel_parser error log + if 'channel_parser' in locals() and hasattr(channel_parser, 'error_log') and len(channel_parser.error_log) > 0: + logger.debug(f"XML parser errors found ({len(channel_parser.error_log)} total):") + for i, error in enumerate(channel_parser.error_log): + logger.debug(f" Error {i+1}: {error}") + if 'channel_parser' in locals(): + del channel_parser + if 'elem' in locals(): + del elem + if 'parent' in locals(): + del parent + + if 'source_file' in locals(): + source_file.close() + del source_file + # Clear remaining large data structures + existing_epgs.clear() + epgs_to_create.clear() + epgs_to_update.clear() + existing_epgs = None + epgs_to_create = None + epgs_to_update = None + if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None: + scanned_tvg_ids.clear() + scanned_tvg_ids = None + cleanup_memory(log_usage=should_log_memory, force_collection=True) + except Exception as e: + logger.warning(f"Cleanup error: {e}") + + try: + if process: + final_memory = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_channels_only] Final memory usage: {final_memory:.2f} MB") + process = None + except: pass - return f"Dispatched jobs complete." -def send_m3u_update(account_id, action, progress, **kwargs): - # Start with the base data dictionary - data = { - "progress": progress, - "type": "m3u_refresh", - "account": account_id, - "action": action, +@shared_task(time_limit=3600, soft_time_limit=3500) +def parse_programs_for_tvg_id(epg_id): + # Skip XMLTV file parsing for Schedules Direct sources. Program data is + # fetched and persisted directly by fetch_schedules_direct(). + try: + from apps.epg.models import EPGData + epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() + if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': + logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") + return "Skipped (Schedules Direct source)" + except Exception as e: + logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") + + if not acquire_task_lock('parse_epg_programs', epg_id): + logger.info(f"Program parse for {epg_id} already in progress, skipping duplicate task") + return "Task already running" + + lock_renewer = TaskLockRenewer('parse_epg_programs', epg_id) + lock_renewer.start() + + source_file = None + program_parser = None + programs_to_create = [] + programs_processed = 0 + try: + # Add memory tracking only in trace mode or higher + try: + process = None + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is TRACE or more verbose or if running in DEBUG mode + should_log_memory = current_log_level <= 5 or settings.DEBUG + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Initial memory usage: {initial_memory:.2f} MB") + mem_before = initial_memory + except ImportError: + process = None + should_log_memory = False + + epg = EPGData.objects.get(id=epg_id) + epg_source = epg.epg_source + + # Skip program parsing for dummy EPG sources - they don't have program data files + if epg_source.source_type == 'dummy': + logger.info(f"Skipping program parsing for dummy EPG source {epg_source.name} (ID: {epg_id})") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + if not Channel.objects.filter(epg_data=epg).exists(): + logger.info(f"No channels matched to EPG {epg.tvg_id}") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") + + # Optimize deletion with a single delete query instead of chunking + # This is faster for most database engines + ProgramData.objects.filter(epg=epg).delete() + + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + if not file_path: + file_path = epg_source.get_cache_file() + + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file not found at: {file_path}") + + if epg_source.url: + # Update the file path in the database + new_path = epg_source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + epg_source.file_path = new_path + epg_source.save(update_fields=['file_path']) + logger.info(f"Fetching new EPG data from URL: {epg_source.url}") + else: + logger.info(f"EPG source does not have a URL, using existing file path: {file_path} to rebuild cache") + + # Fetch new data before continuing + if epg_source: + + # Properly check the return value from fetch_xmltv + fetch_success = fetch_xmltv(epg_source) + + # If fetch was not successful or the file still doesn't exist, abort + if not fetch_success: + logger.error(f"Failed to fetch EPG data, cannot parse programs for tvg_id: {epg.tvg_id}") + # Update status to error if not already set + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data, cannot parse programs" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Also check if the file exists after download + if not os.path.exists(epg_source.file_path): + logger.error(f"Failed to fetch EPG data, file still missing at: {epg_source.file_path}") + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data, file missing after download" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="File not found after download") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Update file_path with the new location + if epg_source.extracted_file_path: + file_path = epg_source.extracted_file_path + else: + file_path = epg_source.file_path + else: + logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") + # Update status to error + epg_source.status = 'error' + epg_source.last_message = f"No URL provided, cannot fetch EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Use streaming parsing to reduce memory usage + # No need to check file type anymore since it's always XML + logger.debug(f"Parsing programs for tvg_id={epg.tvg_id} from {file_path}") + + # Memory usage tracking + if process: + try: + mem_before = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_programs_for_tvg_id] Memory before parsing {epg.tvg_id} - {mem_before:.2f} MB") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + mem_before = 0 + + programs_to_create = [] + batch_size = 1000 # Process in batches to limit memory usage + + try: + # Open the file directly - no need to check compression + logger.debug(f"Opening file for parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + # Stream parse the file using lxml's iterparse + program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) + + for _, elem in program_parser: + if elem.get('channel') == epg.tvg_id: + try: + start_time = parse_xmltv_time(elem.get('start')) + end_time = parse_xmltv_time(elem.get('stop')) + title = None + desc = None + sub_title = None + + # Efficiently process child elements + for child in elem: + if child.tag == 'title': + title = child.text or 'No Title' + elif child.tag == 'desc': + desc = child.text or '' + elif child.tag == 'sub-title': + sub_title = child.text or '' + + if not title: + title = 'No Title' + + # Extract custom properties + custom_props = extract_custom_properties(elem) + custom_properties_json = None + + if custom_props: + logger.trace(f"Number of custom properties: {len(custom_props)}") + custom_properties_json = custom_props + + # Fallback: extract S/E from description when episode-num + # elements didn't provide them + if desc: + has_season = (custom_properties_json or {}).get('season') is not None + has_episode = (custom_properties_json or {}).get('episode') is not None + if not has_season or not has_episode: + d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) + if d_season is not None and d_episode is not None: + if custom_properties_json is None: + custom_properties_json = {} + if not has_season: + custom_properties_json['season'] = d_season + if not has_episode: + custom_properties_json['episode'] = d_episode + custom_properties_json['season_episode_source'] = 'description' + desc = cleaned_desc + + programs_to_create.append(ProgramData( + epg=epg, + start_time=start_time, + end_time=end_time, + title=title[:255], + description=desc, + sub_title=sub_title, + tvg_id=epg.tvg_id, + custom_properties=custom_properties_json + )) + programs_processed += 1 + # Clear the element to free memory + clear_element(elem) + # Batch processing + if len(programs_to_create) >= batch_size: + ProgramData.objects.bulk_create(programs_to_create) + logger.debug(f"Saved batch of {len(programs_to_create)} programs for {epg.tvg_id}") + programs_to_create = [] + # Only call gc.collect() every few batches + if programs_processed % (batch_size * 5) == 0: + gc.collect() + + except Exception as e: + logger.error(f"Error processing program for {epg.tvg_id}: {e}", exc_info=True) + else: + # Immediately clean up non-matching elements to reduce memory pressure + if elem is not None: + clear_element(elem) + continue + + # Make sure to close the file and release parser resources + if source_file: + source_file.close() + source_file = None + + if program_parser: + program_parser = None + + gc.collect() + + except zipfile.BadZipFile as zip_error: + logger.error(f"Bad ZIP file: {zip_error}") + raise + except etree.XMLSyntaxError as xml_error: + logger.error(f"XML syntax error parsing program data: {xml_error}") + raise + except Exception as e: + logger.error(f"Error parsing XML for programs: {e}", exc_info=True) + raise + finally: + # Ensure file is closed even if an exception occurs + if source_file: + source_file.close() + source_file = None + # Memory tracking after processing + if process: + try: + mem_after = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Memory after parsing 1 {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + + # Process any remaining items + if programs_to_create: + ProgramData.objects.bulk_create(programs_to_create) + logger.debug(f"Saved final batch of {len(programs_to_create)} programs for {epg.tvg_id}") + programs_to_create = None + custom_props = None + custom_properties_json = None + + + logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.") + finally: + # Reset internal caches and pools that lxml might be keeping + try: + etree.clear_error_log() + except: + pass + # Explicit cleanup of all potentially large objects + if source_file: + try: + source_file.close() + except: + pass + source_file = None + program_parser = None + programs_to_create = None + + epg_source = None + # Add comprehensive cleanup before releasing lock + cleanup_memory(log_usage=should_log_memory, force_collection=True) + # Memory tracking after processing + if process: + try: + mem_after = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Final memory usage {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + process = None + epg = None + programs_processed = None + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + + + +def parse_programs_for_source(epg_source, tvg_id=None): + """ + Parse programs for all MAPPED channels from an EPG source in a single pass. + + This is an optimized version that: + 1. Only processes EPG entries that are actually mapped to channels + 2. Parses the XML file ONCE instead of once per channel + 3. Skips programmes for unmapped channels entirely during parsing + + This dramatically improves performance when an EPG source has many channels + but only a fraction are mapped. + """ + # Send initial programs parsing notification + send_epg_update(epg_source.id, "parsing_programs", 0) + should_log_memory = False + process = None + initial_memory = 0 + source_file = None + + # Add memory tracking only in trace mode or higher + try: + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is TRACE or more verbose + should_log_memory = current_log_level <= 5 or settings.DEBUG # Assuming TRACE is level 5 or lower + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_source] Initial memory usage: {initial_memory:.2f} MB") + except ImportError: + logger.warning("psutil not available for memory tracking") + process = None + should_log_memory = False + + try: + # Only get EPG entries that are actually mapped to channels + mapped_epg_ids = set( + Channel.objects.filter( + epg_data__epg_source=epg_source, + epg_data__isnull=False + ).values_list('epg_data_id', flat=True) + ) + + if not mapped_epg_ids: + total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() + logger.info(f"No channels mapped to any EPG entries from source: {epg_source.name} " + f"(source has {total_epg_count} EPG entries, 0 mapped)") + # Update status - this is not an error, just no mapped entries + epg_source.status = 'success' + epg_source.last_message = f"No channels mapped to this EPG source ({total_epg_count} entries available)" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="success") + return True + + # Get the mapped EPG entries with their tvg_ids + mapped_epgs = EPGData.objects.filter(id__in=mapped_epg_ids).values('id', 'tvg_id') + tvg_id_to_epg_id = {epg['tvg_id']: epg['id'] for epg in mapped_epgs if epg['tvg_id']} + mapped_tvg_ids = set(tvg_id_to_epg_id.keys()) + + total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() + mapped_count = len(mapped_tvg_ids) + + logger.info(f"Parsing programs for {mapped_count} MAPPED channels from source: {epg_source.name} " + f"(skipping {total_epg_count - mapped_count} unmapped EPG entries)") + + # Get the file path + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + if not file_path: + file_path = epg_source.get_cache_file() + + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file not found at: {file_path}") + + if epg_source.url: + # Update the file path in the database + new_path = epg_source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + epg_source.file_path = new_path + epg_source.save(update_fields=['file_path']) + logger.info(f"Fetching new EPG data from URL: {epg_source.url}") + + # Fetch new data before continuing + fetch_success = fetch_xmltv(epg_source) + + if not fetch_success: + logger.error(f"Failed to fetch EPG data for source: {epg_source.name}") + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") + return False + + # Update file_path with the new location + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + else: + logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") + epg_source.status = 'error' + epg_source.last_message = f"No URL provided, cannot fetch EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") + return False + + # SINGLE PASS PARSING: Parse the XML file once and collect all programs in memory + # We parse FIRST, then do an atomic delete+insert to avoid race conditions + # where clients might see empty/partial EPG data during the transition + all_programs_to_create = [] + programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} # Track count per channel + total_programs = 0 + skipped_programs = 0 + last_progress_update = 0 + + try: + logger.debug(f"Opening file for single-pass parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + # Stream parse the file using lxml's iterparse + program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) + + for _, elem in program_parser: + channel_id = elem.get('channel') + + # Skip programmes for unmapped channels immediately + if channel_id not in mapped_tvg_ids: + skipped_programs += 1 + # Clear element to free memory + clear_element(elem) + continue + + # This programme is for a mapped channel - process it + try: + start_time = parse_xmltv_time(elem.get('start')) + end_time = parse_xmltv_time(elem.get('stop')) + title = None + desc = None + sub_title = None + + # Efficiently process child elements + for child in elem: + if child.tag == 'title': + title = child.text or 'No Title' + elif child.tag == 'desc': + desc = child.text or '' + elif child.tag == 'sub-title': + sub_title = child.text or '' + + if not title: + title = 'No Title' + + # Extract custom properties + custom_props = extract_custom_properties(elem) + custom_properties_json = custom_props if custom_props else None + + # Fallback: extract S/E from description when episode-num + # elements didn't provide them + if desc: + has_season = (custom_properties_json or {}).get('season') is not None + has_episode = (custom_properties_json or {}).get('episode') is not None + if not has_season or not has_episode: + d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) + if d_season is not None and d_episode is not None: + if custom_properties_json is None: + custom_properties_json = {} + if not has_season: + custom_properties_json['season'] = d_season + if not has_episode: + custom_properties_json['episode'] = d_episode + custom_properties_json['season_episode_source'] = 'description' + desc = cleaned_desc + + epg_id = tvg_id_to_epg_id[channel_id] + all_programs_to_create.append(ProgramData( + epg_id=epg_id, + start_time=start_time, + end_time=end_time, + title=title[:255], + description=desc, + sub_title=sub_title, + tvg_id=channel_id, + custom_properties=custom_properties_json + )) + total_programs += 1 + programs_by_channel[channel_id] += 1 + + # Clear the element to free memory + clear_element(elem) + + # Send progress update (estimate based on programs processed) + if total_programs - last_progress_update >= 5000: + last_progress_update = total_programs + # Cap at 70% during parsing phase (save 30% for DB operations) + progress = min(70, 10 + int((total_programs / max(total_programs + 10000, 1)) * 60)) + send_epg_update(epg_source.id, "parsing_programs", progress, + processed=total_programs, channels=mapped_count) + + # Periodic garbage collection during parsing + if total_programs % 5000 == 0: + gc.collect() + + except Exception as e: + logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) + clear_element(elem) + continue + + except etree.XMLSyntaxError as xml_error: + logger.error(f"XML syntax error parsing program data: {xml_error}") + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"XML parsing error: {str(xml_error)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(xml_error)) + return False + except Exception as e: + logger.error(f"Error parsing XML for programs: {e}", exc_info=True) + raise + finally: + if source_file: + source_file.close() + source_file = None + + # Now perform atomic delete + bulk insert + # This ensures clients never see empty/partial EPG data + logger.info(f"Parsed {total_programs} programs, performing atomic database update...") + send_epg_update(epg_source.id, "parsing_programs", 75, message="Updating database...") + + batch_size = 1000 + try: + with transaction.atomic(): + # Kill any individual statement that hangs longer than 10 minutes. + # SET LOCAL automatically resets when this transaction ends (commit or rollback). + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + # Delete existing programs for mapped EPGs + deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] + logger.debug(f"Deleted {deleted_count} existing programs") + + # Clean up orphaned programs for unmapped EPG entries + unmapped_epg_ids = list(EPGData.objects.filter( + epg_source=epg_source + ).exclude(id__in=mapped_epg_ids).values_list('id', flat=True)) + + if unmapped_epg_ids: + orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] + if orphaned_count > 0: + logger.info(f"Cleaned up {orphaned_count} orphaned programs for {len(unmapped_epg_ids)} unmapped EPG entries") + + # Bulk insert all new programs in batches within the same transaction + for i in range(0, len(all_programs_to_create), batch_size): + batch = all_programs_to_create[i:i + batch_size] + ProgramData.objects.bulk_create(batch) + + # Update progress during insertion + progress = 75 + int((i / len(all_programs_to_create)) * 20) if all_programs_to_create else 95 + if i % (batch_size * 5) == 0: + send_epg_update(epg_source.id, "parsing_programs", min(95, progress), + message=f"Inserting programs... {i}/{len(all_programs_to_create)}") + + logger.info(f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs") + + except Exception as db_error: + logger.error(f"Database error during atomic update: {db_error}", exc_info=True) + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"Database error: {str(db_error)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(db_error)) + return False + finally: + # Clear the large list to free memory + all_programs_to_create = None + gc.collect() + + # Count channels that actually got programs + channels_with_programs = sum(1 for count in programs_by_channel.values() if count > 0) + + # Success message + epg_source.status = EPGSource.STATUS_SUCCESS + epg_source.last_message = ( + f"Parsed {total_programs:,} programs for {channels_with_programs} channels " + f"(skipped {skipped_programs:,} programs for {total_epg_count - mapped_count} unmapped channels)" + ) + epg_source.updated_at = timezone.now() + epg_source.save(update_fields=['status', 'last_message', 'updated_at']) + + # Log system event for EPG refresh + log_system_event( + event_type='epg_refresh', + source_name=epg_source.name, + programs=total_programs, + channels=channels_with_programs, + skipped_programs=skipped_programs, + unmapped_channels=total_epg_count - mapped_count, + ) + + # Send completion notification with status + send_epg_update(epg_source.id, "parsing_programs", 100, + status="success", + message=epg_source.last_message, + updated_at=epg_source.updated_at.isoformat()) + + logger.info(f"Completed parsing programs for source: {epg_source.name} - " + f"{total_programs:,} programs for {channels_with_programs} channels, " + f"skipped {skipped_programs:,} programs for unmapped channels") + return True + + except Exception as e: + logger.error(f"Error in parse_programs_for_source: {e}", exc_info=True) + # Update status to error + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"Error parsing programs: {str(e)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, + status="error", + message=epg_source.last_message) + return False + finally: + # Final memory cleanup and tracking + if source_file: + try: + source_file.close() + except: + pass + source_file = None + + # Explicitly release any remaining large data structures + programs_to_create = None + programs_by_channel = None + mapped_epg_ids = None + mapped_tvg_ids = None + tvg_id_to_epg_id = None + gc.collect() + + # Add comprehensive memory cleanup at the end + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + final_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") + # Explicitly clear the process object to prevent potential memory leaks + process = None +@shared_task(bind=True) +def fetch_schedules_direct_stations(self, source_id): + """ + Lightweight Celery task that runs a stations-only Schedules Direct fetch. + Called on initial source creation so EPGData entries exist for auto-matching + before the user commits to a full schedule/program fetch. + """ + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + logger.error(f"EPGSource {source_id} not found for SD stations fetch") + return + fetch_schedules_direct(source, stations_only=True) + + +def fetch_schedules_direct(source, stations_only=False, force=False): + """ + Fetch EPG data from the Schedules Direct JSON API and persist it to the + EPGData / ProgramData models. + + Authentication flow (as required by the SD API specification): + 1. POST credentials to the token endpoint (password must be SHA1-hashed + as required by the Schedules Direct API specification. + 2. Use the returned token for all subsequent requests via the 'token' header. + 3. Tokens are valid for 24 hours; SD returns the current valid token if one + already exists for the account. + + Data flow: + 1. Fetch subscribed lineups for the account. + 2. Fetch station metadata for each lineup. + 3. Persist station metadata to EPGData. + 4. If stations_only=True, stop here. Used on initial source creation so + the user can run Auto-match EPG before the full program fetch. + 5. Fetch schedule grids in 14-day date-batched requests per station. + 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). + 7. Persist channels to EPGData and programs to ProgramData. + + Args: + source: EPGSource instance + stations_only: If True, only fetch and persist station metadata (no schedules/programs). + Used on initial source creation to populate EPGData for auto-matching + before channels are assigned. + """ + import hashlib + from datetime import date + + + logger.info(f"Fetching Schedules Direct data for source: {source.name}") + + # ------------------------------------------------------------------------- + # Validate credentials + # ------------------------------------------------------------------------- + username = (source.username or '').strip() + password = (source.password or '').strip() + + if not username or not password: + msg = "Schedules Direct source requires both a username and password." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Enforce 2-hour minimum interval between full fetches (not stations-only). + # Schedules Direct enforces rate limits of ~200 requests per 2-hour window. + # This prevents automated abuse regardless of how the refresh was triggered. + # + # Exception: if no SDScheduleMD5 records exist yet, this is the first full + # refresh after initial source creation (stations-only runs first and updates + # updated_at, which would otherwise incorrectly trigger this guard). Always + # allow the first full refresh through so guide data is immediately available. + # ------------------------------------------------------------------------- + if not stations_only and not force and source.updated_at: + from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 + has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() + if has_prior_full_refresh: + elapsed = (timezone.now() - source.updated_at).total_seconds() + min_interval_seconds = 2 * 3600 # 2 hours + if elapsed < min_interval_seconds: + remaining_minutes = int((min_interval_seconds - elapsed) / 60) + msg = ( + f"Schedules Direct refresh skipped. Minimum 2-hour interval not reached. " + f"Last refreshed {int(elapsed / 60)} minutes ago. " + f"Please wait {remaining_minutes} more minute(s)." + ) + logger.warning(f"SD source {source.id}: {msg}") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + else: + logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") + elif force and not stations_only: + logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") + + # ------------------------------------------------------------------------- + # Build SD-specific headers + # SD API spec requires the User-Agent to identify the application and version. + # SergeantPanda confirmed Dispatcharr should identify itself properly. + # ------------------------------------------------------------------------- + from version import __version__ as dispatcharr_version + sd_user_agent = f"Dispatcharr/{dispatcharr_version}" + + def _sd_headers(token=None): + h = { + 'Content-Type': 'application/json', + 'User-Agent': sd_user_agent, + } + if token: + h['token'] = token + return h + + # ------------------------------------------------------------------------- + # Step 1: Authenticate and obtain session token + # The SD API requires the password to be SHA1-hashed before transmission. + # This is a requirement of the Schedules Direct API specification, not an + # architectural choice. + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_FETCHING + source.last_message = "Authenticating with Schedules Direct..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") + + try: + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + token_response = requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers=_sd_headers(), + timeout=30, + ) + token_response.raise_for_status() + token_data = token_response.json() + + auth_code = token_data.get('code', 0) + if auth_code != 0: + if auth_code == 4007: + msg = "Schedules Direct: this application is not authorized. Please contact the Dispatcharr maintainers." + elif auth_code == 4004: + msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." + elif auth_code == 4009: + msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours. Check for misconfiguration." + elif auth_code == 4001: + msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." + elif auth_code == 4008: + msg = "Schedules Direct: account is inactive. Please log in to schedulesdirect.org to reactivate." + else: + msg = f"Schedules Direct authentication failed (code {auth_code}): {token_data.get('message', 'Unknown error')}" + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + token = token_data.get('token') + if not token: + msg = "Schedules Direct returned no token." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info("Schedules Direct authentication successful.") + + except requests.exceptions.RequestException as e: + msg = f"Network error authenticating with Schedules Direct: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 2: Check account status (respect OFFLINE system status) + # ------------------------------------------------------------------------- + try: + status_response = requests.get( + f"{SD_BASE_URL}/status", + headers=_sd_headers(token), + timeout=30, + ) + status_response.raise_for_status() + status_data = status_response.json() + system_status = status_data.get('systemStatus', [{}])[0].get('status', 'Online') + if system_status == 'Offline': + # Per SD API spec: if system is offline, disconnect and do not + # retry for 1 hour. We set idle status rather than error since + # this is a temporary SD-side condition. + msg = "Schedules Direct system is currently offline. Per SD guidelines, retrying in 1 hour." + logger.warning(msg) + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.debug(f"Schedules Direct system status: {system_status}") + except requests.exceptions.RequestException as e: + logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") + + # ------------------------------------------------------------------------- + # Step 3: Fetch subscribed lineups and build station map + # ------------------------------------------------------------------------- + _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=_sd_headers(token), + timeout=30, + ) + # SD returns 400 with code 4102 when no lineups are configured. + # This is a valid account state. The user needs to add lineups via + # the Manage Lineups UI. Treat as idle rather than error. + if lineups_response.status_code == 400: + sd_data = lineups_response.json() + if sd_data.get('code') == 4102: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + lineups_response.raise_for_status() + lineups_data = lineups_response.json() + lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] + if not lineups: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no active lineups found.") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.info(f"Found {len(lineups)} lineup(s) in SD account.") + + # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) + sd_lineup_country = None + for l in lineups: + lid = l.get('lineupID') or l.get('lineup') or '' + if '-' in lid: + sd_lineup_country = lid.split('-')[0] + break + logger.debug(f"SD lineup country: {sd_lineup_country}") + except requests.exceptions.RequestException as e: + msg = f"Failed to fetch Schedules Direct lineups: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # Build station metadata map: stationID -> {name, callsign, logo_url} + station_map = {} + _sd_send_ws_sync(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") + for lineup in lineups: + lineup_id = lineup.get('lineupID') or lineup.get('lineup') + if not lineup_id: + continue + try: + detail_response = requests.get( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=_sd_headers(token), + timeout=30, + ) + detail_response.raise_for_status() + detail_data = detail_response.json() + for station in detail_data.get('stations', []): + sid = station.get('stationID') + if not sid: + continue + logo_url = None + logos = station.get('stationLogo') or station.get('logo') or [] + if isinstance(logos, list) and logos: + # Read preferred logo style from source settings; default to 'dark' + logo_style = (source.custom_properties or {}).get('logo_style', 'dark') + preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) + logo_url = preferred.get('URL') or preferred.get('url') + elif isinstance(logos, dict): + logo_url = logos.get('URL') or logos.get('url') + station_map[sid] = { + 'name': station.get('name', sid), + 'callsign': station.get('callsign', ''), + 'logo_url': logo_url, + } + logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") + + if not station_map: + msg = "No stations found across all Schedules Direct lineups." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info(f"Built station map with {len(station_map)} stations.") + + # ------------------------------------------------------------------------- + # Step 4: Persist station metadata to EPGData + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_PARSING + source.last_message = f"Syncing {len(station_map)} stations..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") + + existing_epg_map = { + epg.tvg_id: epg + for epg in EPGData.objects.filter(epg_source=source) } - # Only fetch the account when we actually need to fill in missing fields. - # Many callers in tight loops already pass status/message; skip the DB hit then. - if "status" not in kwargs or "message" not in kwargs: - try: - account = M3UAccount.objects.only("status", "last_message").get(id=account_id) - if "status" not in kwargs: - data["status"] = account.status - if "message" not in kwargs and account.last_message: - data["message"] = account.last_message - except Exception: - pass + epgs_to_create = [] + epgs_to_update = [] + icon_max_length = EPGData._meta.get_field('icon_url').max_length + name_max_length = EPGData._meta.get_field('name').max_length - # Add the additional key-value pairs from kwargs - data.update(kwargs) - send_websocket_update("updates", "update", data, collect_garbage=False) + for sid, info in station_map.items(): + display_name = (info['name'] or sid)[:name_max_length] + logo = info['logo_url'] + if logo and len(logo) > icon_max_length: + logo = None - # Explicitly clear data reference to help garbage collection - data = None - - -def evaluate_profile_expiration_notification(profile): - """ - Evaluate a single M3UAccountProfile's expiration date and create, update, - or delete the corresponding SystemNotification accordingly. - - Returns the notification key that should remain active (warning or expired), - or None if the profile is not expiring soon and any stale notifications were removed. - This return value is used by the bulk task to track active keys for stale cleanup. - """ - from core.models import SystemNotification - from core.utils import send_websocket_notification, send_notification_dismissed - - exp = profile.exp_date - if not exp: - return None - - now = timezone.now() - warning_threshold = now + timezone.timedelta(days=7) - warning_key = f"xc-exp-warning-{profile.id}" - expired_key = f"xc-exp-expired-{profile.id}" - - if exp <= now: - # Already expired — delete warning, create/update expired notification - deleted_warning = list( - SystemNotification.objects.filter(notification_key=warning_key) - .values_list("notification_key", flat=True) - ) - SystemNotification.objects.filter(notification_key=warning_key).delete() - for key in deleted_warning: - send_notification_dismissed(key) - - notification, created = SystemNotification.objects.update_or_create( - notification_key=expired_key, - defaults={ - "notification_type": SystemNotification.NotificationType.WARNING, - "priority": SystemNotification.Priority.HIGH, - "title": f"Account Expired: {profile.name}", - "message": ( - f'Profile "{profile.name}" on M3U account ' - f'"{profile.m3u_account.name}" has expired ' - f"(expired {exp.strftime('%Y-%m-%d %H:%M UTC')})." - ), - "action_data": { - "profile_id": profile.id, - "account_id": profile.m3u_account.id, - "account_name": profile.m3u_account.name, - "profile_name": profile.name, - "exp_date": exp.isoformat(), - }, - "is_active": True, - "admin_only": True, - }, - ) - send_websocket_notification(notification) - return expired_key - - elif exp <= warning_threshold: - # Expiring within 7 days — delete expired notification, create/update warning - deleted_expired = list( - SystemNotification.objects.filter(notification_key=expired_key) - .values_list("notification_key", flat=True) - ) - SystemNotification.objects.filter(notification_key=expired_key).delete() - for key in deleted_expired: - send_notification_dismissed(key) - - days_left = (exp - now).days - if days_left == 0: - expires_in_str = "today" - elif days_left == 1: - expires_in_str = "in 1 day" + if sid in existing_epg_map: + epg_obj = existing_epg_map[sid] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != logo: + epg_obj.icon_url = logo + needs_update = True + if needs_update: + epgs_to_update.append(epg_obj) else: - expires_in_str = f"in {days_left} days" + epgs_to_create.append(EPGData( + tvg_id=sid, + name=display_name, + icon_url=logo, + epg_source=source, + )) - notification, created = SystemNotification.objects.update_or_create( - notification_key=warning_key, - defaults={ - "notification_type": SystemNotification.NotificationType.WARNING, - "priority": SystemNotification.Priority.NORMAL, - "title": f"Account Expiring: {profile.name}", - "message": ( - f'Profile "{profile.name}" on M3U account ' - f'"{profile.m3u_account.name}" expires {expires_in_str} ' - f"(expires {exp.strftime('%Y-%m-%d %H:%M UTC')})." - ), - "action_data": { - "profile_id": profile.id, - "account_id": profile.m3u_account.id, - "account_name": profile.m3u_account.name, - "profile_name": profile.name, - "exp_date": exp.isoformat(), - }, - "is_active": True, - "admin_only": True, - }, + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) + logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") + + gc.collect() + + # Rebuild map with fresh DB ids for all stations + epg_id_map = { + epg.tvg_id: epg.id + for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) + } + + # Station sync complete. Send progress update before continuing into programs phase. + # We deliberately do NOT send parsing_channels at 100 with status=success here + # because that would cause the frontend to mark the source as complete and + # stop rendering progress updates for the subsequent program fetch phases. + _sd_send_ws_sync(source.id, "parsing_programs", 30, + message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") + + # ------------------------------------------------------------------------- + # Stations-only mode. Used on initial source creation. + # Stop here so the user can run Auto-match EPG before the full program fetch. + # ------------------------------------------------------------------------- + if stations_only: + success_msg = ( + f"{len(station_map)} stations loaded from Schedules Direct. " + f"Run Auto-match EPG to map your channels, then use the Refresh " + f"button to populate guide data." ) - send_websocket_notification(notification) - return warning_key + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_channels", 100, status="success", + message=success_msg, channels_count=len(station_map)) + logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") + return - else: - # Not expiring soon — delete any stale notifications - deleted_keys = list( - SystemNotification.objects.filter( - notification_key__in=[warning_key, expired_key] - ).values_list("notification_key", flat=True) - ) - SystemNotification.objects.filter( - notification_key__in=[warning_key, expired_key] - ).delete() - for key in deleted_keys: - send_notification_dismissed(key) - return None + # ------------------------------------------------------------------------- + # Step 5: MD5-delta schedule fetch + # First fetch MD5 hashes for all stations/dates. Compare against our + # locally cached hashes to determine which schedules have changed. + # Only download schedules that have actually changed; this minimises + # API calls against SD's rate-limited endpoints. + # ------------------------------------------------------------------------- + from apps.epg.models import SDScheduleMD5 + from django.utils.dateparse import parse_datetime + _sd_send_ws_sync(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") + station_ids = list(station_map.keys()) + today = date.today() + date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] -@shared_task -def check_account_expirations(): - """ - Daily task: check all account profiles for upcoming expirations. - Creates/updates SystemNotifications for profiles expiring within 7 days. - Uses separate notification keys for warning vs expired so users can - dismiss the 7-day warning and still receive the expired notification. - """ - from apps.m3u.models import M3UAccountProfile - from core.models import SystemNotification - from core.utils import send_notification_dismissed + # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. + # These accumulate one row per station per day and are never useful once past. + pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] + if pruned_sched_md5_count: + logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") - # Find all active profiles with an exp_date that is set - expiring_profiles = ( - M3UAccountProfile.objects.filter( - m3u_account__is_active=True, - is_active=True, - exp_date__isnull=False, - ) - .select_related("m3u_account") - ) + # Fetch MD5 hashes for all stations in batches of 5000 + STATION_BATCH_SIZE = 5000 + server_md5s = {} # (station_id, date) -> {md5, last_modified} - active_notification_keys = set() + logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") - for profile in expiring_profiles: - active_key = evaluate_profile_expiration_notification(profile) - if active_key: - active_notification_keys.add(active_key) + station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] + for batch in station_batches: + try: + md5_response = requests.post( + f"{SD_BASE_URL}/schedules/md5", + json=[{'stationID': sid, 'date': date_list} for sid in batch], + headers=_sd_headers(token), + timeout=120, + ) + md5_response.raise_for_status() + md5_data = md5_response.json() + for sid, dates in md5_data.items(): + for date_str, info in dates.items(): + if info.get('code', 0) == 0: + server_md5s[(sid, date_str)] = { + 'md5': info.get('md5', ''), + 'last_modified': info.get('lastModified', ''), + } + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule MD5s: {e}") - # Delete stale notifications for profiles whose expiration was extended - stale = SystemNotification.objects.filter( - is_active=True, - ).filter( - models.Q(notification_key__startswith="xc-exp-warning-") | - models.Q(notification_key__startswith="xc-exp-expired-") - ).exclude(notification_key__in=active_notification_keys) - stale_keys = list(stale.values_list("notification_key", flat=True)) - stale.delete() - for key in stale_keys: - send_notification_dismissed(key) + # Load our cached MD5s from DB + cached_md5s = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) + } + + # Determine which station/date combinations need downloading + changed_by_station = {} # station_id -> [date_str, ...] + for (sid, date_str), server_info in server_md5s.items(): + if date_str not in date_list: + continue + cached = cached_md5s.get((sid, date_str)) + if cached != server_info['md5']: + changed_by_station.setdefault(sid, []).append(date_str) + + total_changed = sum(len(v) for v in changed_by_station.values()) + total_possible = len(station_ids) * len(date_list) + logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") + _sd_send_ws_sync(source.id, "parsing_programs", 38, + message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") + + # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} + schedules_by_station = {sid: [] for sid in station_ids} + program_ids_needed = set() + + if not changed_by_station: + logger.info("No schedule changes detected, skipping schedule and program downloads.") + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", + message="No schedule changes detected since last refresh. Guide data is up to date.") + source.status = EPGSource.STATUS_SUCCESS + source.last_message = "No schedule changes detected. Guide data is up to date." + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + return + + # Download only changed schedules, batched by 7-day windows per station + SCHEDULE_BATCH_DAYS = 7 + changed_station_ids = list(changed_by_station.keys()) + date_batches = [date_list[i:i + SCHEDULE_BATCH_DAYS] for i in range(0, len(date_list), SCHEDULE_BATCH_DAYS)] + new_md5_records = [] + updated_md5_records = [] + existing_md5_map = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=changed_station_ids) + } + + for batch_idx, date_batch in enumerate(date_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 38 + int((batch_idx / len(date_batches)) * 22) + logger.info(f"Fetching schedule batch {batch_idx + 1} of {len(date_batches)}...") + _sd_send_ws_sync(source.id, "parsing_programs", min(59, pre_progress), + message=f"Fetching schedules: batch {batch_idx + 1} of {len(date_batches)}...") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + # Only include stations that have changes in this date batch + request_body = [ + {'stationID': sid, 'date': [d for d in date_batch if d in changed_by_station.get(sid, [])]} + for sid in changed_station_ids + if any(d in changed_by_station.get(sid, []) for d in date_batch) + ] + if not request_body: + continue + try: + sched_response = requests.post( + f"{SD_BASE_URL}/schedules", + json=request_body, + headers=_sd_headers(token), + timeout=120, + ) + sched_response.raise_for_status() + sched_data = sched_response.json() + + for station_sched in sched_data: + sid = station_sched.get('stationID') + if not sid: + continue + programs = station_sched.get('programs', []) + schedules_by_station.setdefault(sid, []).extend(programs) + for prog in programs: + pid = prog.get('programID') + if pid: + program_ids_needed.add(pid) + + # Update MD5 cache for this station/date + meta = station_sched.get('metadata', {}) + start_date = meta.get('startDate') + md5_val = meta.get('md5', '') + last_mod_str = meta.get('modified', '') + if start_date and md5_val: + key = (sid, start_date) + last_mod = parse_datetime(last_mod_str) if last_mod_str else timezone.now() + if key in existing_md5_map: + rec = existing_md5_map[key] + rec.md5 = md5_val + rec.last_modified = last_mod + updated_md5_records.append(rec) + else: + import datetime as dt_module + try: + date_obj = dt_module.date.fromisoformat(start_date) + new_md5_records.append(SDScheduleMD5( + epg_source=source, + station_id=sid, + date=date_obj, + md5=md5_val, + last_modified=last_mod, + )) + except ValueError: + pass + + progress = 38 + int(((batch_idx + 1) / len(date_batches)) * 22) + _sd_send_ws_sync(source.id, "parsing_programs", min(60, progress), + message=f"Fetching changed schedules: batch {batch_idx + 1}/{len(date_batches)} ({len(program_ids_needed):,} programs found)") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule batch {batch_idx + 1}: {e}") + + # Persist updated MD5 cache + if new_md5_records: + SDScheduleMD5.objects.bulk_create(new_md5_records, ignore_conflicts=True) + logger.info(f"Cached {len(new_md5_records)} new schedule MD5s.") + if updated_md5_records: + SDScheduleMD5.objects.bulk_update(updated_md5_records, ['md5', 'last_modified']) + logger.info(f"Updated {len(updated_md5_records)} existing schedule MD5s.") + + if not program_ids_needed: + msg = "No schedule data returned from Schedules Direct." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 6: MD5-delta program metadata fetch + # The schedule response includes an MD5 hash per program airing. + # Compare against our cached program MD5s to only download programs + # whose metadata has changed since our last fetch. + # ------------------------------------------------------------------------- + + # Build map of programID -> md5 from schedule data + schedule_program_md5s = {} # programID -> md5 from schedule + for sid, airings in schedules_by_station.items(): + for airing in airings: + pid = airing.get('programID') + md5 = airing.get('md5') + if pid and md5: + schedule_program_md5s[pid] = md5 + + # Load cached program MD5s from SDProgramMD5 table, keyed by programID + cached_prog_md5s = { + r.program_id: r.md5 + for r in SDProgramMD5.objects.filter( + epg_source=source, + program_id__in=program_ids_needed, + ).only('program_id', 'md5') + } + + # Only fetch programs where MD5 differs from our cached value + programs_to_fetch = { + pid for pid in program_ids_needed + if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) + } logger.info( - f"Account expiration check complete: {len(active_notification_keys)} active notifications" + f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " + f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") + + program_metadata = {} + program_id_list = list(programs_to_fetch) + total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) + + if program_id_list: + logger.info(f"Fetching metadata for {len(program_id_list)} programs in {total_batches} batch(es).") + for batch_idx in range(total_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 60 + int((batch_idx / total_batches) * 20) + logger.info(f"Fetching program metadata batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)...") + _sd_send_ws_sync(source.id, "parsing_programs", min(79, pre_progress), + message=f"Fetching program data: batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + batch = program_id_list[batch_idx * SD_PROGRAM_BATCH_SIZE:(batch_idx + 1) * SD_PROGRAM_BATCH_SIZE] + try: + prog_response = requests.post( + f"{SD_BASE_URL}/programs", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + prog_response.raise_for_status() + prog_data = prog_response.json() + for prog in prog_data: + pid = prog.get('programID') + if pid: + program_metadata[pid] = prog + + progress = 60 + int(((batch_idx + 1) / total_batches) * 20) + _sd_send_ws_sync(source.id, "parsing_programs", min(80, progress), + message=f"Fetching program details: batch {batch_idx + 1}/{total_batches} ({len(program_metadata):,} programs loaded)") + logger.debug(f"Fetched program metadata batch {batch_idx + 1}/{total_batches}") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch program metadata batch {batch_idx + 1}: {e}") + else: + logger.info("All program metadata unchanged - skipping program download.") + _sd_send_ws_sync(source.id, "parsing_programs", 80, message="Program metadata unchanged - using cached data.") + + gc.collect() + + # ------------------------------------------------------------------------- + # Step 7: Build ProgramData records and persist atomically + # ------------------------------------------------------------------------- + logger.info("Building program records...") + _sd_send_ws_sync(source.id, "parsing_programs", 80) + + # Only process stations that are mapped to channels to match the existing + # XMLTV flow (parse_programs_for_source only processes mapped channels). + from apps.channels.models import Channel as ChannelModel + mapped_epg_ids = set( + ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) ) + mapped_tvg_ids = set( + EPGData.objects.filter( + id__in=mapped_epg_ids, + epg_source=source, + ).values_list('tvg_id', flat=True) + ) + + # Cache existing program data for unchanged programs BEFORE surgical delete. + # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only + # programs with changed program MD5s get metadata re-downloaded. The surgical delete + # wipes ALL ProgramData for changed dates, so unchanged programs lose their titles. + # This cache preserves their data for rebuilding. + unchanged_pids = set() + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + continue + for airing in airings: + pid = airing.get('programID') + if pid and pid not in program_metadata: + unchanged_pids.add(pid) + + existing_program_cache = {} + if unchanged_pids: + for pd in ProgramData.objects.filter( + epg__epg_source=source, + program_id__in=unchanged_pids, + ).only('program_id', 'title', 'description', 'sub_title', 'custom_properties'): + if pd.program_id not in existing_program_cache: + existing_program_cache[pd.program_id] = { + 'title': pd.title, + 'description': pd.description, + 'sub_title': pd.sub_title, + 'custom_properties': pd.custom_properties, + } + logger.info(f"Cached {len(existing_program_cache)} existing program records for unchanged programs.") + + all_programs_to_create = [] + total_programs = 0 + skipped_unmapped = 0 + + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + skipped_unmapped += len(airings) + continue + + epg_db_id = epg_id_map.get(sid) + if not epg_db_id: + continue + + for airing in airings: + pid = airing.get('programID') + air_time = airing.get('airDateTime') + duration_secs = airing.get('duration', 0) + + if not pid or not air_time or not duration_secs: + continue + + try: + start_dt = parse_schedules_direct_time(air_time) + end_dt = start_dt + timedelta(seconds=int(duration_secs)) + except Exception as e: + logger.debug(f"Could not parse air time '{air_time}': {e}") + continue + + meta = program_metadata.get(pid, {}) + cached_prog = existing_program_cache.get(pid) if not meta else None + + if cached_prog: + # Unchanged program — reuse cached data from before surgical delete + title = cached_prog['title'] or 'No Title' + desc = cached_prog['description'] or '' + episode_title = cached_prog['sub_title'] or '' + custom_props = cached_prog['custom_properties'] or {} + else: + titles = meta.get('titles', [{}]) + title = titles[0].get('title120', '') if titles else '' + if not title: + title = meta.get('episodeTitle150', '') or 'No Title' + title = title[:255] + + if not cached_prog: + descriptions = meta.get('descriptions', {}) + desc = '' + for key in ('description1000', 'description255', 'description100'): + candidates = descriptions.get(key, []) + if candidates: + desc = candidates[0].get('description', '') + if desc: + break + + episode_title = meta.get('episodeTitle150', '') + + # Build custom_properties following the same pattern as the XMLTV parser + custom_props = {} + + # Season/Episode — search all metadata entries, not just [0] + metadata_block = meta.get('metadata', []) + gracenote_meta = {} + for md_entry in metadata_block: + if 'Gracenote' in md_entry: + gracenote_meta = md_entry['Gracenote'] + break + if not gracenote_meta: + # Fall back to TVmaze if Gracenote is absent + for md_entry in metadata_block: + if 'TVmaze' in md_entry: + gracenote_meta = md_entry['TVmaze'] + break + season = gracenote_meta.get('season') + episode = gracenote_meta.get('episode') + if season: + custom_props['season'] = int(season) + if episode: + custom_props['episode'] = int(episode) + if season and episode: + custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" + + # Content rating — store full array, pick display rating by lineup country + content_rating = meta.get('contentRating', []) + if content_rating: + custom_props['content_ratings'] = content_rating + selected = None + if sd_lineup_country: + for cr in content_rating: + if cr.get('country', '') == sd_lineup_country: + selected = cr + break + if not selected: + # Fall back to USA, then first available + for cr in content_rating: + if cr.get('country', '') == 'USA': + selected = cr + break + if not selected: + selected = content_rating[0] + custom_props['rating'] = selected.get('code', '') + custom_props['rating_system'] = selected.get('body', '') + + # Content advisory — content warnings + content_advisory = meta.get('contentAdvisory', []) + if content_advisory: + custom_props['content_advisory'] = content_advisory + + # Categories — combine entityType, showType, and genres + categories = [] + entity_type = meta.get('entityType', '') + show_type = meta.get('showType', '') + if entity_type: + categories.append(entity_type) + if show_type and show_type != entity_type: + categories.append(show_type) + genres = meta.get('genres', []) + categories.extend(genres) + if categories: + custom_props['categories'] = categories + + # Cast — top-billed only, store characterName, drop role noise + cast = meta.get('cast', []) + crew = meta.get('crew', []) + credits = {} + if cast: + # Sort by billingOrder and cap at top-billed actors + sorted_cast = sorted( + [p for p in cast if p.get('name')], + key=lambda p: int(p.get('billingOrder', '999')) + ) + # Separate main cast from guest stars + main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] + # Store top-billed main cast (matching XMLTV parity) + credits['actor'] = [ + { + 'name': p.get('name', ''), + **(({'character': p['characterName']} ) if p.get('characterName') else {}), + } + for p in (main_cast[:6] if main_cast else sorted_cast[:6]) + ] + if crew: + for member in crew: + role = member.get('role', '').lower() + name = member.get('name', '') + if not name: + continue + if 'director' in role: + credits.setdefault('director', []).append(name) + elif 'writer' in role or 'screenwriter' in role: + credits.setdefault('writer', []).append(name) + elif 'producer' in role: + credits.setdefault('producer', []).append(name) + if credits: + custom_props['credits'] = credits + + # Airing flags + if airing.get('liveTapeDelay') == 'Live': + custom_props['live'] = True + if airing.get('new'): + custom_props['new'] = True + else: + custom_props['previously_shown'] = True + if airing.get('premiere'): + custom_props['premiere'] = True + + # Original air date — full date, not just year + original_air_date = meta.get('originalAirDate', '') + movie_year = meta.get('movie', {}).get('year', '') + if original_air_date: + custom_props['date'] = original_air_date + elif movie_year: + custom_props['date'] = str(movie_year) + + # Country of production + country = meta.get('country', []) + if country: + custom_props['country'] = country[0] if len(country) == 1 else ', '.join(country) + + # Runtime — program duration without commercials (seconds → store for display) + runtime_secs = meta.get('duration') or meta.get('movie', {}).get('duration') + if runtime_secs: + runtime_mins = int(runtime_secs) // 60 + custom_props['length'] = {'value': str(runtime_mins), 'units': 'minutes'} + + # Movie quality ratings → star_ratings (matches XMLTV key) + movie_data = meta.get('movie', {}) + quality_ratings = movie_data.get('qualityRating', []) + if quality_ratings: + star_ratings = [] + for qr in quality_ratings: + rating_str = qr.get('rating', '') + max_rating = qr.get('maxRating', '') + if rating_str and max_rating: + star_ratings.append({ + 'value': f"{rating_str}/{max_rating}", + 'system': qr.get('ratingsBody', ''), + }) + if star_ratings: + custom_props['star_ratings'] = star_ratings + + # Sports event details + event_details = meta.get('eventDetails', {}) + if event_details: + custom_props['event_details'] = event_details + + all_programs_to_create.append(ProgramData( + epg_id=epg_db_id, + start_time=start_dt, + end_time=end_dt, + title=title, + sub_title=episode_title or None, + description=desc or None, + tvg_id=sid, + program_id=pid, + custom_properties=custom_props or None, + )) + total_programs += 1 + + logger.info(f"Built {total_programs} program records " + f"({skipped_unmapped} skipped for unmapped stations).") + + _sd_send_ws_sync(source.id, "parsing_programs", 88) + + # Build a map of epg_db_id -> list of (day_start_utc, day_end_utc) for each changed date. + # Only programs that fall within changed station/date pairs will be deleted and replaced; + # programs for unchanged stations or unchanged dates are left intact. + import datetime as dt_module + epg_changed_date_ranges = {} + for sid, changed_date_strs in changed_by_station.items(): + epg_db_id = epg_id_map.get(sid) + if not epg_db_id or epg_db_id not in mapped_epg_ids: + continue + ranges = [] + for ds in changed_date_strs: + d = dt_module.date.fromisoformat(ds) + day_start = datetime(d.year, d.month, d.day, tzinfo=dt_timezone.utc) + ranges.append((day_start, day_start + timedelta(days=1))) + if ranges: + epg_changed_date_ranges[epg_db_id] = ranges + + # Atomic delete (surgical) + bulk insert + BATCH_SIZE = 1000 + try: + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + total_deleted = 0 + for epg_db_id, day_ranges in epg_changed_date_ranges.items(): + q = Q() + for day_start, day_end in day_ranges: + q |= Q(start_time__gte=day_start, start_time__lt=day_end) + cnt = ProgramData.objects.filter(epg_id=epg_db_id).filter(q).delete()[0] + total_deleted += cnt + logger.debug(f"Deleted {total_deleted} changed SD programs across {len(epg_changed_date_ranges)} stations.") + for i in range(0, len(all_programs_to_create), BATCH_SIZE): + ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) + progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) + _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) + + logger.info(f"Committed {total_programs} Schedules Direct programs to database.") + + # Upsert SDProgramMD5 records for programs we just downloaded + # This updates the cache so future fetches can skip unchanged programs + if schedule_program_md5s: + md5_records = [ + SDProgramMD5( + epg_source=source, + program_id=pid, + md5=md5, + ) + for pid, md5 in schedule_program_md5s.items() + if pid in program_metadata # Only cache programs that were actually downloaded + ] + if md5_records: + SDProgramMD5.objects.bulk_create( + md5_records, + update_conflicts=True, + unique_fields=['epg_source', 'program_id'], + update_fields=['md5'], + ) + logger.info(f"Cached {len(md5_records)} program MD5s for future delta detection.") + + except Exception as db_error: + msg = f"Database error persisting Schedules Direct programs: {db_error}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + finally: + all_programs_to_create = None + gc.collect() + + # ------------------------------------------------------------------------- + # Step 8: Fetch program artwork (posters) if enabled + # ------------------------------------------------------------------------- + fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) + if fetch_posters and program_metadata: + logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") + _sd_send_ws_sync(source.id, "parsing_programs", 98, + message="Fetching program artwork...") + + try: + # Build a set of unique artwork lookup IDs. + # For episodes (EP...), use the series root (SH...0000) so we get + # series-level artwork — one poster per series instead of per-episode. + artwork_lookup_ids = set() + pid_to_artwork_key = {} # maps original programID -> the key we looked up + + for pid in program_metadata: + if pid.startswith('EP'): + sh_root = 'SH' + pid[2:10] + '0000' + artwork_lookup_ids.add(sh_root) + pid_to_artwork_key[pid] = sh_root + else: + artwork_lookup_ids.add(pid) + pid_to_artwork_key[pid] = pid + + artwork_map = {} # artwork_key -> poster_url + artwork_list = list(artwork_lookup_ids) + SD_ARTWORK_BATCH_SIZE = 500 + + total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) + logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " + f"in {total_art_batches} batch(es).") + + for batch_idx in range(total_art_batches): + batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] + try: + art_response = requests.post( + f"{SD_BASE_URL}/metadata/programs/", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + art_response.raise_for_status() + art_data = art_response.json() + + for entry in art_data: + if not isinstance(entry, dict): + continue + entry_pid = entry.get('programID') + images = entry.get('data') or [] + if not entry_pid or not images: + continue + + # Filter to only dict entries — SD sometimes returns bare strings + images = [img for img in images if isinstance(img, dict)] + if not images: + continue + + # Pick the best poster image: + # Prefer portrait orientation (2x3, 3x4) in larger sizes + # SD categories include: Iconic, Banner-L1, Banner-L2, Logo + # SD uses width/height instead of a "size" field + poster_url = None + + # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic + for min_width in [240, 135, 120]: + for pref_cat in ['Banner-L1', 'Iconic']: + match = next((img for img in images + if img.get('aspect') in ('2x3', '3x4') + and img.get('category') == pref_cat + and (img.get('width', 0) or 0) >= min_width), None) + if match: + poster_url = match.get('uri') + break + if poster_url: + break + + # Fallback: any portrait image at any size + if not poster_url: + portrait = next((img for img in images + if img.get('aspect') in ('2x3', '3x4')), None) + if portrait: + poster_url = portrait.get('uri') + + if poster_url: + # Complete the URL if it's relative + if not poster_url.startswith('http'): + poster_url = f"{SD_BASE_URL}/image/{poster_url}" + artwork_map[entry_pid] = poster_url + + logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " + f"{len(artwork_map)} posters found so far.") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") + + # Bulk-update ProgramData records with poster URLs + if artwork_map: + programs_to_update = [] + for prog in ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).only('id', 'program_id', 'custom_properties'): + art_key = pid_to_artwork_key.get(prog.program_id) + poster = artwork_map.get(art_key) if art_key else None + if poster: + cp = prog.custom_properties or {} + cp['poster_url'] = poster + prog.custom_properties = cp + programs_to_update.append(prog) + + if programs_to_update: + ProgramData.objects.bulk_update( + programs_to_update, ['custom_properties'], batch_size=1000 + ) + logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") + else: + logger.info("No poster artwork matched committed programs.") + else: + logger.info("No poster artwork found from Schedules Direct.") + + except Exception as art_error: + logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) + + elif fetch_posters: + logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") + + # ------------------------------------------------------------------------- + # Step 9: Apply SD station logos to matched channels if enabled + # ------------------------------------------------------------------------- + use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) + if use_sd_logos: + try: + from apps.channels.models import Channel as ChannelModel, Logo + + channels_to_update = [] + logos_created = 0 + + for channel in ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).select_related('epg_data', 'logo'): + icon_url = (channel.epg_data.icon_url or '').strip() + if not icon_url: + continue + + # Skip if channel already has this logo URL + if channel.logo and channel.logo.url == icon_url: + continue + + # Find or create a Logo object for this URL + try: + logo = Logo.objects.get(url=icon_url) + except Logo.DoesNotExist: + logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" + logo = Logo.objects.create(name=logo_name, url=icon_url) + logos_created += 1 + + channel.logo = logo + channels_to_update.append(channel) + + if channels_to_update: + ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) + logger.info(f"Applied SD logos to {len(channels_to_update)} channels " + f"({logos_created} new logos created).") + else: + logger.info("All matched channels already have current SD logos.") + + except Exception as logo_error: + logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) + + # Prune ProgramData whose end_time has passed. With surgical per-date deletes, + # programs from dates that have rolled off the window are never explicitly removed. + today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) + try: + expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] + if expired_count: + logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") + except Exception as prune_err: + logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") + + # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. + try: + live_program_ids = set( + ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) + .values_list('program_id', flat=True) + ) + pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( + program_id__in=live_program_ids + ).delete()[0] + if pruned_prog_md5_count: + logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") + except Exception as prune_err: + logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + + # ------------------------------------------------------------------------- + # Prune stale poster cache files (>30 days old or orphaned) + # ------------------------------------------------------------------------- + try: + cache_dir = '/data/cache/posters' + if os.path.exists(cache_dir): + import time as time_module + # Collect all poster hashes currently referenced by ProgramData + active_hashes = set() + for url in ProgramData.objects.filter( + epg__epg_source=source, + custom_properties__has_key='poster_url', + ).values_list('custom_properties__poster_url', flat=True): + if url: + active_hashes.add(url.rsplit('/', 1)[-1]) + + pruned_posters = 0 + for fname in os.listdir(cache_dir): + fpath = os.path.join(cache_dir, fname) + if not os.path.isfile(fpath): + continue + file_age = time_module.time() - os.path.getmtime(fpath) + # Remove if older than 30 days OR not referenced by any current program + if file_age > 30 * 24 * 3600 or fname not in active_hashes: + os.remove(fpath) + pruned_posters += 1 + if pruned_posters: + logger.info(f"Pruned {pruned_posters} stale poster cache files.") + except Exception as poster_prune_err: + logger.warning(f"Failed to prune poster cache: {poster_prune_err}") + + # ------------------------------------------------------------------------- + # Done + # ------------------------------------------------------------------------- + success_msg = ( + f"Successfully fetched {total_programs:,} programs for " + f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " + f"({skipped_unmapped:,} programs skipped for unmapped stations)." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message=success_msg) + log_system_event( + event_type='epg_refresh', + source_name=source.name, + programs=total_programs, + channels=len(mapped_tvg_ids), + skipped_programs=skipped_unmapped, + ) + logger.info(f"Schedules Direct fetch complete for source: {source.name}") + + +# ------------------------------- +# Helper parse functions +# ------------------------------- +def parse_xmltv_time(time_str): + try: + # Basic format validation + if len(time_str) < 14: + logger.warning(f"XMLTV timestamp too short: '{time_str}', using as-is") + dt_obj = datetime.strptime(time_str, '%Y%m%d%H%M%S') + return timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + + # Parse base datetime + dt_obj = datetime.strptime(time_str[:14], '%Y%m%d%H%M%S') + + # Handle timezone if present + if len(time_str) >= 20: # Has timezone info + tz_sign = time_str[15] + tz_hours = int(time_str[16:18]) + tz_minutes = int(time_str[18:20]) + + # Create a timezone object + if tz_sign == '+': + tz_offset = dt_timezone(timedelta(hours=tz_hours, minutes=tz_minutes)) + elif tz_sign == '-': + tz_offset = dt_timezone(timedelta(hours=-tz_hours, minutes=-tz_minutes)) + else: + tz_offset = dt_timezone.utc + + # Make datetime aware with correct timezone + aware_dt = datetime.replace(dt_obj, tzinfo=tz_offset) + # Convert to UTC + aware_dt = aware_dt.astimezone(dt_timezone.utc) + + logger.trace(f"Parsed XMLTV time '{time_str}' to {aware_dt}") + return aware_dt + else: + # No timezone info, assume UTC + aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + logger.trace(f"Parsed XMLTV time without timezone '{time_str}' as UTC: {aware_dt}") + return aware_dt + + except Exception as e: + logger.error(f"Error parsing XMLTV time '{time_str}': {e}", exc_info=True) + raise + + +def parse_schedules_direct_time(time_str): + try: + dt_obj = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%SZ') + aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + logger.debug(f"Parsed Schedules Direct time '{time_str}' to {aware_dt}") + return aware_dt + except Exception as e: + logger.error(f"Error parsing Schedules Direct time '{time_str}': {e}", exc_info=True) + raise + + +# Re-export from utils to preserve backward compatibility for any callers +from apps.epg.utils import extract_season_episode_from_description, _ONSCREEN_RE # noqa: F401 + + +# Helper function to extract custom properties - moved to a separate function to clean up the code +def extract_custom_properties(prog): + # Create a new dictionary for each call + custom_props = {} + + # Extract categories with a single comprehension to reduce intermediate objects + categories = [cat.text.strip() for cat in prog.findall('category') if cat.text and cat.text.strip()] + if categories: + custom_props['categories'] = categories + + # Extract keywords (new) + keywords = [kw.text.strip() for kw in prog.findall('keyword') if kw.text and kw.text.strip()] + if keywords: + custom_props['keywords'] = keywords + + # Extract episode numbers + for ep_num in prog.findall('episode-num'): + system = ep_num.get('system', '') + if system == 'xmltv_ns' and ep_num.text: + # Parse XMLTV episode-num format (season.episode.part) + parts = ep_num.text.split('.') + if len(parts) >= 2: + if parts[0].strip() != '': + try: + season = int(parts[0]) + 1 # XMLTV format is zero-based + custom_props['season'] = season + except ValueError: + pass + if parts[1].strip() != '': + try: + episode = int(parts[1]) + 1 # XMLTV format is zero-based + custom_props['episode'] = episode + except ValueError: + pass + elif system == 'onscreen' and ep_num.text: + onscreen_text = ep_num.text.strip() + custom_props['onscreen_episode'] = onscreen_text + # Extract season/episode from onscreen format if not already set by xmltv_ns + if 'season' not in custom_props or 'episode' not in custom_props: + match = _ONSCREEN_RE.search(onscreen_text) + if match: + if 'season' not in custom_props: + custom_props['season'] = int(match.group(1)) + if 'episode' not in custom_props: + custom_props['episode'] = int(match.group(2)) + elif system == 'dd_progid' and ep_num.text: + # Store the dd_progid format + custom_props['dd_progid'] = ep_num.text.strip() + # Add support for other systems like thetvdb.com, themoviedb.org, imdb.com + elif system in ['thetvdb.com', 'themoviedb.org', 'imdb.com'] and ep_num.text: + custom_props[f'{system}_id'] = ep_num.text.strip() + + # Extract ratings more efficiently + rating_elem = prog.find('rating') + if rating_elem is not None: + value_elem = rating_elem.find('value') + if value_elem is not None and value_elem.text: + custom_props['rating'] = value_elem.text.strip() + if rating_elem.get('system'): + custom_props['rating_system'] = rating_elem.get('system') + + # Extract star ratings (new) + star_ratings = [] + for star_rating in prog.findall('star-rating'): + value_elem = star_rating.find('value') + if value_elem is not None and value_elem.text: + rating_data = {'value': value_elem.text.strip()} + if star_rating.get('system'): + rating_data['system'] = star_rating.get('system') + star_ratings.append(rating_data) + if star_ratings: + custom_props['star_ratings'] = star_ratings + + # Extract credits more efficiently + credits_elem = prog.find('credits') + if credits_elem is not None: + credits = {} + for credit_type in ['director', 'actor', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if credit_type == 'actor': + # Handle actors with roles and guest status + actors = [] + for actor_elem in credits_elem.findall('actor'): + if actor_elem.text and actor_elem.text.strip(): + actor_data = {'name': actor_elem.text.strip()} + if actor_elem.get('role'): + actor_data['role'] = actor_elem.get('role') + if actor_elem.get('guest') == 'yes': + actor_data['guest'] = True + actors.append(actor_data) + if actors: + credits['actor'] = actors + else: + names = [e.text.strip() for e in credits_elem.findall(credit_type) if e.text and e.text.strip()] + if names: + credits[credit_type] = names + if credits: + custom_props['credits'] = credits + + # Extract other common program metadata + date_elem = prog.find('date') + if date_elem is not None and date_elem.text: + custom_props['date'] = date_elem.text.strip() + + country_elem = prog.find('country') + if country_elem is not None and country_elem.text: + custom_props['country'] = country_elem.text.strip() + + # Extract language information (new) + language_elem = prog.find('language') + if language_elem is not None and language_elem.text: + custom_props['language'] = language_elem.text.strip() + + orig_language_elem = prog.find('orig-language') + if orig_language_elem is not None and orig_language_elem.text: + custom_props['original_language'] = orig_language_elem.text.strip() + + # Extract length (new) + length_elem = prog.find('length') + if length_elem is not None and length_elem.text: + try: + length_value = int(length_elem.text.strip()) + length_units = length_elem.get('units', 'minutes') + custom_props['length'] = {'value': length_value, 'units': length_units} + except ValueError: + pass + + # Extract video information (new) + video_elem = prog.find('video') + if video_elem is not None: + video_info = {} + for video_attr in ['present', 'colour', 'aspect', 'quality']: + attr_elem = video_elem.find(video_attr) + if attr_elem is not None and attr_elem.text: + video_info[video_attr] = attr_elem.text.strip() + if video_info: + custom_props['video'] = video_info + + # Extract audio information (new) + audio_elem = prog.find('audio') + if audio_elem is not None: + audio_info = {} + for audio_attr in ['present', 'stereo']: + attr_elem = audio_elem.find(audio_attr) + if attr_elem is not None and attr_elem.text: + audio_info[audio_attr] = attr_elem.text.strip() + if audio_info: + custom_props['audio'] = audio_info + + # Extract subtitles information (new) + subtitles = [] + for subtitle_elem in prog.findall('subtitles'): + subtitle_data = {} + if subtitle_elem.get('type'): + subtitle_data['type'] = subtitle_elem.get('type') + lang_elem = subtitle_elem.find('language') + if lang_elem is not None and lang_elem.text: + subtitle_data['language'] = lang_elem.text.strip() + if subtitle_data: + subtitles.append(subtitle_data) + + if subtitles: + custom_props['subtitles'] = subtitles + + # Extract reviews (new) + reviews = [] + for review_elem in prog.findall('review'): + if review_elem.text and review_elem.text.strip(): + review_data = {'content': review_elem.text.strip()} + if review_elem.get('type'): + review_data['type'] = review_elem.get('type') + if review_elem.get('source'): + review_data['source'] = review_elem.get('source') + if review_elem.get('reviewer'): + review_data['reviewer'] = review_elem.get('reviewer') + reviews.append(review_data) + if reviews: + custom_props['reviews'] = reviews + + # Extract images (new) + images = [] + for image_elem in prog.findall('image'): + if image_elem.text and image_elem.text.strip(): + image_data = {'url': image_elem.text.strip()} + for attr in ['type', 'size', 'orient', 'system']: + if image_elem.get(attr): + image_data[attr] = image_elem.get(attr) + images.append(image_data) + if images: + custom_props['images'] = images + + icon_elem = prog.find('icon') + if icon_elem is not None and icon_elem.get('src'): + custom_props['icon'] = icon_elem.get('src') + + # Simpler approach for boolean flags - expanded list + for kw in ['previously-shown', 'premiere', 'new', 'live', 'last-chance']: + if prog.find(kw) is not None: + custom_props[kw.replace('-', '_')] = True + + # Extract premiere and last-chance text content if available + premiere_elem = prog.find('premiere') + if premiere_elem is not None: + custom_props['premiere'] = True + if premiere_elem.text and premiere_elem.text.strip(): + custom_props['premiere_text'] = premiere_elem.text.strip() + + last_chance_elem = prog.find('last-chance') + if last_chance_elem is not None: + custom_props['last_chance'] = True + if last_chance_elem.text and last_chance_elem.text.strip(): + custom_props['last_chance_text'] = last_chance_elem.text.strip() + + # Extract previously-shown details + prev_shown_elem = prog.find('previously-shown') + if prev_shown_elem is not None: + custom_props['previously_shown'] = True + prev_shown_data = {} + if prev_shown_elem.get('start'): + prev_shown_data['start'] = prev_shown_elem.get('start') + if prev_shown_elem.get('channel'): + prev_shown_data['channel'] = prev_shown_elem.get('channel') + if prev_shown_data: + custom_props['previously_shown_details'] = prev_shown_data + + return custom_props + + +def clear_element(elem): + """Clear an XML element and its parent to free memory.""" + try: + elem.clear() + parent = elem.getparent() + if parent is not None: + while elem.getprevious() is not None: + del parent[0] + parent.remove(elem) + except Exception as e: + logger.warning(f"Error clearing XML element: {e}", exc_info=True) + + +def detect_file_format(file_path=None, content=None): + """ + Detect file format by examining content or file path. + + Args: + file_path: Path to file (optional) + content: Raw file content bytes (optional) + + Returns: + tuple: (format_type, is_compressed, file_extension) + format_type: 'gzip', 'zip', 'xml', or 'unknown' + is_compressed: Boolean indicating if the file is compressed + file_extension: Appropriate file extension including dot (.gz, .zip, .xml) + """ + # Default return values + format_type = 'unknown' + is_compressed = False + file_extension = '.tmp' + + # First priority: check content magic numbers as they're most reliable + if content: + # We only need the first few bytes for magic number detection + header = content[:20] if len(content) >= 20 else content + + # Check for gzip magic number (1f 8b) + if len(header) >= 2 and header[:2] == b'\x1f\x8b': + return 'gzip', True, '.gz' + + # Check for zip magic number (PK..) + if len(header) >= 2 and header[:2] == b'PK': + return 'zip', True, '.zip' + + # Check for XML - either standard XML header or XMLTV-specific tag + if len(header) >= 5 and (b'' in header): + return 'xml', False, '.xml' + + # Second priority: check file extension - focus on the final extension for compression + if file_path: + logger.debug(f"Detecting file format for: {file_path}") + + # Handle compound extensions like .xml.gz - prioritize compression extensions + lower_path = file_path.lower() + + # Check for compression extensions explicitly + if lower_path.endswith('.gz') or lower_path.endswith('.gzip'): + return 'gzip', True, '.gz' + elif lower_path.endswith('.zip'): + return 'zip', True, '.zip' + elif lower_path.endswith('.xml'): + return 'xml', False, '.xml' + + # Fallback to mimetypes only if direct extension check doesn't work + import mimetypes + mime_type, _ = mimetypes.guess_type(file_path) + logger.debug(f"Guessed MIME type: {mime_type}") + if mime_type: + if mime_type == 'application/gzip' or mime_type == 'application/x-gzip': + return 'gzip', True, '.gz' + elif mime_type == 'application/zip': + return 'zip', True, '.zip' + elif mime_type == 'application/xml' or mime_type == 'text/xml': + return 'xml', False, '.xml' + + # If we reach here, we couldn't reliably determine the format + return format_type, is_compressed, file_extension + + +def generate_dummy_epg(source): + """ + DEPRECATED: This function is no longer used. + + Dummy EPG programs are now generated on-demand when they are requested + (during XMLTV export or EPG grid display), rather than being pre-generated + and stored in the database. + + See: apps/output/views.py - generate_custom_dummy_programs() + + This function remains for backward compatibility but should not be called. + """ + logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " + f"Dummy EPG programs are now generated on-demand.") + return True diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..582d6a1b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +services: + dispatcharr: + build: + context: . + dockerfile: docker/Dockerfile + container_name: dispatcharr_dev + restart: unless-stopped + ports: + - 9191:9191 + - 8001:8001 + volumes: + - dispatcharr_data:/data + environment: + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - POSTGRES_DB=dispatcharr + - POSTGRES_USER=dispatch + - POSTGRES_PASSWORD=secret + - DJANGO_SECRET_KEY=dev-secret-key-not-for-production + - DISPATCHARR_LOG_LEVEL=info + depends_on: + db: + condition: service_healthy + + db: + image: postgres:17 + container_name: dispatcharr_dev_db + restart: unless-stopped + environment: + - POSTGRES_DB=dispatcharr + - POSTGRES_USER=dispatch + - POSTGRES_PASSWORD=secret + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dispatch -d dispatcharr"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + dispatcharr_data: + postgres_data: diff --git a/serializers.py b/serializers.py new file mode 100644 index 00000000..e9ac8581 --- /dev/null +++ b/serializers.py @@ -0,0 +1,866 @@ +import json +from datetime import datetime + +from rest_framework import serializers +from .models import ( + Stream, + Channel, + ChannelGroup, + ChannelOverride, + ChannelStream, + ChannelGroupM3UAccount, + Logo, + ChannelProfile, + ChannelProfileMembership, + Recording, + RecurringRecordingRule, +) +from apps.epg.serializers import EPGDataSerializer +from core.models import StreamProfile +from apps.epg.models import EPGData +from django.db import connection, transaction +from django.urls import reverse +from rest_framework import serializers +from django.utils import timezone +from core.utils import validate_flexible_url + + +class LogoSerializer(serializers.ModelSerializer): + cache_url = serializers.SerializerMethodField() + channel_count = serializers.SerializerMethodField() + is_used = serializers.SerializerMethodField() + channel_names = serializers.SerializerMethodField() + + class Meta: + model = Logo + fields = ["id", "name", "url", "cache_url", "channel_count", "is_used", "channel_names"] + + def validate_url(self, value): + """Validate that the URL is unique for creation or update""" + if self.instance and self.instance.url == value: + return value + + if Logo.objects.filter(url=value).exists(): + raise serializers.ValidationError("A logo with this URL already exists.") + + return value + + def create(self, validated_data): + """Handle logo creation with proper URL validation""" + return Logo.objects.create(**validated_data) + + def update(self, instance, validated_data): + """Handle logo updates""" + for attr, value in validated_data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + def get_cache_url(self, obj): + # Cache-busting: append a short hash of the logo's source URL so the browser + # fetches fresh when the logo changes (e.g., M3U logo replaced by SD logo). + # The backend ignores the 'v' parameter — it's purely for browser cache invalidation. + # See SD integration PR notes for context on why this was added. + import hashlib + url_hash = hashlib.md5((obj.url or '').encode()).hexdigest()[:8] + base_path = reverse("api:channels:logo-cache", args=[obj.id]) + cache_url = f"{base_path}?v={url_hash}" + request = self.context.get("request") + if request: + return request.build_absolute_uri(cache_url) + return cache_url + + def get_channel_count(self, obj): + """Get the number of channels using this logo""" + # `channel_count` is provided as an annotation in LogoViewSet.get_queryset(). + # Fall back to a query only when serializing a single un-annotated Logo + # (e.g. nested inside ChannelSerializer.get_logo()). + annotated = getattr(obj, "channel_count", None) + if annotated is not None: + return annotated + return obj.channels.count() + + def get_is_used(self, obj): + """Check if this logo is used by any channels""" + return self.get_channel_count(obj) > 0 + + def get_channel_names(self, obj): + """Get the names of channels using this logo (limited to first 5)""" + names = [] + + # When LogoViewSet.get_queryset() prefetches `channels`, iterating + # obj.channels.all() reuses the cached set; slicing happens in Python. + channels = list(obj.channels.all()[:5]) + for channel in channels: + names.append(f"Channel: {channel.name}") + + total_count = self.get_channel_count(obj) + if total_count > 5: + names.append(f"...and {total_count - 5} more") + + return names + + +# +# Stream +# +class StreamSerializer(serializers.ModelSerializer): + url = serializers.CharField( + required=False, + allow_blank=True, + allow_null=True, + validators=[validate_flexible_url] + ) + stream_profile_id = serializers.PrimaryKeyRelatedField( + queryset=StreamProfile.objects.all(), + source="stream_profile", + allow_null=True, + required=False, + ) + read_only_fields = ["is_custom", "m3u_account", "stream_hash", "stream_id", "stream_chno"] + + class Meta: + model = Stream + fields = [ + "id", + "name", + "url", + "m3u_account", # Uncomment if using M3U fields + "logo_url", + "tvg_id", + "local_file", + "current_viewers", + "updated_at", + "last_seen", + "is_stale", + "is_adult", + "stream_profile_id", + "is_custom", + "channel_group", + "stream_hash", + "stream_stats", + "stream_stats_updated_at", + "stream_id", + "stream_chno", + ] + + def get_fields(self): + fields = super().get_fields() + + # Unable to edit specific properties if this stream was created from an M3U account + if ( + self.instance + and getattr(self.instance, "m3u_account", None) + and not self.instance.is_custom + ): + fields["id"].read_only = True + fields["name"].read_only = True + fields["url"].read_only = True + fields["m3u_account"].read_only = True + fields["tvg_id"].read_only = True + fields["channel_group"].read_only = True + + return fields + + +class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): + m3u_accounts = serializers.IntegerField(source="m3u_accounts.id", read_only=True) + enabled = serializers.BooleanField() + auto_channel_sync = serializers.BooleanField(default=False) + auto_sync_channel_start = serializers.FloatField( + allow_null=True, required=False, min_value=1 + ) + auto_sync_channel_end = serializers.FloatField( + allow_null=True, required=False, min_value=1 + ) + custom_properties = serializers.JSONField(required=False) + # Provider stream count for this group+account. Lets users size an + # optional end-range without first running a blind sync. + stream_count = serializers.SerializerMethodField() + + class Meta: + model = ChannelGroupM3UAccount + fields = [ + "m3u_accounts", + "channel_group", + "enabled", + "auto_channel_sync", + "auto_sync_channel_start", + "auto_sync_channel_end", + "custom_properties", + "is_stale", + "last_seen", + "stream_count", + ] + + def get_stream_count(self, obj): + """ + Return the number of streams for this (m3u_account, channel_group) + pair. A parent serializer (e.g. M3UAccountSerializer) may seed + ``context["stream_counts"]`` with a pre-aggregated dict keyed by + ``(m3u_account_id, channel_group_id)`` to avoid one COUNT per row; + when present, it is used as the source of truth. The per-row + COUNT fallback is correct for stand-alone serialization (rare, + low-volume) and exists so direct ChannelGroupM3UAccount queries + do not require callers to know the seeding pattern. + """ + counts = self.context.get("stream_counts") + if counts is not None: + return counts.get((obj.m3u_account_id, obj.channel_group_id), 0) + from apps.channels.models import Stream + + return Stream.objects.filter( + m3u_account_id=obj.m3u_account_id, + channel_group_id=obj.channel_group_id, + ).count() + + def to_representation(self, instance): + data = super().to_representation(instance) + + custom_props = instance.custom_properties or {} + + return data + + def to_internal_value(self, data): + # Accept both dict and JSON string for custom_properties (for backward compatibility) + val = data.get("custom_properties") + if isinstance(val, str): + try: + data["custom_properties"] = json.loads(val) + except Exception: + pass + + return super().to_internal_value(data) + + def validate(self, attrs): + # Partial PATCHes only carry submitted fields; fill missing + # start/end from the instance so the validator catches a PATCH + # that lowers end past the existing start. + start = attrs.get("auto_sync_channel_start") + end = attrs.get("auto_sync_channel_end") + if start is None and self.instance is not None: + start = self.instance.auto_sync_channel_start + if end is None and self.instance is not None: + end = self.instance.auto_sync_channel_end + if start is not None and end is not None and end < start: + raise serializers.ValidationError( + { + "auto_sync_channel_end": ( + "End must be greater than or equal to start." + ) + } + ) + return super().validate(attrs) + +# +# Channel Group +# +class ChannelGroupSerializer(serializers.ModelSerializer): + channel_count = serializers.SerializerMethodField() + m3u_account_count = serializers.SerializerMethodField() + m3u_accounts = ChannelGroupM3UAccountSerializer( + many=True, + read_only=True + ) + + class Meta: + model = ChannelGroup + fields = ["id", "name", "channel_count", "m3u_account_count", "m3u_accounts"] + + def get_channel_count(self, obj): + # Use the queryset annotation when available (list path); fall back + # to a live query for retrieve/create/update where it isn't set. + v = getattr(obj, 'channel_count', None) + return v if v is not None else obj.channels.count() + + def get_m3u_account_count(self, obj): + v = getattr(obj, 'm3u_account_count', None) + return v if v is not None else obj.m3u_accounts.count() + + +class ChannelProfileSerializer(serializers.ModelSerializer): + channels = serializers.SerializerMethodField() + + class Meta: + model = ChannelProfile + fields = ["id", "name", "channels"] + + def get_channels(self, obj): + # Use prefetched attr when available, fall back to a direct query. + memberships = getattr(obj, 'enabled_memberships', None) + if memberships is not None: + return [m.channel_id for m in memberships] + return list( + ChannelProfileMembership.objects.filter( + channel_profile=obj, enabled=True + ).values_list('channel_id', flat=True) + ) + + +class ChannelProfileMembershipSerializer(serializers.ModelSerializer): + class Meta: + model = ChannelProfileMembership + fields = ["channel", "enabled"] + + +class ChanneProfilelMembershipUpdateSerializer(serializers.Serializer): + channel_id = serializers.IntegerField() # Ensure channel_id is an integer + enabled = serializers.BooleanField() + + +class BulkChannelProfileMembershipSerializer(serializers.Serializer): + channels = serializers.ListField( + child=ChanneProfilelMembershipUpdateSerializer(), # Use the nested serializer + allow_empty=False, + ) + + def validate_channels(self, value): + if not value: + raise serializers.ValidationError("At least one channel must be provided.") + return value + + +# +# Channel override +# +# Nullable per-field overrides resolved over the parent Channel in read +# paths. Embedded in ChannelSerializer so clients can upsert/clear in the +# same PATCH that targets direct channel fields. +class ChannelOverrideSerializer(serializers.ModelSerializer): + # HDHR clients reject negative GuideNumber and zero is not a real + # provider value, so reject both at the API boundary. + channel_number = serializers.FloatField( + allow_null=True, required=False, min_value=0.0001 + ) + channel_group_id = serializers.PrimaryKeyRelatedField( + queryset=ChannelGroup.objects.all(), + source="channel_group", + allow_null=True, + required=False, + ) + logo_id = serializers.PrimaryKeyRelatedField( + queryset=Logo.objects.all(), + source="logo", + allow_null=True, + required=False, + ) + epg_data_id = serializers.PrimaryKeyRelatedField( + queryset=EPGData.objects.all(), + source="epg_data", + allow_null=True, + required=False, + ) + stream_profile_id = serializers.PrimaryKeyRelatedField( + queryset=StreamProfile.objects.all(), + source="stream_profile", + allow_null=True, + required=False, + ) + + class Meta: + model = ChannelOverride + fields = [ + "name", + "channel_number", + "channel_group_id", + "logo_id", + "tvg_id", + "tvc_guide_stationid", + "epg_data_id", + "stream_profile_id", + ] + extra_kwargs = { + "name": {"allow_null": True, "required": False}, + "tvg_id": {"allow_null": True, "required": False}, + "tvc_guide_stationid": {"allow_null": True, "required": False}, + } + + +# +# Channel +# +class ChannelSerializer(serializers.ModelSerializer): + # Show nested group data, or ID + # Ensure channel_number is explicitly typed as FloatField and properly validated + channel_number = serializers.FloatField( + allow_null=True, + required=False, + error_messages={"invalid": "Channel number must be a valid decimal number."}, + ) + channel_group_id = serializers.PrimaryKeyRelatedField( + queryset=ChannelGroup.objects.all(), source="channel_group", required=False + ) + epg_data_id = serializers.PrimaryKeyRelatedField( + queryset=EPGData.objects.all(), + source="epg_data", + required=False, + allow_null=True, + ) + + stream_profile_id = serializers.PrimaryKeyRelatedField( + queryset=StreamProfile.objects.all(), + source="stream_profile", + allow_null=True, + required=False, + ) + + streams = serializers.PrimaryKeyRelatedField( + queryset=Stream.objects.all(), many=True, required=False + ) + + logo_id = serializers.PrimaryKeyRelatedField( + queryset=Logo.objects.all(), + source="logo", + allow_null=True, + required=False, + ) + + auto_created_by_name = serializers.SerializerMethodField() + override = ChannelOverrideSerializer( + required=False, + allow_null=True, + help_text=( + "Per-field overrides for an auto-created channel. " + 'Send {"override": {"name": "ESPN"}} to upsert the listed ' + 'fields, {"override": {"name": null}} to clear specific fields ' + 'while leaving others, or {"override": null} to delete the ' + "override row entirely. Omitting the key leaves any existing " + "override unchanged. Only valid for auto_created=True channels. " + "Duplicate channel_number values across channels are permitted; " + "downstream client behavior on duplicates varies by client." + ), + ) + source_stream = serializers.SerializerMethodField() + # Effective fields coalesce override over channel column. Consumers + # display these; raw fields remain in the response so the edit form + # can show them as "Provider: X" subtext. + effective_name = serializers.SerializerMethodField() + effective_channel_number = serializers.SerializerMethodField() + effective_channel_group_id = serializers.SerializerMethodField() + effective_logo_id = serializers.SerializerMethodField() + effective_tvg_id = serializers.SerializerMethodField() + effective_tvc_guide_stationid = serializers.SerializerMethodField() + effective_epg_data_id = serializers.SerializerMethodField() + effective_stream_profile_id = serializers.SerializerMethodField() + + class Meta: + model = Channel + fields = [ + "id", + "channel_number", + "name", + "channel_group_id", + "tvg_id", + "tvc_guide_stationid", + "epg_data_id", + "streams", + "stream_profile_id", + "uuid", + "logo_id", + "user_level", + "is_adult", + "hidden_from_output", + "auto_created", + "auto_created_by", + "auto_created_by_name", + "override", + "source_stream", + "effective_name", + "effective_channel_number", + "effective_channel_group_id", + "effective_logo_id", + "effective_tvg_id", + "effective_tvc_guide_stationid", + "effective_epg_data_id", + "effective_stream_profile_id", + ] + + def _effective_value(self, obj, field_name): + override = getattr(obj, "_channel_override_cache", None) + if override is None: + try: + override = obj.override + except ChannelOverride.DoesNotExist: + override = None + obj._channel_override_cache = override + if override is not None: + value = getattr(override, field_name, None) + if value is not None: + return value + return getattr(obj, field_name, None) + + def get_effective_name(self, obj): + return self._effective_value(obj, "name") + + def get_effective_channel_number(self, obj): + return self._effective_value(obj, "channel_number") + + def get_effective_channel_group_id(self, obj): + return self._effective_value(obj, "channel_group_id") + + def get_effective_logo_id(self, obj): + return self._effective_value(obj, "logo_id") + + def get_effective_tvg_id(self, obj): + return self._effective_value(obj, "tvg_id") + + def get_effective_tvc_guide_stationid(self, obj): + return self._effective_value(obj, "tvc_guide_stationid") + + def get_effective_epg_data_id(self, obj): + return self._effective_value(obj, "epg_data_id") + + def get_effective_stream_profile_id(self, obj): + return self._effective_value(obj, "stream_profile_id") + + def get_source_stream(self, obj): + """ + Return the originating provider stream for an auto-created channel. + + Surfaces the provider stream's name and owning M3U account so the + frontend can render "Auto-created from: / " + in the channel edit form. Returns None for manual channels. + """ + if not self.context.get("include_source_stream", False): + return None + if not obj.auto_created: + return None + # Viewset prefetches `channelstream_set` ordered by `order`, so + # `.all()[0]` reuses the cache and returns the lowest-order entry. + prefetched_list = list(obj.channelstream_set.all()) + if not prefetched_list: + return None + cs = prefetched_list[0] + if not cs.stream: + return None + stream = cs.stream + return { + "id": stream.id, + "name": stream.name, + "account_id": stream.m3u_account_id, + "account_name": getattr(stream.m3u_account, "name", None), + } + + def to_representation(self, instance): + include_streams = self.context.get("include_streams", False) + + if include_streams: + self.fields["streams"] = serializers.SerializerMethodField() + return super().to_representation(instance) + else: + # Read from the prefetched channelstream_set (ordered by the + # viewset's Prefetch); chaining .order_by() rebuilds the + # queryset and fires one SELECT per row in list responses. + representation = super().to_representation(instance) + if "streams" in representation: + representation["streams"] = [ + cs.stream_id for cs in instance.channelstream_set.all() + ] + return representation + + def get_logo(self, obj): + return LogoSerializer(obj.logo).data + + def get_streams(self, obj): + """Retrieve ordered stream IDs for GET requests.""" + return StreamSerializer( + obj.streams.all().order_by("channelstream__order"), many=True + ).data + + def create(self, validated_data): + streams = validated_data.pop("streams", []) + override_data = validated_data.pop("override", None) + channel_number = validated_data.pop( + "channel_number", Channel.get_next_available_channel_number() + ) + validated_data["channel_number"] = channel_number + + # Auto-assign Default Group if no channel_group is specified + if "channel_group" not in validated_data or validated_data.get("channel_group") is None: + from apps.channels.models import ChannelGroup + default_group, _ = ChannelGroup.objects.get_or_create(name="Default Group") + validated_data["channel_group"] = default_group + + # Atomic wrapper keeps the channel insert and its override row + # in the same transaction so a failure on either rolls both back. + with transaction.atomic(): + channel = Channel.objects.create(**validated_data) + + # Add streams in the specified order + for index, stream in enumerate(streams): + ChannelStream.objects.create( + channel=channel, stream_id=stream.id, order=index + ) + + if override_data: + # Manual channels (auto_created=False) have no provider + # value to override; reject the override payload here so a + # programmatic client can't write a semantically meaningless + # row that the frontend would then surface as "Overrides + # active". + if not channel.auto_created: + raise serializers.ValidationError( + { + "override": ( + "Cannot set override on a manual channel; " + "overrides only apply to auto-created channels." + ) + } + ) + obj = ChannelOverride.objects.create(channel=channel, **override_data) + # Drop an all-null override row; an empty override would + # falsely surface as active in the UI. + if not obj.has_any_override(): + obj.delete() + + return channel + + def update(self, instance, validated_data): + """ + PATCH handler for Channel rows. The ``override`` key carries + per-field user overrides for auto-created channels and follows + these rules: + + * key absent from payload: no change to existing overrides + * ``{"override": {"field": value}}``: upsert those fields + * ``{"override": {"field": null}}``: clear those specific fields + * ``{"override": null}``: delete the override row entirely + + Key presence is what distinguishes "no change" from "delete"; + an explicit null means delete. Override mutations are rejected + on manual channels (auto_created=False) since there is no + provider value to override. + """ + streams = validated_data.pop("streams", None) + has_override_key = "override" in self.initial_data + override_data = validated_data.pop("override", None) + + # Block override mutations on manual channels (no provider + # value to override). Clearing is a tolerated no-op. + if ( + has_override_key + and override_data is not None + and override_data != {} + and not instance.auto_created + ): + raise serializers.ValidationError( + { + "override": ( + "Cannot set override on a manual channel; " + "overrides only apply to auto-created channels." + ) + } + ) + + # Atomic so a failure on the override row rolls back the + # channel update too. + with transaction.atomic(): + # Skip save() when only override keys were submitted; a + # no-op UPDATE would bump updated_at and bust caches. + if validated_data: + for attr, value in validated_data.items(): + setattr(instance, attr, value) + instance.save() + + if has_override_key: + if override_data is None: + # Explicit null: remove the override row. + ChannelOverride.objects.filter(channel=instance).delete() + elif override_data == {}: + # Empty dict has no field intent; no-op. + pass + else: + obj, _ = ChannelOverride.objects.update_or_create( + channel=instance, defaults=override_data + ) + # Drop an all-null override; would falsely surface + # as active in the UI. + if not obj.has_any_override(): + obj.delete() + # Queryset writes leave the reverse-OneToOne cache stale; + # clear it so to_representation reads the new state. + try: + instance._state.fields_cache.pop("override", None) + except AttributeError: + pass + if hasattr(instance, "_channel_override_cache"): + delattr(instance, "_channel_override_cache") + + if streams is not None: + # Normalize stream IDs + normalized_ids = [ + stream.id if hasattr(stream, "id") else stream for stream in streams + ] + + # Get current mapping of stream_id -> ChannelStream + current_links = { + cs.stream_id: cs for cs in instance.channelstream_set.all() + } + + # Track existing stream IDs + existing_ids = set(current_links.keys()) + new_ids = set(normalized_ids) + + # Delete any links not in the new list + to_remove = existing_ids - new_ids + if to_remove: + instance.channelstream_set.filter(stream_id__in=to_remove).delete() + + # Update or create with new order + to_update = [] + for order, stream_id in enumerate(normalized_ids): + if stream_id in current_links: + cs = current_links[stream_id] + if cs.order != order: + cs.order = order + to_update.append(cs) + else: + ChannelStream.objects.create( + channel=instance, stream_id=stream_id, order=order + ) + + if to_update: + ChannelStream.objects.bulk_update(to_update, ["order"]) + + return instance + + def validate_channel_number(self, value): + """Ensure channel_number is properly processed as a float""" + if value is None: + return value + + try: + # Ensure it's processed as a float + return float(value) + except (ValueError, TypeError): + raise serializers.ValidationError( + "Channel number must be a valid decimal number." + ) + + def validate_stream_profile(self, value): + """Handle special case where empty/0 values mean 'use default' (null)""" + if value == "0" or value == 0 or value == "" or value is None: + return None + return value # PrimaryKeyRelatedField will handle the conversion to object + + def get_auto_created_by_name(self, obj): + """Get the name of the M3U account that auto-created this channel.""" + if obj.auto_created_by: + return obj.auto_created_by.name + return None + + +class RecordingSerializer(serializers.ModelSerializer): + class Meta: + model = Recording + fields = "__all__" + read_only_fields = ["task_id"] + + def validate(self, data): + from core.models import CoreSettings + start_time = data.get("start_time") + end_time = data.get("end_time") + + if start_time and timezone.is_naive(start_time): + start_time = timezone.make_aware(start_time, timezone.get_current_timezone()) + data["start_time"] = start_time + if end_time and timezone.is_naive(end_time): + end_time = timezone.make_aware(end_time, timezone.get_current_timezone()) + data["end_time"] = end_time + + # If this is an EPG-based recording (program provided), apply global pre/post offsets + try: + cp = data.get("custom_properties") or {} + is_epg_based = isinstance(cp, dict) and isinstance(cp.get("program"), (dict,)) + except Exception: + is_epg_based = False + + if is_epg_based and start_time and end_time: + try: + pre_min = int(CoreSettings.get_dvr_pre_offset_minutes()) + except Exception: + pre_min = 0 + try: + post_min = int(CoreSettings.get_dvr_post_offset_minutes()) + except Exception: + post_min = 0 + from datetime import timedelta + try: + if pre_min and pre_min > 0: + start_time = start_time - timedelta(minutes=pre_min) + except Exception: + pass + try: + if post_min and post_min > 0: + end_time = end_time + timedelta(minutes=post_min) + except Exception: + pass + # write back adjusted times so scheduling uses them + data["start_time"] = start_time + data["end_time"] = end_time + + now = timezone.now() # timezone-aware current time + + if end_time < now: + raise serializers.ValidationError("End time must be in the future.") + + if start_time < now: + # Optional: Adjust start_time if it's in the past but end_time is in the future + data["start_time"] = now # or: timezone.now() + timedelta(seconds=1) + if end_time <= data["start_time"]: + raise serializers.ValidationError("End time must be after start time.") + + return data + + +class RecurringRecordingRuleSerializer(serializers.ModelSerializer): + class Meta: + model = RecurringRecordingRule + fields = "__all__" + read_only_fields = ["created_at", "updated_at"] + + def validate_days_of_week(self, value): + if not value: + raise serializers.ValidationError("Select at least one day of the week") + cleaned = [] + for entry in value: + try: + iv = int(entry) + except (TypeError, ValueError): + raise serializers.ValidationError("Days of week must be integers 0-6") + if iv < 0 or iv > 6: + raise serializers.ValidationError("Days of week must be between 0 (Monday) and 6 (Sunday)") + cleaned.append(iv) + return sorted(set(cleaned)) + + def validate(self, attrs): + start = attrs.get("start_time") or getattr(self.instance, "start_time", None) + end = attrs.get("end_time") or getattr(self.instance, "end_time", None) + start_date = attrs.get("start_date") if "start_date" in attrs else getattr(self.instance, "start_date", None) + end_date = attrs.get("end_date") if "end_date" in attrs else getattr(self.instance, "end_date", None) + if start_date is None: + existing_start = getattr(self.instance, "start_date", None) + if existing_start is None: + raise serializers.ValidationError("Start date is required") + if start_date and end_date and end_date < start_date: + raise serializers.ValidationError("End date must be on or after start date") + if end_date is None: + existing_end = getattr(self.instance, "end_date", None) + if existing_end is None: + raise serializers.ValidationError("End date is required") + if start and end and start_date and end_date: + start_dt = datetime.combine(start_date, start) + end_dt = datetime.combine(end_date, end) + if end_dt <= start_dt: + raise serializers.ValidationError("End datetime must be after start datetime") + elif start and end and end == start: + raise serializers.ValidationError("End time must be different from start time") + # Normalize empty strings to None for dates + if attrs.get("end_date") == "": + attrs["end_date"] = None + if attrs.get("start_date") == "": + attrs["start_date"] = None + return super().validate(attrs) + + def create(self, validated_data): + return super().create(validated_data) diff --git a/tasks.py b/tasks.py new file mode 100644 index 00000000..fc43c36d --- /dev/null +++ b/tasks.py @@ -0,0 +1,3594 @@ +# apps/epg/tasks.py + +import logging +import gzip +import html.entities +import os +import uuid +import requests +import time # Add import for tracking download progress +from datetime import datetime, timedelta, timezone as dt_timezone +import gc # Add garbage collection module +import json +from lxml import etree # Using lxml exclusively +import psutil # Add import for memory tracking +import zipfile + +from celery import shared_task +from django.conf import settings +from django.db import connection, transaction +from django.db.models import Q +from django.utils import timezone +from apps.channels.models import Channel +from core.models import UserAgent, CoreSettings + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer + +from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 +from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event + +logger = logging.getLogger(__name__) + +SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' +SD_DAYS_TO_FETCH = 20 +SD_PROGRAM_BATCH_SIZE = 5000 + +# DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named +# entities so lxml/libxml2 can resolve references like é correctly +# instead of silently dropping them in recovery mode. +# The 5 XML-predefined entities (amp, lt, gt, quot, apos) are always +# recognised by the XML spec and must not be redeclared. +_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'}) + + +def _build_html_entity_doctype() -> bytes: + """Build a DOCTYPE internal subset declaring all HTML 4 named entities.""" + lines = [b'\n'.encode('ascii')) + lines.append(b']>\n') + return b''.join(lines) + + +_HTML_ENTITY_DOCTYPE = _build_html_entity_doctype() + + +class _PrependStream: + """Wraps an open binary file and prepends a bytes prefix to its content. + + Used by _open_xmltv_file to inject a DOCTYPE entity block before the + file content reaches lxml's iterparse, with zero disk I/O. + """ + + __slots__ = ('_prefix', '_prefix_pos', '_file') + + def __init__(self, prefix: bytes, file_obj): + self._prefix = prefix + self._prefix_pos = 0 + self._file = file_obj + + def read(self, size=-1): + prefix_len = len(self._prefix) + if self._prefix_pos >= prefix_len: + return self._file.read(size) + remaining = prefix_len - self._prefix_pos + if size < 0: + chunk = self._prefix[self._prefix_pos:] + self._file.read() + self._prefix_pos = prefix_len + return chunk + if size <= remaining: + chunk = self._prefix[self._prefix_pos:self._prefix_pos + size] + self._prefix_pos += size + return chunk + chunk = self._prefix[self._prefix_pos:] + self._prefix_pos = prefix_len + return chunk + self._file.read(size - remaining) + + def close(self): + self._file.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +def _open_xmltv_file(file_path: str): + """Open an XMLTV file for lxml iterparse, injecting an HTML entity DOCTYPE. + + Prepends a block that declares all 252 HTML 4 named + entities so lxml/libxml2 resolves references like é correctly + instead of silently dropping them in recovery mode. This involves zero + disk I/O (the DOCTYPE is streamed in-memory before the file content). + + If the file already contains a declaration the file is returned + unchanged; a second DOCTYPE would be invalid XML. + + The caller is responsible for closing the returned object. + """ + f = open(file_path, 'rb') + start = f.read(512) + + # Do not inject if the file already declares a DOCTYPE. + if b'= 0: + decl_end = start.find(b'?>', xml_pos) + if decl_end >= 0: + xml_decl = start[:decl_end + 2] + f.seek(decl_end + 2) + return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f) + + # No XML declaration found; insert DOCTYPE at the very start of the file. + f.seek(0) + return _PrependStream(_HTML_ENTITY_DOCTYPE, f) + + +def validate_icon_url_fast(icon_url, max_length=None): + """ + Fast validation for icon URLs during parsing. + Returns None if URL is too long, original URL otherwise. + If max_length is None, gets it dynamically from the EPGData model field. + """ + if max_length is None: + # Get max_length dynamically from the model field + max_length = EPGData._meta.get_field('icon_url').max_length + + if icon_url and len(icon_url) > max_length: + logger.warning(f"Icon URL too long ({len(icon_url)} > {max_length}), skipping: {icon_url[:100]}...") + return None + return icon_url + + +MAX_EXTRACT_CHUNK_SIZE = 65536 # 64kb (base2) + + +def send_epg_update(source_id, action, progress, **kwargs): + """Send WebSocket update about EPG download/parsing progress""" + # Start with the base data dictionary + data = { + "progress": progress, + "type": "epg_refresh", + "source": source_id, + "action": action, + } + + # Add the additional key-value pairs from kwargs + data.update(kwargs) + + # Use the standardized update function with garbage collection for program parsing + # This is a high-frequency operation that needs more aggressive memory management + collect_garbage = action == "parsing_programs" and progress % 10 == 0 + send_websocket_update('updates', 'update', data, collect_garbage=collect_garbage) + + # Explicitly clear references + data = None + + # For high-frequency parsing, occasionally force additional garbage collection + # to prevent memory buildup + if action == "parsing_programs" and progress % 50 == 0: + gc.collect() + + +def _sd_send_ws_sync(source_id, action, progress, **kwargs): + """ + Sends a WebSocket progress update synchronously via Redis, bypassing gevent.spawn. + + In Celery prefork workers that inherit gevent monkey-patching from uWSGI, + gevent.spawn schedules coroutines that never execute because there is no + running gevent hub in the worker process. This function writes directly to + Redis using the channels_redis 4.x wire format, guaranteeing delivery + regardless of the execution context. + """ + try: + import msgpack + import redis as redis_lib + from django.conf import settings + + data = {"progress": progress, "type": "epg_refresh", "source": source_id, "action": action} + data.update(kwargs) + message = {"type": "update", "data": data} + + redis_url = getattr(settings, "CELERY_BROKER_URL", "redis://localhost:6379/0") + r = redis_lib.from_url(redis_url, decode_responses=False) + + prefix = "asgi" + group_name = "updates" + group_key = f"{prefix}:group:{group_name}" + group_expiry = 86400 + channel_expiry = 60 + rand_len = 12 + now = time.time() + + r.zremrangebyscore(group_key, 0, now - group_expiry) + raw = r.zrange(group_key, 0, -1) + if not raw: + return + + channels = [m.decode("utf-8") if isinstance(m, bytes) else m for m in raw] + nonlocal_map = {} + for ch in channels: + pos = ch.find("!") + nl = ch[:pos + 1] if pos >= 0 else ch + nonlocal_map.setdefault(nl, []).append(ch) + + pipe = r.pipeline(transaction=False) + for nl, chs in nonlocal_map.items(): + channel_key = prefix + nl + msg = dict(message) + msg["__asgi_channel__"] = chs + serialized = os.urandom(rand_len) + msgpack.packb(msg) + pipe.zadd(channel_key, {serialized: now}) + pipe.expire(channel_key, channel_expiry) + pipe.execute() + r.close() + except Exception as e: + logger.warning(f"SD WebSocket sync send failed: {e}") + # Fall back to standard path + send_epg_update(source_id, action, progress, **kwargs) + + +def delete_epg_refresh_task_by_id(epg_id): + """ + Delete the periodic task associated with an EPG source ID. + Can be called directly or from the post_delete signal. + Returns True if a task was found and deleted, False otherwise. + """ + try: + task = None + task_name = f"epg_source-refresh-{epg_id}" + + # Look for task by name + try: + from django_celery_beat.models import PeriodicTask, IntervalSchedule + task = PeriodicTask.objects.get(name=task_name) + logger.info(f"Found task by name: {task.id} for EPGSource {epg_id}") + except PeriodicTask.DoesNotExist: + logger.warning(f"No PeriodicTask found with name {task_name}") + return False + + # Now delete the task and its interval + if task: + # Store interval info before deleting the task + interval_id = None + if hasattr(task, 'interval') and task.interval: + interval_id = task.interval.id + + # Count how many TOTAL tasks use this interval (including this one) + tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() + logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") + + # Delete the task first + task_id = task.id + task.delete() + logger.info(f"Successfully deleted periodic task {task_id}") + + # Now check if we should delete the interval + # We only delete if it was the ONLY task using this interval + if interval_id and tasks_with_same_interval == 1: + try: + interval = IntervalSchedule.objects.get(id=interval_id) + logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") + interval.delete() + logger.info(f"Successfully deleted interval {interval_id}") + except IntervalSchedule.DoesNotExist: + logger.warning(f"Interval {interval_id} no longer exists") + elif interval_id: + logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") + + return True + return False + except Exception as e: + logger.error(f"Error deleting periodic task for EPGSource {epg_id}: {str(e)}", exc_info=True) + return False + + +@shared_task +def refresh_all_epg_data(): + logger.info("Starting refresh_epg_data task.") + # Exclude dummy EPG sources from refresh - they don't need refreshing + active_sources = EPGSource.objects.filter(is_active=True).exclude(source_type='dummy') + logger.debug(f"Found {active_sources.count()} active EPGSource(s) (excluding dummy EPGs).") + + for source in active_sources: + refresh_epg_data(source.id) + # Force garbage collection between sources + gc.collect() + + logger.info("Finished refresh_epg_data task.") + return "EPG data refreshed." + + +@shared_task(time_limit=14400) +def refresh_epg_data(source_id, force=False): + if not acquire_task_lock('refresh_epg_data', source_id): + logger.debug(f"EPG refresh for {source_id} already running") + return + + lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) + lock_renewer.start() + + source = None + try: + # Try to get the EPG source + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + # The EPG source doesn't exist, so delete the periodic task if it exists + logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") + + # Call the shared function to delete the task + if delete_epg_refresh_task_by_id(source_id): + logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") + else: + logger.info(f"No orphaned task found for EPG source {source_id}") + + # Release the lock and exit + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return f"EPG source {source_id} does not exist, task cleaned up" + + # The source exists but is not active, just skip processing + if not source.is_active: + logger.info(f"EPG source {source_id} is not active. Skipping.") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return + + # Skip refresh for dummy EPG sources - they don't need refreshing + if source.source_type == 'dummy': + logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + gc.collect() + return + + # Continue with the normal processing... + logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") + if source.source_type == 'xmltv': + fetch_success = fetch_xmltv(source) + if not fetch_success: + logger.error(f"Failed to fetch XMLTV for source {source.name}") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return + + parse_channels_success = parse_channels_only(source) + if not parse_channels_success: + logger.error(f"Failed to parse channels for source {source.name}") + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + # Force garbage collection before exit + gc.collect() + return + + parse_programs_for_source(source) + + elif source.source_type == 'schedules_direct': + fetch_schedules_direct(source, force=force) + + source.save(update_fields=['updated_at']) + # After successful EPG refresh, evaluate DVR series rules to schedule new episodes + try: + from apps.channels.tasks import evaluate_series_rules + evaluate_series_rules.delay() + except Exception: + pass + except Exception as e: + logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) + try: + if source: + source.status = 'error' + source.last_message = f"Error refreshing EPG data: {str(e)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) + except Exception as inner_e: + logger.error(f"Error updating source status: {inner_e}") + finally: + # Clear references to ensure proper garbage collection + source = None + # Force garbage collection before releasing the lock + gc.collect() + lock_renewer.stop() + release_task_lock('refresh_epg_data', source_id) + + +def fetch_xmltv(source): + # Handle cases with local file but no URL + if not source.url and source.file_path and os.path.exists(source.file_path): + logger.info(f"Using existing local file for EPG source: {source.name} at {source.file_path}") + + # Check if the existing file is compressed and we need to extract it + if source.file_path.endswith(('.gz', '.zip')) and not source.file_path.endswith('.xml'): + try: + # Define the path for the extracted file in the cache directory + cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") + os.makedirs(cache_dir, exist_ok=True) + xml_path = os.path.join(cache_dir, f"{source.id}.xml") + + # Extract to the cache location keeping the original + extracted_path = extract_compressed_file(source.file_path, xml_path, delete_original=False) + + if extracted_path: + logger.info(f"Extracted mapped compressed file to: {extracted_path}") + # Update to use extracted_file_path instead of changing file_path + source.extracted_file_path = extracted_path + source.save(update_fields=['extracted_file_path']) + else: + logger.error(f"Failed to extract mapped compressed file. Using original file: {source.file_path}") + except Exception as e: + logger.error(f"Failed to extract existing compressed file: {e}") + # Continue with the original file if extraction fails + + # Set the status to success in the database + source.status = 'success' + source.save(update_fields=['status']) + + # Send a download complete notification + send_epg_update(source.id, "downloading", 100, status="success") + + # Return True to indicate successful fetch, processing will continue with parse_channels_only + return True + + # Handle cases where no URL is provided and no valid file path exists + if not source.url: + # Update source status for missing URL + source.status = 'error' + source.last_message = "No URL provided and no valid local file exists" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "downloading", 100, status="error", error="No URL provided and no valid local file exists") + return False + + logger.info(f"Fetching XMLTV data from source: {source.name}") + try: + # Get default user agent from settings + stream_settings = CoreSettings.get_stream_settings() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + default_user_agent_id = stream_settings.get('default_user_agent') + if default_user_agent_id: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + + headers = { + 'User-Agent': user_agent + } + + # Update status to fetching before starting download + source.status = 'fetching' + source.save(update_fields=['status']) + + # Send initial download notification + send_epg_update(source.id, "downloading", 0) + + # Use streaming response to track download progress + with requests.get(source.url, headers=headers, stream=True, timeout=60) as response: + # Handle 404 specifically + if response.status_code == 404: + logger.error(f"EPG URL not found (404): {source.url}") + # Update status to error in the database + source.status = 'error' + source.last_message = f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" + source.save(update_fields=['status', 'last_message']) + + # Notify users through the WebSocket about the EPG fetch failure + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": 404, + "message": f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" + } + } + ) + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error="URL not found (404)") + return False + + # For all other error status codes + if response.status_code >= 400: + error_message = f"HTTP error {response.status_code}" + user_message = f"EPG source '{source.name}' encountered HTTP error {response.status_code}" + + # Update status to error in the database + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Notify users through the WebSocket + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": response.status_code, + "message": user_message + } + } + ) + # Update download progress + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + + response.raise_for_status() + logger.debug("XMLTV data fetched successfully.") + + # Define base paths for consistent file naming + cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") + os.makedirs(cache_dir, exist_ok=True) + + # Create temporary download file with .tmp extension + temp_download_path = os.path.join(cache_dir, f"{source.id}.tmp") + + # Check if we have content length for progress tracking + total_size = int(response.headers.get('content-length', 0)) + downloaded = 0 + start_time = time.time() + last_update_time = start_time + update_interval = 0.5 # Only update every 0.5 seconds + + # Download to temporary file + with open(temp_download_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=16384): + f.write(chunk) + + downloaded += len(chunk) + elapsed_time = time.time() - start_time + + # Calculate download speed in KB/s + speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0 + + # Calculate progress percentage + if total_size and total_size > 0: + progress = min(100, int((downloaded / total_size) * 100)) + else: + # If no content length header, estimate progress + progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown + + # Time remaining (in seconds) + time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0 + + # Only send updates at specified intervals to avoid flooding + current_time = time.time() + if current_time - last_update_time >= update_interval and progress > 0: + last_update_time = current_time + send_epg_update( + source.id, + "downloading", + progress, + speed=round(speed, 2), + elapsed_time=round(elapsed_time, 1), + time_remaining=round(time_remaining, 1), + downloaded=f"{downloaded / (1024 * 1024):.2f} MB" + ) + + # Explicitly delete the chunk to free memory immediately + del chunk + + # Send completion notification + send_epg_update(source.id, "downloading", 100) + + # Determine the appropriate file extension based on content detection + with open(temp_download_path, 'rb') as f: + content_sample = f.read(1024) # Just need the first 1KB to detect format + + # Use our helper function to detect the format + format_type, is_compressed, file_extension = detect_file_format( + file_path=source.url, # Original URL as a hint + content=content_sample # Actual file content for detection + ) + + logger.debug(f"File format detection results: type={format_type}, compressed={is_compressed}, extension={file_extension}") + + # Ensure consistent final paths + compressed_path = os.path.join(cache_dir, f"{source.id}{file_extension}" if is_compressed else f"{source.id}.compressed") + xml_path = os.path.join(cache_dir, f"{source.id}.xml") + + # Clean up old files before saving new ones + if os.path.exists(compressed_path): + try: + os.remove(compressed_path) + logger.debug(f"Removed old compressed file: {compressed_path}") + except OSError as e: + logger.warning(f"Failed to remove old compressed file: {e}") + + if os.path.exists(xml_path): + try: + os.remove(xml_path) + logger.debug(f"Removed old XML file: {xml_path}") + except OSError as e: + logger.warning(f"Failed to remove old XML file: {e}") + + # Rename the temp file to appropriate final path + if is_compressed: + try: + os.rename(temp_download_path, compressed_path) + logger.debug(f"Renamed temp file to compressed file: {compressed_path}") + current_file_path = compressed_path + except OSError as e: + logger.error(f"Failed to rename temp file to compressed file: {e}") + current_file_path = temp_download_path # Fall back to using temp file + else: + try: + os.rename(temp_download_path, xml_path) + logger.debug(f"Renamed temp file to XML file: {xml_path}") + current_file_path = xml_path + except OSError as e: + logger.error(f"Failed to rename temp file to XML file: {e}") + current_file_path = temp_download_path # Fall back to using temp file + + # Now extract the file if it's compressed + if is_compressed: + try: + logger.info(f"Extracting compressed file {current_file_path}") + send_epg_update(source.id, "extracting", 0, message="Extracting downloaded file") + + # Always extract to the standard XML path - set delete_original to True to clean up + extracted = extract_compressed_file(current_file_path, xml_path, delete_original=True) + + if extracted: + logger.info(f"Successfully extracted to {xml_path}, compressed file deleted") + send_epg_update(source.id, "extracting", 100, message=f"File extracted successfully, temporary file removed") + # Update to store only the extracted file path since the compressed file is now gone + source.file_path = xml_path + source.extracted_file_path = None + else: + logger.error("Extraction failed, using compressed file") + send_epg_update(source.id, "extracting", 100, status="error", message="Extraction failed, using compressed file") + # Use the compressed file + source.file_path = current_file_path + source.extracted_file_path = None + except Exception as e: + logger.error(f"Error extracting file: {str(e)}", exc_info=True) + send_epg_update(source.id, "extracting", 100, status="error", message=f"Error during extraction: {str(e)}") + # Use the compressed file if extraction fails + source.file_path = current_file_path + source.extracted_file_path = None + else: + # It's already an XML file + source.file_path = current_file_path + source.extracted_file_path = None + + # Update the source's file paths + source.save(update_fields=['file_path', 'status', 'extracted_file_path']) + + # Update status to parsing + source.status = 'parsing' + source.save(update_fields=['status']) + + logger.info(f"Cached EPG file saved to {source.file_path}") + + return True + + except requests.exceptions.HTTPError as e: + logger.error(f"HTTP Error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Get error details + status_code = e.response.status_code if hasattr(e, 'response') and e.response else 'unknown' + error_message = str(e) + + # Create a user-friendly message + user_message = f"EPG source '{source.name}' encountered HTTP error {status_code}" + + # Add specific handling for common HTTP errors + if status_code == 404: + user_message = f"EPG source '{source.name}' URL not found (404) - will retry on next scheduled run" + elif status_code == 401 or status_code == 403: + user_message = f"EPG source '{source.name}' access denied (HTTP {status_code}) - check credentials" + elif status_code == 429: + user_message = f"EPG source '{source.name}' rate limited (429) - try again later" + elif status_code >= 500: + user_message = f"EPG source '{source.name}' server error (HTTP {status_code}) - will retry later" + + # Update source status to error with the error message + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Notify users through the WebSocket about the EPG fetch failure + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": status_code, + "message": user_message, + "details": error_message + } + } + ) + + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except requests.exceptions.ConnectionError as e: + # Handle connection errors separately + error_message = str(e) + user_message = f"Connection error: Unable to connect to EPG source '{source.name}'" + logger.error(f"Connection error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Update source status + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Send notifications + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + 'data': { + "success": False, + "type": "epg_fetch_error", + "source_id": source.id, + "source_name": source.name, + "error_code": "connection_error", + "message": user_message + } + } + ) + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except requests.exceptions.Timeout as e: + # Handle timeout errors specifically + error_message = str(e) + user_message = f"Timeout error: EPG source '{source.name}' took too long to respond" + logger.error(f"Timeout error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Update source status + source.status = 'error' + source.last_message = user_message + source.save(update_fields=['status', 'last_message']) + + # Send notifications + send_epg_update(source.id, "downloading", 100, status="error", error=user_message) + return False + except Exception as e: + error_message = str(e) + logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True) + + # Update source status for general exceptions too + source.status = 'error' + source.last_message = f"Error: {error_message}" + source.save(update_fields=['status', 'last_message']) + + # Ensure we update the download progress to 100 with error status + send_epg_update(source.id, "downloading", 100, status="error", error=f"Error: {error_message}") + return False + + +def extract_compressed_file(file_path, output_path=None, delete_original=False): + """ + Extracts a compressed file (.gz or .zip) to an XML file. + + Args: + file_path: Path to the compressed file + output_path: Specific path where the file should be extracted (optional) + delete_original: Whether to delete the original compressed file after successful extraction + + Returns: + Path to the extracted XML file, or None if extraction failed + """ + try: + if output_path is None: + base_path = os.path.splitext(file_path)[0] + extracted_path = f"{base_path}.xml" + else: + extracted_path = output_path + + # Make sure the output path doesn't already exist + if os.path.exists(extracted_path): + try: + os.remove(extracted_path) + logger.info(f"Removed existing extracted file: {extracted_path}") + except Exception as e: + logger.warning(f"Failed to remove existing extracted file: {e}") + # If we can't delete the existing file and no specific output was requested, + # create a unique filename instead + if output_path is None: + base_path = os.path.splitext(file_path)[0] + extracted_path = f"{base_path}_{uuid.uuid4().hex[:8]}.xml" + + # Use our detection helper to determine the file format instead of relying on extension + with open(file_path, 'rb') as f: + content_sample = f.read(4096) # Read a larger sample to ensure accurate detection + + format_type, is_compressed, _ = detect_file_format(file_path=file_path, content=content_sample) + + if format_type == 'gzip': + logger.debug(f"Extracting gzip file: {file_path}") + try: + # First check if the content is XML by reading a sample + with gzip.open(file_path, 'rb') as gz_file: + content_sample = gz_file.read(4096) # Read first 4KB for detection + detected_format, _, _ = detect_file_format(content=content_sample) + + if detected_format != 'xml': + logger.warning(f"GZIP file does not appear to contain XML content: {file_path} (detected as: {detected_format})") + # Continue anyway since GZIP only contains one file + + # Reset file pointer and extract the content + gz_file.seek(0) + with open(extracted_path, 'wb') as out_file: + while True: + chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk or len(chunk) == 0: + break + out_file.write(chunk) + except Exception as e: + logger.error(f"Error extracting GZIP file: {e}", exc_info=True) + return None + + logger.info(f"Successfully extracted gzip file to: {extracted_path}") + + # Delete original compressed file if requested + if delete_original: + try: + os.remove(file_path) + logger.info(f"Deleted original compressed file: {file_path}") + except Exception as e: + logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + + return extracted_path + + elif format_type == 'zip': + logger.debug(f"Extracting zip file: {file_path}") + with zipfile.ZipFile(file_path, 'r') as zip_file: + # Find the first XML file in the ZIP archive + xml_files = [f for f in zip_file.namelist() if f.lower().endswith('.xml')] + + if not xml_files: + logger.info("No files with .xml extension found in ZIP archive, checking content of all files") + # Check content of each file to see if any are XML without proper extension + for filename in zip_file.namelist(): + if not filename.endswith('/'): # Skip directories + try: + # Read a sample of the file content + content_sample = zip_file.read(filename, 4096) # Read up to 4KB for detection + format_type, _, _ = detect_file_format(content=content_sample) + if format_type == 'xml': + logger.info(f"Found XML content in file without .xml extension: {filename}") + xml_files = [filename] + break + except Exception as e: + logger.warning(f"Error reading file {filename} from ZIP: {e}") + + if not xml_files: + logger.error("No XML file found in ZIP archive") + return None + + # Extract the first XML file + with open(extracted_path, 'wb') as out_file: + with zip_file.open(xml_files[0], "r") as xml_file: + while True: + chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk or len(chunk) == 0: + break + out_file.write(chunk) + + logger.info(f"Successfully extracted zip file to: {extracted_path}") + + # Delete original compressed file if requested + if delete_original: + try: + os.remove(file_path) + logger.info(f"Deleted original compressed file: {file_path}") + except Exception as e: + logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + + return extracted_path + + else: + logger.error(f"Unsupported or unrecognized compressed file format: {file_path} (detected as: {format_type})") + return None + + except Exception as e: + logger.error(f"Error extracting {file_path}: {str(e)}", exc_info=True) + return None + + +def parse_channels_only(source): + # Use extracted file if available, otherwise use the original file path + file_path = source.extracted_file_path if source.extracted_file_path else source.file_path + if not file_path: + file_path = source.get_cache_file() + + # Send initial parsing notification + send_epg_update(source.id, "parsing_channels", 0) + + process = None + should_log_memory = False + + try: + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file does not exist at path: {file_path}") + + # Update the source's file_path to the default cache location + new_path = source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + source.file_path = new_path + source.save(update_fields=['file_path']) + + # If the source has a URL, fetch the data before continuing + if source.url: + logger.info(f"Fetching new EPG data from URL: {source.url}") + fetch_success = fetch_xmltv(source) # Store the result + + # Only proceed if fetch was successful AND file exists + if not fetch_success: + logger.error(f"Failed to fetch EPG data from URL: {source.url}") + # Update status to error + source.status = 'error' + source.last_message = f"Failed to fetch EPG data from URL" + source.save(update_fields=['status', 'last_message']) + # Send error notification + send_epg_update(source.id, "parsing_channels", 100, status="error", error="Failed to fetch EPG data") + return False + + # Verify the file was downloaded successfully + if not os.path.exists(source.file_path): + logger.error(f"Failed to fetch EPG data, file still missing at: {source.file_path}") + # Update status to error + source.status = 'error' + source.last_message = f"Failed to fetch EPG data, file missing after download" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found after download") + return False + + # Update file_path with the new location + file_path = source.file_path + else: + logger.error(f"No URL provided for EPG source {source.name}, cannot fetch new data") + # Update status to error + source.status = 'error' + source.last_message = f"No URL provided, cannot fetch EPG data" + source.save(update_fields=['updated_at']) + + # Initialize process variable for memory tracking only in debug mode + try: + process = None + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is DEBUG (10) or more verbose + # This is more future-proof than string comparisons + should_log_memory = current_log_level <= logging.DEBUG or settings.DEBUG + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_channels_only] Initial memory usage: {initial_memory:.2f} MB") + except (ImportError, NameError): + process = None + should_log_memory = False + logger.warning("psutil not available for memory tracking") + + # Replace full dictionary load with more efficient lookup set + existing_tvg_ids = set() + existing_epgs = {} + scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup + last_id = 0 + chunk_size = 5000 + + while True: + tvg_id_chunk = set(EPGData.objects.filter( + epg_source=source, + id__gt=last_id + ).order_by('id').values_list('tvg_id', flat=True)[:chunk_size]) + + if not tvg_id_chunk: + break + + existing_tvg_ids.update(tvg_id_chunk) + last_id = EPGData.objects.filter(tvg_id__in=tvg_id_chunk).order_by('-id')[0].id + # Update progress to show file read starting + send_epg_update(source.id, "parsing_channels", 10) + + # Stream parsing instead of loading entire file at once + # This can be simplified since we now always have XML files + epgs_to_create = [] + epgs_to_update = [] + total_channels = 0 + processed_channels = 0 + batch_size = 500 # Process in batches to limit memory usage + progress = 0 # Initialize progress variable here + icon_url_max_length = EPGData._meta.get_field('icon_url').max_length # Get max length for icon_url field + name_max_length = EPGData._meta.get_field('name').max_length # Get max length for name field + + # Track memory at key points + if process: + logger.debug(f"[parse_channels_only] Memory before opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + try: + # Attempt to count existing channels in the database + try: + total_channels = EPGData.objects.filter(epg_source=source).count() + logger.info(f"Found {total_channels} existing channels for this source") + except Exception as e: + logger.error(f"Error counting channels: {e}") + total_channels = 500 # Default estimate + if process: + logger.debug(f"[parse_channels_only] Memory after closing initial file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Update progress after counting + send_epg_update(source.id, "parsing_channels", 25, total_channels=total_channels) + + # Open the file - no need to check file type since it's always XML now + logger.debug(f"Opening file for channel parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + if process: + logger.debug(f"[parse_channels_only] Memory after opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Change iterparse to look for both channel and programme elements + logger.debug(f"Creating iterparse context for channels and programmes") + channel_parser = etree.iterparse(source_file, events=('end',), tag=('channel', 'programme'), remove_blank_text=True, recover=True) + if process: + logger.debug(f"[parse_channels_only] Memory after creating iterparse: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + channel_count = 0 + total_elements_processed = 0 # Track total elements processed, not just channels + for _, elem in channel_parser: + total_elements_processed += 1 + # Only process channel elements + if elem.tag == 'channel': + channel_count += 1 + tvg_id = elem.get('id', '').strip() + if tvg_id: + scanned_tvg_ids.add(tvg_id) + display_name = None + icon_url = None + for child in elem: + if display_name is None and child.tag == 'display-name' and child.text: + display_name = child.text.strip() + elif child.tag == 'icon': + raw_icon_url = child.get('src', '').strip() + icon_url = validate_icon_url_fast(raw_icon_url, icon_url_max_length) + if display_name and icon_url: + break # No need to continue if we have both + + if not display_name: + display_name = tvg_id + + if display_name and len(display_name) > name_max_length: + logger.warning(f"EPG display name too long ({len(display_name)} > {name_max_length}), truncating: {display_name[:80]}...") + display_name = display_name[:name_max_length] + + # Use lazy loading approach to reduce memory usage + if tvg_id in existing_tvg_ids: + # Only fetch the object if we need to update it and it hasn't been loaded yet + if tvg_id not in existing_epgs: + try: + # This loads the full EPG object from the database and caches it + existing_epgs[tvg_id] = EPGData.objects.get(tvg_id=tvg_id, epg_source=source) + except EPGData.DoesNotExist: + # Handle race condition where record was deleted + existing_tvg_ids.remove(tvg_id) + epgs_to_create.append(EPGData( + tvg_id=tvg_id, + name=display_name, + icon_url=icon_url, + epg_source=source, + )) + logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 1: {tvg_id} - {display_name}") + processed_channels += 1 + continue + + # We use the cached object to check if the name or icon_url has changed + epg_obj = existing_epgs[tvg_id] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != icon_url: + epg_obj.icon_url = icon_url + needs_update = True + + if needs_update: + epgs_to_update.append(epg_obj) + logger.debug(f"[parse_channels_only] Added channel to update to epgs_to_update: {tvg_id} - {display_name}") + else: + # No changes needed, just clear the element + logger.debug(f"[parse_channels_only] No changes needed for channel {tvg_id} - {display_name}") + else: + # This is a new channel that doesn't exist in our database + epgs_to_create.append(EPGData( + tvg_id=tvg_id, + name=display_name, + icon_url=icon_url, + epg_source=source, + )) + logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 2: {tvg_id} - {display_name}") + + processed_channels += 1 + + # Batch processing + if len(epgs_to_create) >= batch_size: + logger.info(f"[parse_channels_only] Bulk creating {len(epgs_to_create)} EPG entries") + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + if process: + logger.info(f"[parse_channels_only] Memory after bulk_create: {process.memory_info().rss / 1024 / 1024:.2f} MB") + del epgs_to_create # Explicit deletion + epgs_to_create = [] + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + logger.info(f"[parse_channels_only] Memory after gc.collect(): {process.memory_info().rss / 1024 / 1024:.2f} MB") + + if len(epgs_to_update) >= batch_size: + logger.info(f"[parse_channels_only] Bulk updating {len(epgs_to_update)} EPG entries") + if process: + logger.info(f"[parse_channels_only] Memory before bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) + if process: + logger.info(f"[parse_channels_only] Memory after bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") + epgs_to_update = [] + # Force garbage collection + cleanup_memory(log_usage=should_log_memory, force_collection=True) + + # Periodically clear the existing_epgs cache to prevent memory buildup + if processed_channels % 1000 == 0: + logger.info(f"[parse_channels_only] Clearing existing_epgs cache at {processed_channels} channels") + existing_epgs.clear() + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + logger.info(f"[parse_channels_only] Memory after clearing cache: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Send progress updates + if processed_channels % 100 == 0 or processed_channels == total_channels: + progress = 25 + int((processed_channels / total_channels) * 65) if total_channels > 0 else 90 + send_epg_update( + source.id, + "parsing_channels", + progress, + processed=processed_channels, + total=total_channels + ) + if processed_channels > total_channels: + logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels - total_channels} additional channels") + else: + logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels}/{total_channels}") + if process: + logger.debug(f"[parse_channels_only] Memory before elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + # Clear memory + try: + # First clear the element's content + clear_element(elem) + + except Exception as e: + # Just log the error and continue - don't let cleanup errors stop processing + logger.debug(f"[parse_channels_only] Non-critical error during XML element cleanup: {e}") + if process: + logger.debug(f"[parse_channels_only] Memory after elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + logger.debug(f"[parse_channels_only] Total elements processed: {total_elements_processed}") + + else: + logger.trace(f"[parse_channels_only] Skipping non-channel element: {elem.get('channel', 'unknown')} - {elem.get('start', 'unknown')} {elem.tag}") + clear_element(elem) + continue + + except (etree.XMLSyntaxError, Exception) as xml_error: + logger.error(f"[parse_channels_only] XML parsing failed: {xml_error}") + # Update status to error + source.status = 'error' + source.last_message = f"Error parsing XML file: {str(xml_error)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(xml_error)) + return False + if process: + logger.info(f"[parse_channels_only] Processed {processed_channels} channels current memory: {process.memory_info().rss / 1024 / 1024:.2f} MB") + else: + logger.info(f"[parse_channels_only] Processed {processed_channels} channels") + # Process any remaining items + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.debug(f"[parse_channels_only] Created final batch of {len(epgs_to_create)} EPG entries") + + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) + logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") + + # Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel. + # Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list. + potentially_stale = existing_tvg_ids - scanned_tvg_ids + if potentially_stale: + stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True) + deleted_count, _ = stale_qs.delete() + if deleted_count: + logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel") + + if process: + logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") + + # Update source status with channel count + source.status = 'success' + source.last_message = f"Successfully parsed {processed_channels} channels" + source.save(update_fields=['status', 'last_message']) + + # Send completion notification + send_epg_update( + source.id, + "parsing_channels", + 100, + status="success", + channels_count=processed_channels + ) + + send_websocket_update('updates', 'update', {"success": True, "type": "epg_channels"}) + + logger.info(f"Finished parsing channel info. Found {processed_channels} channels.") + + return True + + except FileNotFoundError: + logger.error(f"EPG file not found at: {file_path}") + # Update status to error + source.status = 'error' + source.last_message = f"EPG file not found: {file_path}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found") + return False + except Exception as e: + logger.error(f"Error reading EPG file {file_path}: {e}", exc_info=True) + # Update status to error + source.status = 'error' + source.last_message = f"Error parsing EPG file: {str(e)}" + source.save(update_fields=['status', 'last_message']) + send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(e)) + return False + finally: + # Cleanup memory and close file + if process: + logger.debug(f"[parse_channels_only] Memory before cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") + try: + # Output any errors in the channel_parser error log + if 'channel_parser' in locals() and hasattr(channel_parser, 'error_log') and len(channel_parser.error_log) > 0: + logger.debug(f"XML parser errors found ({len(channel_parser.error_log)} total):") + for i, error in enumerate(channel_parser.error_log): + logger.debug(f" Error {i+1}: {error}") + if 'channel_parser' in locals(): + del channel_parser + if 'elem' in locals(): + del elem + if 'parent' in locals(): + del parent + + if 'source_file' in locals(): + source_file.close() + del source_file + # Clear remaining large data structures + existing_epgs.clear() + epgs_to_create.clear() + epgs_to_update.clear() + existing_epgs = None + epgs_to_create = None + epgs_to_update = None + if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None: + scanned_tvg_ids.clear() + scanned_tvg_ids = None + cleanup_memory(log_usage=should_log_memory, force_collection=True) + except Exception as e: + logger.warning(f"Cleanup error: {e}") + + try: + if process: + final_memory = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_channels_only] Final memory usage: {final_memory:.2f} MB") + process = None + except: + pass + + + +@shared_task(time_limit=3600, soft_time_limit=3500) +def parse_programs_for_tvg_id(epg_id): + # Skip XMLTV file parsing for Schedules Direct sources. Program data is + # fetched and persisted directly by fetch_schedules_direct(). + try: + from apps.epg.models import EPGData + epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() + if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': + logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") + return "Skipped (Schedules Direct source)" + except Exception as e: + logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") + + if not acquire_task_lock('parse_epg_programs', epg_id): + logger.info(f"Program parse for {epg_id} already in progress, skipping duplicate task") + return "Task already running" + + lock_renewer = TaskLockRenewer('parse_epg_programs', epg_id) + lock_renewer.start() + + source_file = None + program_parser = None + programs_to_create = [] + programs_processed = 0 + try: + # Add memory tracking only in trace mode or higher + try: + process = None + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is TRACE or more verbose or if running in DEBUG mode + should_log_memory = current_log_level <= 5 or settings.DEBUG + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Initial memory usage: {initial_memory:.2f} MB") + mem_before = initial_memory + except ImportError: + process = None + should_log_memory = False + + epg = EPGData.objects.get(id=epg_id) + epg_source = epg.epg_source + + # Skip program parsing for dummy EPG sources - they don't have program data files + if epg_source.source_type == 'dummy': + logger.info(f"Skipping program parsing for dummy EPG source {epg_source.name} (ID: {epg_id})") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + if not Channel.objects.filter(epg_data=epg).exists(): + logger.info(f"No channels matched to EPG {epg.tvg_id}") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") + + # Optimize deletion with a single delete query instead of chunking + # This is faster for most database engines + ProgramData.objects.filter(epg=epg).delete() + + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + if not file_path: + file_path = epg_source.get_cache_file() + + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file not found at: {file_path}") + + if epg_source.url: + # Update the file path in the database + new_path = epg_source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + epg_source.file_path = new_path + epg_source.save(update_fields=['file_path']) + logger.info(f"Fetching new EPG data from URL: {epg_source.url}") + else: + logger.info(f"EPG source does not have a URL, using existing file path: {file_path} to rebuild cache") + + # Fetch new data before continuing + if epg_source: + + # Properly check the return value from fetch_xmltv + fetch_success = fetch_xmltv(epg_source) + + # If fetch was not successful or the file still doesn't exist, abort + if not fetch_success: + logger.error(f"Failed to fetch EPG data, cannot parse programs for tvg_id: {epg.tvg_id}") + # Update status to error if not already set + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data, cannot parse programs" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Also check if the file exists after download + if not os.path.exists(epg_source.file_path): + logger.error(f"Failed to fetch EPG data, file still missing at: {epg_source.file_path}") + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data, file missing after download" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="File not found after download") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Update file_path with the new location + if epg_source.extracted_file_path: + file_path = epg_source.extracted_file_path + else: + file_path = epg_source.file_path + else: + logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") + # Update status to error + epg_source.status = 'error' + epg_source.last_message = f"No URL provided, cannot fetch EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + return + + # Use streaming parsing to reduce memory usage + # No need to check file type anymore since it's always XML + logger.debug(f"Parsing programs for tvg_id={epg.tvg_id} from {file_path}") + + # Memory usage tracking + if process: + try: + mem_before = process.memory_info().rss / 1024 / 1024 + logger.debug(f"[parse_programs_for_tvg_id] Memory before parsing {epg.tvg_id} - {mem_before:.2f} MB") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + mem_before = 0 + + programs_to_create = [] + batch_size = 1000 # Process in batches to limit memory usage + + try: + # Open the file directly - no need to check compression + logger.debug(f"Opening file for parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + # Stream parse the file using lxml's iterparse + program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) + + for _, elem in program_parser: + if elem.get('channel') == epg.tvg_id: + try: + start_time = parse_xmltv_time(elem.get('start')) + end_time = parse_xmltv_time(elem.get('stop')) + title = None + desc = None + sub_title = None + + # Efficiently process child elements + for child in elem: + if child.tag == 'title': + title = child.text or 'No Title' + elif child.tag == 'desc': + desc = child.text or '' + elif child.tag == 'sub-title': + sub_title = child.text or '' + + if not title: + title = 'No Title' + + # Extract custom properties + custom_props = extract_custom_properties(elem) + custom_properties_json = None + + if custom_props: + logger.trace(f"Number of custom properties: {len(custom_props)}") + custom_properties_json = custom_props + + # Fallback: extract S/E from description when episode-num + # elements didn't provide them + if desc: + has_season = (custom_properties_json or {}).get('season') is not None + has_episode = (custom_properties_json or {}).get('episode') is not None + if not has_season or not has_episode: + d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) + if d_season is not None and d_episode is not None: + if custom_properties_json is None: + custom_properties_json = {} + if not has_season: + custom_properties_json['season'] = d_season + if not has_episode: + custom_properties_json['episode'] = d_episode + custom_properties_json['season_episode_source'] = 'description' + desc = cleaned_desc + + programs_to_create.append(ProgramData( + epg=epg, + start_time=start_time, + end_time=end_time, + title=title[:255], + description=desc, + sub_title=sub_title, + tvg_id=epg.tvg_id, + custom_properties=custom_properties_json + )) + programs_processed += 1 + # Clear the element to free memory + clear_element(elem) + # Batch processing + if len(programs_to_create) >= batch_size: + ProgramData.objects.bulk_create(programs_to_create) + logger.debug(f"Saved batch of {len(programs_to_create)} programs for {epg.tvg_id}") + programs_to_create = [] + # Only call gc.collect() every few batches + if programs_processed % (batch_size * 5) == 0: + gc.collect() + + except Exception as e: + logger.error(f"Error processing program for {epg.tvg_id}: {e}", exc_info=True) + else: + # Immediately clean up non-matching elements to reduce memory pressure + if elem is not None: + clear_element(elem) + continue + + # Make sure to close the file and release parser resources + if source_file: + source_file.close() + source_file = None + + if program_parser: + program_parser = None + + gc.collect() + + except zipfile.BadZipFile as zip_error: + logger.error(f"Bad ZIP file: {zip_error}") + raise + except etree.XMLSyntaxError as xml_error: + logger.error(f"XML syntax error parsing program data: {xml_error}") + raise + except Exception as e: + logger.error(f"Error parsing XML for programs: {e}", exc_info=True) + raise + finally: + # Ensure file is closed even if an exception occurs + if source_file: + source_file.close() + source_file = None + # Memory tracking after processing + if process: + try: + mem_after = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Memory after parsing 1 {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + + # Process any remaining items + if programs_to_create: + ProgramData.objects.bulk_create(programs_to_create) + logger.debug(f"Saved final batch of {len(programs_to_create)} programs for {epg.tvg_id}") + programs_to_create = None + custom_props = None + custom_properties_json = None + + + logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.") + finally: + # Reset internal caches and pools that lxml might be keeping + try: + etree.clear_error_log() + except: + pass + # Explicit cleanup of all potentially large objects + if source_file: + try: + source_file.close() + except: + pass + source_file = None + program_parser = None + programs_to_create = None + + epg_source = None + # Add comprehensive cleanup before releasing lock + cleanup_memory(log_usage=should_log_memory, force_collection=True) + # Memory tracking after processing + if process: + try: + mem_after = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_tvg_id] Final memory usage {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") + except Exception as e: + logger.warning(f"Error tracking memory: {e}") + process = None + epg = None + programs_processed = None + lock_renewer.stop() + release_task_lock('parse_epg_programs', epg_id) + + + +def parse_programs_for_source(epg_source, tvg_id=None): + """ + Parse programs for all MAPPED channels from an EPG source in a single pass. + + This is an optimized version that: + 1. Only processes EPG entries that are actually mapped to channels + 2. Parses the XML file ONCE instead of once per channel + 3. Skips programmes for unmapped channels entirely during parsing + + This dramatically improves performance when an EPG source has many channels + but only a fraction are mapped. + """ + # Send initial programs parsing notification + send_epg_update(epg_source.id, "parsing_programs", 0) + should_log_memory = False + process = None + initial_memory = 0 + source_file = None + + # Add memory tracking only in trace mode or higher + try: + # Get current log level as a number + current_log_level = logger.getEffectiveLevel() + + # Only track memory usage when log level is TRACE or more verbose + should_log_memory = current_log_level <= 5 or settings.DEBUG # Assuming TRACE is level 5 or lower + + if should_log_memory: + process = psutil.Process() + initial_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_source] Initial memory usage: {initial_memory:.2f} MB") + except ImportError: + logger.warning("psutil not available for memory tracking") + process = None + should_log_memory = False + + try: + # Only get EPG entries that are actually mapped to channels + mapped_epg_ids = set( + Channel.objects.filter( + epg_data__epg_source=epg_source, + epg_data__isnull=False + ).values_list('epg_data_id', flat=True) + ) + + if not mapped_epg_ids: + total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() + logger.info(f"No channels mapped to any EPG entries from source: {epg_source.name} " + f"(source has {total_epg_count} EPG entries, 0 mapped)") + # Update status - this is not an error, just no mapped entries + epg_source.status = 'success' + epg_source.last_message = f"No channels mapped to this EPG source ({total_epg_count} entries available)" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="success") + return True + + # Get the mapped EPG entries with their tvg_ids + mapped_epgs = EPGData.objects.filter(id__in=mapped_epg_ids).values('id', 'tvg_id') + tvg_id_to_epg_id = {epg['tvg_id']: epg['id'] for epg in mapped_epgs if epg['tvg_id']} + mapped_tvg_ids = set(tvg_id_to_epg_id.keys()) + + total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() + mapped_count = len(mapped_tvg_ids) + + logger.info(f"Parsing programs for {mapped_count} MAPPED channels from source: {epg_source.name} " + f"(skipping {total_epg_count - mapped_count} unmapped EPG entries)") + + # Get the file path + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + if not file_path: + file_path = epg_source.get_cache_file() + + # Check if the file exists + if not os.path.exists(file_path): + logger.error(f"EPG file not found at: {file_path}") + + if epg_source.url: + # Update the file path in the database + new_path = epg_source.get_cache_file() + logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") + epg_source.file_path = new_path + epg_source.save(update_fields=['file_path']) + logger.info(f"Fetching new EPG data from URL: {epg_source.url}") + + # Fetch new data before continuing + fetch_success = fetch_xmltv(epg_source) + + if not fetch_success: + logger.error(f"Failed to fetch EPG data for source: {epg_source.name}") + epg_source.status = 'error' + epg_source.last_message = f"Failed to download EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") + return False + + # Update file_path with the new location + file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path + else: + logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") + epg_source.status = 'error' + epg_source.last_message = f"No URL provided, cannot fetch EPG data" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") + return False + + # SINGLE PASS PARSING: Parse the XML file once and collect all programs in memory + # We parse FIRST, then do an atomic delete+insert to avoid race conditions + # where clients might see empty/partial EPG data during the transition + all_programs_to_create = [] + programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} # Track count per channel + total_programs = 0 + skipped_programs = 0 + last_progress_update = 0 + + try: + logger.debug(f"Opening file for single-pass parsing: {file_path}") + source_file = _open_xmltv_file(file_path) + + # Stream parse the file using lxml's iterparse + program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) + + for _, elem in program_parser: + channel_id = elem.get('channel') + + # Skip programmes for unmapped channels immediately + if channel_id not in mapped_tvg_ids: + skipped_programs += 1 + # Clear element to free memory + clear_element(elem) + continue + + # This programme is for a mapped channel - process it + try: + start_time = parse_xmltv_time(elem.get('start')) + end_time = parse_xmltv_time(elem.get('stop')) + title = None + desc = None + sub_title = None + + # Efficiently process child elements + for child in elem: + if child.tag == 'title': + title = child.text or 'No Title' + elif child.tag == 'desc': + desc = child.text or '' + elif child.tag == 'sub-title': + sub_title = child.text or '' + + if not title: + title = 'No Title' + + # Extract custom properties + custom_props = extract_custom_properties(elem) + custom_properties_json = custom_props if custom_props else None + + # Fallback: extract S/E from description when episode-num + # elements didn't provide them + if desc: + has_season = (custom_properties_json or {}).get('season') is not None + has_episode = (custom_properties_json or {}).get('episode') is not None + if not has_season or not has_episode: + d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) + if d_season is not None and d_episode is not None: + if custom_properties_json is None: + custom_properties_json = {} + if not has_season: + custom_properties_json['season'] = d_season + if not has_episode: + custom_properties_json['episode'] = d_episode + custom_properties_json['season_episode_source'] = 'description' + desc = cleaned_desc + + epg_id = tvg_id_to_epg_id[channel_id] + all_programs_to_create.append(ProgramData( + epg_id=epg_id, + start_time=start_time, + end_time=end_time, + title=title[:255], + description=desc, + sub_title=sub_title, + tvg_id=channel_id, + custom_properties=custom_properties_json + )) + total_programs += 1 + programs_by_channel[channel_id] += 1 + + # Clear the element to free memory + clear_element(elem) + + # Send progress update (estimate based on programs processed) + if total_programs - last_progress_update >= 5000: + last_progress_update = total_programs + # Cap at 70% during parsing phase (save 30% for DB operations) + progress = min(70, 10 + int((total_programs / max(total_programs + 10000, 1)) * 60)) + send_epg_update(epg_source.id, "parsing_programs", progress, + processed=total_programs, channels=mapped_count) + + # Periodic garbage collection during parsing + if total_programs % 5000 == 0: + gc.collect() + + except Exception as e: + logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) + clear_element(elem) + continue + + except etree.XMLSyntaxError as xml_error: + logger.error(f"XML syntax error parsing program data: {xml_error}") + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"XML parsing error: {str(xml_error)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(xml_error)) + return False + except Exception as e: + logger.error(f"Error parsing XML for programs: {e}", exc_info=True) + raise + finally: + if source_file: + source_file.close() + source_file = None + + # Now perform atomic delete + bulk insert + # This ensures clients never see empty/partial EPG data + logger.info(f"Parsed {total_programs} programs, performing atomic database update...") + send_epg_update(epg_source.id, "parsing_programs", 75, message="Updating database...") + + batch_size = 1000 + try: + with transaction.atomic(): + # Kill any individual statement that hangs longer than 10 minutes. + # SET LOCAL automatically resets when this transaction ends (commit or rollback). + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + # Delete existing programs for mapped EPGs + deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] + logger.debug(f"Deleted {deleted_count} existing programs") + + # Clean up orphaned programs for unmapped EPG entries + unmapped_epg_ids = list(EPGData.objects.filter( + epg_source=epg_source + ).exclude(id__in=mapped_epg_ids).values_list('id', flat=True)) + + if unmapped_epg_ids: + orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] + if orphaned_count > 0: + logger.info(f"Cleaned up {orphaned_count} orphaned programs for {len(unmapped_epg_ids)} unmapped EPG entries") + + # Bulk insert all new programs in batches within the same transaction + for i in range(0, len(all_programs_to_create), batch_size): + batch = all_programs_to_create[i:i + batch_size] + ProgramData.objects.bulk_create(batch) + + # Update progress during insertion + progress = 75 + int((i / len(all_programs_to_create)) * 20) if all_programs_to_create else 95 + if i % (batch_size * 5) == 0: + send_epg_update(epg_source.id, "parsing_programs", min(95, progress), + message=f"Inserting programs... {i}/{len(all_programs_to_create)}") + + logger.info(f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs") + + except Exception as db_error: + logger.error(f"Database error during atomic update: {db_error}", exc_info=True) + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"Database error: {str(db_error)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(db_error)) + return False + finally: + # Clear the large list to free memory + all_programs_to_create = None + gc.collect() + + # Count channels that actually got programs + channels_with_programs = sum(1 for count in programs_by_channel.values() if count > 0) + + # Success message + epg_source.status = EPGSource.STATUS_SUCCESS + epg_source.last_message = ( + f"Parsed {total_programs:,} programs for {channels_with_programs} channels " + f"(skipped {skipped_programs:,} programs for {total_epg_count - mapped_count} unmapped channels)" + ) + epg_source.updated_at = timezone.now() + epg_source.save(update_fields=['status', 'last_message', 'updated_at']) + + # Log system event for EPG refresh + log_system_event( + event_type='epg_refresh', + source_name=epg_source.name, + programs=total_programs, + channels=channels_with_programs, + skipped_programs=skipped_programs, + unmapped_channels=total_epg_count - mapped_count, + ) + + # Send completion notification with status + send_epg_update(epg_source.id, "parsing_programs", 100, + status="success", + message=epg_source.last_message, + updated_at=epg_source.updated_at.isoformat()) + + logger.info(f"Completed parsing programs for source: {epg_source.name} - " + f"{total_programs:,} programs for {channels_with_programs} channels, " + f"skipped {skipped_programs:,} programs for unmapped channels") + return True + + except Exception as e: + logger.error(f"Error in parse_programs_for_source: {e}", exc_info=True) + # Update status to error + epg_source.status = EPGSource.STATUS_ERROR + epg_source.last_message = f"Error parsing programs: {str(e)}" + epg_source.save(update_fields=['status', 'last_message']) + send_epg_update(epg_source.id, "parsing_programs", 100, + status="error", + message=epg_source.last_message) + return False + finally: + # Final memory cleanup and tracking + if source_file: + try: + source_file.close() + except: + pass + source_file = None + + # Explicitly release any remaining large data structures + programs_to_create = None + programs_by_channel = None + mapped_epg_ids = None + mapped_tvg_ids = None + tvg_id_to_epg_id = None + gc.collect() + + # Add comprehensive memory cleanup at the end + cleanup_memory(log_usage=should_log_memory, force_collection=True) + if process: + final_memory = process.memory_info().rss / 1024 / 1024 + logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") + # Explicitly clear the process object to prevent potential memory leaks + process = None +@shared_task(bind=True) +def fetch_schedules_direct_stations(self, source_id): + """ + Lightweight Celery task that runs a stations-only Schedules Direct fetch. + Called on initial source creation so EPGData entries exist for auto-matching + before the user commits to a full schedule/program fetch. + """ + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + logger.error(f"EPGSource {source_id} not found for SD stations fetch") + return + fetch_schedules_direct(source, stations_only=True) + + +def fetch_schedules_direct(source, stations_only=False, force=False): + """ + Fetch EPG data from the Schedules Direct JSON API and persist it to the + EPGData / ProgramData models. + + Authentication flow (as required by the SD API specification): + 1. POST credentials to the token endpoint (password must be SHA1-hashed + as required by the Schedules Direct API specification. + 2. Use the returned token for all subsequent requests via the 'token' header. + 3. Tokens are valid for 24 hours; SD returns the current valid token if one + already exists for the account. + + Data flow: + 1. Fetch subscribed lineups for the account. + 2. Fetch station metadata for each lineup. + 3. Persist station metadata to EPGData. + 4. If stations_only=True, stop here. Used on initial source creation so + the user can run Auto-match EPG before the full program fetch. + 5. Fetch schedule grids in 14-day date-batched requests per station. + 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). + 7. Persist channels to EPGData and programs to ProgramData. + + Args: + source: EPGSource instance + stations_only: If True, only fetch and persist station metadata (no schedules/programs). + Used on initial source creation to populate EPGData for auto-matching + before channels are assigned. + """ + import hashlib + from datetime import date + + + logger.info(f"Fetching Schedules Direct data for source: {source.name}") + + # ------------------------------------------------------------------------- + # Validate credentials + # ------------------------------------------------------------------------- + username = (source.username or '').strip() + password = (source.password or '').strip() + + if not username or not password: + msg = "Schedules Direct source requires both a username and password." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Enforce 2-hour minimum interval between full fetches (not stations-only). + # Schedules Direct enforces rate limits of ~200 requests per 2-hour window. + # This prevents automated abuse regardless of how the refresh was triggered. + # + # Exception: if no SDScheduleMD5 records exist yet, this is the first full + # refresh after initial source creation (stations-only runs first and updates + # updated_at, which would otherwise incorrectly trigger this guard). Always + # allow the first full refresh through so guide data is immediately available. + # ------------------------------------------------------------------------- + if not stations_only and not force and source.updated_at: + from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 + has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() + if has_prior_full_refresh: + elapsed = (timezone.now() - source.updated_at).total_seconds() + min_interval_seconds = 2 * 3600 # 2 hours + if elapsed < min_interval_seconds: + remaining_minutes = int((min_interval_seconds - elapsed) / 60) + msg = ( + f"Schedules Direct refresh skipped. Minimum 2-hour interval not reached. " + f"Last refreshed {int(elapsed / 60)} minutes ago. " + f"Please wait {remaining_minutes} more minute(s)." + ) + logger.warning(f"SD source {source.id}: {msg}") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + else: + logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") + elif force and not stations_only: + logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") + + # ------------------------------------------------------------------------- + # Build SD-specific headers + # SD API spec requires the User-Agent to identify the application and version. + # SergeantPanda confirmed Dispatcharr should identify itself properly. + # ------------------------------------------------------------------------- + from version import __version__ as dispatcharr_version + sd_user_agent = f"Dispatcharr/{dispatcharr_version}" + + def _sd_headers(token=None): + h = { + 'Content-Type': 'application/json', + 'User-Agent': sd_user_agent, + } + if token: + h['token'] = token + return h + + # ------------------------------------------------------------------------- + # Step 1: Authenticate and obtain session token + # The SD API requires the password to be SHA1-hashed before transmission. + # This is a requirement of the Schedules Direct API specification, not an + # architectural choice. + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_FETCHING + source.last_message = "Authenticating with Schedules Direct..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") + + try: + sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() + token_response = requests.post( + f"{SD_BASE_URL}/token", + json={'username': username, 'password': sha1_password}, + headers=_sd_headers(), + timeout=30, + ) + token_response.raise_for_status() + token_data = token_response.json() + + auth_code = token_data.get('code', 0) + if auth_code != 0: + if auth_code == 4007: + msg = "Schedules Direct: this application is not authorized. Please contact the Dispatcharr maintainers." + elif auth_code == 4004: + msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." + elif auth_code == 4009: + msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours. Check for misconfiguration." + elif auth_code == 4001: + msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." + elif auth_code == 4008: + msg = "Schedules Direct: account is inactive. Please log in to schedulesdirect.org to reactivate." + else: + msg = f"Schedules Direct authentication failed (code {auth_code}): {token_data.get('message', 'Unknown error')}" + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + token = token_data.get('token') + if not token: + msg = "Schedules Direct returned no token." + logger.error(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info("Schedules Direct authentication successful.") + + except requests.exceptions.RequestException as e: + msg = f"Network error authenticating with Schedules Direct: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 2: Check account status (respect OFFLINE system status) + # ------------------------------------------------------------------------- + try: + status_response = requests.get( + f"{SD_BASE_URL}/status", + headers=_sd_headers(token), + timeout=30, + ) + status_response.raise_for_status() + status_data = status_response.json() + system_status = status_data.get('systemStatus', [{}])[0].get('status', 'Online') + if system_status == 'Offline': + # Per SD API spec: if system is offline, disconnect and do not + # retry for 1 hour. We set idle status rather than error since + # this is a temporary SD-side condition. + msg = "Schedules Direct system is currently offline. Per SD guidelines, retrying in 1 hour." + logger.warning(msg) + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.debug(f"Schedules Direct system status: {system_status}") + except requests.exceptions.RequestException as e: + logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") + + # ------------------------------------------------------------------------- + # Step 3: Fetch subscribed lineups and build station map + # ------------------------------------------------------------------------- + _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") + try: + lineups_response = requests.get( + f"{SD_BASE_URL}/lineups", + headers=_sd_headers(token), + timeout=30, + ) + # SD returns 400 with code 4102 when no lineups are configured. + # This is a valid account state. The user needs to add lineups via + # the Manage Lineups UI. Treat as idle rather than error. + if lineups_response.status_code == 400: + sd_data = lineups_response.json() + if sd_data.get('code') == 4102: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + lineups_response.raise_for_status() + lineups_data = lineups_response.json() + lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] + if not lineups: + msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." + logger.warning(f"SD source {source.id}: no active lineups found.") + source.status = EPGSource.STATUS_IDLE + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) + return + logger.info(f"Found {len(lineups)} lineup(s) in SD account.") + + # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) + sd_lineup_country = None + for l in lineups: + lid = l.get('lineupID') or l.get('lineup') or '' + if '-' in lid: + sd_lineup_country = lid.split('-')[0] + break + logger.debug(f"SD lineup country: {sd_lineup_country}") + except requests.exceptions.RequestException as e: + msg = f"Failed to fetch Schedules Direct lineups: {e}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + # Build station metadata map: stationID -> {name, callsign, logo_url} + station_map = {} + _sd_send_ws_sync(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") + for lineup in lineups: + lineup_id = lineup.get('lineupID') or lineup.get('lineup') + if not lineup_id: + continue + try: + detail_response = requests.get( + f"{SD_BASE_URL}/lineups/{lineup_id}", + headers=_sd_headers(token), + timeout=30, + ) + detail_response.raise_for_status() + detail_data = detail_response.json() + for station in detail_data.get('stations', []): + sid = station.get('stationID') + if not sid: + continue + logo_url = None + logos = station.get('stationLogo') or station.get('logo') or [] + if isinstance(logos, list) and logos: + # Read preferred logo style from source settings; default to 'dark' + logo_style = (source.custom_properties or {}).get('logo_style', 'dark') + preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) + logo_url = preferred.get('URL') or preferred.get('url') + elif isinstance(logos, dict): + logo_url = logos.get('URL') or logos.get('url') + station_map[sid] = { + 'name': station.get('name', sid), + 'callsign': station.get('callsign', ''), + 'logo_url': logo_url, + } + logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") + + if not station_map: + msg = "No stations found across all Schedules Direct lineups." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) + return + + logger.info(f"Built station map with {len(station_map)} stations.") + + # ------------------------------------------------------------------------- + # Step 4: Persist station metadata to EPGData + # ------------------------------------------------------------------------- + source.status = EPGSource.STATUS_PARSING + source.last_message = f"Syncing {len(station_map)} stations..." + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") + + existing_epg_map = { + epg.tvg_id: epg + for epg in EPGData.objects.filter(epg_source=source) + } + + epgs_to_create = [] + epgs_to_update = [] + icon_max_length = EPGData._meta.get_field('icon_url').max_length + name_max_length = EPGData._meta.get_field('name').max_length + + for sid, info in station_map.items(): + display_name = (info['name'] or sid)[:name_max_length] + logo = info['logo_url'] + if logo and len(logo) > icon_max_length: + logo = None + + if sid in existing_epg_map: + epg_obj = existing_epg_map[sid] + needs_update = False + if epg_obj.name != display_name: + epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != logo: + epg_obj.icon_url = logo + needs_update = True + if needs_update: + epgs_to_update.append(epg_obj) + else: + epgs_to_create.append(EPGData( + tvg_id=sid, + name=display_name, + icon_url=logo, + epg_source=source, + )) + + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) + logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") + + gc.collect() + + # Rebuild map with fresh DB ids for all stations + epg_id_map = { + epg.tvg_id: epg.id + for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) + } + + # Station sync complete. Send progress update before continuing into programs phase. + # We deliberately do NOT send parsing_channels at 100 with status=success here + # because that would cause the frontend to mark the source as complete and + # stop rendering progress updates for the subsequent program fetch phases. + _sd_send_ws_sync(source.id, "parsing_programs", 30, + message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") + + # ------------------------------------------------------------------------- + # Stations-only mode. Used on initial source creation. + # Stop here so the user can run Auto-match EPG before the full program fetch. + # ------------------------------------------------------------------------- + if stations_only: + success_msg = ( + f"{len(station_map)} stations loaded from Schedules Direct. " + f"Run Auto-match EPG to map your channels, then use the Refresh " + f"button to populate guide data." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_channels", 100, status="success", + message=success_msg, channels_count=len(station_map)) + logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") + return + + # ------------------------------------------------------------------------- + # Step 5: MD5-delta schedule fetch + # First fetch MD5 hashes for all stations/dates. Compare against our + # locally cached hashes to determine which schedules have changed. + # Only download schedules that have actually changed; this minimises + # API calls against SD's rate-limited endpoints. + # ------------------------------------------------------------------------- + from apps.epg.models import SDScheduleMD5 + from django.utils.dateparse import parse_datetime + + _sd_send_ws_sync(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") + station_ids = list(station_map.keys()) + today = date.today() + date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] + + # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. + # These accumulate one row per station per day and are never useful once past. + pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] + if pruned_sched_md5_count: + logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") + + # Fetch MD5 hashes for all stations in batches of 5000 + STATION_BATCH_SIZE = 5000 + server_md5s = {} # (station_id, date) -> {md5, last_modified} + + logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") + + station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] + for batch in station_batches: + try: + md5_response = requests.post( + f"{SD_BASE_URL}/schedules/md5", + json=[{'stationID': sid, 'date': date_list} for sid in batch], + headers=_sd_headers(token), + timeout=120, + ) + md5_response.raise_for_status() + md5_data = md5_response.json() + for sid, dates in md5_data.items(): + for date_str, info in dates.items(): + if info.get('code', 0) == 0: + server_md5s[(sid, date_str)] = { + 'md5': info.get('md5', ''), + 'last_modified': info.get('lastModified', ''), + } + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule MD5s: {e}") + + # Load our cached MD5s from DB + cached_md5s = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) + } + + # Determine which station/date combinations need downloading + changed_by_station = {} # station_id -> [date_str, ...] + for (sid, date_str), server_info in server_md5s.items(): + if date_str not in date_list: + continue + cached = cached_md5s.get((sid, date_str)) + if cached != server_info['md5']: + changed_by_station.setdefault(sid, []).append(date_str) + + total_changed = sum(len(v) for v in changed_by_station.values()) + total_possible = len(station_ids) * len(date_list) + logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") + _sd_send_ws_sync(source.id, "parsing_programs", 38, + message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") + + # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} + schedules_by_station = {sid: [] for sid in station_ids} + program_ids_needed = set() + + if not changed_by_station: + logger.info("No schedule changes detected, skipping schedule and program downloads.") + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", + message="No schedule changes detected since last refresh. Guide data is up to date.") + source.status = EPGSource.STATUS_SUCCESS + source.last_message = "No schedule changes detected. Guide data is up to date." + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + return + + # Download only changed schedules, batched by 7-day windows per station + SCHEDULE_BATCH_DAYS = 7 + changed_station_ids = list(changed_by_station.keys()) + date_batches = [date_list[i:i + SCHEDULE_BATCH_DAYS] for i in range(0, len(date_list), SCHEDULE_BATCH_DAYS)] + new_md5_records = [] + updated_md5_records = [] + existing_md5_map = { + (r.station_id, r.date.strftime('%Y-%m-%d')): r + for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=changed_station_ids) + } + + for batch_idx, date_batch in enumerate(date_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 38 + int((batch_idx / len(date_batches)) * 22) + logger.info(f"Fetching schedule batch {batch_idx + 1} of {len(date_batches)}...") + _sd_send_ws_sync(source.id, "parsing_programs", min(59, pre_progress), + message=f"Fetching schedules: batch {batch_idx + 1} of {len(date_batches)}...") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + # Only include stations that have changes in this date batch + request_body = [ + {'stationID': sid, 'date': [d for d in date_batch if d in changed_by_station.get(sid, [])]} + for sid in changed_station_ids + if any(d in changed_by_station.get(sid, []) for d in date_batch) + ] + if not request_body: + continue + try: + sched_response = requests.post( + f"{SD_BASE_URL}/schedules", + json=request_body, + headers=_sd_headers(token), + timeout=120, + ) + sched_response.raise_for_status() + sched_data = sched_response.json() + + for station_sched in sched_data: + sid = station_sched.get('stationID') + if not sid: + continue + programs = station_sched.get('programs', []) + schedules_by_station.setdefault(sid, []).extend(programs) + for prog in programs: + pid = prog.get('programID') + if pid: + program_ids_needed.add(pid) + + # Update MD5 cache for this station/date + meta = station_sched.get('metadata', {}) + start_date = meta.get('startDate') + md5_val = meta.get('md5', '') + last_mod_str = meta.get('modified', '') + if start_date and md5_val: + key = (sid, start_date) + last_mod = parse_datetime(last_mod_str) if last_mod_str else timezone.now() + if key in existing_md5_map: + rec = existing_md5_map[key] + rec.md5 = md5_val + rec.last_modified = last_mod + updated_md5_records.append(rec) + else: + import datetime as dt_module + try: + date_obj = dt_module.date.fromisoformat(start_date) + new_md5_records.append(SDScheduleMD5( + epg_source=source, + station_id=sid, + date=date_obj, + md5=md5_val, + last_modified=last_mod, + )) + except ValueError: + pass + + progress = 38 + int(((batch_idx + 1) / len(date_batches)) * 22) + _sd_send_ws_sync(source.id, "parsing_programs", min(60, progress), + message=f"Fetching changed schedules: batch {batch_idx + 1}/{len(date_batches)} ({len(program_ids_needed):,} programs found)") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch schedule batch {batch_idx + 1}: {e}") + + # Persist updated MD5 cache + if new_md5_records: + SDScheduleMD5.objects.bulk_create(new_md5_records, ignore_conflicts=True) + logger.info(f"Cached {len(new_md5_records)} new schedule MD5s.") + if updated_md5_records: + SDScheduleMD5.objects.bulk_update(updated_md5_records, ['md5', 'last_modified']) + logger.info(f"Updated {len(updated_md5_records)} existing schedule MD5s.") + + if not program_ids_needed: + msg = "No schedule data returned from Schedules Direct." + logger.warning(msg) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + + # ------------------------------------------------------------------------- + # Step 6: MD5-delta program metadata fetch + # The schedule response includes an MD5 hash per program airing. + # Compare against our cached program MD5s to only download programs + # whose metadata has changed since our last fetch. + # ------------------------------------------------------------------------- + + # Build map of programID -> md5 from schedule data + schedule_program_md5s = {} # programID -> md5 from schedule + for sid, airings in schedules_by_station.items(): + for airing in airings: + pid = airing.get('programID') + md5 = airing.get('md5') + if pid and md5: + schedule_program_md5s[pid] = md5 + + # Load cached program MD5s from SDProgramMD5 table, keyed by programID + cached_prog_md5s = { + r.program_id: r.md5 + for r in SDProgramMD5.objects.filter( + epg_source=source, + program_id__in=program_ids_needed, + ).only('program_id', 'md5') + } + + # Only fetch programs where MD5 differs from our cached value + programs_to_fetch = { + pid for pid in program_ids_needed + if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) + } + + logger.info( + f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " + f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") + + program_metadata = {} + program_id_list = list(programs_to_fetch) + total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) + + if program_id_list: + logger.info(f"Fetching metadata for {len(program_id_list)} programs in {total_batches} batch(es).") + for batch_idx in range(total_batches): + # Notify frontend at the start of each batch so progress updates immediately + pre_progress = 60 + int((batch_idx / total_batches) * 20) + logger.info(f"Fetching program metadata batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)...") + _sd_send_ws_sync(source.id, "parsing_programs", min(79, pre_progress), + message=f"Fetching program data: batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)") + # Yield to gevent hub so the WebSocket update is delivered before the blocking request + try: + import gevent; gevent.sleep(0) + except ImportError: + pass + batch = program_id_list[batch_idx * SD_PROGRAM_BATCH_SIZE:(batch_idx + 1) * SD_PROGRAM_BATCH_SIZE] + try: + prog_response = requests.post( + f"{SD_BASE_URL}/programs", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + prog_response.raise_for_status() + prog_data = prog_response.json() + for prog in prog_data: + pid = prog.get('programID') + if pid: + program_metadata[pid] = prog + + progress = 60 + int(((batch_idx + 1) / total_batches) * 20) + _sd_send_ws_sync(source.id, "parsing_programs", min(80, progress), + message=f"Fetching program details: batch {batch_idx + 1}/{total_batches} ({len(program_metadata):,} programs loaded)") + logger.debug(f"Fetched program metadata batch {batch_idx + 1}/{total_batches}") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch program metadata batch {batch_idx + 1}: {e}") + else: + logger.info("All program metadata unchanged - skipping program download.") + _sd_send_ws_sync(source.id, "parsing_programs", 80, message="Program metadata unchanged - using cached data.") + + gc.collect() + + # ------------------------------------------------------------------------- + # Step 7: Build ProgramData records and persist atomically + # ------------------------------------------------------------------------- + logger.info("Building program records...") + _sd_send_ws_sync(source.id, "parsing_programs", 80) + + # Only process stations that are mapped to channels to match the existing + # XMLTV flow (parse_programs_for_source only processes mapped channels). + from apps.channels.models import Channel as ChannelModel + mapped_epg_ids = set( + ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) + ) + mapped_tvg_ids = set( + EPGData.objects.filter( + id__in=mapped_epg_ids, + epg_source=source, + ).values_list('tvg_id', flat=True) + ) + + # Cache existing program data for unchanged programs BEFORE surgical delete. + # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only + # programs with changed program MD5s get metadata re-downloaded. The surgical delete + # wipes ALL ProgramData for changed dates, so unchanged programs lose their titles. + # This cache preserves their data for rebuilding. + unchanged_pids = set() + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + continue + for airing in airings: + pid = airing.get('programID') + if pid and pid not in program_metadata: + unchanged_pids.add(pid) + + existing_program_cache = {} + if unchanged_pids: + for pd in ProgramData.objects.filter( + epg__epg_source=source, + program_id__in=unchanged_pids, + ).only('program_id', 'title', 'description', 'sub_title', 'custom_properties'): + if pd.program_id not in existing_program_cache: + existing_program_cache[pd.program_id] = { + 'title': pd.title, + 'description': pd.description, + 'sub_title': pd.sub_title, + 'custom_properties': pd.custom_properties, + } + logger.info(f"Cached {len(existing_program_cache)} existing program records for unchanged programs.") + + all_programs_to_create = [] + total_programs = 0 + skipped_unmapped = 0 + + for sid, airings in schedules_by_station.items(): + if sid not in mapped_tvg_ids: + skipped_unmapped += len(airings) + continue + + epg_db_id = epg_id_map.get(sid) + if not epg_db_id: + continue + + for airing in airings: + pid = airing.get('programID') + air_time = airing.get('airDateTime') + duration_secs = airing.get('duration', 0) + + if not pid or not air_time or not duration_secs: + continue + + try: + start_dt = parse_schedules_direct_time(air_time) + end_dt = start_dt + timedelta(seconds=int(duration_secs)) + except Exception as e: + logger.debug(f"Could not parse air time '{air_time}': {e}") + continue + + meta = program_metadata.get(pid, {}) + cached_prog = existing_program_cache.get(pid) if not meta else None + + if cached_prog: + # Unchanged program — reuse cached data from before surgical delete + title = cached_prog['title'] or 'No Title' + desc = cached_prog['description'] or '' + episode_title = cached_prog['sub_title'] or '' + custom_props = cached_prog['custom_properties'] or {} + else: + titles = meta.get('titles', [{}]) + title = titles[0].get('title120', '') if titles else '' + if not title: + title = meta.get('episodeTitle150', '') or 'No Title' + title = title[:255] + + if not cached_prog: + descriptions = meta.get('descriptions', {}) + desc = '' + for key in ('description1000', 'description255', 'description100'): + candidates = descriptions.get(key, []) + if candidates: + desc = candidates[0].get('description', '') + if desc: + break + + episode_title = meta.get('episodeTitle150', '') + + # Build custom_properties following the same pattern as the XMLTV parser + custom_props = {} + + # Season/Episode — search all metadata entries, not just [0] + metadata_block = meta.get('metadata', []) + gracenote_meta = {} + for md_entry in metadata_block: + if 'Gracenote' in md_entry: + gracenote_meta = md_entry['Gracenote'] + break + if not gracenote_meta: + # Fall back to TVmaze if Gracenote is absent + for md_entry in metadata_block: + if 'TVmaze' in md_entry: + gracenote_meta = md_entry['TVmaze'] + break + season = gracenote_meta.get('season') + episode = gracenote_meta.get('episode') + if season: + custom_props['season'] = int(season) + if episode: + custom_props['episode'] = int(episode) + if season and episode: + custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" + + # Content rating — store full array, pick display rating by lineup country + content_rating = meta.get('contentRating', []) + if content_rating: + custom_props['content_ratings'] = content_rating + selected = None + if sd_lineup_country: + for cr in content_rating: + if cr.get('country', '') == sd_lineup_country: + selected = cr + break + if not selected: + # Fall back to USA, then first available + for cr in content_rating: + if cr.get('country', '') == 'USA': + selected = cr + break + if not selected: + selected = content_rating[0] + custom_props['rating'] = selected.get('code', '') + custom_props['rating_system'] = selected.get('body', '') + + # Content advisory — content warnings + content_advisory = meta.get('contentAdvisory', []) + if content_advisory: + custom_props['content_advisory'] = content_advisory + + # Categories — combine entityType, showType, and genres + categories = [] + entity_type = meta.get('entityType', '') + show_type = meta.get('showType', '') + if entity_type: + categories.append(entity_type) + if show_type and show_type != entity_type: + categories.append(show_type) + genres = meta.get('genres', []) + categories.extend(genres) + if categories: + custom_props['categories'] = categories + + # Cast — top-billed only, store characterName, drop role noise + cast = meta.get('cast', []) + crew = meta.get('crew', []) + credits = {} + if cast: + # Sort by billingOrder and cap at top-billed actors + sorted_cast = sorted( + [p for p in cast if p.get('name')], + key=lambda p: int(p.get('billingOrder', '999')) + ) + # Separate main cast from guest stars + main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] + # Store top-billed main cast (matching XMLTV parity) + credits['actor'] = [ + { + 'name': p.get('name', ''), + **(({'character': p['characterName']} ) if p.get('characterName') else {}), + } + for p in (main_cast[:6] if main_cast else sorted_cast[:6]) + ] + if crew: + for member in crew: + role = member.get('role', '').lower() + name = member.get('name', '') + if not name: + continue + if 'director' in role: + credits.setdefault('director', []).append(name) + elif 'writer' in role or 'screenwriter' in role: + credits.setdefault('writer', []).append(name) + elif 'producer' in role: + credits.setdefault('producer', []).append(name) + if credits: + custom_props['credits'] = credits + + # Airing flags + if airing.get('liveTapeDelay') == 'Live': + custom_props['live'] = True + if airing.get('new'): + custom_props['new'] = True + else: + custom_props['previously_shown'] = True + if airing.get('premiere'): + custom_props['premiere'] = True + + # Original air date — full date, not just year + original_air_date = meta.get('originalAirDate', '') + movie_year = meta.get('movie', {}).get('year', '') + if original_air_date: + custom_props['date'] = original_air_date + elif movie_year: + custom_props['date'] = str(movie_year) + + # Country of production + country = meta.get('country', []) + if country: + custom_props['country'] = country[0] if len(country) == 1 else ', '.join(country) + + # Runtime — program duration without commercials (seconds → store for display) + runtime_secs = meta.get('duration') or meta.get('movie', {}).get('duration') + if runtime_secs: + runtime_mins = int(runtime_secs) // 60 + custom_props['length'] = {'value': str(runtime_mins), 'units': 'minutes'} + + # Movie quality ratings → star_ratings (matches XMLTV key) + movie_data = meta.get('movie', {}) + quality_ratings = movie_data.get('qualityRating', []) + if quality_ratings: + star_ratings = [] + for qr in quality_ratings: + rating_str = qr.get('rating', '') + max_rating = qr.get('maxRating', '') + if rating_str and max_rating: + star_ratings.append({ + 'value': f"{rating_str}/{max_rating}", + 'system': qr.get('ratingsBody', ''), + }) + if star_ratings: + custom_props['star_ratings'] = star_ratings + + # Sports event details + event_details = meta.get('eventDetails', {}) + if event_details: + custom_props['event_details'] = event_details + + all_programs_to_create.append(ProgramData( + epg_id=epg_db_id, + start_time=start_dt, + end_time=end_dt, + title=title, + sub_title=episode_title or None, + description=desc or None, + tvg_id=sid, + program_id=pid, + custom_properties=custom_props or None, + )) + total_programs += 1 + + logger.info(f"Built {total_programs} program records " + f"({skipped_unmapped} skipped for unmapped stations).") + + _sd_send_ws_sync(source.id, "parsing_programs", 88) + + # Build a map of epg_db_id -> list of (day_start_utc, day_end_utc) for each changed date. + # Only programs that fall within changed station/date pairs will be deleted and replaced; + # programs for unchanged stations or unchanged dates are left intact. + import datetime as dt_module + epg_changed_date_ranges = {} + for sid, changed_date_strs in changed_by_station.items(): + epg_db_id = epg_id_map.get(sid) + if not epg_db_id or epg_db_id not in mapped_epg_ids: + continue + ranges = [] + for ds in changed_date_strs: + d = dt_module.date.fromisoformat(ds) + day_start = datetime(d.year, d.month, d.day, tzinfo=dt_timezone.utc) + ranges.append((day_start, day_start + timedelta(days=1))) + if ranges: + epg_changed_date_ranges[epg_db_id] = ranges + + # Atomic delete (surgical) + bulk insert + BATCH_SIZE = 1000 + try: + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '10min'") + total_deleted = 0 + for epg_db_id, day_ranges in epg_changed_date_ranges.items(): + q = Q() + for day_start, day_end in day_ranges: + q |= Q(start_time__gte=day_start, start_time__lt=day_end) + cnt = ProgramData.objects.filter(epg_id=epg_db_id).filter(q).delete()[0] + total_deleted += cnt + logger.debug(f"Deleted {total_deleted} changed SD programs across {len(epg_changed_date_ranges)} stations.") + for i in range(0, len(all_programs_to_create), BATCH_SIZE): + ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) + progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) + _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) + + logger.info(f"Committed {total_programs} Schedules Direct programs to database.") + + # Upsert SDProgramMD5 records for programs we just downloaded + # This updates the cache so future fetches can skip unchanged programs + if schedule_program_md5s: + md5_records = [ + SDProgramMD5( + epg_source=source, + program_id=pid, + md5=md5, + ) + for pid, md5 in schedule_program_md5s.items() + if pid in program_metadata # Only cache programs that were actually downloaded + ] + if md5_records: + SDProgramMD5.objects.bulk_create( + md5_records, + update_conflicts=True, + unique_fields=['epg_source', 'program_id'], + update_fields=['md5'], + ) + logger.info(f"Cached {len(md5_records)} program MD5s for future delta detection.") + + except Exception as db_error: + msg = f"Database error persisting Schedules Direct programs: {db_error}" + logger.error(msg, exc_info=True) + source.status = EPGSource.STATUS_ERROR + source.last_message = msg + source.save(update_fields=['status', 'last_message']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) + return + finally: + all_programs_to_create = None + gc.collect() + + # ------------------------------------------------------------------------- + # Step 8: Fetch program artwork (posters) if enabled + # ------------------------------------------------------------------------- + fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) + if fetch_posters and program_metadata: + logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") + _sd_send_ws_sync(source.id, "parsing_programs", 98, + message="Fetching program artwork...") + + try: + # Build a set of unique artwork lookup IDs. + # For episodes (EP...), use the series root (SH...0000) so we get + # series-level artwork — one poster per series instead of per-episode. + artwork_lookup_ids = set() + pid_to_artwork_key = {} # maps original programID -> the key we looked up + + for pid in program_metadata: + if pid.startswith('EP'): + sh_root = 'SH' + pid[2:10] + '0000' + artwork_lookup_ids.add(sh_root) + pid_to_artwork_key[pid] = sh_root + else: + artwork_lookup_ids.add(pid) + pid_to_artwork_key[pid] = pid + + artwork_map = {} # artwork_key -> poster_url + artwork_list = list(artwork_lookup_ids) + SD_ARTWORK_BATCH_SIZE = 500 + + total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) + logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " + f"in {total_art_batches} batch(es).") + + for batch_idx in range(total_art_batches): + batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] + try: + art_response = requests.post( + f"{SD_BASE_URL}/metadata/programs/", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + art_response.raise_for_status() + art_data = art_response.json() + + for entry in art_data: + if not isinstance(entry, dict): + continue + entry_pid = entry.get('programID') + images = entry.get('data') or [] + if not entry_pid or not images: + continue + + # Filter to only dict entries — SD sometimes returns bare strings + images = [img for img in images if isinstance(img, dict)] + if not images: + continue + + # Pick the best poster image: + # Prefer portrait orientation (2x3, 3x4) in larger sizes + # SD categories include: Iconic, Banner-L1, Banner-L2, Logo + # SD uses width/height instead of a "size" field + poster_url = None + + # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic + for min_width in [240, 135, 120]: + for pref_cat in ['Banner-L1', 'Iconic']: + match = next((img for img in images + if img.get('aspect') in ('2x3', '3x4') + and img.get('category') == pref_cat + and (img.get('width', 0) or 0) >= min_width), None) + if match: + poster_url = match.get('uri') + break + if poster_url: + break + + # Fallback: any portrait image at any size + if not poster_url: + portrait = next((img for img in images + if img.get('aspect') in ('2x3', '3x4')), None) + if portrait: + poster_url = portrait.get('uri') + + if poster_url: + # Complete the URL if it's relative + if not poster_url.startswith('http'): + poster_url = f"{SD_BASE_URL}/image/{poster_url}" + artwork_map[entry_pid] = poster_url + + logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " + f"{len(artwork_map)} posters found so far.") + + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") + + # Bulk-update ProgramData records with poster URLs + if artwork_map: + programs_to_update = [] + for prog in ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).only('id', 'program_id', 'custom_properties'): + art_key = pid_to_artwork_key.get(prog.program_id) + poster = artwork_map.get(art_key) if art_key else None + if poster: + cp = prog.custom_properties or {} + cp['poster_url'] = poster + prog.custom_properties = cp + programs_to_update.append(prog) + + if programs_to_update: + ProgramData.objects.bulk_update( + programs_to_update, ['custom_properties'], batch_size=1000 + ) + logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") + else: + logger.info("No poster artwork matched committed programs.") + else: + logger.info("No poster artwork found from Schedules Direct.") + + except Exception as art_error: + logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) + + elif fetch_posters: + logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") + + # ------------------------------------------------------------------------- + # Step 9: Apply SD station logos to matched channels if enabled + # ------------------------------------------------------------------------- + use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) + if use_sd_logos: + try: + from apps.channels.models import Channel as ChannelModel, Logo + + channels_to_update = [] + logos_created = 0 + + for channel in ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).select_related('epg_data', 'logo'): + icon_url = (channel.epg_data.icon_url or '').strip() + if not icon_url: + continue + + # Skip if channel already has this logo URL + if channel.logo and channel.logo.url == icon_url: + continue + + # Find or create a Logo object for this URL + try: + logo = Logo.objects.get(url=icon_url) + except Logo.DoesNotExist: + logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" + logo = Logo.objects.create(name=logo_name, url=icon_url) + logos_created += 1 + + channel.logo = logo + channels_to_update.append(channel) + + if channels_to_update: + ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) + logger.info(f"Applied SD logos to {len(channels_to_update)} channels " + f"({logos_created} new logos created).") + else: + logger.info("All matched channels already have current SD logos.") + + except Exception as logo_error: + logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) + + # Prune ProgramData whose end_time has passed. With surgical per-date deletes, + # programs from dates that have rolled off the window are never explicitly removed. + today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) + try: + expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] + if expired_count: + logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") + except Exception as prune_err: + logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") + + # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. + try: + live_program_ids = set( + ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) + .values_list('program_id', flat=True) + ) + pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( + program_id__in=live_program_ids + ).delete()[0] + if pruned_prog_md5_count: + logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") + except Exception as prune_err: + logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + + # ------------------------------------------------------------------------- + # Prune stale poster cache files (>30 days old or orphaned) + # ------------------------------------------------------------------------- + try: + cache_dir = '/data/cache/posters' + if os.path.exists(cache_dir): + import time as time_module + # Collect all poster hashes currently referenced by ProgramData + active_hashes = set() + for url in ProgramData.objects.filter( + epg__epg_source=source, + custom_properties__has_key='poster_url', + ).values_list('custom_properties__poster_url', flat=True): + if url: + active_hashes.add(url.rsplit('/', 1)[-1]) + + pruned_posters = 0 + for fname in os.listdir(cache_dir): + fpath = os.path.join(cache_dir, fname) + if not os.path.isfile(fpath): + continue + file_age = time_module.time() - os.path.getmtime(fpath) + # Remove if older than 30 days OR not referenced by any current program + if file_age > 30 * 24 * 3600 or fname not in active_hashes: + os.remove(fpath) + pruned_posters += 1 + if pruned_posters: + logger.info(f"Pruned {pruned_posters} stale poster cache files.") + except Exception as poster_prune_err: + logger.warning(f"Failed to prune poster cache: {poster_prune_err}") + + # ------------------------------------------------------------------------- + # Done + # ------------------------------------------------------------------------- + success_msg = ( + f"Successfully fetched {total_programs:,} programs for " + f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " + f"({skipped_unmapped:,} programs skipped for unmapped stations)." + ) + source.status = EPGSource.STATUS_SUCCESS + source.last_message = success_msg + source.updated_at = timezone.now() + source.save(update_fields=['status', 'last_message', 'updated_at']) + _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message=success_msg) + log_system_event( + event_type='epg_refresh', + source_name=source.name, + programs=total_programs, + channels=len(mapped_tvg_ids), + skipped_programs=skipped_unmapped, + ) + logger.info(f"Schedules Direct fetch complete for source: {source.name}") + + +# ------------------------------- +# Helper parse functions +# ------------------------------- +def parse_xmltv_time(time_str): + try: + # Basic format validation + if len(time_str) < 14: + logger.warning(f"XMLTV timestamp too short: '{time_str}', using as-is") + dt_obj = datetime.strptime(time_str, '%Y%m%d%H%M%S') + return timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + + # Parse base datetime + dt_obj = datetime.strptime(time_str[:14], '%Y%m%d%H%M%S') + + # Handle timezone if present + if len(time_str) >= 20: # Has timezone info + tz_sign = time_str[15] + tz_hours = int(time_str[16:18]) + tz_minutes = int(time_str[18:20]) + + # Create a timezone object + if tz_sign == '+': + tz_offset = dt_timezone(timedelta(hours=tz_hours, minutes=tz_minutes)) + elif tz_sign == '-': + tz_offset = dt_timezone(timedelta(hours=-tz_hours, minutes=-tz_minutes)) + else: + tz_offset = dt_timezone.utc + + # Make datetime aware with correct timezone + aware_dt = datetime.replace(dt_obj, tzinfo=tz_offset) + # Convert to UTC + aware_dt = aware_dt.astimezone(dt_timezone.utc) + + logger.trace(f"Parsed XMLTV time '{time_str}' to {aware_dt}") + return aware_dt + else: + # No timezone info, assume UTC + aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + logger.trace(f"Parsed XMLTV time without timezone '{time_str}' as UTC: {aware_dt}") + return aware_dt + + except Exception as e: + logger.error(f"Error parsing XMLTV time '{time_str}': {e}", exc_info=True) + raise + + +def parse_schedules_direct_time(time_str): + try: + dt_obj = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%SZ') + aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) + logger.debug(f"Parsed Schedules Direct time '{time_str}' to {aware_dt}") + return aware_dt + except Exception as e: + logger.error(f"Error parsing Schedules Direct time '{time_str}': {e}", exc_info=True) + raise + + +# Re-export from utils to preserve backward compatibility for any callers +from apps.epg.utils import extract_season_episode_from_description, _ONSCREEN_RE # noqa: F401 + + +# Helper function to extract custom properties - moved to a separate function to clean up the code +def extract_custom_properties(prog): + # Create a new dictionary for each call + custom_props = {} + + # Extract categories with a single comprehension to reduce intermediate objects + categories = [cat.text.strip() for cat in prog.findall('category') if cat.text and cat.text.strip()] + if categories: + custom_props['categories'] = categories + + # Extract keywords (new) + keywords = [kw.text.strip() for kw in prog.findall('keyword') if kw.text and kw.text.strip()] + if keywords: + custom_props['keywords'] = keywords + + # Extract episode numbers + for ep_num in prog.findall('episode-num'): + system = ep_num.get('system', '') + if system == 'xmltv_ns' and ep_num.text: + # Parse XMLTV episode-num format (season.episode.part) + parts = ep_num.text.split('.') + if len(parts) >= 2: + if parts[0].strip() != '': + try: + season = int(parts[0]) + 1 # XMLTV format is zero-based + custom_props['season'] = season + except ValueError: + pass + if parts[1].strip() != '': + try: + episode = int(parts[1]) + 1 # XMLTV format is zero-based + custom_props['episode'] = episode + except ValueError: + pass + elif system == 'onscreen' and ep_num.text: + onscreen_text = ep_num.text.strip() + custom_props['onscreen_episode'] = onscreen_text + # Extract season/episode from onscreen format if not already set by xmltv_ns + if 'season' not in custom_props or 'episode' not in custom_props: + match = _ONSCREEN_RE.search(onscreen_text) + if match: + if 'season' not in custom_props: + custom_props['season'] = int(match.group(1)) + if 'episode' not in custom_props: + custom_props['episode'] = int(match.group(2)) + elif system == 'dd_progid' and ep_num.text: + # Store the dd_progid format + custom_props['dd_progid'] = ep_num.text.strip() + # Add support for other systems like thetvdb.com, themoviedb.org, imdb.com + elif system in ['thetvdb.com', 'themoviedb.org', 'imdb.com'] and ep_num.text: + custom_props[f'{system}_id'] = ep_num.text.strip() + + # Extract ratings more efficiently + rating_elem = prog.find('rating') + if rating_elem is not None: + value_elem = rating_elem.find('value') + if value_elem is not None and value_elem.text: + custom_props['rating'] = value_elem.text.strip() + if rating_elem.get('system'): + custom_props['rating_system'] = rating_elem.get('system') + + # Extract star ratings (new) + star_ratings = [] + for star_rating in prog.findall('star-rating'): + value_elem = star_rating.find('value') + if value_elem is not None and value_elem.text: + rating_data = {'value': value_elem.text.strip()} + if star_rating.get('system'): + rating_data['system'] = star_rating.get('system') + star_ratings.append(rating_data) + if star_ratings: + custom_props['star_ratings'] = star_ratings + + # Extract credits more efficiently + credits_elem = prog.find('credits') + if credits_elem is not None: + credits = {} + for credit_type in ['director', 'actor', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if credit_type == 'actor': + # Handle actors with roles and guest status + actors = [] + for actor_elem in credits_elem.findall('actor'): + if actor_elem.text and actor_elem.text.strip(): + actor_data = {'name': actor_elem.text.strip()} + if actor_elem.get('role'): + actor_data['role'] = actor_elem.get('role') + if actor_elem.get('guest') == 'yes': + actor_data['guest'] = True + actors.append(actor_data) + if actors: + credits['actor'] = actors + else: + names = [e.text.strip() for e in credits_elem.findall(credit_type) if e.text and e.text.strip()] + if names: + credits[credit_type] = names + if credits: + custom_props['credits'] = credits + + # Extract other common program metadata + date_elem = prog.find('date') + if date_elem is not None and date_elem.text: + custom_props['date'] = date_elem.text.strip() + + country_elem = prog.find('country') + if country_elem is not None and country_elem.text: + custom_props['country'] = country_elem.text.strip() + + # Extract language information (new) + language_elem = prog.find('language') + if language_elem is not None and language_elem.text: + custom_props['language'] = language_elem.text.strip() + + orig_language_elem = prog.find('orig-language') + if orig_language_elem is not None and orig_language_elem.text: + custom_props['original_language'] = orig_language_elem.text.strip() + + # Extract length (new) + length_elem = prog.find('length') + if length_elem is not None and length_elem.text: + try: + length_value = int(length_elem.text.strip()) + length_units = length_elem.get('units', 'minutes') + custom_props['length'] = {'value': length_value, 'units': length_units} + except ValueError: + pass + + # Extract video information (new) + video_elem = prog.find('video') + if video_elem is not None: + video_info = {} + for video_attr in ['present', 'colour', 'aspect', 'quality']: + attr_elem = video_elem.find(video_attr) + if attr_elem is not None and attr_elem.text: + video_info[video_attr] = attr_elem.text.strip() + if video_info: + custom_props['video'] = video_info + + # Extract audio information (new) + audio_elem = prog.find('audio') + if audio_elem is not None: + audio_info = {} + for audio_attr in ['present', 'stereo']: + attr_elem = audio_elem.find(audio_attr) + if attr_elem is not None and attr_elem.text: + audio_info[audio_attr] = attr_elem.text.strip() + if audio_info: + custom_props['audio'] = audio_info + + # Extract subtitles information (new) + subtitles = [] + for subtitle_elem in prog.findall('subtitles'): + subtitle_data = {} + if subtitle_elem.get('type'): + subtitle_data['type'] = subtitle_elem.get('type') + lang_elem = subtitle_elem.find('language') + if lang_elem is not None and lang_elem.text: + subtitle_data['language'] = lang_elem.text.strip() + if subtitle_data: + subtitles.append(subtitle_data) + + if subtitles: + custom_props['subtitles'] = subtitles + + # Extract reviews (new) + reviews = [] + for review_elem in prog.findall('review'): + if review_elem.text and review_elem.text.strip(): + review_data = {'content': review_elem.text.strip()} + if review_elem.get('type'): + review_data['type'] = review_elem.get('type') + if review_elem.get('source'): + review_data['source'] = review_elem.get('source') + if review_elem.get('reviewer'): + review_data['reviewer'] = review_elem.get('reviewer') + reviews.append(review_data) + if reviews: + custom_props['reviews'] = reviews + + # Extract images (new) + images = [] + for image_elem in prog.findall('image'): + if image_elem.text and image_elem.text.strip(): + image_data = {'url': image_elem.text.strip()} + for attr in ['type', 'size', 'orient', 'system']: + if image_elem.get(attr): + image_data[attr] = image_elem.get(attr) + images.append(image_data) + if images: + custom_props['images'] = images + + icon_elem = prog.find('icon') + if icon_elem is not None and icon_elem.get('src'): + custom_props['icon'] = icon_elem.get('src') + + # Simpler approach for boolean flags - expanded list + for kw in ['previously-shown', 'premiere', 'new', 'live', 'last-chance']: + if prog.find(kw) is not None: + custom_props[kw.replace('-', '_')] = True + + # Extract premiere and last-chance text content if available + premiere_elem = prog.find('premiere') + if premiere_elem is not None: + custom_props['premiere'] = True + if premiere_elem.text and premiere_elem.text.strip(): + custom_props['premiere_text'] = premiere_elem.text.strip() + + last_chance_elem = prog.find('last-chance') + if last_chance_elem is not None: + custom_props['last_chance'] = True + if last_chance_elem.text and last_chance_elem.text.strip(): + custom_props['last_chance_text'] = last_chance_elem.text.strip() + + # Extract previously-shown details + prev_shown_elem = prog.find('previously-shown') + if prev_shown_elem is not None: + custom_props['previously_shown'] = True + prev_shown_data = {} + if prev_shown_elem.get('start'): + prev_shown_data['start'] = prev_shown_elem.get('start') + if prev_shown_elem.get('channel'): + prev_shown_data['channel'] = prev_shown_elem.get('channel') + if prev_shown_data: + custom_props['previously_shown_details'] = prev_shown_data + + return custom_props + + +def clear_element(elem): + """Clear an XML element and its parent to free memory.""" + try: + elem.clear() + parent = elem.getparent() + if parent is not None: + while elem.getprevious() is not None: + del parent[0] + parent.remove(elem) + except Exception as e: + logger.warning(f"Error clearing XML element: {e}", exc_info=True) + + +def detect_file_format(file_path=None, content=None): + """ + Detect file format by examining content or file path. + + Args: + file_path: Path to file (optional) + content: Raw file content bytes (optional) + + Returns: + tuple: (format_type, is_compressed, file_extension) + format_type: 'gzip', 'zip', 'xml', or 'unknown' + is_compressed: Boolean indicating if the file is compressed + file_extension: Appropriate file extension including dot (.gz, .zip, .xml) + """ + # Default return values + format_type = 'unknown' + is_compressed = False + file_extension = '.tmp' + + # First priority: check content magic numbers as they're most reliable + if content: + # We only need the first few bytes for magic number detection + header = content[:20] if len(content) >= 20 else content + + # Check for gzip magic number (1f 8b) + if len(header) >= 2 and header[:2] == b'\x1f\x8b': + return 'gzip', True, '.gz' + + # Check for zip magic number (PK..) + if len(header) >= 2 and header[:2] == b'PK': + return 'zip', True, '.zip' + + # Check for XML - either standard XML header or XMLTV-specific tag + if len(header) >= 5 and (b'' in header): + return 'xml', False, '.xml' + + # Second priority: check file extension - focus on the final extension for compression + if file_path: + logger.debug(f"Detecting file format for: {file_path}") + + # Handle compound extensions like .xml.gz - prioritize compression extensions + lower_path = file_path.lower() + + # Check for compression extensions explicitly + if lower_path.endswith('.gz') or lower_path.endswith('.gzip'): + return 'gzip', True, '.gz' + elif lower_path.endswith('.zip'): + return 'zip', True, '.zip' + elif lower_path.endswith('.xml'): + return 'xml', False, '.xml' + + # Fallback to mimetypes only if direct extension check doesn't work + import mimetypes + mime_type, _ = mimetypes.guess_type(file_path) + logger.debug(f"Guessed MIME type: {mime_type}") + if mime_type: + if mime_type == 'application/gzip' or mime_type == 'application/x-gzip': + return 'gzip', True, '.gz' + elif mime_type == 'application/zip': + return 'zip', True, '.zip' + elif mime_type == 'application/xml' or mime_type == 'text/xml': + return 'xml', False, '.xml' + + # If we reach here, we couldn't reliably determine the format + return format_type, is_compressed, file_extension + + +def generate_dummy_epg(source): + """ + DEPRECATED: This function is no longer used. + + Dummy EPG programs are now generated on-demand when they are requested + (during XMLTV export or EPG grid display), rather than being pre-generated + and stored in the database. + + See: apps/output/views.py - generate_custom_dummy_programs() + + This function remains for backward compatibility but should not be called. + """ + logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " + f"Dummy EPG programs are now generated on-demand.") + return True From 763d1430fdd10801f59c52c960f1751f9737dc9c Mon Sep 17 00:00:00 2001 From: mwhit Date: Thu, 4 Jun 2026 04:14:23 +0000 Subject: [PATCH 33/76] fix: add channel_group_id to M3U stream bulk_update fields Streams matched by hash during refresh were not updating channel_group_id, causing cleanup to delete all streams when group IDs change (e.g., after migration reset). Adds channel_group_id to comparison, assignment, only() fetch, and bulk_update in both standard and XC processing paths. --- apps/m3u/tasks.py | 7078 +++++++++++++++++++++++---------------------- 1 file changed, 3650 insertions(+), 3428 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index fc43c36d..3b504ebf 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1,256 +1,1643 @@ -# apps/epg/tasks.py - +# apps/m3u/tasks.py import logging -import gzip -import html.entities -import os -import uuid +import re +import regex import requests -import time # Add import for tracking download progress -from datetime import datetime, timedelta, timezone as dt_timezone -import gc # Add garbage collection module -import json -from lxml import etree # Using lxml exclusively -import psutil # Add import for memory tracking -import zipfile - -from celery import shared_task +import os +import gc +import gzip, zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from celery.app.control import Inspect +from celery.result import AsyncResult +from celery import shared_task, current_app, group from django.conf import settings -from django.db import connection, transaction -from django.db.models import Q -from django.utils import timezone -from apps.channels.models import Channel -from core.models import UserAgent, CoreSettings - +from django.core.cache import cache +from django.db import models, transaction +from .models import M3UAccount +from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount from asgiref.sync import async_to_sync from channels.layers import get_channel_layer - -from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 -from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event +from django.utils import timezone +import time +import json +from core.utils import ( + RedisClient, + acquire_task_lock, + release_task_lock, + TaskLockRenewer, + natural_sort_key, + log_system_event, +) +from core.models import CoreSettings, UserAgent +from asgiref.sync import async_to_sync +from core.xtream_codes import Client as XCClient +from core.utils import send_websocket_update +from .utils import normalize_stream_url logger = logging.getLogger(__name__) -SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' -SD_DAYS_TO_FETCH = 20 -SD_PROGRAM_BATCH_SIZE = 5000 +BATCH_SIZE = 1500 # Optimized batch size for threading +m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") -# DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named -# entities so lxml/libxml2 can resolve references like é correctly -# instead of silently dropping them in recovery mode. -# The 5 XML-predefined entities (amp, lt, gt, quot, apos) are always -# recognised by the XML spec and must not be redeclared. -_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'}) +_EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') -def _build_html_entity_doctype() -> bytes: - """Build a DOCTYPE internal subset declaring all HTML 4 named entities.""" - lines = [b'\n'.encode('ascii')) - lines.append(b']>\n') - return b''.join(lines) +def fetch_m3u_lines(account, use_cache=False): + os.makedirs(m3u_dir, exist_ok=True) + file_path = os.path.join(m3u_dir, f"{account.id}.m3u") + + """Fetch M3U file lines efficiently.""" + if account.server_url: + if not use_cache or not os.path.exists(file_path): + try: + # Try to get account-specific user agent first + user_agent_obj = account.get_user_agent() + user_agent = ( + user_agent_obj.user_agent + if user_agent_obj + else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + ) + + logger.debug( + f"Using user agent: {user_agent} for M3U account: {account.name}" + ) + headers = {"User-Agent": user_agent} + logger.info(f"Fetching from URL {account.server_url}") + + # Set account status to FETCHING before starting download + account.status = M3UAccount.Status.FETCHING + account.last_message = "Starting download..." + account.save(update_fields=["status", "last_message"]) + + response = requests.get( + account.server_url, headers=headers, stream=True, + timeout=(30, 60), # 30s connect, 60s read between chunks + ) + + # Log the actual response details for debugging + logger.debug(f"HTTP Response: {response.status_code} from {account.server_url}") + logger.debug(f"Content-Type: {response.headers.get('content-type', 'Not specified')}") + logger.debug(f"Content-Length: {response.headers.get('content-length', 'Not specified')}") + logger.debug(f"Response headers: {dict(response.headers)}") + + # Check if we've been redirected to a different URL + if hasattr(response, 'url') and response.url != account.server_url: + logger.warning(f"Request was redirected from {account.server_url} to {response.url}") + + # Check for ANY non-success status code FIRST (before raise_for_status) + if response.status_code < 200 or response.status_code >= 300: + # For error responses, read the content immediately (not streaming) + try: + response_content = response.text[:1000] # Capture up to 1000 characters + logger.error(f"Error response content: {response_content!r}") + except Exception as e: + logger.error(f"Could not read error response content: {e}") + response_content = "Could not read error response content" + + # Provide specific messages for known non-standard codes + if response.status_code == 884: + error_msg = f"Server returned HTTP 884 (authentication/authorization failure) from URL: {account.server_url}. Server message: {response_content}" + elif response.status_code >= 800: + error_msg = f"Server returned non-standard HTTP status {response.status_code} from URL: {account.server_url}. Server message: {response_content}" + elif response.status_code == 404: + error_msg = f"M3U file not found (404) at URL: {account.server_url}. Server message: {response_content}" + elif response.status_code == 403: + error_msg = f"Access forbidden (403) to M3U file at URL: {account.server_url}. Server message: {response_content}" + elif response.status_code == 401: + error_msg = f"Authentication required (401) for M3U file at URL: {account.server_url}. Server message: {response_content}" + elif response.status_code == 500: + error_msg = f"Server error (500) while fetching M3U file from URL: {account.server_url}. Server message: {response_content}" + else: + error_msg = f"HTTP error ({response.status_code}) while fetching M3U file from URL: {account.server_url}. Server message: {response_content}" + + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account.id, + "downloading", + 100, + status="error", + error=error_msg, + ) + return [], False + + # Only call raise_for_status if we have a success code (this should not raise now) + response.raise_for_status() + + total_size = int(response.headers.get("Content-Length", 0)) + downloaded = 0 + start_time = time.time() + last_update_time = start_time + progress = 0 + has_content = False + + # Stream directly to a temp file to avoid holding the entire + # M3U in memory (large files can be 100MB+, which would use + # ~3x that in RAM in certain approaches). + temp_path = file_path + ".tmp" + try: + send_m3u_update(account.id, "downloading", 0) + with open(temp_path, "wb") as tmp_file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + tmp_file.write(chunk) + has_content = True + + downloaded += len(chunk) + elapsed_time = time.time() - start_time + + # Calculate download speed in KB/s + speed = downloaded / elapsed_time / 1024 # in KB/s + + # Calculate progress percentage + if total_size and total_size > 0: + progress = (downloaded / total_size) * 100 + + # Time remaining (in seconds) + time_remaining = ( + (total_size - downloaded) / (speed * 1024) + if speed > 0 + else 0 + ) + + current_time = time.time() + if current_time - last_update_time >= 0.5: + last_update_time = current_time + if progress > 0: + # Update the account's last_message with detailed progress info + progress_msg = f"Downloading: {progress:.1f}% - {speed:.1f} KB/s - {time_remaining:.1f}s remaining" + account.last_message = progress_msg + account.save(update_fields=["last_message"]) + + send_m3u_update( + account.id, + "downloading", + progress, + speed=speed, + elapsed_time=elapsed_time, + time_remaining=time_remaining, + message=progress_msg, + ) + + # Check if we actually received any content + logger.info(f"Download completed. Has content: {has_content}, Content length: {downloaded} bytes") + if not has_content or downloaded == 0: + error_msg = f"Server responded successfully (HTTP {response.status_code}) but provided empty M3U file from URL: {account.server_url}" + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account.id, + "downloading", + 100, + status="error", + error=error_msg, + ) + return [], False + + # Validate the file by reading only the first portion from + # disk — no need to load the entire file into memory just + # to check the header. + VALIDATION_READ_SIZE = 32768 # 32KB covers headers comfortably + try: + with open(temp_path, "rb") as vf: + head_bytes = vf.read(VALIDATION_READ_SIZE) + head_str = head_bytes.decode('utf-8', errors='ignore') + head_lines = head_str.strip().split('\n') + + # Count total lines efficiently without loading full file + with open(temp_path, "rb") as vf: + total_lines = sum(1 for _ in vf) + + # Log first few lines for debugging (be careful not to log too much) + preview_lines = head_lines[:5] + logger.info(f"Content preview (first 5 lines): {preview_lines}") + logger.info(f"Total lines in content: {total_lines}") + + # Check if it's a valid M3U file (should start with #EXTM3U or contain M3U-like content) + is_valid_m3u = False + + # First, check if this looks like an error response disguised as 200 OK + head_lower = head_str.lower() + if any(error_indicator in head_lower for error_indicator in [ + ' dict: """ + Parse an EXTINF line from an M3U file. + This function removes the "#EXTINF:" prefix, then extracts all key="value" attributes, + and treats everything after the last attribute as the display name. - __slots__ = ('_prefix', '_prefix_pos', '_file') - - def __init__(self, prefix: bytes, file_obj): - self._prefix = prefix - self._prefix_pos = 0 - self._file = file_obj - - def read(self, size=-1): - prefix_len = len(self._prefix) - if self._prefix_pos >= prefix_len: - return self._file.read(size) - remaining = prefix_len - self._prefix_pos - if size < 0: - chunk = self._prefix[self._prefix_pos:] + self._file.read() - self._prefix_pos = prefix_len - return chunk - if size <= remaining: - chunk = self._prefix[self._prefix_pos:self._prefix_pos + size] - self._prefix_pos += size - return chunk - chunk = self._prefix[self._prefix_pos:] - self._prefix_pos = prefix_len - return chunk + self._file.read(size - remaining) - - def close(self): - self._file.close() - - def __enter__(self): - return self - - def __exit__(self, *_): - self.close() - - -def _open_xmltv_file(file_path: str): - """Open an XMLTV file for lxml iterparse, injecting an HTML entity DOCTYPE. - - Prepends a block that declares all 252 HTML 4 named - entities so lxml/libxml2 resolves references like é correctly - instead of silently dropping them in recovery mode. This involves zero - disk I/O (the DOCTYPE is streamed in-memory before the file content). - - If the file already contains a declaration the file is returned - unchanged; a second DOCTYPE would be invalid XML. - - The caller is responsible for closing the returned object. + Returns a dictionary with: + - 'attributes': a dict of attribute key/value pairs (e.g. tvg-id, tvg-logo, group-title) + - 'display_name': the text after the attributes (the fallback display name) + - 'name': the value from tvg-name (if present) or the display name otherwise. """ - f = open(file_path, 'rb') - start = f.read(512) - - # Do not inject if the file already declares a DOCTYPE. - if b'= 0: - decl_end = start.find(b'?>', xml_pos) - if decl_end >= 0: - xml_decl = start[:decl_end + 2] - f.seek(decl_end + 2) - return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f) - - # No XML declaration found; insert DOCTYPE at the very start of the file. - f.seek(0) - return _PrependStream(_HTML_ENTITY_DOCTYPE, f) - - -def validate_icon_url_fast(icon_url, max_length=None): - """ - Fast validation for icon URLs during parsing. - Returns None if URL is too long, original URL otherwise. - If max_length is None, gets it dynamically from the EPGData model field. - """ - if max_length is None: - # Get max_length dynamically from the model field - max_length = EPGData._meta.get_field('icon_url').max_length - - if icon_url and len(icon_url) > max_length: - logger.warning(f"Icon URL too long ({len(icon_url)} > {max_length}), skipping: {icon_url[:100]}...") + if not line.startswith("#EXTINF:"): return None - return icon_url + content = line[len("#EXTINF:") :].strip() + + # Single pass: extract all attributes AND track the last attribute position. + # Keys are normalised to lowercase so downstream code can use plain dict.get() + attrs = {} + last_attr_end = 0 + + for match in _EXTINF_ATTR_RE.finditer(content): + attrs[match.group(1).lower()] = match.group(3) + last_attr_end = match.end() + + # Everything after the last attribute (skipping leading comma and whitespace) is the display name + if last_attr_end > 0: + remaining = content[last_attr_end:].strip() + # Remove leading comma if present + if remaining.startswith(','): + remaining = remaining[1:].strip() + display_name = remaining + else: + # No attributes found, try the old comma-split method as fallback + parts = content.split(',', 1) + if len(parts) == 2: + display_name = parts[1].strip() + else: + display_name = content.strip() + + # Per the base #EXTINF spec, the comma text is the canonical human-readable title. + # Fall back to tvc-guide-title, then tvg-name (which some providers use as an EPG key, + # not a display label), and finally the raw content if everything else is empty. + name = display_name or attrs.get("tvc-guide-title") or attrs.get("tvg-name") or content.strip() + return {"attributes": attrs, "display_name": display_name, "name": name} -MAX_EXTRACT_CHUNK_SIZE = 65536 # 64kb (base2) +def iter_m3u_entries(lines): + """ + Generator that yields fully-assembled M3U stream entries from raw lines. + + Each yielded dict is guaranteed to contain a ``url`` key in addition to the + fields produced by :func:`parse_extinf_line` (``attributes``, ``display_name``, + ``name``). Recognised extended-tag lines that appear *between* an ``#EXTINF`` + and its URL are accumulated into the pending entry so they are available for + downstream processing: + + - ``#EXTGRP`` — sets ``attributes["group-title"]`` when no ``group-title`` + attribute was present on the ``#EXTINF`` line (explicit attribute wins). + - ``#EXTVLCOPT`` — stored as a list under the ``vlc_opts`` key. + + Unknown directives (``#KODIPROP``, etc.) and blank lines are + silently skipped while keeping the pending entry intact. A second ``#EXTINF`` + before a URL discards the first entry with a warning. A trailing ``#EXTINF`` + at end-of-file with no URL is also discarded. + + Adding support for a new directive requires only a new ``elif`` branch here; + no other code needs to change. + """ + pending = None + pending_line_num = None + for line_num, raw_line in enumerate(lines, 1): + line = raw_line.strip() + if not line: + continue + + if line.startswith("#EXTINF"): + if pending is not None: + logger.warning( + f"Line {pending_line_num}: #EXTINF had no URL (next #EXTINF at line {line_num}); " + f"discarding entry: {list(pending['attributes'].items())[:3]}" + ) + parsed = parse_extinf_line(line) + if parsed is None: + logger.warning(f"Line {line_num}: Failed to parse #EXTINF: {line[:200]}") + pending = parsed # None if malformed; URL branch guards on `pending is not None` + pending_line_num = line_num + + elif line.startswith("#EXTGRP:"): + # Only apply when group-title is absent — explicit attribute wins. + if pending is not None and "group-title" not in pending["attributes"]: + pending["attributes"]["group-title"] = line[len("#EXTGRP:"):].strip() + # else: #EXTGRP outside an entry, or group-title already set — silently skip + + elif line.startswith("#EXTVLCOPT:"): + if pending is not None: + pending.setdefault("vlc_opts", []).append(line[len("#EXTVLCOPT:"):]) + # else: #EXTVLCOPT outside an entry — silently skip + + elif pending is not None and line.startswith(("http", "rtsp", "rtp", "udp")): + pending["url"] = normalize_stream_url(line) if line.startswith("udp") else line + yield pending + pending = None + pending_line_num = None + + # else: unknown directive or bare content — skip, keeping pending intact + + if pending is not None: + logger.warning( + f"Line {pending_line_num}: #EXTINF at end of file had no URL; " + f"discarding entry: {list(pending['attributes'].items())[:3]}" + ) -def send_epg_update(source_id, action, progress, **kwargs): - """Send WebSocket update about EPG download/parsing progress""" - # Start with the base data dictionary - data = { - "progress": progress, - "type": "epg_refresh", - "source": source_id, - "action": action, +@shared_task +def refresh_m3u_accounts(): + """Queue background parse for all active M3UAccounts.""" + active_accounts = M3UAccount.objects.filter(is_active=True) + count = 0 + for account in active_accounts: + refresh_single_m3u_account.delay(account.id) + count += 1 + + msg = f"Queued M3U refresh for {count} active account(s)." + logger.info(msg) + return msg + + +def check_field_lengths(streams_to_create): + for stream in streams_to_create: + for field, value in stream.__dict__.items(): + if isinstance(value, str) and len(value) > 255: + print(f"{field} --- {value}") + + print("") + print("") + + +@shared_task +def process_groups(account, groups, scan_start_time=None): + """Process groups and update their relationships with the M3U account. + + Args: + account: M3UAccount instance + groups: Dict of group names to custom properties + scan_start_time: Timestamp when the scan started (for consistent last_seen marking) + """ + # Use scan_start_time if provided, otherwise current time + # This ensures consistency with stream processing and cleanup logic + if scan_start_time is None: + scan_start_time = timezone.now() + + existing_groups = { + group.name: group + for group in ChannelGroup.objects.filter(name__in=groups.keys()) + } + logger.info(f"Currently {len(existing_groups)} existing groups") + + # Check if we should auto-enable new groups based on account settings + account_custom_props = account.custom_properties or {} + auto_enable_new_groups_live = account_custom_props.get("auto_enable_new_groups_live", True) + + # Separate existing groups from groups that need to be created + existing_group_objs = [] + groups_to_create = [] + + for group_name, custom_props in groups.items(): + if group_name in existing_groups: + existing_group_objs.append(existing_groups[group_name]) + else: + groups_to_create.append(ChannelGroup(name=group_name)) + + # Create new groups and fetch them back with IDs + newly_created_group_objs = [] + if groups_to_create: + logger.info(f"Creating {len(groups_to_create)} new groups for account {account.id}") + newly_created_group_objs = list(ChannelGroup.bulk_create_and_fetch(groups_to_create)) + logger.debug(f"Successfully created {len(newly_created_group_objs)} new groups") + + # Combine all groups + all_group_objs = existing_group_objs + newly_created_group_objs + + # Get existing relationships for this account + existing_relationships = { + rel.channel_group.name: rel + for rel in ChannelGroupM3UAccount.objects.filter( + m3u_account=account, + channel_group__name__in=groups.keys() + ).select_related('channel_group') } - # Add the additional key-value pairs from kwargs - data.update(kwargs) + relations_to_create = [] + relations_to_update = [] - # Use the standardized update function with garbage collection for program parsing - # This is a high-frequency operation that needs more aggressive memory management - collect_garbage = action == "parsing_programs" and progress % 10 == 0 - send_websocket_update('updates', 'update', data, collect_garbage=collect_garbage) + for group in all_group_objs: + custom_props = groups.get(group.name, {}) - # Explicitly clear references - data = None + if group.name in existing_relationships: + # Update existing relationship if xc_id has changed (preserve other custom properties) + existing_rel = existing_relationships[group.name] - # For high-frequency parsing, occasionally force additional garbage collection - # to prevent memory buildup - if action == "parsing_programs" and progress % 50 == 0: - gc.collect() + # Get existing custom properties (now JSONB, no need to parse) + existing_custom_props = existing_rel.custom_properties or {} + + # Get the new xc_id from groups data + new_xc_id = custom_props.get("xc_id") + existing_xc_id = existing_custom_props.get("xc_id") + + # Only update if xc_id has changed + if new_xc_id != existing_xc_id: + # Merge new xc_id with existing custom properties to preserve user settings + updated_custom_props = existing_custom_props.copy() + if new_xc_id is not None: + updated_custom_props["xc_id"] = new_xc_id + elif "xc_id" in updated_custom_props: + # Remove xc_id if it's no longer provided (e.g., converting from XC to standard) + del updated_custom_props["xc_id"] + + existing_rel.custom_properties = updated_custom_props + existing_rel.last_seen = scan_start_time + existing_rel.is_stale = False + relations_to_update.append(existing_rel) + logger.debug(f"Updated xc_id for group '{group.name}' from '{existing_xc_id}' to '{new_xc_id}' - account {account.id}") + else: + # Update last_seen even if xc_id hasn't changed + existing_rel.last_seen = scan_start_time + existing_rel.is_stale = False + relations_to_update.append(existing_rel) + logger.debug(f"xc_id unchanged for group '{group.name}' - account {account.id}") + else: + # Create new relationship - this group is new to this M3U account + # Use the auto_enable setting to determine if it should start enabled + if not auto_enable_new_groups_live: + logger.info(f"Group '{group.name}' is new to account {account.id} - creating relationship but DISABLED (auto_enable_new_groups_live=False)") + + relations_to_create.append( + ChannelGroupM3UAccount( + channel_group=group, + m3u_account=account, + custom_properties=custom_props, + enabled=auto_enable_new_groups_live, + last_seen=scan_start_time, + is_stale=False, + ) + ) + + # Bulk create new relationships + if relations_to_create: + ChannelGroupM3UAccount.objects.bulk_create(relations_to_create, ignore_conflicts=True) + logger.debug(f"Created {len(relations_to_create)} new group relationships for account {account.id}") + + # Bulk update existing relationships + if relations_to_update: + ChannelGroupM3UAccount.objects.bulk_update(relations_to_update, ['custom_properties', 'last_seen', 'is_stale']) + logger.info(f"Updated {len(relations_to_update)} existing group relationships for account {account.id}") -def _sd_send_ws_sync(source_id, action, progress, **kwargs): +def cleanup_stale_group_relationships(account, scan_start_time): """ - Sends a WebSocket progress update synchronously via Redis, bypassing gevent.spawn. - - In Celery prefork workers that inherit gevent monkey-patching from uWSGI, - gevent.spawn schedules coroutines that never execute because there is no - running gevent hub in the worker process. This function writes directly to - Redis using the channels_redis 4.x wire format, guaranteeing delivery - regardless of the execution context. + Remove group relationships that haven't been seen since the stale retention period. + This follows the same logic as stream cleanup for consistency. """ + # Calculate cutoff date for stale group relationships + stale_cutoff = scan_start_time - timezone.timedelta(days=account.stale_stream_days) + logger.info( + f"Removing group relationships not seen since {stale_cutoff} for M3U account {account.id}" + ) + + # Find stale relationships + stale_relationships = ChannelGroupM3UAccount.objects.filter( + m3u_account=account, + last_seen__lt=stale_cutoff + ).select_related('channel_group') + + relations_to_delete = list(stale_relationships) + deleted_count = len(relations_to_delete) + + if deleted_count > 0: + logger.info( + f"Found {deleted_count} stale group relationships for account {account.id}: " + f"{[rel.channel_group.name for rel in relations_to_delete]}" + ) + + # Delete the stale relationships + stale_relationships.delete() + + # Check if any of the deleted relationships left groups with no remaining associations + orphaned_group_ids = [] + for rel in relations_to_delete: + group = rel.channel_group + + # Check if this group has any remaining M3U account relationships + remaining_m3u_relationships = ChannelGroupM3UAccount.objects.filter( + channel_group=group + ).exists() + + # Check if this group has any direct channels (not through M3U accounts) + has_direct_channels = group.related_channels().exists() + + # If no relationships and no direct channels, it's safe to delete + if not remaining_m3u_relationships and not has_direct_channels: + orphaned_group_ids.append(group.id) + logger.debug(f"Group '{group.name}' has no remaining associations and will be deleted") + + # Delete truly orphaned groups + if orphaned_group_ids: + deleted_groups = list(ChannelGroup.objects.filter(id__in=orphaned_group_ids).values_list('name', flat=True)) + ChannelGroup.objects.filter(id__in=orphaned_group_ids).delete() + logger.info(f"Deleted {len(orphaned_group_ids)} orphaned groups that had no remaining associations: {deleted_groups}") + else: + logger.debug(f"No stale group relationships found for account {account.id}") + + return deleted_count + + +def collect_xc_streams(account_id, enabled_groups): + """Collect all XC streams in a single API call and filter by enabled groups.""" + account = M3UAccount.objects.get(id=account_id) + all_streams = [] + + # Create a mapping from category_id to group info for filtering + enabled_category_ids = {} + for group_name, props in enabled_groups.items(): + if "xc_id" in props: + enabled_category_ids[str(props["xc_id"])] = { + "name": group_name, + "props": props + } + try: - import msgpack - import redis as redis_lib - from django.conf import settings + with XCClient( + account.server_url, + account.username, + account.password, + account.get_user_agent(), + ) as xc_client: - data = {"progress": progress, "type": "epg_refresh", "source": source_id, "action": action} - data.update(kwargs) - message = {"type": "update", "data": data} + # Fetch ALL live streams in a single API call (much more efficient) + logger.info("Fetching ALL live streams from XC provider...") + all_xc_streams = xc_client.get_all_live_streams() # Get all streams without category filter - redis_url = getattr(settings, "CELERY_BROKER_URL", "redis://localhost:6379/0") - r = redis_lib.from_url(redis_url, decode_responses=False) + if not all_xc_streams: + logger.warning("No live streams returned from XC provider") + return [] - prefix = "asgi" - group_name = "updates" - group_key = f"{prefix}:group:{group_name}" - group_expiry = 86400 - channel_expiry = 60 - rand_len = 12 - now = time.time() + logger.info(f"Retrieved {len(all_xc_streams)} total live streams from provider") - r.zremrangebyscore(group_key, 0, now - group_expiry) - raw = r.zrange(group_key, 0, -1) - if not raw: - return + # Filter streams based on enabled categories + filtered_count = 0 + for stream in all_xc_streams: + # Fall back to a generated name if the provider returns null/empty + stream_name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" + if not stream.get("name"): + logger.warning( + f"XC stream has null/empty name; using generated name '{stream_name}' " + f"(stream_id={stream.get('stream_id', 'unknown')})" + ) - channels = [m.decode("utf-8") if isinstance(m, bytes) else m for m in raw] - nonlocal_map = {} - for ch in channels: - pos = ch.find("!") - nl = ch[:pos + 1] if pos >= 0 else ch - nonlocal_map.setdefault(nl, []).append(ch) + # Get the category_id for this stream + category_id = str(stream.get("category_id", "")) + + # Only include streams from enabled categories + if category_id in enabled_category_ids: + group_info = enabled_category_ids[category_id] + + # Convert XC stream to our standard format with all properties preserved + stream_data = { + "name": stream_name, + "url": xc_client.get_stream_url(stream["stream_id"]), + "attributes": { + "tvg-id": stream.get("epg_channel_id", ""), + "tvg-logo": stream.get("stream_icon", ""), + "group-title": group_info["name"], + # Preserve all XC stream properties as custom attributes + "stream_id": str(stream.get("stream_id", "")), + "num": stream.get("num"), + "category_id": category_id, + "stream_type": stream.get("stream_type", ""), + "added": stream.get("added", ""), + "is_adult": str(stream.get("is_adult", "0")), + "custom_sid": stream.get("custom_sid", ""), + # Include any other properties that might be present + **{k: str(v) for k, v in stream.items() if k not in [ + "name", "stream_id", "epg_channel_id", "stream_icon", + "category_id", "stream_type", "added", "is_adult", "custom_sid", "num" + ] and v is not None} + } + } + all_streams.append(stream_data) + filtered_count += 1 - pipe = r.pipeline(transaction=False) - for nl, chs in nonlocal_map.items(): - channel_key = prefix + nl - msg = dict(message) - msg["__asgi_channel__"] = chs - serialized = os.urandom(rand_len) + msgpack.packb(msg) - pipe.zadd(channel_key, {serialized: now}) - pipe.expire(channel_key, channel_expiry) - pipe.execute() - r.close() except Exception as e: - logger.warning(f"SD WebSocket sync send failed: {e}") - # Fall back to standard path - send_epg_update(source_id, action, progress, **kwargs) + logger.error(f"Failed to fetch XC streams: {str(e)}") + return [] + + logger.info(f"Filtered {filtered_count} streams from {len(enabled_category_ids)} enabled categories") + return all_streams + +def process_xc_category_direct(account_id, batch, groups, hash_keys): + from django.db import connections + + # Ensure clean database connections for threading + connections.close_all() + + account = M3UAccount.objects.get(id=account_id) + + streams_to_create = [] + streams_to_update = [] + stream_hashes = {} + + try: + with XCClient( + account.server_url, + account.username, + account.password, + account.get_user_agent(), + ) as xc_client: + # Log the batch details to help with debugging + logger.debug(f"Processing XC batch: {batch}") + + for group_name, props in batch.items(): + # Check if we have a valid xc_id for this group + if "xc_id" not in props: + logger.error( + f"Missing xc_id for group {group_name} in batch {batch}" + ) + continue + + # Get actual group ID from the mapping + group_id = groups.get(group_name) + if not group_id: + logger.error(f"Group {group_name} not found in enabled groups") + continue + + try: + logger.debug( + f"Fetching streams for XC category: {group_name} (ID: {props['xc_id']})" + ) + streams = xc_client.get_live_category_streams(props["xc_id"]) + + if not streams: + logger.warning( + f"No streams found for XC category {group_name} (ID: {props['xc_id']})" + ) + continue + + logger.debug( + f"Found {len(streams)} streams for category {group_name}" + ) + + for stream in streams: + name = stream.get("name") or f"{account.name} - {stream.get('stream_id', 'Unknown')}" + if not stream.get("name"): + logger.warning( + f"XC stream has null/empty name in category {group_name}; " + f"using generated name '{name}' (stream_id={stream.get('stream_id', 'unknown')})" + ) + raw_stream_id = stream.get("stream_id", "") + provider_stream_id = None + if raw_stream_id: + try: + provider_stream_id = int(raw_stream_id) + except (ValueError, TypeError): + pass + url = xc_client.get_stream_url(stream["stream_id"]) + tvg_id = stream.get("epg_channel_id", "") + tvg_logo = stream.get("stream_icon", "") + group_title = group_name + stream_chno = stream.get("num") + # Convert stream_chno to float if valid, otherwise None + if stream_chno is not None: + try: + stream_chno = float(stream_chno) + except (ValueError, TypeError): + stream_chno = None + + stream_hash = Stream.generate_hash_key( + name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, + account_type='XC', stream_id=provider_stream_id + ) + stream_props = { + "name": name, + "url": url, + "logo_url": tvg_logo, + "tvg_id": tvg_id, + "m3u_account": account, + "channel_group_id": int(group_id), + "stream_hash": stream_hash, + "custom_properties": stream, + "is_adult": parse_is_adult(stream.get("is_adult", 0)), + "is_stale": False, + "stream_id": provider_stream_id, + "stream_chno": stream_chno, + } + + if stream_hash not in stream_hashes: + stream_hashes[stream_hash] = stream_props + except Exception as e: + logger.error( + f"Error processing XC category {group_name} (ID: {props['xc_id']}): {str(e)}" + ) + continue + + # Process all found streams + existing_streams = { + s.stream_hash: s + for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( + 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id' + ) + } + + for stream_hash, stream_props in stream_hashes.items(): + if stream_hash in existing_streams: + obj = existing_streams[stream_hash] + # Optimized field comparison for XC streams + changed = ( + obj.name != stream_props["name"] or + obj.url != stream_props["url"] or + obj.logo_url != stream_props["logo_url"] or + obj.tvg_id != stream_props["tvg_id"] or + obj.custom_properties != stream_props["custom_properties"] or + obj.is_adult != stream_props["is_adult"] or + obj.stream_id != stream_props["stream_id"] or + obj.stream_chno != stream_props["stream_chno"] or + obj.channel_group_id != stream_props["channel_group_id"] + ) + + if changed: + for key, value in stream_props.items(): + setattr(obj, key, value) + obj.last_seen = timezone.now() + obj.updated_at = timezone.now() # Update timestamp only for changed streams + obj.is_stale = False + streams_to_update.append(obj) + else: + # Always update last_seen, even if nothing else changed + obj.last_seen = timezone.now() + obj.is_stale = False + # Don't update updated_at for unchanged streams + streams_to_update.append(obj) + + # Remove from existing_streams since we've processed it + del existing_streams[stream_hash] + else: + stream_props["last_seen"] = timezone.now() + stream_props["updated_at"] = ( + timezone.now() + ) # Set initial updated_at for new streams + stream_props["is_stale"] = False + streams_to_create.append(Stream(**stream_props)) + + try: + with transaction.atomic(): + if streams_to_create: + Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) + + if streams_to_update: + # Simplified bulk update for better performance + Stream.objects.bulk_update( + streams_to_update, + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'channel_group_id'], + batch_size=150 # Smaller batch size for XC processing + ) + + # Update last_seen for any remaining existing streams that weren't processed + if len(existing_streams.keys()) > 0: + Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) + except Exception as e: + logger.error(f"Bulk operation failed for XC streams: {str(e)}") + + retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + + except Exception as e: + logger.error(f"XC category processing error: {str(e)}") + retval = f"Error processing XC batch: {str(e)}" + finally: + # Clean up database connections for threading + connections.close_all() + + # Aggressive garbage collection + del streams_to_create, streams_to_update, stream_hashes, existing_streams + gc.collect() + + return retval -def delete_epg_refresh_task_by_id(epg_id): +def process_m3u_batch_direct(account_id, batch, groups, hash_keys): + """Processes a batch of M3U streams using bulk operations with thread-safe DB connections.""" + from django.db import connections + + # Ensure clean database connections for threading + connections.close_all() + + account = M3UAccount.objects.get(id=account_id) + + compiled_filters = [ + ( + re.compile( + f.regex_pattern, + ( + re.IGNORECASE + if (f.custom_properties or {}).get( + "case_sensitive", True + ) + == False + else 0 + ), + ), + f, + ) + for f in account.filters.order_by("order") + ] + + streams_to_create = [] + streams_to_update = [] + stream_hashes = {} + + name_max_length = Stream._meta.get_field('name').max_length + + logger.debug(f"Processing batch of {len(batch)} for M3U account {account_id}") + if compiled_filters: + logger.debug(f"Using compiled filters: {[f[1].regex_pattern for f in compiled_filters]}") + for stream_info in batch: + try: + name, url = stream_info["name"], stream_info["url"] + + # Validate URL length - maximum of 4096 characters + if url and len(url) > 4096: + logger.warning(f"Skipping stream '{name}': URL too long ({len(url)} characters, max 4096)") + continue + + # Truncate name if it exceeds the model field limit + if name and len(name) > name_max_length: + logger.warning(f"Stream name too long ({len(name)} > {name_max_length}), truncating: {name[:80]}...") + name = name[:name_max_length] + + tvg_id, tvg_logo = get_case_insensitive_attr( + stream_info["attributes"], "tvg-id", "" + ), get_case_insensitive_attr(stream_info["attributes"], "tvg-logo", "") + group_title = get_case_insensitive_attr( + stream_info["attributes"], "group-title", "Default Group" + ) + logger.debug(f"Processing stream: {name} - {url} in group {group_title}") + include = True + for pattern, filter in compiled_filters: + logger.trace(f"Checking filter pattern {pattern}") + target = name + if filter.filter_type == "url": + target = url + elif filter.filter_type == "group": + target = group_title + + if pattern.search(target or ""): + logger.debug( + f"Stream {name} - {url} matches filter pattern {filter.regex_pattern}" + ) + include = not filter.exclude + break + + if not include: + logger.debug(f"Stream excluded by filter, skipping.") + continue + + # Filter out disabled groups for this account + if group_title not in groups: + logger.debug( + f"Skipping stream in disabled or excluded group: {group_title}" + ) + continue + + # Determine provider-specific fields first + provider_stream_id = None + channel_num = None + account_type_for_hash = None + + if account.account_type == M3UAccount.Types.XC: + account_type_for_hash = 'XC' + raw_stream_id = stream_info["attributes"].get("stream_id", "") + if raw_stream_id: + try: + provider_stream_id = int(raw_stream_id) + except (ValueError, TypeError): + pass + raw_num = stream_info["attributes"].get("num") + if raw_num is not None: + try: + channel_num = float(raw_num) + except (ValueError, TypeError): + pass + else: + # For standard M3U accounts, check for tvg-chno or channel-number + tvg_chno = get_case_insensitive_attr(stream_info["attributes"], "tvg-chno", None) + if tvg_chno is None: + tvg_chno = get_case_insensitive_attr(stream_info["attributes"], "channel-number", None) + if tvg_chno is not None: + try: + channel_num = float(tvg_chno) + except (ValueError, TypeError): + pass + + # Generate hash once with all parameters + stream_hash = Stream.generate_hash_key( + name, url, tvg_id, hash_keys, m3u_id=account_id, group=group_title, + account_type=account_type_for_hash, stream_id=provider_stream_id + ) + + stream_props = { + "name": name, + "url": url, + "logo_url": tvg_logo, + "tvg_id": tvg_id, + "m3u_account": account, + "channel_group_id": int(groups.get(group_title)), + "stream_hash": stream_hash, + "custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"], + "is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)), + "is_stale": False, + "stream_id": provider_stream_id, + "stream_chno": channel_num, + } + + if stream_hash not in stream_hashes: + stream_hashes[stream_hash] = stream_props + except Exception as e: + logger.error(f"Failed to process stream {name}: {e}") + logger.error(json.dumps(stream_info)) + + existing_streams = { + s.stream_hash: s + for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys()).select_related('m3u_account').only( + 'id', 'stream_hash', 'name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'last_seen', 'updated_at', 'm3u_account', 'stream_id', 'stream_chno', 'channel_group_id' + ) + } + + for stream_hash, stream_props in stream_hashes.items(): + if stream_hash in existing_streams: + obj = existing_streams[stream_hash] + # Optimized field comparison + changed = ( + obj.name != stream_props["name"] or + obj.url != stream_props["url"] or + obj.logo_url != stream_props["logo_url"] or + obj.tvg_id != stream_props["tvg_id"] or + obj.custom_properties != stream_props["custom_properties"] or + obj.is_adult != stream_props["is_adult"] or + obj.stream_id != stream_props["stream_id"] or + obj.stream_chno != stream_props["stream_chno"] or + obj.channel_group_id != stream_props["channel_group_id"] + ) + + # Always update last_seen + obj.last_seen = timezone.now() + + if changed: + # Only update fields that changed and set updated_at + obj.name = stream_props["name"] + obj.url = stream_props["url"] + obj.logo_url = stream_props["logo_url"] + obj.tvg_id = stream_props["tvg_id"] + obj.custom_properties = stream_props["custom_properties"] + obj.is_adult = stream_props["is_adult"] + obj.stream_id = stream_props["stream_id"] + obj.stream_chno = stream_props["stream_chno"] + obj.channel_group_id = stream_props["channel_group_id"] + obj.updated_at = timezone.now() + + # Always mark as not stale since we saw it in this refresh + obj.is_stale = False + + streams_to_update.append(obj) + else: + # New stream + stream_props["last_seen"] = timezone.now() + stream_props["updated_at"] = timezone.now() + stream_props["is_stale"] = False + streams_to_create.append(Stream(**stream_props)) + + try: + with transaction.atomic(): + if streams_to_create: + Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) + + if streams_to_update: + # Update all streams in a single bulk operation + Stream.objects.bulk_update( + streams_to_update, + ['name', 'url', 'logo_url', 'tvg_id', 'custom_properties', 'is_adult', 'last_seen', 'updated_at', 'is_stale', 'stream_id', 'stream_chno', 'channel_group_id'], + batch_size=200 + ) + except Exception as e: + logger.error(f"Bulk operation failed: {str(e)}") + + retval = f"M3U account: {account_id}, Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + + # Clean up database connections for threading + connections.close_all() + + # Free batch data structures (reference-counted deallocation) + del streams_to_create, streams_to_update, stream_hashes, existing_streams + + return retval + + +def cleanup_streams(account_id, scan_start_time=timezone.now): + account = M3UAccount.objects.get(id=account_id, is_active=True) + existing_groups = ChannelGroup.objects.filter( + m3u_accounts__m3u_account=account, + m3u_accounts__enabled=True, + ).values_list("id", flat=True) + logger.info( + f"Found {len(existing_groups)} active groups for M3U account {account_id}" + ) + + # Calculate cutoff date for stale streams + stale_cutoff = scan_start_time - timezone.timedelta(days=account.stale_stream_days) + logger.info( + f"Removing streams not seen since {stale_cutoff} for M3U account {account_id}" + ) + + # Delete streams that are not in active groups + streams_to_delete = Stream.objects.filter(m3u_account=account).exclude( + channel_group__in=existing_groups + ) + + # Also delete streams that haven't been seen for longer than stale_stream_days + stale_streams = Stream.objects.filter( + m3u_account=account, last_seen__lt=stale_cutoff + ) + + deleted_count = streams_to_delete.count() + stale_count = stale_streams.count() + + streams_to_delete.delete() + stale_streams.delete() + + total_deleted = deleted_count + stale_count + logger.info( + f"Cleanup for M3U account {account_id} complete: {deleted_count} streams removed due to group filter, {stale_count} removed as stale" + ) + + # Return the total count of deleted streams + return total_deleted + + +@shared_task +def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_start_time=None): + """Refresh M3U groups for an account. + + Args: + account_id: ID of the M3U account + use_cache: Whether to use cached M3U file + full_refresh: Whether this is part of a full refresh + scan_start_time: Timestamp when the scan started (for consistent last_seen marking) """ - Delete the periodic task associated with an EPG source ID. + if not acquire_task_lock("refresh_m3u_account_groups", account_id): + return f"Task already running for account_id={account_id}.", None + + lock_renewer = TaskLockRenewer("refresh_m3u_account_groups", account_id) + lock_renewer.start() + + try: + account = M3UAccount.objects.get(id=account_id, is_active=True) + except M3UAccount.DoesNotExist: + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return f"M3UAccount with ID={account_id} not found or inactive.", None + + extinf_data = [] + groups = {"Default Group": {}} + + if account.account_type == M3UAccount.Types.XC: + # Log detailed information about the account + logger.info( + f"Processing XC account {account_id} with URL: {account.server_url}" + ) + logger.debug( + f"Username: {account.username}, Has password: {'Yes' if account.password else 'No'}" + ) + + # Validate required fields + if not account.server_url: + error_msg = "Missing server URL for Xtream Codes account" + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, "processing_groups", 100, status="error", error=error_msg + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + + if not account.username or not account.password: + error_msg = "Missing username or password for Xtream Codes account" + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, "processing_groups", 100, status="error", error=error_msg + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + + try: + # Ensure server URL is properly formatted + server_url = account.server_url.rstrip("/") + if not ( + server_url.startswith("http://") or server_url.startswith("https://") + ): + server_url = f"http://{server_url}" + + # User agent handling - completely rewritten + try: + # Debug the user agent issue + logger.debug(f"Getting user agent for account {account.id}") + + # Use a hardcoded user agent string to avoid any issues with object structure + user_agent_string = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + ) + + try: + # Try to get the user agent directly from the database + if account.user_agent_id: + ua_obj = UserAgent.objects.get(id=account.user_agent_id) + if ( + ua_obj + and hasattr(ua_obj, "user_agent") + and ua_obj.user_agent + ): + user_agent_string = ua_obj.user_agent + logger.debug( + f"Using user agent from account: {user_agent_string}" + ) + else: + # Get default user agent from CoreSettings + default_ua_id = CoreSettings.get_default_user_agent_id() + logger.debug( + f"Default user agent ID from settings: {default_ua_id}" + ) + if default_ua_id: + ua_obj = UserAgent.objects.get(id=default_ua_id) + if ( + ua_obj + and hasattr(ua_obj, "user_agent") + and ua_obj.user_agent + ): + user_agent_string = ua_obj.user_agent + logger.debug( + f"Using default user agent: {user_agent_string}" + ) + except Exception as e: + logger.warning( + f"Error getting user agent, using fallback: {str(e)}" + ) + + logger.debug(f"Final user agent string: {user_agent_string}") + except Exception as e: + user_agent_string = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + ) + logger.warning( + f"Exception in user agent handling, using fallback: {str(e)}" + ) + + logger.info( + f"Creating XCClient with URL: {account.server_url}, Username: {account.username}, User-Agent: {user_agent_string}" + ) + + # Create XCClient with explicit error handling + try: + with XCClient( + account.server_url, account.username, account.password, user_agent_string + ) as xc_client: + logger.info(f"XCClient instance created successfully") + + # Queue async profile refresh task to run in background + # This prevents any delay in the main refresh process + try: + logger.info(f"Queueing background profile refresh for account {account.name}") + refresh_account_profiles.delay(account.id) + except Exception as e: + logger.warning(f"Failed to queue profile refresh task: {str(e)}") + # Don't fail the main refresh if profile refresh can't be queued + + # Get categories with detailed error handling + try: + logger.info(f"Getting live categories from XC server") + xc_categories = xc_client.get_live_categories() + logger.info( + f"Found {len(xc_categories)} categories: {xc_categories}" + ) + + # Validate response + if not isinstance(xc_categories, list): + error_msg = ( + f"Unexpected response from XC server: {xc_categories}" + ) + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, + "processing_groups", + 100, + status="error", + error=error_msg, + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + + if len(xc_categories) == 0: + logger.warning("No categories found in XC server response") + + for category in xc_categories: + cat_name = category.get("category_name", "Unknown Category") + cat_id = category.get("category_id", "0") + logger.info(f"Adding category: {cat_name} (ID: {cat_id})") + groups[cat_name] = { + "xc_id": cat_id, + } + except Exception as e: + # Determine if this is an authentication error or category retrieval error + error_str = str(e).lower() + # Check for authentication-related keywords or HTTP status codes commonly used for auth failures + is_auth_error = any(keyword in error_str for keyword in [ + 'auth', 'credential', 'login', 'unauthorized', 'forbidden', + '401', '403', '512', '513' # HTTP status codes: 401 Unauthorized, 403 Forbidden, 512-513 (non-standard auth failure) + ]) + + if is_auth_error: + error_msg = f"Failed to authenticate with XC server: {str(e)}" + else: + error_msg = f"Failed to get categories from XC server: {str(e)}" + + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, + "processing_groups", + 100, + status="error", + error=error_msg, + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + + except Exception as e: + error_msg = f"Failed to create XC Client: {str(e)}" + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, + "processing_groups", + 100, + status="error", + error=error_msg, + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + except Exception as e: + error_msg = f"Unexpected error occurred in XC Client: {str(e)}" + logger.error(error_msg) + account.status = M3UAccount.Status.ERROR + account.last_message = error_msg + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, "processing_groups", 100, status="error", error=error_msg + ) + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return error_msg, None + else: + lines, success = fetch_m3u_lines(account, use_cache) + if not success: + # If fetch failed, don't continue processing + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + return f"Failed to fetch M3U data for account_id={account_id}.", None + + # Log basic file structure for debugging + logger.debug(f"Processing {len(lines)} lines from M3U file") + + valid_stream_count = 0 + + for entry in iter_m3u_entries(lines): + valid_stream_count += 1 + group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") + if group_title_attr and group_title_attr not in groups: + logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") + groups[group_title_attr] = {} + extinf_data.append(entry) + + if valid_stream_count % 1000 == 0: + logger.debug(f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}") + + logger.info(f"M3U parsing complete - Valid streams: {valid_stream_count}") + + # Log group statistics + logger.info( + f"Found {len(groups)} groups in M3U file: {', '.join(list(groups.keys())[:20])}" + + ("..." if len(groups) > 20 else "") + ) + + # Cache processed data + cache_path = os.path.join(m3u_dir, f"{account_id}.json") + with open(cache_path, "w", encoding="utf-8") as f: + json.dump( + { + "extinf_data": extinf_data, + "groups": groups, + }, + f, + ) + logger.debug(f"Cached parsed M3U data to {cache_path}") + + send_m3u_update(account_id, "processing_groups", 0) + + process_groups(account, groups, scan_start_time) + + lock_renewer.stop() + release_task_lock("refresh_m3u_account_groups", account_id) + + if not full_refresh: + # Use update() instead of save() to avoid triggering signals + M3UAccount.objects.filter(id=account_id).update( + status=M3UAccount.Status.PENDING_SETUP, + last_message="M3U groups loaded. Please select groups or refresh M3U to complete setup.", + ) + send_m3u_update( + account_id, + "processing_groups", + 100, + status="pending_setup", + message="M3U groups loaded. Please select groups or refresh M3U to complete setup.", + ) + + return extinf_data, groups + + +def delete_m3u_refresh_task_by_id(account_id): + """ + Delete the periodic task associated with an M3U account ID. Can be called directly or from the post_delete signal. Returns True if a task was found and deleted, False otherwise. """ try: task = None - task_name = f"epg_source-refresh-{epg_id}" + task_name = f"m3u_account-refresh-{account_id}" # Look for task by name try: from django_celery_beat.models import PeriodicTask, IntervalSchedule + task = PeriodicTask.objects.get(name=task_name) - logger.info(f"Found task by name: {task.id} for EPGSource {epg_id}") + logger.debug(f"Found task by name: {task.id} for M3UAccount {account_id}") except PeriodicTask.DoesNotExist: logger.warning(f"No PeriodicTask found with name {task_name}") return False @@ -259,3336 +1646,2171 @@ def delete_epg_refresh_task_by_id(epg_id): if task: # Store interval info before deleting the task interval_id = None - if hasattr(task, 'interval') and task.interval: + if hasattr(task, "interval") and task.interval: interval_id = task.interval.id # Count how many TOTAL tasks use this interval (including this one) - tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() - logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") + tasks_with_same_interval = PeriodicTask.objects.filter( + interval_id=interval_id + ).count() + logger.debug( + f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total" + ) # Delete the task first task_id = task.id task.delete() - logger.info(f"Successfully deleted periodic task {task_id}") + logger.debug(f"Successfully deleted periodic task {task_id}") # Now check if we should delete the interval # We only delete if it was the ONLY task using this interval if interval_id and tasks_with_same_interval == 1: try: interval = IntervalSchedule.objects.get(id=interval_id) - logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") + logger.debug( + f"Deleting interval schedule {interval_id} (not shared with other tasks)" + ) interval.delete() - logger.info(f"Successfully deleted interval {interval_id}") + logger.debug(f"Successfully deleted interval {interval_id}") except IntervalSchedule.DoesNotExist: logger.warning(f"Interval {interval_id} no longer exists") elif interval_id: - logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") + logger.debug( + f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks" + ) return True return False except Exception as e: - logger.error(f"Error deleting periodic task for EPGSource {epg_id}: {str(e)}", exc_info=True) + logger.error( + f"Error deleting periodic task for M3UAccount {account_id}: {str(e)}", + exc_info=True, + ) return False +def _next_available_number(used_numbers, start, end=None): + """ + Return the smallest integer >= start that is not present in `used_numbers`. + + When `end` is provided (inclusive upper bound for range-constrained groups), + returns None if the search exceeds that bound instead of running + indefinitely. The search is O(cluster size) per call against the set; + maintaining the cursor monotonically across calls keeps the + "next_available" numbering mode from becoming O(N^2) on large groups. + """ + n = start + while n in used_numbers: + n += 1 + if end is not None and n > end: + return None + if end is not None and n > end: + return None + return n + + +def _pick_target_number( + mode, + stream, + used_numbers, + fixed_cursor, + fallback_start, + end_number=None, + range_start=None, +): + """ + Return the channel number a given stream should claim under the group's + numbering mode, or None if the configured range is exhausted. + + Shared by the existing-channel renumber pass and the new-channel create + pass so both honor identical mode semantics: provider-supplied number + when available and free, otherwise fall back; or always next-available; + or fixed-cursor sequential. + + `range_start`, when provided, is the inclusive lower bound for the + group's configured numbering range. Provider-supplied numbers below + this bound fall back to the next-available picker so freshly-created + channels never land outside the configured range. + """ + if mode == "provider": + chno = stream.stream_chno + if ( + chno is not None + and chno not in used_numbers + and (range_start is None or chno >= range_start) + and (end_number is None or chno <= end_number) + ): + return chno + # No usable provider number: walk from fallback_start, bumped up + # to range_start when set so the fallback never lands below the + # configured range. + effective_start = ( + max(fallback_start, range_start) + if range_start is not None + else fallback_start + ) + return _next_available_number(used_numbers, effective_start, end=end_number) + if mode == "next_available": + return _next_available_number(used_numbers, 1, end=end_number) + return _next_available_number(used_numbers, fixed_cursor, end=end_number) + + +def _custom_properties_as_dict(value): + """ + Normalize a JSONField-backed custom_properties value into a dict. + + Historical data has rows where the field holds a JSON-encoded string + instead of a dict. Django's JSONField serializes whatever it gets, so + `.get()` on one of those rows raises AttributeError and aborts the + entire sync. Treat string values as JSON to parse, and fall back to an + empty dict for anything that isn't a dict after parsing. + """ + import json + + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except (ValueError, TypeError): + logger.warning( + "custom_properties stored as non-JSON string; ignoring: %r", + value[:100], + ) + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + +def _classify_sync_failure(exc): + """ + Map an exception raised during per-stream sync to a coarse typed + reason used by the completion notification's grouped failure list. + Keeps the bucket count small so the modal stays readable; the + underlying exception text is preserved verbatim in ``error``. + """ + from django.db import IntegrityError + + if isinstance(exc, IntegrityError): + return "INTEGRITY_ERROR" + return "OTHER" + + @shared_task -def refresh_all_epg_data(): - logger.info("Starting refresh_epg_data task.") - # Exclude dummy EPG sources from refresh - they don't need refreshing - active_sources = EPGSource.objects.filter(is_active=True).exclude(source_type='dummy') - logger.debug(f"Found {active_sources.count()} active EPGSource(s) (excluding dummy EPGs).") +def sync_auto_channels(account_id, scan_start_time=None): + """ + Automatically create/update/delete channels to match streams in groups with auto_channel_sync enabled. + Preserves existing channel UUIDs to maintain M3U link integrity. + Called after M3U refresh completes successfully. + """ + from apps.channels.models import ( + Channel, + ChannelGroup, + ChannelGroupM3UAccount, + Stream, + ChannelStream, + ) + from apps.epg.models import EPGData + from django.utils import timezone - for source in active_sources: - refresh_epg_data(source.id) - # Force garbage collection between sources - gc.collect() - - logger.info("Finished refresh_epg_data task.") - return "EPG data refreshed." - - -@shared_task(time_limit=14400) -def refresh_epg_data(source_id, force=False): - if not acquire_task_lock('refresh_epg_data', source_id): - logger.debug(f"EPG refresh for {source_id} already running") - return - - lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) - lock_renewer.start() - - source = None try: - # Try to get the EPG source - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - # The EPG source doesn't exist, so delete the periodic task if it exists - logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") + account = M3UAccount.objects.get(id=account_id) + logger.info(f"Starting auto channel sync for M3U account {account.name}") - # Call the shared function to delete the task - if delete_epg_refresh_task_by_id(source_id): - logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") - else: - logger.info(f"No orphaned task found for EPG source {source_id}") + # Always use scan_start_time as the cutoff for last_seen + if scan_start_time is not None: + if isinstance(scan_start_time, str): + scan_start_time = timezone.datetime.fromisoformat(scan_start_time) + else: + scan_start_time = timezone.now() - # Release the lock and exit - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return f"EPG source {source_id} does not exist, task cleaned up" + # Get groups with auto sync enabled for this account + auto_sync_groups = ChannelGroupM3UAccount.objects.filter( + m3u_account=account, enabled=True, auto_channel_sync=True + ).select_related("channel_group") - # The source exists but is not active, just skip processing - if not source.is_active: - logger.info(f"EPG source {source_id} is not active. Skipping.") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return + channels_created = 0 + channels_updated = 0 + channels_deleted = 0 + channels_failed = 0 + # Per-failure context for the completion notification. Each entry + # carries a typed ``reason`` so the modal can group counts by + # cause; the cap keeps the WebSocket payload bounded but is sized + # generously to cover realistic multi-provider failure sets. + # Full per-stream detail still goes to ``logger.warning`` for + # power-user diagnostics regardless of the cap. + failed_stream_details = [] + FAILURE_LOG_LIMIT = 1000 - # Skip refresh for dummy EPG sources - they don't need refreshing - if source.source_type == 'dummy': - logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - gc.collect() - return + # Group range reservations (start+end) are advisory and NOT seeded + # here: two groups with overlapping ranges must cooperate, so only + # actually-occupied numbers constrain assignment. + # Hidden auto-created channels stay in the seed because the renumber + # loop iterates current provider streams (which excludes hidden + # ones); excluding them here would let sync reclaim their numbers. + used_numbers = set( + Channel.objects.exclude( + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ).values_list("channel_number", flat=True) + ) + # Override pins are global reservations: effective_channel_number + # uses the override, so the picker must treat those numbers as + # taken or sync can produce duplicate effective channel numbers. + from apps.channels.models import ChannelOverride - # Continue with the normal processing... - logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") - if source.source_type == 'xmltv': - fetch_success = fetch_xmltv(source) - if not fetch_success: - logger.error(f"Failed to fetch XMLTV for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return + used_numbers.update( + ChannelOverride.objects.filter( + channel_number__isnull=False + ).values_list("channel_number", flat=True) + ) + used_numbers.discard(None) - parse_channels_success = parse_channels_only(source) - if not parse_channels_success: - logger.error(f"Failed to parse channels for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return + for group_relation in auto_sync_groups: + channel_group = group_relation.channel_group + start_number = group_relation.auto_sync_channel_start or 1.0 + # Optional upper bound; _next_available_number returns None when + # exhausted, which the per-stream loop converts to a failure. + end_number = group_relation.auto_sync_channel_end - parse_programs_for_source(source) - - elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source, force=force) - - source.save(update_fields=['updated_at']) - # After successful EPG refresh, evaluate DVR series rules to schedule new episodes - try: - from apps.channels.tasks import evaluate_series_rules - evaluate_series_rules.delay() - except Exception: - pass - except Exception as e: - logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) - try: - if source: - source.status = 'error' - source.last_message = f"Error refreshing EPG data: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) - except Exception as inner_e: - logger.error(f"Error updating source status: {inner_e}") - finally: - # Clear references to ensure proper garbage collection - source = None - # Force garbage collection before releasing the lock - gc.collect() - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - - -def fetch_xmltv(source): - # Handle cases with local file but no URL - if not source.url and source.file_path and os.path.exists(source.file_path): - logger.info(f"Using existing local file for EPG source: {source.name} at {source.file_path}") - - # Check if the existing file is compressed and we need to extract it - if source.file_path.endswith(('.gz', '.zip')) and not source.file_path.endswith('.xml'): - try: - # Define the path for the extracted file in the cache directory - cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") - os.makedirs(cache_dir, exist_ok=True) - xml_path = os.path.join(cache_dir, f"{source.id}.xml") - - # Extract to the cache location keeping the original - extracted_path = extract_compressed_file(source.file_path, xml_path, delete_original=False) - - if extracted_path: - logger.info(f"Extracted mapped compressed file to: {extracted_path}") - # Update to use extracted_file_path instead of changing file_path - source.extracted_file_path = extracted_path - source.save(update_fields=['extracted_file_path']) - else: - logger.error(f"Failed to extract mapped compressed file. Using original file: {source.file_path}") - except Exception as e: - logger.error(f"Failed to extract existing compressed file: {e}") - # Continue with the original file if extraction fails - - # Set the status to success in the database - source.status = 'success' - source.save(update_fields=['status']) - - # Send a download complete notification - send_epg_update(source.id, "downloading", 100, status="success") - - # Return True to indicate successful fetch, processing will continue with parse_channels_only - return True - - # Handle cases where no URL is provided and no valid file path exists - if not source.url: - # Update source status for missing URL - source.status = 'error' - source.last_message = "No URL provided and no valid local file exists" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "downloading", 100, status="error", error="No URL provided and no valid local file exists") - return False - - logger.info(f"Fetching XMLTV data from source: {source.name}") - try: - # Get default user agent from settings - stream_settings = CoreSettings.get_stream_settings() - user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default - default_user_agent_id = stream_settings.get('default_user_agent') - if default_user_agent_id: - try: - user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() - if user_agent_obj and user_agent_obj.user_agent: - user_agent = user_agent_obj.user_agent - logger.debug(f"Using default user agent: {user_agent}") - except (ValueError, Exception) as e: - logger.warning(f"Error retrieving default user agent, using fallback: {e}") - - headers = { - 'User-Agent': user_agent - } - - # Update status to fetching before starting download - source.status = 'fetching' - source.save(update_fields=['status']) - - # Send initial download notification - send_epg_update(source.id, "downloading", 0) - - # Use streaming response to track download progress - with requests.get(source.url, headers=headers, stream=True, timeout=60) as response: - # Handle 404 specifically - if response.status_code == 404: - logger.error(f"EPG URL not found (404): {source.url}") - # Update status to error in the database - source.status = 'error' - source.last_message = f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket about the EPG fetch failure - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": 404, - "message": f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" - } - } + # Get force_dummy_epg, group_override, and regex patterns from group custom_properties + group_custom_props = {} + force_dummy_epg = False # Backward compatibility: legacy option to disable EPG + override_group_id = None + name_regex_pattern = None + name_replace_pattern = None + name_match_regex = None + name_match_exclude_regex = None + channel_profile_ids = None + channel_sort_order = None + channel_sort_reverse = False + stream_profile_id = None + custom_logo_id = None + custom_epg_id = None # New option: select specific EPG source (takes priority over force_dummy_epg) + channel_numbering_mode = "fixed" # Default mode + channel_numbering_fallback = 1 # Default fallback for provider mode + group_custom_props = _custom_properties_as_dict( + group_relation.custom_properties + ) + if group_custom_props: + force_dummy_epg = group_custom_props.get("force_dummy_epg", False) + override_group_id = group_custom_props.get("group_override") + name_regex_pattern = group_custom_props.get("name_regex_pattern") + name_replace_pattern = group_custom_props.get( + "name_replace_pattern" ) - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error="URL not found (404)") - return False - - # For all other error status codes - if response.status_code >= 400: - error_message = f"HTTP error {response.status_code}" - user_message = f"EPG source '{source.name}' encountered HTTP error {response.status_code}" - - # Update status to error in the database - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": response.status_code, - "message": user_message - } - } + name_match_regex = group_custom_props.get("name_match_regex") + name_match_exclude_regex = group_custom_props.get( + "name_match_exclude_regex" ) - # Update download progress - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False + channel_profile_ids = group_custom_props.get("channel_profile_ids") + custom_epg_id = group_custom_props.get("custom_epg_id") + channel_sort_order = group_custom_props.get("channel_sort_order") + channel_sort_reverse = group_custom_props.get( + "channel_sort_reverse", False + ) + stream_profile_id = group_custom_props.get("stream_profile_id") + custom_logo_id = group_custom_props.get("custom_logo_id") + channel_numbering_mode = group_custom_props.get("channel_numbering_mode", "fixed") + channel_numbering_fallback = group_custom_props.get("channel_numbering_fallback", 1) - response.raise_for_status() - logger.debug("XMLTV data fetched successfully.") + # Determine which group to use for created channels + target_group = channel_group + if override_group_id: + try: + target_group = ChannelGroup.objects.get(id=override_group_id) + logger.info( + f"Using override group '{target_group.name}' instead of '{channel_group.name}' for auto-created channels" + ) + except ChannelGroup.DoesNotExist: + logger.warning( + f"Override group with ID {override_group_id} not found, using original group '{channel_group.name}'" + ) - # Define base paths for consistent file naming - cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") - os.makedirs(cache_dir, exist_ok=True) - - # Create temporary download file with .tmp extension - temp_download_path = os.path.join(cache_dir, f"{source.id}.tmp") - - # Check if we have content length for progress tracking - total_size = int(response.headers.get('content-length', 0)) - downloaded = 0 - start_time = time.time() - last_update_time = start_time - update_interval = 0.5 # Only update every 0.5 seconds - - # Download to temporary file - with open(temp_download_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=16384): - f.write(chunk) - - downloaded += len(chunk) - elapsed_time = time.time() - start_time - - # Calculate download speed in KB/s - speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0 - - # Calculate progress percentage - if total_size and total_size > 0: - progress = min(100, int((downloaded / total_size) * 100)) - else: - # If no content length header, estimate progress - progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown - - # Time remaining (in seconds) - time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0 - - # Only send updates at specified intervals to avoid flooding - current_time = time.time() - if current_time - last_update_time >= update_interval and progress > 0: - last_update_time = current_time - send_epg_update( - source.id, - "downloading", - progress, - speed=round(speed, 2), - elapsed_time=round(elapsed_time, 1), - time_remaining=round(time_remaining, 1), - downloaded=f"{downloaded / (1024 * 1024):.2f} MB" - ) - - # Explicitly delete the chunk to free memory immediately - del chunk - - # Send completion notification - send_epg_update(source.id, "downloading", 100) - - # Determine the appropriate file extension based on content detection - with open(temp_download_path, 'rb') as f: - content_sample = f.read(1024) # Just need the first 1KB to detect format - - # Use our helper function to detect the format - format_type, is_compressed, file_extension = detect_file_format( - file_path=source.url, # Original URL as a hint - content=content_sample # Actual file content for detection + logger.info( + f"Processing auto sync for group: {channel_group.name} (mode: {channel_numbering_mode}, start: {start_number})" ) - logger.debug(f"File format detection results: type={format_type}, compressed={is_compressed}, extension={file_extension}") + # Get all current streams in this group for this M3U account, filter out stale streams + current_streams = Stream.objects.filter( + m3u_account=account, + channel_group=channel_group, + last_seen__gte=scan_start_time, + ) - # Ensure consistent final paths - compressed_path = os.path.join(cache_dir, f"{source.id}{file_extension}" if is_compressed else f"{source.id}.compressed") - xml_path = os.path.join(cache_dir, f"{source.id}.xml") - - # Clean up old files before saving new ones - if os.path.exists(compressed_path): + # Filter streams in Python using the same `regex` module as the + # preview API. This ensures auto-sync accepts the same patterns + # the user tested in the frontend (e.g. `(?)` as a no-op inline + # modifier), and avoids passing potentially incompatible syntax + # to PostgreSQL's regex engine. + streams_is_list = False + if name_match_regex: try: - os.remove(compressed_path) - logger.debug(f"Removed old compressed file: {compressed_path}") - except OSError as e: - logger.warning(f"Failed to remove old compressed file: {e}") + match_re = regex.compile(name_match_regex, regex.IGNORECASE) + current_streams = [s for s in current_streams if match_re.search(s.name)] + streams_is_list = True + except regex.error as e: + logger.warning( + f"Invalid name_match_regex '{name_match_regex}' for group '{channel_group.name}': {e}. Skipping name filter." + ) - if os.path.exists(xml_path): + # Exclude regex runs after the include filter so the two + # compose: include narrows, exclude removes from what's left. + if name_match_exclude_regex: try: - os.remove(xml_path) - logger.debug(f"Removed old XML file: {xml_path}") - except OSError as e: - logger.warning(f"Failed to remove old XML file: {e}") + exclude_re = regex.compile(name_match_exclude_regex, regex.IGNORECASE) + current_streams = [s for s in current_streams if not exclude_re.search(s.name)] + streams_is_list = True + except regex.error as e: + logger.warning( + f"Invalid name_match_exclude_regex '{name_match_exclude_regex}' for group '{channel_group.name}': {e}. Skipping exclude filter." + ) - # Rename the temp file to appropriate final path - if is_compressed: - try: - os.rename(temp_download_path, compressed_path) - logger.debug(f"Renamed temp file to compressed file: {compressed_path}") - current_file_path = compressed_path - except OSError as e: - logger.error(f"Failed to rename temp file to compressed file: {e}") - current_file_path = temp_download_path # Fall back to using temp file - else: - try: - os.rename(temp_download_path, xml_path) - logger.debug(f"Renamed temp file to XML file: {xml_path}") - current_file_path = xml_path - except OSError as e: - logger.error(f"Failed to rename temp file to XML file: {e}") - current_file_path = temp_download_path # Fall back to using temp file - - # Now extract the file if it's compressed - if is_compressed: - try: - logger.info(f"Extracting compressed file {current_file_path}") - send_epg_update(source.id, "extracting", 0, message="Extracting downloaded file") - - # Always extract to the standard XML path - set delete_original to True to clean up - extracted = extract_compressed_file(current_file_path, xml_path, delete_original=True) - - if extracted: - logger.info(f"Successfully extracted to {xml_path}, compressed file deleted") - send_epg_update(source.id, "extracting", 100, message=f"File extracted successfully, temporary file removed") - # Update to store only the extracted file path since the compressed file is now gone - source.file_path = xml_path - source.extracted_file_path = None + # --- APPLY CHANNEL SORT ORDER --- + if channel_sort_order and channel_sort_order != "": + if channel_sort_order == "name": + if not streams_is_list: + current_streams = list(current_streams) + streams_is_list = True + current_streams.sort( + key=lambda stream: natural_sort_key(stream.name), + reverse=channel_sort_reverse, + ) + elif channel_sort_order == "tvg_id": + if streams_is_list: + current_streams.sort( + key=lambda s: (s.tvg_id or ""), + reverse=channel_sort_reverse, + ) else: - logger.error("Extraction failed, using compressed file") - send_epg_update(source.id, "extracting", 100, status="error", message="Extraction failed, using compressed file") - # Use the compressed file - source.file_path = current_file_path - source.extracted_file_path = None - except Exception as e: - logger.error(f"Error extracting file: {str(e)}", exc_info=True) - send_epg_update(source.id, "extracting", 100, status="error", message=f"Error during extraction: {str(e)}") - # Use the compressed file if extraction fails - source.file_path = current_file_path - source.extracted_file_path = None + order_prefix = "-" if channel_sort_reverse else "" + current_streams = current_streams.order_by(f"{order_prefix}tvg_id") + elif channel_sort_order == "updated_at": + if streams_is_list: + current_streams.sort( + key=lambda s: (s.updated_at or ""), + reverse=channel_sort_reverse, + ) + else: + order_prefix = "-" if channel_sort_reverse else "" + current_streams = current_streams.order_by( + f"{order_prefix}updated_at" + ) + else: + logger.warning( + f"Unknown channel_sort_order '{channel_sort_order}' for group '{channel_group.name}'. Using provider order." + ) + if streams_is_list: + current_streams.sort( + key=lambda s: s.id, + reverse=channel_sort_reverse, + ) + else: + order_prefix = "-" if channel_sort_reverse else "" + current_streams = current_streams.order_by(f"{order_prefix}id") else: - # It's already an XML file - source.file_path = current_file_path - source.extracted_file_path = None + # Provider order (default) - can still be reversed + if streams_is_list: + current_streams.sort( + key=lambda s: s.id, + reverse=channel_sort_reverse, + ) + else: + order_prefix = "-" if channel_sort_reverse else "" + current_streams = current_streams.order_by(f"{order_prefix}id") - # Update the source's file paths - source.save(update_fields=['file_path', 'status', 'extracted_file_path']) + # Scoped to this group so the loop below runs in O(group size). + # Multi-stream channels are deduped by channel_id so every + # stream_id maps to the same in-memory Channel instance and + # post-loop bulk_update writes the merged state. + existing_channel_map = {} + existing_channels_by_id = {} + existing_channel_streams = ( + ChannelStream.objects.filter( + channel__auto_created=True, + channel__auto_created_by=account, + stream__m3u_account=account, + stream__channel_group=channel_group, + ) + .select_related("channel") + ) + for cs in existing_channel_streams: + if cs.stream_id and cs.channel_id: + canonical = existing_channels_by_id.setdefault( + cs.channel_id, cs.channel + ) + existing_channel_map[cs.stream_id] = canonical - # Update status to parsing - source.status = 'parsing' - source.save(update_fields=['status']) + # Track which streams we've processed + processed_stream_ids = set() - logger.info(f"Cached EPG file saved to {source.file_path}") + # Check if we have streams - handle both QuerySet and list cases + has_streams = ( + len(current_streams) > 0 + if streams_is_list + else current_streams.exists() + ) - return True + # Bulk pre-fetch collapses N+1 Logo/EPGData lookups into a + # pair of in_bulk() calls. + from apps.channels.models import Logo + from apps.epg.models import EPGSource - except requests.exceptions.HTTPError as e: - logger.error(f"HTTP Error fetching XMLTV from {source.name}: {e}", exc_info=True) + # Resolve the group's custom EPG source once. + custom_epg_source = None + custom_dummy_epg_data = None + if custom_epg_id: + try: + custom_epg_source = EPGSource.objects.get(id=custom_epg_id) + if custom_epg_source.source_type == "dummy": + custom_dummy_epg_data = ( + EPGData.objects.filter( + epg_source=custom_epg_source + ).first() + ) + if not custom_dummy_epg_data: + logger.warning( + f"No EPGData found for dummy EPG source " + f"{custom_epg_source.name} (ID: {custom_epg_id})" + ) + except EPGSource.DoesNotExist: + logger.warning( + f"Custom EPG source with ID {custom_epg_id} not found " + f"for group '{channel_group.name}', falling back to " + f"auto-match" + ) - # Get error details - status_code = e.response.status_code if hasattr(e, 'response') and e.response else 'unknown' - error_message = str(e) + # Resolve the group's custom logo once. + custom_logo = None + if custom_logo_id: + try: + custom_logo = Logo.objects.get(id=custom_logo_id) + except Logo.DoesNotExist: + logger.warning( + f"Custom logo with ID {custom_logo_id} not found for " + f"group '{channel_group.name}', falling back to stream " + f"logos" + ) - # Create a user-friendly message - user_message = f"EPG source '{source.name}' encountered HTTP error {status_code}" - - # Add specific handling for common HTTP errors - if status_code == 404: - user_message = f"EPG source '{source.name}' URL not found (404) - will retry on next scheduled run" - elif status_code == 401 or status_code == 403: - user_message = f"EPG source '{source.name}' access denied (HTTP {status_code}) - check credentials" - elif status_code == 429: - user_message = f"EPG source '{source.name}' rate limited (429) - try again later" - elif status_code >= 500: - user_message = f"EPG source '{source.name}' server error (HTTP {status_code}) - will retry later" - - # Update source status to error with the error message - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket about the EPG fetch failure - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": status_code, - "message": user_message, - "details": error_message + logo_cache_by_url = {} + epg_cache_by_tvg_id = {} + if has_streams: + # Collect unique URLs / tvg_ids in one DB call each. + stream_iter = ( + current_streams + if streams_is_list + else list(current_streams.values("logo_url", "tvg_id")) + ) + unique_logo_urls = { + s.get("logo_url") if isinstance(s, dict) else getattr(s, "logo_url", None) + for s in stream_iter } - } - ) + unique_logo_urls.discard(None) + unique_logo_urls.discard("") + if unique_logo_urls: + logo_cache_by_url = { + lg.url: lg + for lg in Logo.objects.filter(url__in=unique_logo_urls) + } - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - except requests.exceptions.ConnectionError as e: - # Handle connection errors separately - error_message = str(e) - user_message = f"Connection error: Unable to connect to EPG source '{source.name}'" - logger.error(f"Connection error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Update source status - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Send notifications - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": "connection_error", - "message": user_message + unique_tvg_ids = { + s.get("tvg_id") if isinstance(s, dict) else getattr(s, "tvg_id", None) + for s in stream_iter } - } + unique_tvg_ids.discard(None) + unique_tvg_ids.discard("") + # Skip the EPG cache when force_dummy_epg with no + # custom source override; the resolver always returns None. + want_epg_cache = unique_tvg_ids and ( + not force_dummy_epg or custom_epg_id + ) + if want_epg_cache: + # Scope to the group's pinned source so foreign-source + # tvg_id matches do not leak in. + epg_q = EPGData.objects.filter(tvg_id__in=unique_tvg_ids) + if ( + custom_epg_source is not None + and custom_epg_source.source_type != "dummy" + ): + epg_q = epg_q.filter(epg_source=custom_epg_source) + epg_cache_by_tvg_id = {d.tvg_id: d for d in epg_q} + + def _resolve_logo_for_stream(stream): + """Return a Logo for stream.logo_url, creating it once if needed.""" + url = getattr(stream, "logo_url", None) + if not url: + return None + cached = logo_cache_by_url.get(url) + if cached is not None: + return cached + created, _ = Logo.objects.get_or_create( + url=url, + defaults={"name": stream.name or stream.tvg_id or "Unknown"}, + ) + logo_cache_by_url[url] = created + return created + + def _resolve_epg_for_stream(stream): + """Return the EPGData row that should be assigned to this + stream's channel. Encodes all four group-level EPG modes: + + 1. custom dummy source: single shared EPGData + 2. custom non-dummy source: cache lookup, scoped to + that source + 3. force_dummy_epg with no custom: None (clear EPG) + 4. default auto-match: cache lookup, any source + + The cache (epg_cache_by_tvg_id) is built once above with the + correct scope so the per-stream lookup is a dict access. + """ + if custom_epg_source is not None: + if custom_epg_source.source_type == "dummy": + return custom_dummy_epg_data + tvg_id = getattr(stream, "tvg_id", None) + if not tvg_id: + return None + return epg_cache_by_tvg_id.get(tvg_id) + if force_dummy_epg: + return None + tvg_id = getattr(stream, "tvg_id", None) + if not tvg_id: + return None + return epg_cache_by_tvg_id.get(tvg_id) + + if not has_streams: + logger.debug(f"No streams found in group {channel_group.name}") + # No streams left in the group: drop the visible auto + # channels. Hidden channels are preserved so the hide + # flag survives temporary provider drops (event/PPV). + channels_to_delete = [ + ch + for ch in existing_channel_map.values() + if not ch.hidden_from_output + ] + if channels_to_delete: + deleted_count = len(channels_to_delete) + Channel.objects.filter( + id__in=[ch.id for ch in channels_to_delete] + ).delete() + channels_deleted += deleted_count + logger.debug( + f"Deleted {deleted_count} auto channels (no streams remaining)" + ) + continue + + # Prepare profiles to assign to new channels + from apps.channels.models import ChannelProfile, ChannelProfileMembership + + if ( + channel_profile_ids + and isinstance(channel_profile_ids, list) + and len(channel_profile_ids) > 0 + ): + # Convert all to int (in case they're strings) + try: + profile_ids = [int(pid) for pid in channel_profile_ids] + except Exception: + profile_ids = [] + profiles_to_assign = list( + ChannelProfile.objects.filter(id__in=profile_ids) + ) + else: + profiles_to_assign = list(ChannelProfile.objects.all()) + + # Get stream profile to assign if specified + from core.models import StreamProfile + stream_profile_to_assign = None + if stream_profile_id: + try: + stream_profile_to_assign = StreamProfile.objects.get(id=int(stream_profile_id)) + logger.info( + f"Will assign stream profile '{stream_profile_to_assign.name}' to auto-synced streams in group '{channel_group.name}'" + ) + except (StreamProfile.DoesNotExist, ValueError, TypeError): + logger.warning( + f"Stream profile with ID {stream_profile_id} not found for group '{channel_group.name}', streams will use default profile" + ) + stream_profile_to_assign = None + + current_channel_number = start_number + + # Renumber existing channels to match sort order. Compact + # mode skips this; the end-of-iteration pack is the source + # of truth and would overwrite the renumber. + compact_mode = bool(group_custom_props.get("compact_numbering")) + channels_to_renumber = [] + temp_channel_number = start_number + + for stream in current_streams if not compact_mode else []: + if stream.id in existing_channel_map: + channel = existing_channel_map[stream.id] + + target_number = _pick_target_number( + channel_numbering_mode, + stream, + used_numbers, + temp_channel_number, + channel_numbering_fallback, + end_number=end_number, + range_start=start_number, + ) + + # Range exhausted: leave the channel at its existing + # number. The renumber pass is sort-optimization only; + # no failure record needed. + if target_number is None: + # Preserve the channel's current number in used_numbers + if channel.channel_number is not None: + used_numbers.add(channel.channel_number) + continue + + # Add this number to used_numbers so we don't reuse it in this batch + used_numbers.add(target_number) + + if channel.channel_number != target_number: + channel.channel_number = target_number + channels_to_renumber.append(channel) + logger.debug( + f"Will renumber channel '{channel.name}' to {target_number}" + ) + + # Only increment temp_channel_number in fixed mode + if channel_numbering_mode == "fixed": + temp_channel_number += 1.0 + if temp_channel_number % 1 != 0: # Has decimal + temp_channel_number = int(temp_channel_number) + 1.0 + + # Bulk update channel numbers if any need renumbering + if channels_to_renumber: + Channel.objects.bulk_update( + channels_to_renumber, ["channel_number"], batch_size=500 + ) + logger.info( + f"Renumbered {len(channels_to_renumber)} channels to maintain sort order" + ) + + # When the group's configured range is narrower than its existing + # channels span, any non-hidden auto-created channel whose number + # falls outside [start, end] gets deleted. The new-channel + # creation loop below picks up the freed stream and re-creates + # the channel at a slot inside the new range, so the net user + # outcome is a renumber, not a failure. Counted in + # channels_deleted; the replacement counts in channels_created. + # Hidden channels are preserved. Runs BEFORE new-channel creation + # so slots freed by the deletions are available to incoming + # streams. + if end_number is not None: + overflow_delete_ids = [] + for stream_id, ch in list(existing_channel_map.items()): + if ch.hidden_from_output: + continue + num = ch.channel_number + if num is None: + continue + if num < start_number or num > end_number: + overflow_delete_ids.append(ch.id) + existing_channel_map.pop(stream_id, None) + used_numbers.discard(num) + if overflow_delete_ids: + deleted = Channel.objects.filter( + id__in=overflow_delete_ids + ).delete() + removed_count = ( + deleted[1].get("dispatcharr_channels.Channel", 0) + if isinstance(deleted, tuple) and len(deleted) > 1 + else len(overflow_delete_ids) + ) + channels_deleted += removed_count + logger.info( + f"Deleted {removed_count} channels outside the " + f"range {int(start_number)}-{int(end_number)} for " + f"group '{channel_group.name}'" + ) + + # Reset channel number counter for processing new channels + current_channel_number = start_number + + # Per-channel changes are buffered and bulk_update'd once after the + # loop. update_fields is set explicitly so post_save signals only + # fire for receivers whose tracked field actually changed. + existing_dirty_channels = [] + existing_dirty_ids = set() + existing_dirty_field_set = set() + # Subset of channels whose epg_data actually changed in this + # pass. Used by the dispatcher below to fire the EPG parse + # task only for those, not for every channel in + # existing_dirty_channels. + epg_dirty_channel_ids = set() + + # New channels are buffered and bulk_create'd after the loop. + # bulk_create skips post_save, so the EPG parse task is dispatched + # once per unique epg_data_id below rather than per channel. + # Pairs are (Channel(), Stream) so the post-loop step can attach + # ChannelStream rows using the IDs Postgres returns. + new_channels_pending = [] + + for stream in current_streams: + processed_stream_ids.add(stream.id) + try: + # Parse custom properties for additional info + stream_custom_props = stream.custom_properties or {} + tvc_guide_stationid = stream_custom_props.get("tvc-guide-stationid") + + # --- REGEX FIND/REPLACE LOGIC --- + original_name = stream.name + new_name = original_name + if name_regex_pattern is not None: + # If replace is None, treat as empty string (remove match) + replace = ( + name_replace_pattern + if name_replace_pattern is not None + else "" + ) + try: + # Convert $1, $2, etc. to \1, \2, etc. for consistency with M3U profiles + safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace) + new_name = re.sub( + name_regex_pattern, safe_replace_pattern, original_name + ) + except re.error as e: + logger.warning( + f"Regex error for group '{channel_group.name}': {e}. Using original name." + ) + new_name = original_name + + # Check if we already have a channel for this stream + existing_channel = existing_channel_map.get(stream.id) + + if existing_channel: + # Track only the fields that actually changed, so the + # eventual UPDATE writes one column per change instead + # of every column on every channel. The dirty list is + # accumulated and bulk_update'd after the loop - + # which avoids issuing an UPDATE per channel and + # avoids firing the EPG post_save signal on saves + # that didn't touch epg_data. + dirty_fields = [] + + if existing_channel.name != new_name: + existing_channel.name = new_name + dirty_fields.append("name") + + if existing_channel.tvg_id != stream.tvg_id: + existing_channel.tvg_id = stream.tvg_id + dirty_fields.append("tvg_id") + + if existing_channel.tvc_guide_stationid != tvc_guide_stationid: + existing_channel.tvc_guide_stationid = tvc_guide_stationid + dirty_fields.append("tvc_guide_stationid") + + # The group override may direct sync to a different + # ChannelGroup than the one currently on the row. + if existing_channel.channel_group_id != target_group.id: + existing_channel.channel_group = target_group + dirty_fields.append("channel_group") + + # Logo: custom group setting wins; otherwise stream logo + current_logo = ( + custom_logo + if custom_logo_id and custom_logo is not None + else _resolve_logo_for_stream(stream) + ) + current_logo_id = current_logo.id if current_logo else None + if existing_channel.logo_id != current_logo_id: + existing_channel.logo = current_logo + dirty_fields.append("logo") + + # EPG: handled centrally by _resolve_epg_for_stream + current_epg_data = _resolve_epg_for_stream(stream) + current_epg_id = ( + current_epg_data.id if current_epg_data else None + ) + if existing_channel.epg_data_id != current_epg_id: + existing_channel.epg_data = current_epg_data + dirty_fields.append("epg_data") + if current_epg_id is not None: + epg_dirty_channel_ids.add(existing_channel.id) + + # Stream profile: only set if group has one configured + if ( + stream_profile_to_assign is not None + and existing_channel.stream_profile_id + != stream_profile_to_assign.id + ): + existing_channel.stream_profile = stream_profile_to_assign + dirty_fields.append("stream_profile") + + if dirty_fields: + # Multi-stream channels appear once per stream; + # dedupe by id so bulk_update does not double-fire + # and channels_updated does not double-count. + if existing_channel.id not in existing_dirty_ids: + existing_dirty_channels.append(existing_channel) + existing_dirty_ids.add(existing_channel.id) + channels_updated += 1 + existing_dirty_field_set.update(dirty_fields) + + else: + # Range exhaustion is surfaced to the user via the + # completion notification, not swallowed. + target_number = _pick_target_number( + channel_numbering_mode, + stream, + used_numbers, + current_channel_number, + channel_numbering_fallback, + end_number=end_number, + range_start=start_number, + ) + + if target_number is None: + channels_failed += 1 + if len(failed_stream_details) < FAILURE_LOG_LIMIT: + failed_stream_details.append({ + "stream_name": stream.name, + "stream_id": stream.id, + "group": channel_group.name, + "reason": "RANGE_EXHAUSTED", + "error": ( + f"Channel number range " + f"{int(start_number)}-{int(end_number)} is full" + ), + }) + processed_stream_ids.add(stream.id) + continue + + # Add this number to used_numbers + used_numbers.add(target_number) + + # Resolve every FK BEFORE the create call so the + # initial INSERT carries the complete row. + new_logo = ( + custom_logo + if custom_logo_id and custom_logo is not None + else _resolve_logo_for_stream(stream) + ) + new_epg_data = _resolve_epg_for_stream(stream) + + new_channels_pending.append( + ( + Channel( + channel_number=target_number, + name=new_name, + tvg_id=stream.tvg_id, + tvc_guide_stationid=tvc_guide_stationid, + channel_group=target_group, + user_level=0, + auto_created=True, + auto_created_by=account, + logo=new_logo, + epg_data=new_epg_data, + stream_profile=stream_profile_to_assign, + ), + stream, + ) + ) + + # Increment channel number for next iteration (only in fixed mode) + if channel_numbering_mode == "fixed": + current_channel_number += 1.0 + if current_channel_number % 1 != 0: # Has decimal + current_channel_number = int(current_channel_number) + 1.0 + + except Exception as e: + logger.error( + f"Error processing auto channel for stream {stream.name}: {str(e)}" + ) + channels_failed += 1 + if len(failed_stream_details) < FAILURE_LOG_LIMIT: + failed_stream_details.append({ + "stream_name": stream.name, + "stream_id": stream.id, + "group": channel_group.name, + "reason": _classify_sync_failure(e), + "error": str(e), + }) + continue + + # Bulk-create channels, then dependent rows using the IDs + # Postgres returns. bulk_create skips post_save, so the EPG + # parse task is dispatched explicitly per-epg_data_id below + # to avoid flooding Celery at scale. + if new_channels_pending: + channel_objs = [pair[0] for pair in new_channels_pending] + streams_for_new = [pair[1] for pair in new_channels_pending] + Channel.objects.bulk_create(channel_objs, batch_size=500) + + ChannelStream.objects.bulk_create( + [ + ChannelStream( + channel_id=channel_objs[i].id, + stream_id=streams_for_new[i].id, + order=0, + ) + for i in range(len(channel_objs)) + ], + batch_size=500, + ) + + if profiles_to_assign: + ChannelProfileMembership.objects.bulk_create( + [ + ChannelProfileMembership( + channel_id=ch.id, + channel_profile_id=profile.id, + enabled=True, + ) + for ch in channel_objs + for profile in profiles_to_assign + ], + ignore_conflicts=True, + batch_size=500, + ) + + channels_created += len(channel_objs) + + # One EPG parse task per unique EPGData replaces the + # per-channel post_save dispatch bypassed by bulk_create. + from apps.epg.tasks import parse_programs_for_tvg_id + + unique_epg_ids = { + ch.epg_data_id for ch in channel_objs if ch.epg_data_id + } + for epg_id in unique_epg_ids: + parse_programs_for_tvg_id.delay(epg_id) + + logger.debug( + f"Bulk created {len(channel_objs)} channels in group " + f"'{channel_group.name}'; dispatched " + f"{len(unique_epg_ids)} unique EPG parse task(s)" + ) + + # bulk_update writes only the columns named in `fields` and + # bypasses post_save, so the EPG refresh signal cannot fire here. + # Dispatch one parse task per unique EPGData id when epg_data was + # in the dirty set, mirroring the new-channel path above. + if existing_dirty_channels: + Channel.objects.bulk_update( + existing_dirty_channels, + fields=list(existing_dirty_field_set), + batch_size=500, + ) + if epg_dirty_channel_ids: + from apps.epg.tasks import parse_programs_for_tvg_id + + # Dispatch only for channels whose epg_data_id changed. + # Other dirty channels would queue redundant parses. + unique_epg_ids = { + ch.epg_data_id + for ch in existing_dirty_channels + if ch.id in epg_dirty_channel_ids and ch.epg_data_id + } + for epg_id in unique_epg_ids: + parse_programs_for_tvg_id.delay(epg_id) + logger.debug( + f"Bulk updated {len(existing_dirty_channels)} existing " + f"channels (fields: {sorted(existing_dirty_field_set)})" + ) + + # Reconcile ChannelProfileMembership in two writes: one + # bulk_update for enable-flips, one bulk_create for missing + # rows. Avoids a per-channel save loop. + existing_channel_ids = [ + c.id for c in existing_channel_map.values() + ] + target_profile_ids = {p.id for p in profiles_to_assign} + if existing_channel_ids: + membership_rows = list( + ChannelProfileMembership.objects.filter( + channel_id__in=existing_channel_ids + ).only("id", "channel_id", "channel_profile_id", "enabled") + ) + memberships_by_channel = {} + for m in membership_rows: + memberships_by_channel.setdefault(m.channel_id, []).append(m) + + rows_to_flip = [] + rows_to_create = [] + for ch_id in existing_channel_ids: + rows = memberships_by_channel.get(ch_id, []) + have_for_target = set() + for m in rows: + if m.channel_profile_id in target_profile_ids: + have_for_target.add(m.channel_profile_id) + if not m.enabled: + m.enabled = True + rows_to_flip.append(m) + else: + if m.enabled: + m.enabled = False + rows_to_flip.append(m) + missing = target_profile_ids - have_for_target + for pid in missing: + rows_to_create.append( + ChannelProfileMembership( + channel_id=ch_id, + channel_profile_id=pid, + enabled=True, + ) + ) + + if rows_to_flip: + ChannelProfileMembership.objects.bulk_update( + rows_to_flip, ["enabled"], batch_size=500 + ) + if rows_to_create: + ChannelProfileMembership.objects.bulk_create( + rows_to_create, ignore_conflicts=True, batch_size=500 + ) + if rows_to_flip or rows_to_create: + logger.debug( + f"Reconciled memberships for " + f"{len(existing_channel_ids)} channels " + f"({len(rows_to_flip)} flipped, " + f"{len(rows_to_create)} created)" + ) + + # Delete channels whose streams have all disappeared. + # Hidden channels are preserved so event/PPV holds across + # provider drops. + channel_streams_in_group = {} + for stream_id, channel in existing_channel_map.items(): + channel_streams_in_group.setdefault(channel.id, []).append( + (stream_id, channel) + ) + channels_to_delete = [] + for ch_id, pairs in channel_streams_in_group.items(): + channel = pairs[0][1] + if channel.hidden_from_output: + continue + stream_ids = {sid for sid, _ in pairs} + if not (stream_ids & processed_stream_ids): + channels_to_delete.append(channel) + + if channels_to_delete: + deleted_count = len(channels_to_delete) + Channel.objects.filter( + id__in=[ch.id for ch in channels_to_delete] + ).delete() + channels_deleted += deleted_count + logger.debug( + f"Deleted {deleted_count} auto channels for removed streams" + ) + + # Compact-mode pack: hidden channels release their number and + # visible channels pack contiguously into [start, end]. Runs + # after create/update/delete so the channel set is stable. + if compact_mode: + from apps.channels.compact_numbering import repack_group + + pack_result = repack_group(group_relation) + if pack_result["failed"]: + channels_failed += pack_result["failed"] + if ( + len(failed_stream_details) < FAILURE_LOG_LIMIT + ): + failed_stream_details.append( + { + "stream_name": None, + "stream_id": None, + "group": channel_group.name, + "reason": "RANGE_EXHAUSTED", + "error": ( + f"Compact pack: {pack_result['failed']} " + f"visible channel(s) could not fit in range " + f"{int(start_number)}" + + ( + f"-{int(end_number)}" + if end_number + else "+" + ) + ), + } + ) + logger.debug( + f"Compact pack for group '{channel_group.name}': " + f"{pack_result['assigned']} assigned, " + f"{pack_result['released']} released, " + f"{pack_result['failed']} failed" + ) + + # Cleanup mode read from account.custom_properties.orphan_channel_cleanup: + # "always" (default; key absent) removes every orphan auto channel; + # "preserve_customized" keeps those with a ChannelOverride row; + # "never" disables cleanup. Hidden channels are preserved across all + # modes so event/PPV channels that come and go are not silently lost. + cleanup_mode = (account.custom_properties or {}).get( + "orphan_channel_cleanup", "always" ) - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - except requests.exceptions.Timeout as e: - # Handle timeout errors specifically - error_message = str(e) - user_message = f"Timeout error: EPG source '{source.name}' took too long to respond" - logger.error(f"Timeout error fetching XMLTV from {source.name}: {e}", exc_info=True) + if cleanup_mode != "never": + orphaned_channels = Channel.objects.filter( + auto_created=True, + auto_created_by=account, + hidden_from_output=False, + ).exclude( + id__in=ChannelStream.objects.filter( + stream__m3u_account=account, + stream__isnull=False, + ).values_list("channel_id", flat=True) + ) + if cleanup_mode == "preserve_customized": + orphaned_channels = orphaned_channels.filter(override__isnull=True) - # Update source status - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) + _, per_model = orphaned_channels.delete() + deleted_channels = per_model.get("dispatcharr_channels.Channel", 0) + if deleted_channels: + channels_deleted += deleted_channels + logger.info( + f"Deleted {deleted_channels} orphaned auto channels with no valid streams (mode={cleanup_mode})" + ) + + logger.info( + f"Auto channel sync complete for account {account.name}: " + f"{channels_created} created, {channels_updated} updated, " + f"{channels_deleted} deleted, {channels_failed} failed" + ) + return { + "status": "ok", + "channels_created": channels_created, + "channels_updated": channels_updated, + "channels_deleted": channels_deleted, + "channels_failed": channels_failed, + "failed_stream_details": failed_stream_details, + } - # Send notifications - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False except Exception as e: - error_message = str(e) - logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Update source status for general exceptions too - source.status = 'error' - source.last_message = f"Error: {error_message}" - source.save(update_fields=['status', 'last_message']) - - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error=f"Error: {error_message}") - return False + logger.error(f"Error in auto channel sync for account {account_id}: {str(e)}") + return { + "status": "error", + "error": str(e), + "channels_created": 0, + "channels_updated": 0, + "channels_deleted": 0, + "channels_failed": 0, + "failed_stream_details": [], + } -def extract_compressed_file(file_path, output_path=None, delete_original=False): +def get_transformed_credentials(account, profile=None): """ - Extracts a compressed file (.gz or .zip) to an XML file. + Get transformed credentials for XtreamCodes API calls. Args: - file_path: Path to the compressed file - output_path: Specific path where the file should be extracted (optional) - delete_original: Whether to delete the original compressed file after successful extraction + account: M3UAccount instance + profile: M3UAccountProfile instance (optional, if not provided will use primary profile) Returns: - Path to the extracted XML file, or None if extraction failed + tuple: (transformed_url, transformed_username, transformed_password) """ + import re + import urllib.parse + + # If no profile is provided, find the primary active profile + if profile is None: + try: + from apps.m3u.models import M3UAccountProfile + profile = M3UAccountProfile.objects.filter( + m3u_account=account, + is_active=True + ).first() + if profile: + logger.debug(f"Using primary profile '{profile.name}' for URL transformation") + else: + logger.debug(f"No active profiles found for account {account.name}, using base credentials") + except Exception as e: + logger.warning(f"Could not get primary profile for account {account.name}: {e}") + profile = None + + base_url = account.server_url + base_username = account.username + base_password = account.password # Build a complete URL with credentials (similar to how IPTV URLs are structured) + # Format: http://server.com:port/live/username/password/1234.ts + if base_url and base_username and base_password: + # Remove trailing slash from server URL if present + clean_server_url = base_url.rstrip('/') + + # Build the complete URL with embedded credentials + complete_url = f"{clean_server_url}/live/{base_username}/{base_password}/1234.ts" + logger.debug(f"Built complete URL: {complete_url}") + + # Apply profile-specific transformations if profile is provided + if profile and profile.search_pattern and profile.replace_pattern: + try: + # Handle backreferences: convert JS-style $ -> \g, $1 -> \1 + # regex module accepts JS-style (?...) named groups natively + safe_replace_pattern = regex.sub(r'\$<([^>]+)>', r'\\g<\1>', profile.replace_pattern) + safe_replace_pattern = regex.sub(r'\$(\d+)', r'\\\1', safe_replace_pattern) + + # Apply transformation to the complete URL + transformed_complete_url = regex.sub(profile.search_pattern, safe_replace_pattern, complete_url) + logger.info(f"Transformed complete URL: {complete_url} -> {transformed_complete_url}") + + # Extract components from the transformed URL + # Pattern: http://server.com:port/live/username/password/1234.ts + parsed_url = urllib.parse.urlparse(transformed_complete_url) + path_parts = [part for part in parsed_url.path.split('/') if part] + + if len(path_parts) >= 4 and path_parts[-1] == '1234.ts': + # Extract username and password from the known structure: + # .../{live}/{username}/{password}/1234.ts + # Using negative indices so sub-paths in the server URL don't shift extraction + transformed_username = path_parts[-3] + transformed_password = path_parts[-2] + + # Rebuild server URL: preserve any sub-path that precedes + # /live/username/password/1234.ts (path_parts[:-4]). + base_path_parts = path_parts[:-4] + base_path = ('/' + '/'.join(base_path_parts)) if base_path_parts else '' + transformed_url = f"{parsed_url.scheme}://{parsed_url.netloc}{base_path}" + + logger.debug(f"Extracted transformed credentials:") + logger.debug(f" Server URL: {transformed_url}") + logger.debug(f" Username: {transformed_username}") + logger.debug(f" Password: {transformed_password}") + + return transformed_url, transformed_username, transformed_password + else: + logger.warning(f"Could not extract credentials from transformed URL: {transformed_complete_url}") + return base_url, base_username, base_password + + except Exception as e: + logger.error(f"Error transforming URL for profile {profile.name if profile else 'unknown'}: {e}") + return base_url, base_username, base_password + else: + # No profile or no transformation patterns + return base_url, base_username, base_password + else: + logger.warning(f"Missing credentials for account {account.name}") + return base_url, base_username, base_password + + +@shared_task +def refresh_account_profiles(account_id): + """Refresh account information for all active profiles of an XC account. + + This task runs asynchronously in the background after account refresh completes. + It includes rate limiting delays between profile authentications to prevent provider bans. + """ + from django.conf import settings + import time + try: - if output_path is None: - base_path = os.path.splitext(file_path)[0] - extracted_path = f"{base_path}.xml" + account = M3UAccount.objects.get(id=account_id, is_active=True) + + if account.account_type != M3UAccount.Types.XC: + logger.debug(f"Account {account_id} is not XC type, skipping profile refresh") + return f"Account {account_id} is not an XtreamCodes account" + + from apps.m3u.models import M3UAccountProfile + + profiles = M3UAccountProfile.objects.filter( + m3u_account=account, + is_active=True + ) + + if not profiles.exists(): + logger.info(f"No active profiles found for account {account.name}") + return f"No active profiles for account {account_id}" + + # Get user agent for this account + try: + user_agent_string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + if account.user_agent_id: + from core.models import UserAgent + ua_obj = UserAgent.objects.get(id=account.user_agent_id) + if ua_obj and hasattr(ua_obj, "user_agent") and ua_obj.user_agent: + user_agent_string = ua_obj.user_agent + except Exception as e: + logger.warning(f"Error getting user agent, using fallback: {str(e)}") + logger.debug(f"Using user agent for profile refresh: {user_agent_string}") + # Get rate limiting delay from settings + profile_delay = getattr(settings, 'XC_PROFILE_REFRESH_DELAY', 2.5) + + profiles_updated = 0 + profiles_failed = 0 + + logger.info(f"Starting background refresh for {profiles.count()} profiles of account {account.name}") + + for idx, profile in enumerate(profiles): + try: + # Add delay between profiles to prevent rate limiting (except for first profile) + if idx > 0: + logger.info(f"Waiting {profile_delay}s before refreshing next profile to avoid rate limiting") + time.sleep(profile_delay) + + # Get transformed credentials for this specific profile + profile_url, profile_username, profile_password = get_transformed_credentials(account, profile) + + # Create a separate XC client for this profile's credentials + with XCClient( + profile_url, + profile_username, + profile_password, + user_agent_string + ) as profile_client: + # Authenticate with this profile's credentials + if profile_client.authenticate(): + # Get account information specific to this profile's credentials + profile_account_info = profile_client.get_account_info() + + # Merge with existing custom_properties if they exist + existing_props = profile.custom_properties or {} + existing_props.update(profile_account_info) + profile.custom_properties = existing_props + profile.save(update_fields=['custom_properties']) + + profiles_updated += 1 + logger.info(f"Updated account information for profile '{profile.name}' ({profiles_updated}/{profiles.count()})") + else: + profiles_failed += 1 + logger.warning(f"Failed to authenticate profile '{profile.name}' with transformed credentials") + + except Exception as profile_error: + profiles_failed += 1 + logger.error(f"Failed to update account information for profile '{profile.name}': {str(profile_error)}") + # Continue with other profiles even if one fails + + result_msg = f"Profile refresh complete for account {account.name}: {profiles_updated} updated, {profiles_failed} failed" + logger.info(result_msg) + return result_msg + + except M3UAccount.DoesNotExist: + error_msg = f"Account {account_id} not found" + logger.error(error_msg) + return error_msg + except Exception as e: + error_msg = f"Error refreshing profiles for account {account_id}: {str(e)}" + logger.error(error_msg) + return error_msg + + +@shared_task +def refresh_account_info(profile_id): + """Refresh only the account information for a specific M3U profile.""" + if not acquire_task_lock("refresh_account_info", profile_id): + return f"Account info refresh task already running for profile_id={profile_id}." + + try: + from apps.m3u.models import M3UAccountProfile + import re + + profile = M3UAccountProfile.objects.get(id=profile_id) + account = profile.m3u_account + + if account.account_type != M3UAccount.Types.XC: + release_task_lock("refresh_account_info", profile_id) + return f"Profile {profile_id} belongs to account {account.id} which is not an XtreamCodes account." + + # Get transformed credentials using the helper function + transformed_url, transformed_username, transformed_password = get_transformed_credentials(account, profile) + + # Initialize XtreamCodes client with extracted/transformed credentials + client = XCClient( + transformed_url, + transformed_username, + transformed_password, + account.get_user_agent(), + ) # Authenticate and get account info + auth_result = client.authenticate() + if not auth_result: + error_msg = f"Authentication failed for profile {profile.name} ({profile_id})" + logger.error(error_msg) + + # Send error notification to frontend via websocket + send_websocket_update( + "updates", + "update", + { + "type": "account_info_refresh_error", + "profile_id": profile_id, + "profile_name": profile.name, + "error": "Authentication failed with the provided credentials", + "message": f"Failed to authenticate profile '{profile.name}'. Please check the credentials." + } + ) + + release_task_lock("refresh_account_info", profile_id) + return error_msg + + # Get account information + account_info = client.get_account_info() + + # Update only this specific profile with the new account info + if not profile.custom_properties: + profile.custom_properties = {} + profile.custom_properties.update(account_info) + profile.save() + + # Send success notification to frontend via websocket + send_websocket_update( + "updates", + "update", + { + "type": "account_info_refresh_success", + "profile_id": profile_id, + "profile_name": profile.name, + "message": f"Account information successfully refreshed for profile '{profile.name}'" + } + ) + + release_task_lock("refresh_account_info", profile_id) + return f"Account info refresh completed for profile {profile_id} ({profile.name})." + + except M3UAccountProfile.DoesNotExist: + error_msg = f"Profile {profile_id} not found" + logger.error(error_msg) + + send_websocket_update( + "updates", + "update", + { + "type": "account_refresh_error", + "profile_id": profile_id, + "error": "Profile not found", + "message": f"Profile {profile_id} not found" + } + ) + + release_task_lock("refresh_account_info", profile_id) + return error_msg + except Exception as e: + error_msg = f"Error refreshing account info for profile {profile_id}: {str(e)}" + logger.error(error_msg) + + send_websocket_update( + "updates", + "update", + { + "type": "account_refresh_error", + "profile_id": profile_id, + "error": str(e), + "message": f"Failed to refresh account info: {str(e)}" + } + ) + + release_task_lock("refresh_account_info", profile_id) + return error_msg +@shared_task(time_limit=3600, soft_time_limit=3500) +def refresh_single_m3u_account(account_id): + """Splits M3U processing into chunks and dispatches them as parallel tasks.""" + if not acquire_task_lock("refresh_single_m3u_account", account_id): + return f"Task already running for account_id={account_id}." + + # Keep the lock alive while this long-running task is working. + # Without renewal, the 300s lock TTL can expire during large + # downloads/parses, allowing duplicate tasks to start. + lock_renewer = TaskLockRenewer("refresh_single_m3u_account", account_id) + lock_renewer.start() + + try: + return _refresh_single_m3u_account_impl(account_id) + finally: + # Guaranteed cleanup on all exit paths (success, exception, early return) + lock_renewer.stop() + release_task_lock("refresh_single_m3u_account", account_id) + + +def _refresh_single_m3u_account_impl(account_id): + """Implementation of M3U account refresh with guaranteed memory cleanup.""" + # Record start time + refresh_start_timestamp = timezone.now() # For the cleanup function + start_time = time.time() # For tracking elapsed time as float + streams_created = 0 + streams_updated = 0 + streams_deleted = 0 + + try: + account = M3UAccount.objects.get(id=account_id, is_active=True) + if not account.is_active: + logger.debug(f"Account {account_id} is not active, skipping.") + return + + # Set status to fetching + account.status = M3UAccount.Status.FETCHING + account.save(update_fields=['status']) + + filters = list(account.filters.all()) + + # Check if VOD is enabled for this account + vod_enabled = False + if account.custom_properties: + custom_props = account.custom_properties or {} + vod_enabled = custom_props.get('enable_vod', False) + + except M3UAccount.DoesNotExist: + # The M3U account doesn't exist, so delete the periodic task if it exists + logger.warning( + f"M3U account with ID {account_id} not found, but task was triggered. Cleaning up orphaned task." + ) + + # Call the helper function to delete the task + if delete_m3u_refresh_task_by_id(account_id): + logger.info( + f"Successfully cleaned up orphaned task for M3U account {account_id}" + ) else: - extracted_path = output_path + logger.debug(f"No orphaned task found for M3U account {account_id}") - # Make sure the output path doesn't already exist - if os.path.exists(extracted_path): + return f"M3UAccount with ID={account_id} not found or inactive, task cleaned up" + + # Fetch M3U lines and handle potential issues + extinf_data = [] + groups = None + + cache_path = os.path.join(m3u_dir, f"{account_id}.json") + if os.path.exists(cache_path): + try: + with open(cache_path, "r") as file: + data = json.load(file) + + extinf_data = data["extinf_data"] + groups = data["groups"] + del data # Free top-level dict; extinf_data/groups retain their references + except json.JSONDecodeError as e: + # Handle corrupted JSON file + logger.error( + f"Error parsing cached M3U data for account {account_id}: {str(e)}" + ) + + # Backup the corrupted file for potential analysis + backup_path = f"{cache_path}.corrupted" try: - os.remove(extracted_path) - logger.info(f"Removed existing extracted file: {extracted_path}") - except Exception as e: - logger.warning(f"Failed to remove existing extracted file: {e}") - # If we can't delete the existing file and no specific output was requested, - # create a unique filename instead - if output_path is None: - base_path = os.path.splitext(file_path)[0] - extracted_path = f"{base_path}_{uuid.uuid4().hex[:8]}.xml" + os.rename(cache_path, backup_path) + logger.info(f"Renamed corrupted cache file to {backup_path}") + except OSError as rename_err: + logger.warning( + f"Failed to rename corrupted cache file: {str(rename_err)}" + ) - # Use our detection helper to determine the file format instead of relying on extension - with open(file_path, 'rb') as f: - content_sample = f.read(4096) # Read a larger sample to ensure accurate detection + # Reset the data to empty structures + extinf_data = [] + groups = None + except Exception as e: + logger.error(f"Unexpected error reading cached M3U data: {str(e)}") + extinf_data = [] + groups = None - format_type, is_compressed, _ = detect_file_format(file_path=file_path, content=content_sample) + if not extinf_data: + try: + logger.info(f"Calling refresh_m3u_groups for account {account_id}") + result = refresh_m3u_groups(account_id, full_refresh=True, scan_start_time=refresh_start_timestamp) + logger.trace(f"refresh_m3u_groups result: {result}") - if format_type == 'gzip': - logger.debug(f"Extracting gzip file: {file_path}") + # Check for completely empty result or missing groups + if not result or result[1] is None: + logger.error( + f"Failed to refresh M3U groups for account {account_id}: {result}" + ) + return "Failed to update m3u account - download failed or other error" + + extinf_data, groups = result + + # XC accounts can have empty extinf_data but valid groups try: - # First check if the content is XML by reading a sample - with gzip.open(file_path, 'rb') as gz_file: - content_sample = gz_file.read(4096) # Read first 4KB for detection - detected_format, _, _ = detect_file_format(content=content_sample) + account = M3UAccount.objects.get(id=account_id) + is_xc_account = account.account_type == M3UAccount.Types.XC + except M3UAccount.DoesNotExist: + is_xc_account = False - if detected_format != 'xml': - logger.warning(f"GZIP file does not appear to contain XML content: {file_path} (detected as: {detected_format})") - # Continue anyway since GZIP only contains one file + # For XC accounts, empty extinf_data is normal at this stage + if not extinf_data and not is_xc_account: + logger.error(f"No streams found for non-XC account {account_id}") + account.status = M3UAccount.Status.ERROR + account.last_message = "No streams found in M3U source" + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, "parsing", 100, status="error", error="No streams found" + ) + except Exception as e: + logger.error(f"Exception in refresh_m3u_groups: {str(e)}", exc_info=True) + account.status = M3UAccount.Status.ERROR + account.last_message = f"Error refreshing M3U groups: {str(e)}" + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, + "parsing", + 100, + status="error", + error=f"Error refreshing M3U groups: {str(e)}", + ) + return "Failed to update m3u account" - # Reset file pointer and extract the content - gz_file.seek(0) - with open(extracted_path, 'wb') as out_file: - while True: - chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk or len(chunk) == 0: - break - out_file.write(chunk) - except Exception as e: - logger.error(f"Error extracting GZIP file: {e}", exc_info=True) - return None + # Only proceed with parsing if we actually have data and no errors were encountered + # Get account type to handle XC accounts differently + try: + is_xc_account = account.account_type == M3UAccount.Types.XC + except Exception: + is_xc_account = False - logger.info(f"Successfully extracted gzip file to: {extracted_path}") + # Modified validation logic for different account types + if (not groups) or (not is_xc_account and not extinf_data): + logger.error(f"No data to process for account {account_id}") + account.status = M3UAccount.Status.ERROR + account.last_message = "No data available for processing" + account.save(update_fields=["status", "last_message"]) + send_m3u_update( + account_id, + "parsing", + 100, + status="error", + error="No data available for processing", + ) + return "Failed to update m3u account, no data available" - # Delete original compressed file if requested - if delete_original: - try: - os.remove(file_path) - logger.info(f"Deleted original compressed file: {file_path}") - except Exception as e: - logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + hash_keys = CoreSettings.get_m3u_hash_key().split(",") - return extracted_path + existing_groups = { + group.name: group.id + for group in ChannelGroup.objects.filter( + m3u_accounts__m3u_account=account, # Filter by the M3UAccount + m3u_accounts__enabled=True, # Filter by the enabled flag in the join table + ) + } - elif format_type == 'zip': - logger.debug(f"Extracting zip file: {file_path}") - with zipfile.ZipFile(file_path, 'r') as zip_file: - # Find the first XML file in the ZIP archive - xml_files = [f for f in zip_file.namelist() if f.lower().endswith('.xml')] + try: + # Set status to parsing + account.status = M3UAccount.Status.PARSING + account.save(update_fields=["status"]) - if not xml_files: - logger.info("No files with .xml extension found in ZIP archive, checking content of all files") - # Check content of each file to see if any are XML without proper extension - for filename in zip_file.namelist(): - if not filename.endswith('/'): # Skip directories + # Commit any pending transactions before threading + from django.db import transaction + transaction.commit() + + # Initialize stream counters + streams_created = 0 + streams_updated = 0 + + if account.account_type == M3UAccount.Types.STADNARD: + logger.debug( + f"Processing Standard account ({account_id}) with groups: {existing_groups}" + ) + # Break into batches and process with threading - use global batch size + batches = [ + extinf_data[i : i + BATCH_SIZE] + for i in range(0, len(extinf_data), BATCH_SIZE) + ] + + logger.info(f"Processing {len(extinf_data)} streams in {len(batches)} thread batches") + + # Use 2 threads for optimal database connection handling + max_workers = min(2, len(batches)) + logger.debug(f"Using {max_workers} threads for processing") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit batch processing tasks using direct functions (now thread-safe) + future_to_batch = { + executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i + for i, batch in enumerate(batches) + } + + completed_batches = 0 + total_batches = len(batches) + + # Process completed batches as they finish + for future in as_completed(future_to_batch): + batch_idx = future_to_batch[future] + try: + result = future.result() + completed_batches += 1 + + # Extract stream counts from result + if isinstance(result, str): try: - # Read a sample of the file content - content_sample = zip_file.read(filename, 4096) # Read up to 4KB for detection - format_type, _, _ = detect_file_format(content=content_sample) - if format_type == 'xml': - logger.info(f"Found XML content in file without .xml extension: {filename}") - xml_files = [filename] - break - except Exception as e: - logger.warning(f"Error reading file {filename} from ZIP: {e}") + created_match = re.search(r"(\d+) created", result) + updated_match = re.search(r"(\d+) updated", result) + if created_match and updated_match: + created_count = int(created_match.group(1)) + updated_count = int(updated_match.group(1)) + streams_created += created_count + streams_updated += updated_count + except (AttributeError, ValueError): + pass - if not xml_files: - logger.error("No XML file found in ZIP archive") - return None + # Send progress update + progress = int((completed_batches / total_batches) * 100) + current_elapsed = time.time() - start_time - # Extract the first XML file - with open(extracted_path, 'wb') as out_file: - with zip_file.open(xml_files[0], "r") as xml_file: - while True: - chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk or len(chunk) == 0: - break - out_file.write(chunk) + if progress > 0: + estimated_total = (current_elapsed / progress) * 100 + time_remaining = max(0, estimated_total - current_elapsed) + else: + time_remaining = 0 - logger.info(f"Successfully extracted zip file to: {extracted_path}") + send_m3u_update( + account_id, + "parsing", + progress, + elapsed_time=current_elapsed, + time_remaining=time_remaining, + streams_processed=streams_created + streams_updated, + ) - # Delete original compressed file if requested - if delete_original: - try: - os.remove(file_path) - logger.info(f"Deleted original compressed file: {file_path}") - except Exception as e: - logger.warning(f"Failed to delete original compressed file {file_path}: {e}") + logger.debug(f"Thread batch {completed_batches}/{total_batches} completed") - return extracted_path + except Exception as e: + logger.error(f"Error in thread batch {batch_idx}: {str(e)}") + completed_batches += 1 # Still count it to avoid hanging + logger.info(f"Thread-based processing completed for account {account_id}") else: - logger.error(f"Unsupported or unrecognized compressed file format: {file_path} (detected as: {format_type})") - return None + # For XC accounts, get the groups with their custom properties containing xc_id + logger.debug(f"Processing XC account with groups: {existing_groups}") + + # Get the ChannelGroupM3UAccount entries with their custom_properties + channel_group_relationships = ChannelGroupM3UAccount.objects.filter( + m3u_account=account, enabled=True + ).select_related("channel_group") + + filtered_groups = {} + for rel in channel_group_relationships: + group_name = rel.channel_group.name + group_id = rel.channel_group.id + + # Load the custom properties with the xc_id + custom_props = rel.custom_properties or {} + if "xc_id" in custom_props: + filtered_groups[group_name] = { + "xc_id": custom_props["xc_id"], + "channel_group_id": group_id, + } + logger.debug( + f"Added group {group_name} with xc_id {custom_props['xc_id']}" + ) + else: + logger.warning( + f"No xc_id found in custom properties for group {group_name}" + ) + + logger.info( + f"Filtered {len(filtered_groups)} groups for processing: {filtered_groups}" + ) + + # Collect all XC streams in a single API call and filter by enabled categories + logger.info("Fetching all XC streams from provider and filtering by enabled categories...") + all_xc_streams = collect_xc_streams(account_id, filtered_groups) + + if not all_xc_streams: + logger.warning("No streams collected from XC groups") + else: + # Now batch by stream count (like standard M3U processing) + batches = [ + all_xc_streams[i : i + BATCH_SIZE] + for i in range(0, len(all_xc_streams), BATCH_SIZE) + ] + + logger.info(f"Processing {len(all_xc_streams)} XC streams in {len(batches)} batches") + + # Free the original list; batches hold independent sliced copies + del all_xc_streams + + # Use threading for XC stream processing - now with consistent batch sizes + max_workers = min(4, len(batches)) + logger.debug(f"Using {max_workers} threads for XC stream processing") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit stream batch processing tasks (reuse standard M3U processing) + future_to_batch = { + executor.submit(process_m3u_batch_direct, account_id, batch, existing_groups, hash_keys): i + for i, batch in enumerate(batches) + } + + completed_batches = 0 + total_batches = len(batches) + + # Process completed batches as they finish + for future in as_completed(future_to_batch): + batch_idx = future_to_batch[future] + try: + result = future.result() + completed_batches += 1 + + # Extract stream counts from result + if isinstance(result, str): + try: + created_match = re.search(r"(\d+) created", result) + updated_match = re.search(r"(\d+) updated", result) + if created_match and updated_match: + created_count = int(created_match.group(1)) + updated_count = int(updated_match.group(1)) + streams_created += created_count + streams_updated += updated_count + except (AttributeError, ValueError): + pass + + # Send progress update + progress = int((completed_batches / total_batches) * 100) + current_elapsed = time.time() - start_time + + if progress > 0: + estimated_total = (current_elapsed / progress) * 100 + time_remaining = max(0, estimated_total - current_elapsed) + else: + time_remaining = 0 + + send_m3u_update( + account_id, + "parsing", + progress, + elapsed_time=current_elapsed, + time_remaining=time_remaining, + streams_processed=streams_created + streams_updated, + ) + + logger.debug(f"XC thread batch {completed_batches}/{total_batches} completed") + + except Exception as e: + logger.error(f"Error in XC thread batch {batch_idx}: {str(e)}") + completed_batches += 1 # Still count it to avoid hanging + + logger.info(f"XC thread-based processing completed for account {account_id}") + + # Ensure all database transactions are committed before cleanup + logger.info( + f"All thread processing completed, ensuring DB transactions are committed before cleanup" + ) + # Force a simple DB query to ensure connection sync + Stream.objects.filter( + id=-1 + ).exists() # This will never find anything but ensures DB sync + + # Mark streams that weren't seen in this refresh as stale (pending deletion) + stale_stream_count = Stream.objects.filter( + m3u_account=account, + last_seen__lt=refresh_start_timestamp + ).update(is_stale=True) + logger.info(f"Marked {stale_stream_count} streams as stale for account {account_id}") + + # Mark group relationships that weren't seen in this refresh as stale (pending deletion) + stale_group_count = ChannelGroupM3UAccount.objects.filter( + m3u_account=account, + last_seen__lt=refresh_start_timestamp + ).update(is_stale=True) + logger.info(f"Marked {stale_group_count} group relationships as stale for account {account_id}") + + # Now run cleanup + streams_deleted = cleanup_streams(account_id, refresh_start_timestamp) + + # Cleanup stale group relationships (follows same retention policy as streams) + cleanup_stale_group_relationships(account, refresh_start_timestamp) + + # Run auto channel sync after successful refresh + auto_sync_message = "" + auto_sync_result = {} + try: + auto_sync_result = sync_auto_channels( + account_id, scan_start_time=str(refresh_start_timestamp) + ) or {} + logger.info( + f"Auto channel sync result for account {account_id}: {auto_sync_result}" + ) + if auto_sync_result.get("status") == "ok": + created = auto_sync_result.get("channels_created", 0) + updated = auto_sync_result.get("channels_updated", 0) + deleted = auto_sync_result.get("channels_deleted", 0) + failed = auto_sync_result.get("channels_failed", 0) + if created or updated or deleted or failed: + parts = [] + if created: + parts.append(f"{created} channel(s) created") + if updated: + parts.append(f"{updated} updated") + if deleted: + parts.append(f"{deleted} deleted") + if failed: + parts.append(f"{failed} failed") + auto_sync_message = f" Auto-sync: {', '.join(parts)}." + elif auto_sync_result.get("status") == "error": + auto_sync_message = ( + f" Auto-sync error: {auto_sync_result.get('error', 'unknown')}." + ) + except Exception as e: + logger.error( + f"Error running auto channel sync for account {account_id}: {str(e)}" + ) + + # Calculate elapsed time + elapsed_time = time.time() - start_time + + # Calculate total streams processed + streams_processed = streams_created + streams_updated + + # Set status to success and update timestamp BEFORE sending the final update + account.status = M3UAccount.Status.SUCCESS + account.last_message = ( + f"Processing completed in {elapsed_time:.1f} seconds. " + f"Streams: {streams_created} created, {streams_updated} updated, {streams_deleted} removed. " + f"Total processed: {streams_processed}.{auto_sync_message}" + ) + account.updated_at = timezone.now() + account.save(update_fields=["status", "last_message", "updated_at"]) + + # Log system event for M3U refresh + log_system_event( + event_type='m3u_refresh', + account_name=account.name, + elapsed_time=round(elapsed_time, 2), + streams_created=streams_created, + streams_updated=streams_updated, + streams_deleted=streams_deleted, + total_processed=streams_processed, + ) + + # Send final update with complete metrics and explicitly include success status + send_m3u_update( + account_id, + "parsing", + 100, + status="success", # Explicitly set status to success + elapsed_time=elapsed_time, + time_remaining=0, + streams_processed=streams_processed, + streams_created=streams_created, + streams_updated=streams_updated, + streams_deleted=streams_deleted, + # Structured auto-sync counts so the frontend can render a + # warning card when anything failed, without parsing the + # free-text last_message. + channels_created=auto_sync_result.get("channels_created", 0), + channels_updated=auto_sync_result.get("channels_updated", 0), + channels_deleted=auto_sync_result.get("channels_deleted", 0), + channels_failed=auto_sync_result.get("channels_failed", 0), + failed_stream_details=auto_sync_result.get("failed_stream_details", []), + message=account.last_message, + ) + + # Trigger VOD refresh if enabled and account is XtreamCodes type + if vod_enabled and account.account_type == M3UAccount.Types.XC: + logger.info(f"VOD is enabled for account {account_id}, triggering VOD refresh") + try: + from apps.vod.tasks import refresh_vod_content + refresh_vod_content.delay(account_id) + logger.info(f"VOD refresh task queued for account {account_id}") + except Exception as e: + logger.error(f"Failed to queue VOD refresh for account {account_id}: {str(e)}") except Exception as e: - logger.error(f"Error extracting {file_path}: {str(e)}", exc_info=True) + logger.error(f"Error processing M3U for account {account_id}: {str(e)}") + try: + account.status = M3UAccount.Status.ERROR + account.last_message = f"Error processing M3U: {str(e)}" + account.save(update_fields=["status", "last_message"]) + except Exception: + logger.debug(f"Failed to update account {account_id} status during error handling") + raise # Re-raise the exception for Celery to handle + finally: + # Free large data structures regardless of success or failure + if 'existing_groups' in locals(): + del existing_groups + if 'extinf_data' in locals(): + del extinf_data + if 'groups' in locals(): + del groups + if 'batches' in locals(): + del batches + if 'all_xc_streams' in locals(): + del all_xc_streams + if 'data' in locals(): + del data + if 'filtered_groups' in locals(): + del filtered_groups + if 'channel_group_relationships' in locals(): + del channel_group_relationships + + # Remove cache file after processing (success or failure) + cache_path = os.path.join(m3u_dir, f"{account_id}.json") + try: + os.remove(cache_path) + except OSError: + pass + + return f"Dispatched jobs complete." + + +def send_m3u_update(account_id, action, progress, **kwargs): + # Start with the base data dictionary + data = { + "progress": progress, + "type": "m3u_refresh", + "account": account_id, + "action": action, + } + + # Only fetch the account when we actually need to fill in missing fields. + # Many callers in tight loops already pass status/message; skip the DB hit then. + if "status" not in kwargs or "message" not in kwargs: + try: + account = M3UAccount.objects.only("status", "last_message").get(id=account_id) + if "status" not in kwargs: + data["status"] = account.status + if "message" not in kwargs and account.last_message: + data["message"] = account.last_message + except Exception: + pass + + # Add the additional key-value pairs from kwargs + data.update(kwargs) + send_websocket_update("updates", "update", data, collect_garbage=False) + + # Explicitly clear data reference to help garbage collection + data = None + + +def evaluate_profile_expiration_notification(profile): + """ + Evaluate a single M3UAccountProfile's expiration date and create, update, + or delete the corresponding SystemNotification accordingly. + + Returns the notification key that should remain active (warning or expired), + or None if the profile is not expiring soon and any stale notifications were removed. + This return value is used by the bulk task to track active keys for stale cleanup. + """ + from core.models import SystemNotification + from core.utils import send_websocket_notification, send_notification_dismissed + + exp = profile.exp_date + if not exp: + return None + + now = timezone.now() + warning_threshold = now + timezone.timedelta(days=7) + warning_key = f"xc-exp-warning-{profile.id}" + expired_key = f"xc-exp-expired-{profile.id}" + + if exp <= now: + # Already expired — delete warning, create/update expired notification + deleted_warning = list( + SystemNotification.objects.filter(notification_key=warning_key) + .values_list("notification_key", flat=True) + ) + SystemNotification.objects.filter(notification_key=warning_key).delete() + for key in deleted_warning: + send_notification_dismissed(key) + + notification, created = SystemNotification.objects.update_or_create( + notification_key=expired_key, + defaults={ + "notification_type": SystemNotification.NotificationType.WARNING, + "priority": SystemNotification.Priority.HIGH, + "title": f"Account Expired: {profile.name}", + "message": ( + f'Profile "{profile.name}" on M3U account ' + f'"{profile.m3u_account.name}" has expired ' + f"(expired {exp.strftime('%Y-%m-%d %H:%M UTC')})." + ), + "action_data": { + "profile_id": profile.id, + "account_id": profile.m3u_account.id, + "account_name": profile.m3u_account.name, + "profile_name": profile.name, + "exp_date": exp.isoformat(), + }, + "is_active": True, + "admin_only": True, + }, + ) + send_websocket_notification(notification) + return expired_key + + elif exp <= warning_threshold: + # Expiring within 7 days — delete expired notification, create/update warning + deleted_expired = list( + SystemNotification.objects.filter(notification_key=expired_key) + .values_list("notification_key", flat=True) + ) + SystemNotification.objects.filter(notification_key=expired_key).delete() + for key in deleted_expired: + send_notification_dismissed(key) + + days_left = (exp - now).days + if days_left == 0: + expires_in_str = "today" + elif days_left == 1: + expires_in_str = "in 1 day" + else: + expires_in_str = f"in {days_left} days" + + notification, created = SystemNotification.objects.update_or_create( + notification_key=warning_key, + defaults={ + "notification_type": SystemNotification.NotificationType.WARNING, + "priority": SystemNotification.Priority.NORMAL, + "title": f"Account Expiring: {profile.name}", + "message": ( + f'Profile "{profile.name}" on M3U account ' + f'"{profile.m3u_account.name}" expires {expires_in_str} ' + f"(expires {exp.strftime('%Y-%m-%d %H:%M UTC')})." + ), + "action_data": { + "profile_id": profile.id, + "account_id": profile.m3u_account.id, + "account_name": profile.m3u_account.name, + "profile_name": profile.name, + "exp_date": exp.isoformat(), + }, + "is_active": True, + "admin_only": True, + }, + ) + send_websocket_notification(notification) + return warning_key + + else: + # Not expiring soon — delete any stale notifications + deleted_keys = list( + SystemNotification.objects.filter( + notification_key__in=[warning_key, expired_key] + ).values_list("notification_key", flat=True) + ) + SystemNotification.objects.filter( + notification_key__in=[warning_key, expired_key] + ).delete() + for key in deleted_keys: + send_notification_dismissed(key) return None -def parse_channels_only(source): - # Use extracted file if available, otherwise use the original file path - file_path = source.extracted_file_path if source.extracted_file_path else source.file_path - if not file_path: - file_path = source.get_cache_file() - - # Send initial parsing notification - send_epg_update(source.id, "parsing_channels", 0) - - process = None - should_log_memory = False - - try: - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file does not exist at path: {file_path}") - - # Update the source's file_path to the default cache location - new_path = source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - source.file_path = new_path - source.save(update_fields=['file_path']) - - # If the source has a URL, fetch the data before continuing - if source.url: - logger.info(f"Fetching new EPG data from URL: {source.url}") - fetch_success = fetch_xmltv(source) # Store the result - - # Only proceed if fetch was successful AND file exists - if not fetch_success: - logger.error(f"Failed to fetch EPG data from URL: {source.url}") - # Update status to error - source.status = 'error' - source.last_message = f"Failed to fetch EPG data from URL" - source.save(update_fields=['status', 'last_message']) - # Send error notification - send_epg_update(source.id, "parsing_channels", 100, status="error", error="Failed to fetch EPG data") - return False - - # Verify the file was downloaded successfully - if not os.path.exists(source.file_path): - logger.error(f"Failed to fetch EPG data, file still missing at: {source.file_path}") - # Update status to error - source.status = 'error' - source.last_message = f"Failed to fetch EPG data, file missing after download" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found after download") - return False - - # Update file_path with the new location - file_path = source.file_path - else: - logger.error(f"No URL provided for EPG source {source.name}, cannot fetch new data") - # Update status to error - source.status = 'error' - source.last_message = f"No URL provided, cannot fetch EPG data" - source.save(update_fields=['updated_at']) - - # Initialize process variable for memory tracking only in debug mode - try: - process = None - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is DEBUG (10) or more verbose - # This is more future-proof than string comparisons - should_log_memory = current_log_level <= logging.DEBUG or settings.DEBUG - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_channels_only] Initial memory usage: {initial_memory:.2f} MB") - except (ImportError, NameError): - process = None - should_log_memory = False - logger.warning("psutil not available for memory tracking") - - # Replace full dictionary load with more efficient lookup set - existing_tvg_ids = set() - existing_epgs = {} - scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup - last_id = 0 - chunk_size = 5000 - - while True: - tvg_id_chunk = set(EPGData.objects.filter( - epg_source=source, - id__gt=last_id - ).order_by('id').values_list('tvg_id', flat=True)[:chunk_size]) - - if not tvg_id_chunk: - break - - existing_tvg_ids.update(tvg_id_chunk) - last_id = EPGData.objects.filter(tvg_id__in=tvg_id_chunk).order_by('-id')[0].id - # Update progress to show file read starting - send_epg_update(source.id, "parsing_channels", 10) - - # Stream parsing instead of loading entire file at once - # This can be simplified since we now always have XML files - epgs_to_create = [] - epgs_to_update = [] - total_channels = 0 - processed_channels = 0 - batch_size = 500 # Process in batches to limit memory usage - progress = 0 # Initialize progress variable here - icon_url_max_length = EPGData._meta.get_field('icon_url').max_length # Get max length for icon_url field - name_max_length = EPGData._meta.get_field('name').max_length # Get max length for name field - - # Track memory at key points - if process: - logger.debug(f"[parse_channels_only] Memory before opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - try: - # Attempt to count existing channels in the database - try: - total_channels = EPGData.objects.filter(epg_source=source).count() - logger.info(f"Found {total_channels} existing channels for this source") - except Exception as e: - logger.error(f"Error counting channels: {e}") - total_channels = 500 # Default estimate - if process: - logger.debug(f"[parse_channels_only] Memory after closing initial file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Update progress after counting - send_epg_update(source.id, "parsing_channels", 25, total_channels=total_channels) - - # Open the file - no need to check file type since it's always XML now - logger.debug(f"Opening file for channel parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - if process: - logger.debug(f"[parse_channels_only] Memory after opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Change iterparse to look for both channel and programme elements - logger.debug(f"Creating iterparse context for channels and programmes") - channel_parser = etree.iterparse(source_file, events=('end',), tag=('channel', 'programme'), remove_blank_text=True, recover=True) - if process: - logger.debug(f"[parse_channels_only] Memory after creating iterparse: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - channel_count = 0 - total_elements_processed = 0 # Track total elements processed, not just channels - for _, elem in channel_parser: - total_elements_processed += 1 - # Only process channel elements - if elem.tag == 'channel': - channel_count += 1 - tvg_id = elem.get('id', '').strip() - if tvg_id: - scanned_tvg_ids.add(tvg_id) - display_name = None - icon_url = None - for child in elem: - if display_name is None and child.tag == 'display-name' and child.text: - display_name = child.text.strip() - elif child.tag == 'icon': - raw_icon_url = child.get('src', '').strip() - icon_url = validate_icon_url_fast(raw_icon_url, icon_url_max_length) - if display_name and icon_url: - break # No need to continue if we have both - - if not display_name: - display_name = tvg_id - - if display_name and len(display_name) > name_max_length: - logger.warning(f"EPG display name too long ({len(display_name)} > {name_max_length}), truncating: {display_name[:80]}...") - display_name = display_name[:name_max_length] - - # Use lazy loading approach to reduce memory usage - if tvg_id in existing_tvg_ids: - # Only fetch the object if we need to update it and it hasn't been loaded yet - if tvg_id not in existing_epgs: - try: - # This loads the full EPG object from the database and caches it - existing_epgs[tvg_id] = EPGData.objects.get(tvg_id=tvg_id, epg_source=source) - except EPGData.DoesNotExist: - # Handle race condition where record was deleted - existing_tvg_ids.remove(tvg_id) - epgs_to_create.append(EPGData( - tvg_id=tvg_id, - name=display_name, - icon_url=icon_url, - epg_source=source, - )) - logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 1: {tvg_id} - {display_name}") - processed_channels += 1 - continue - - # We use the cached object to check if the name or icon_url has changed - epg_obj = existing_epgs[tvg_id] - needs_update = False - if epg_obj.name != display_name: - epg_obj.name = display_name - needs_update = True - if epg_obj.icon_url != icon_url: - epg_obj.icon_url = icon_url - needs_update = True - - if needs_update: - epgs_to_update.append(epg_obj) - logger.debug(f"[parse_channels_only] Added channel to update to epgs_to_update: {tvg_id} - {display_name}") - else: - # No changes needed, just clear the element - logger.debug(f"[parse_channels_only] No changes needed for channel {tvg_id} - {display_name}") - else: - # This is a new channel that doesn't exist in our database - epgs_to_create.append(EPGData( - tvg_id=tvg_id, - name=display_name, - icon_url=icon_url, - epg_source=source, - )) - logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 2: {tvg_id} - {display_name}") - - processed_channels += 1 - - # Batch processing - if len(epgs_to_create) >= batch_size: - logger.info(f"[parse_channels_only] Bulk creating {len(epgs_to_create)} EPG entries") - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - if process: - logger.info(f"[parse_channels_only] Memory after bulk_create: {process.memory_info().rss / 1024 / 1024:.2f} MB") - del epgs_to_create # Explicit deletion - epgs_to_create = [] - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - logger.info(f"[parse_channels_only] Memory after gc.collect(): {process.memory_info().rss / 1024 / 1024:.2f} MB") - - if len(epgs_to_update) >= batch_size: - logger.info(f"[parse_channels_only] Bulk updating {len(epgs_to_update)} EPG entries") - if process: - logger.info(f"[parse_channels_only] Memory before bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") - EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) - if process: - logger.info(f"[parse_channels_only] Memory after bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") - epgs_to_update = [] - # Force garbage collection - cleanup_memory(log_usage=should_log_memory, force_collection=True) - - # Periodically clear the existing_epgs cache to prevent memory buildup - if processed_channels % 1000 == 0: - logger.info(f"[parse_channels_only] Clearing existing_epgs cache at {processed_channels} channels") - existing_epgs.clear() - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - logger.info(f"[parse_channels_only] Memory after clearing cache: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Send progress updates - if processed_channels % 100 == 0 or processed_channels == total_channels: - progress = 25 + int((processed_channels / total_channels) * 65) if total_channels > 0 else 90 - send_epg_update( - source.id, - "parsing_channels", - progress, - processed=processed_channels, - total=total_channels - ) - if processed_channels > total_channels: - logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels - total_channels} additional channels") - else: - logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels}/{total_channels}") - if process: - logger.debug(f"[parse_channels_only] Memory before elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - # Clear memory - try: - # First clear the element's content - clear_element(elem) - - except Exception as e: - # Just log the error and continue - don't let cleanup errors stop processing - logger.debug(f"[parse_channels_only] Non-critical error during XML element cleanup: {e}") - if process: - logger.debug(f"[parse_channels_only] Memory after elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - logger.debug(f"[parse_channels_only] Total elements processed: {total_elements_processed}") - - else: - logger.trace(f"[parse_channels_only] Skipping non-channel element: {elem.get('channel', 'unknown')} - {elem.get('start', 'unknown')} {elem.tag}") - clear_element(elem) - continue - - except (etree.XMLSyntaxError, Exception) as xml_error: - logger.error(f"[parse_channels_only] XML parsing failed: {xml_error}") - # Update status to error - source.status = 'error' - source.last_message = f"Error parsing XML file: {str(xml_error)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(xml_error)) - return False - if process: - logger.info(f"[parse_channels_only] Processed {processed_channels} channels current memory: {process.memory_info().rss / 1024 / 1024:.2f} MB") - else: - logger.info(f"[parse_channels_only] Processed {processed_channels} channels") - # Process any remaining items - if epgs_to_create: - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - logger.debug(f"[parse_channels_only] Created final batch of {len(epgs_to_create)} EPG entries") - - if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) - logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") - - # Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel. - # Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list. - potentially_stale = existing_tvg_ids - scanned_tvg_ids - if potentially_stale: - stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True) - deleted_count, _ = stale_qs.delete() - if deleted_count: - logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel") - - if process: - logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Update source status with channel count - source.status = 'success' - source.last_message = f"Successfully parsed {processed_channels} channels" - source.save(update_fields=['status', 'last_message']) - - # Send completion notification - send_epg_update( - source.id, - "parsing_channels", - 100, - status="success", - channels_count=processed_channels - ) - - send_websocket_update('updates', 'update', {"success": True, "type": "epg_channels"}) - - logger.info(f"Finished parsing channel info. Found {processed_channels} channels.") - - return True - - except FileNotFoundError: - logger.error(f"EPG file not found at: {file_path}") - # Update status to error - source.status = 'error' - source.last_message = f"EPG file not found: {file_path}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found") - return False - except Exception as e: - logger.error(f"Error reading EPG file {file_path}: {e}", exc_info=True) - # Update status to error - source.status = 'error' - source.last_message = f"Error parsing EPG file: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(e)) - return False - finally: - # Cleanup memory and close file - if process: - logger.debug(f"[parse_channels_only] Memory before cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - try: - # Output any errors in the channel_parser error log - if 'channel_parser' in locals() and hasattr(channel_parser, 'error_log') and len(channel_parser.error_log) > 0: - logger.debug(f"XML parser errors found ({len(channel_parser.error_log)} total):") - for i, error in enumerate(channel_parser.error_log): - logger.debug(f" Error {i+1}: {error}") - if 'channel_parser' in locals(): - del channel_parser - if 'elem' in locals(): - del elem - if 'parent' in locals(): - del parent - - if 'source_file' in locals(): - source_file.close() - del source_file - # Clear remaining large data structures - existing_epgs.clear() - epgs_to_create.clear() - epgs_to_update.clear() - existing_epgs = None - epgs_to_create = None - epgs_to_update = None - if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None: - scanned_tvg_ids.clear() - scanned_tvg_ids = None - cleanup_memory(log_usage=should_log_memory, force_collection=True) - except Exception as e: - logger.warning(f"Cleanup error: {e}") - - try: - if process: - final_memory = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_channels_only] Final memory usage: {final_memory:.2f} MB") - process = None - except: - pass - - - -@shared_task(time_limit=3600, soft_time_limit=3500) -def parse_programs_for_tvg_id(epg_id): - # Skip XMLTV file parsing for Schedules Direct sources. Program data is - # fetched and persisted directly by fetch_schedules_direct(). - try: - from apps.epg.models import EPGData - epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() - if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': - logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") - return "Skipped (Schedules Direct source)" - except Exception as e: - logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") - - if not acquire_task_lock('parse_epg_programs', epg_id): - logger.info(f"Program parse for {epg_id} already in progress, skipping duplicate task") - return "Task already running" - - lock_renewer = TaskLockRenewer('parse_epg_programs', epg_id) - lock_renewer.start() - - source_file = None - program_parser = None - programs_to_create = [] - programs_processed = 0 - try: - # Add memory tracking only in trace mode or higher - try: - process = None - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is TRACE or more verbose or if running in DEBUG mode - should_log_memory = current_log_level <= 5 or settings.DEBUG - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Initial memory usage: {initial_memory:.2f} MB") - mem_before = initial_memory - except ImportError: - process = None - should_log_memory = False - - epg = EPGData.objects.get(id=epg_id) - epg_source = epg.epg_source - - # Skip program parsing for dummy EPG sources - they don't have program data files - if epg_source.source_type == 'dummy': - logger.info(f"Skipping program parsing for dummy EPG source {epg_source.name} (ID: {epg_id})") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - if not Channel.objects.filter(epg_data=epg).exists(): - logger.info(f"No channels matched to EPG {epg.tvg_id}") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") - - # Optimize deletion with a single delete query instead of chunking - # This is faster for most database engines - ProgramData.objects.filter(epg=epg).delete() - - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - if not file_path: - file_path = epg_source.get_cache_file() - - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file not found at: {file_path}") - - if epg_source.url: - # Update the file path in the database - new_path = epg_source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - epg_source.file_path = new_path - epg_source.save(update_fields=['file_path']) - logger.info(f"Fetching new EPG data from URL: {epg_source.url}") - else: - logger.info(f"EPG source does not have a URL, using existing file path: {file_path} to rebuild cache") - - # Fetch new data before continuing - if epg_source: - - # Properly check the return value from fetch_xmltv - fetch_success = fetch_xmltv(epg_source) - - # If fetch was not successful or the file still doesn't exist, abort - if not fetch_success: - logger.error(f"Failed to fetch EPG data, cannot parse programs for tvg_id: {epg.tvg_id}") - # Update status to error if not already set - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data, cannot parse programs" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Also check if the file exists after download - if not os.path.exists(epg_source.file_path): - logger.error(f"Failed to fetch EPG data, file still missing at: {epg_source.file_path}") - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data, file missing after download" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="File not found after download") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Update file_path with the new location - if epg_source.extracted_file_path: - file_path = epg_source.extracted_file_path - else: - file_path = epg_source.file_path - else: - logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") - # Update status to error - epg_source.status = 'error' - epg_source.last_message = f"No URL provided, cannot fetch EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Use streaming parsing to reduce memory usage - # No need to check file type anymore since it's always XML - logger.debug(f"Parsing programs for tvg_id={epg.tvg_id} from {file_path}") - - # Memory usage tracking - if process: - try: - mem_before = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_programs_for_tvg_id] Memory before parsing {epg.tvg_id} - {mem_before:.2f} MB") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - mem_before = 0 - - programs_to_create = [] - batch_size = 1000 # Process in batches to limit memory usage - - try: - # Open the file directly - no need to check compression - logger.debug(f"Opening file for parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - # Stream parse the file using lxml's iterparse - program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) - - for _, elem in program_parser: - if elem.get('channel') == epg.tvg_id: - try: - start_time = parse_xmltv_time(elem.get('start')) - end_time = parse_xmltv_time(elem.get('stop')) - title = None - desc = None - sub_title = None - - # Efficiently process child elements - for child in elem: - if child.tag == 'title': - title = child.text or 'No Title' - elif child.tag == 'desc': - desc = child.text or '' - elif child.tag == 'sub-title': - sub_title = child.text or '' - - if not title: - title = 'No Title' - - # Extract custom properties - custom_props = extract_custom_properties(elem) - custom_properties_json = None - - if custom_props: - logger.trace(f"Number of custom properties: {len(custom_props)}") - custom_properties_json = custom_props - - # Fallback: extract S/E from description when episode-num - # elements didn't provide them - if desc: - has_season = (custom_properties_json or {}).get('season') is not None - has_episode = (custom_properties_json or {}).get('episode') is not None - if not has_season or not has_episode: - d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) - if d_season is not None and d_episode is not None: - if custom_properties_json is None: - custom_properties_json = {} - if not has_season: - custom_properties_json['season'] = d_season - if not has_episode: - custom_properties_json['episode'] = d_episode - custom_properties_json['season_episode_source'] = 'description' - desc = cleaned_desc - - programs_to_create.append(ProgramData( - epg=epg, - start_time=start_time, - end_time=end_time, - title=title[:255], - description=desc, - sub_title=sub_title, - tvg_id=epg.tvg_id, - custom_properties=custom_properties_json - )) - programs_processed += 1 - # Clear the element to free memory - clear_element(elem) - # Batch processing - if len(programs_to_create) >= batch_size: - ProgramData.objects.bulk_create(programs_to_create) - logger.debug(f"Saved batch of {len(programs_to_create)} programs for {epg.tvg_id}") - programs_to_create = [] - # Only call gc.collect() every few batches - if programs_processed % (batch_size * 5) == 0: - gc.collect() - - except Exception as e: - logger.error(f"Error processing program for {epg.tvg_id}: {e}", exc_info=True) - else: - # Immediately clean up non-matching elements to reduce memory pressure - if elem is not None: - clear_element(elem) - continue - - # Make sure to close the file and release parser resources - if source_file: - source_file.close() - source_file = None - - if program_parser: - program_parser = None - - gc.collect() - - except zipfile.BadZipFile as zip_error: - logger.error(f"Bad ZIP file: {zip_error}") - raise - except etree.XMLSyntaxError as xml_error: - logger.error(f"XML syntax error parsing program data: {xml_error}") - raise - except Exception as e: - logger.error(f"Error parsing XML for programs: {e}", exc_info=True) - raise - finally: - # Ensure file is closed even if an exception occurs - if source_file: - source_file.close() - source_file = None - # Memory tracking after processing - if process: - try: - mem_after = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Memory after parsing 1 {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - - # Process any remaining items - if programs_to_create: - ProgramData.objects.bulk_create(programs_to_create) - logger.debug(f"Saved final batch of {len(programs_to_create)} programs for {epg.tvg_id}") - programs_to_create = None - custom_props = None - custom_properties_json = None - - - logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.") - finally: - # Reset internal caches and pools that lxml might be keeping - try: - etree.clear_error_log() - except: - pass - # Explicit cleanup of all potentially large objects - if source_file: - try: - source_file.close() - except: - pass - source_file = None - program_parser = None - programs_to_create = None - - epg_source = None - # Add comprehensive cleanup before releasing lock - cleanup_memory(log_usage=should_log_memory, force_collection=True) - # Memory tracking after processing - if process: - try: - mem_after = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Final memory usage {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - process = None - epg = None - programs_processed = None - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - - - -def parse_programs_for_source(epg_source, tvg_id=None): +@shared_task +def check_account_expirations(): """ - Parse programs for all MAPPED channels from an EPG source in a single pass. - - This is an optimized version that: - 1. Only processes EPG entries that are actually mapped to channels - 2. Parses the XML file ONCE instead of once per channel - 3. Skips programmes for unmapped channels entirely during parsing - - This dramatically improves performance when an EPG source has many channels - but only a fraction are mapped. + Daily task: check all account profiles for upcoming expirations. + Creates/updates SystemNotifications for profiles expiring within 7 days. + Uses separate notification keys for warning vs expired so users can + dismiss the 7-day warning and still receive the expired notification. """ - # Send initial programs parsing notification - send_epg_update(epg_source.id, "parsing_programs", 0) - should_log_memory = False - process = None - initial_memory = 0 - source_file = None + from apps.m3u.models import M3UAccountProfile + from core.models import SystemNotification + from core.utils import send_notification_dismissed - # Add memory tracking only in trace mode or higher - try: - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is TRACE or more verbose - should_log_memory = current_log_level <= 5 or settings.DEBUG # Assuming TRACE is level 5 or lower - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_source] Initial memory usage: {initial_memory:.2f} MB") - except ImportError: - logger.warning("psutil not available for memory tracking") - process = None - should_log_memory = False - - try: - # Only get EPG entries that are actually mapped to channels - mapped_epg_ids = set( - Channel.objects.filter( - epg_data__epg_source=epg_source, - epg_data__isnull=False - ).values_list('epg_data_id', flat=True) + # Find all active profiles with an exp_date that is set + expiring_profiles = ( + M3UAccountProfile.objects.filter( + m3u_account__is_active=True, + is_active=True, + exp_date__isnull=False, ) - - if not mapped_epg_ids: - total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() - logger.info(f"No channels mapped to any EPG entries from source: {epg_source.name} " - f"(source has {total_epg_count} EPG entries, 0 mapped)") - # Update status - this is not an error, just no mapped entries - epg_source.status = 'success' - epg_source.last_message = f"No channels mapped to this EPG source ({total_epg_count} entries available)" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="success") - return True - - # Get the mapped EPG entries with their tvg_ids - mapped_epgs = EPGData.objects.filter(id__in=mapped_epg_ids).values('id', 'tvg_id') - tvg_id_to_epg_id = {epg['tvg_id']: epg['id'] for epg in mapped_epgs if epg['tvg_id']} - mapped_tvg_ids = set(tvg_id_to_epg_id.keys()) - - total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() - mapped_count = len(mapped_tvg_ids) - - logger.info(f"Parsing programs for {mapped_count} MAPPED channels from source: {epg_source.name} " - f"(skipping {total_epg_count - mapped_count} unmapped EPG entries)") - - # Get the file path - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - if not file_path: - file_path = epg_source.get_cache_file() - - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file not found at: {file_path}") - - if epg_source.url: - # Update the file path in the database - new_path = epg_source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - epg_source.file_path = new_path - epg_source.save(update_fields=['file_path']) - logger.info(f"Fetching new EPG data from URL: {epg_source.url}") - - # Fetch new data before continuing - fetch_success = fetch_xmltv(epg_source) - - if not fetch_success: - logger.error(f"Failed to fetch EPG data for source: {epg_source.name}") - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") - return False - - # Update file_path with the new location - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - else: - logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") - epg_source.status = 'error' - epg_source.last_message = f"No URL provided, cannot fetch EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") - return False - - # SINGLE PASS PARSING: Parse the XML file once and collect all programs in memory - # We parse FIRST, then do an atomic delete+insert to avoid race conditions - # where clients might see empty/partial EPG data during the transition - all_programs_to_create = [] - programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} # Track count per channel - total_programs = 0 - skipped_programs = 0 - last_progress_update = 0 - - try: - logger.debug(f"Opening file for single-pass parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - # Stream parse the file using lxml's iterparse - program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) - - for _, elem in program_parser: - channel_id = elem.get('channel') - - # Skip programmes for unmapped channels immediately - if channel_id not in mapped_tvg_ids: - skipped_programs += 1 - # Clear element to free memory - clear_element(elem) - continue - - # This programme is for a mapped channel - process it - try: - start_time = parse_xmltv_time(elem.get('start')) - end_time = parse_xmltv_time(elem.get('stop')) - title = None - desc = None - sub_title = None - - # Efficiently process child elements - for child in elem: - if child.tag == 'title': - title = child.text or 'No Title' - elif child.tag == 'desc': - desc = child.text or '' - elif child.tag == 'sub-title': - sub_title = child.text or '' - - if not title: - title = 'No Title' - - # Extract custom properties - custom_props = extract_custom_properties(elem) - custom_properties_json = custom_props if custom_props else None - - # Fallback: extract S/E from description when episode-num - # elements didn't provide them - if desc: - has_season = (custom_properties_json or {}).get('season') is not None - has_episode = (custom_properties_json or {}).get('episode') is not None - if not has_season or not has_episode: - d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) - if d_season is not None and d_episode is not None: - if custom_properties_json is None: - custom_properties_json = {} - if not has_season: - custom_properties_json['season'] = d_season - if not has_episode: - custom_properties_json['episode'] = d_episode - custom_properties_json['season_episode_source'] = 'description' - desc = cleaned_desc - - epg_id = tvg_id_to_epg_id[channel_id] - all_programs_to_create.append(ProgramData( - epg_id=epg_id, - start_time=start_time, - end_time=end_time, - title=title[:255], - description=desc, - sub_title=sub_title, - tvg_id=channel_id, - custom_properties=custom_properties_json - )) - total_programs += 1 - programs_by_channel[channel_id] += 1 - - # Clear the element to free memory - clear_element(elem) - - # Send progress update (estimate based on programs processed) - if total_programs - last_progress_update >= 5000: - last_progress_update = total_programs - # Cap at 70% during parsing phase (save 30% for DB operations) - progress = min(70, 10 + int((total_programs / max(total_programs + 10000, 1)) * 60)) - send_epg_update(epg_source.id, "parsing_programs", progress, - processed=total_programs, channels=mapped_count) - - # Periodic garbage collection during parsing - if total_programs % 5000 == 0: - gc.collect() - - except Exception as e: - logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) - clear_element(elem) - continue - - except etree.XMLSyntaxError as xml_error: - logger.error(f"XML syntax error parsing program data: {xml_error}") - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"XML parsing error: {str(xml_error)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(xml_error)) - return False - except Exception as e: - logger.error(f"Error parsing XML for programs: {e}", exc_info=True) - raise - finally: - if source_file: - source_file.close() - source_file = None - - # Now perform atomic delete + bulk insert - # This ensures clients never see empty/partial EPG data - logger.info(f"Parsed {total_programs} programs, performing atomic database update...") - send_epg_update(epg_source.id, "parsing_programs", 75, message="Updating database...") - - batch_size = 1000 - try: - with transaction.atomic(): - # Kill any individual statement that hangs longer than 10 minutes. - # SET LOCAL automatically resets when this transaction ends (commit or rollback). - with connection.cursor() as cursor: - cursor.execute("SET LOCAL statement_timeout = '10min'") - # Delete existing programs for mapped EPGs - deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] - logger.debug(f"Deleted {deleted_count} existing programs") - - # Clean up orphaned programs for unmapped EPG entries - unmapped_epg_ids = list(EPGData.objects.filter( - epg_source=epg_source - ).exclude(id__in=mapped_epg_ids).values_list('id', flat=True)) - - if unmapped_epg_ids: - orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] - if orphaned_count > 0: - logger.info(f"Cleaned up {orphaned_count} orphaned programs for {len(unmapped_epg_ids)} unmapped EPG entries") - - # Bulk insert all new programs in batches within the same transaction - for i in range(0, len(all_programs_to_create), batch_size): - batch = all_programs_to_create[i:i + batch_size] - ProgramData.objects.bulk_create(batch) - - # Update progress during insertion - progress = 75 + int((i / len(all_programs_to_create)) * 20) if all_programs_to_create else 95 - if i % (batch_size * 5) == 0: - send_epg_update(epg_source.id, "parsing_programs", min(95, progress), - message=f"Inserting programs... {i}/{len(all_programs_to_create)}") - - logger.info(f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs") - - except Exception as db_error: - logger.error(f"Database error during atomic update: {db_error}", exc_info=True) - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"Database error: {str(db_error)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(db_error)) - return False - finally: - # Clear the large list to free memory - all_programs_to_create = None - gc.collect() - - # Count channels that actually got programs - channels_with_programs = sum(1 for count in programs_by_channel.values() if count > 0) - - # Success message - epg_source.status = EPGSource.STATUS_SUCCESS - epg_source.last_message = ( - f"Parsed {total_programs:,} programs for {channels_with_programs} channels " - f"(skipped {skipped_programs:,} programs for {total_epg_count - mapped_count} unmapped channels)" - ) - epg_source.updated_at = timezone.now() - epg_source.save(update_fields=['status', 'last_message', 'updated_at']) - - # Log system event for EPG refresh - log_system_event( - event_type='epg_refresh', - source_name=epg_source.name, - programs=total_programs, - channels=channels_with_programs, - skipped_programs=skipped_programs, - unmapped_channels=total_epg_count - mapped_count, - ) - - # Send completion notification with status - send_epg_update(epg_source.id, "parsing_programs", 100, - status="success", - message=epg_source.last_message, - updated_at=epg_source.updated_at.isoformat()) - - logger.info(f"Completed parsing programs for source: {epg_source.name} - " - f"{total_programs:,} programs for {channels_with_programs} channels, " - f"skipped {skipped_programs:,} programs for unmapped channels") - return True - - except Exception as e: - logger.error(f"Error in parse_programs_for_source: {e}", exc_info=True) - # Update status to error - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"Error parsing programs: {str(e)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, - status="error", - message=epg_source.last_message) - return False - finally: - # Final memory cleanup and tracking - if source_file: - try: - source_file.close() - except: - pass - source_file = None - - # Explicitly release any remaining large data structures - programs_to_create = None - programs_by_channel = None - mapped_epg_ids = None - mapped_tvg_ids = None - tvg_id_to_epg_id = None - gc.collect() - - # Add comprehensive memory cleanup at the end - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - final_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") - # Explicitly clear the process object to prevent potential memory leaks - process = None -@shared_task(bind=True) -def fetch_schedules_direct_stations(self, source_id): - """ - Lightweight Celery task that runs a stations-only Schedules Direct fetch. - Called on initial source creation so EPGData entries exist for auto-matching - before the user commits to a full schedule/program fetch. - """ - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - logger.error(f"EPGSource {source_id} not found for SD stations fetch") - return - fetch_schedules_direct(source, stations_only=True) - - -def fetch_schedules_direct(source, stations_only=False, force=False): - """ - Fetch EPG data from the Schedules Direct JSON API and persist it to the - EPGData / ProgramData models. - - Authentication flow (as required by the SD API specification): - 1. POST credentials to the token endpoint (password must be SHA1-hashed - as required by the Schedules Direct API specification. - 2. Use the returned token for all subsequent requests via the 'token' header. - 3. Tokens are valid for 24 hours; SD returns the current valid token if one - already exists for the account. - - Data flow: - 1. Fetch subscribed lineups for the account. - 2. Fetch station metadata for each lineup. - 3. Persist station metadata to EPGData. - 4. If stations_only=True, stop here. Used on initial source creation so - the user can run Auto-match EPG before the full program fetch. - 5. Fetch schedule grids in 14-day date-batched requests per station. - 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). - 7. Persist channels to EPGData and programs to ProgramData. - - Args: - source: EPGSource instance - stations_only: If True, only fetch and persist station metadata (no schedules/programs). - Used on initial source creation to populate EPGData for auto-matching - before channels are assigned. - """ - import hashlib - from datetime import date - - - logger.info(f"Fetching Schedules Direct data for source: {source.name}") - - # ------------------------------------------------------------------------- - # Validate credentials - # ------------------------------------------------------------------------- - username = (source.username or '').strip() - password = (source.password or '').strip() - - if not username or not password: - msg = "Schedules Direct source requires both a username and password." - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Enforce 2-hour minimum interval between full fetches (not stations-only). - # Schedules Direct enforces rate limits of ~200 requests per 2-hour window. - # This prevents automated abuse regardless of how the refresh was triggered. - # - # Exception: if no SDScheduleMD5 records exist yet, this is the first full - # refresh after initial source creation (stations-only runs first and updates - # updated_at, which would otherwise incorrectly trigger this guard). Always - # allow the first full refresh through so guide data is immediately available. - # ------------------------------------------------------------------------- - if not stations_only and not force and source.updated_at: - from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 - has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() - if has_prior_full_refresh: - elapsed = (timezone.now() - source.updated_at).total_seconds() - min_interval_seconds = 2 * 3600 # 2 hours - if elapsed < min_interval_seconds: - remaining_minutes = int((min_interval_seconds - elapsed) / 60) - msg = ( - f"Schedules Direct refresh skipped. Minimum 2-hour interval not reached. " - f"Last refreshed {int(elapsed / 60)} minutes ago. " - f"Please wait {remaining_minutes} more minute(s)." - ) - logger.warning(f"SD source {source.id}: {msg}") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - else: - logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") - elif force and not stations_only: - logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") - - # ------------------------------------------------------------------------- - # Build SD-specific headers - # SD API spec requires the User-Agent to identify the application and version. - # SergeantPanda confirmed Dispatcharr should identify itself properly. - # ------------------------------------------------------------------------- - from version import __version__ as dispatcharr_version - sd_user_agent = f"Dispatcharr/{dispatcharr_version}" - - def _sd_headers(token=None): - h = { - 'Content-Type': 'application/json', - 'User-Agent': sd_user_agent, - } - if token: - h['token'] = token - return h - - # ------------------------------------------------------------------------- - # Step 1: Authenticate and obtain session token - # The SD API requires the password to be SHA1-hashed before transmission. - # This is a requirement of the Schedules Direct API specification, not an - # architectural choice. - # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_FETCHING - source.last_message = "Authenticating with Schedules Direct..." - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") - - try: - sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() - token_response = requests.post( - f"{SD_BASE_URL}/token", - json={'username': username, 'password': sha1_password}, - headers=_sd_headers(), - timeout=30, - ) - token_response.raise_for_status() - token_data = token_response.json() - - auth_code = token_data.get('code', 0) - if auth_code != 0: - if auth_code == 4007: - msg = "Schedules Direct: this application is not authorized. Please contact the Dispatcharr maintainers." - elif auth_code == 4004: - msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." - elif auth_code == 4009: - msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours. Check for misconfiguration." - elif auth_code == 4001: - msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." - elif auth_code == 4008: - msg = "Schedules Direct: account is inactive. Please log in to schedulesdirect.org to reactivate." - else: - msg = f"Schedules Direct authentication failed (code {auth_code}): {token_data.get('message', 'Unknown error')}" - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - token = token_data.get('token') - if not token: - msg = "Schedules Direct returned no token." - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - logger.info("Schedules Direct authentication successful.") - - except requests.exceptions.RequestException as e: - msg = f"Network error authenticating with Schedules Direct: {e}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Step 2: Check account status (respect OFFLINE system status) - # ------------------------------------------------------------------------- - try: - status_response = requests.get( - f"{SD_BASE_URL}/status", - headers=_sd_headers(token), - timeout=30, - ) - status_response.raise_for_status() - status_data = status_response.json() - system_status = status_data.get('systemStatus', [{}])[0].get('status', 'Online') - if system_status == 'Offline': - # Per SD API spec: if system is offline, disconnect and do not - # retry for 1 hour. We set idle status rather than error since - # this is a temporary SD-side condition. - msg = "Schedules Direct system is currently offline. Per SD guidelines, retrying in 1 hour." - logger.warning(msg) - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - logger.debug(f"Schedules Direct system status: {system_status}") - except requests.exceptions.RequestException as e: - logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") - - # ------------------------------------------------------------------------- - # Step 3: Fetch subscribed lineups and build station map - # ------------------------------------------------------------------------- - _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") - try: - lineups_response = requests.get( - f"{SD_BASE_URL}/lineups", - headers=_sd_headers(token), - timeout=30, - ) - # SD returns 400 with code 4102 when no lineups are configured. - # This is a valid account state. The user needs to add lineups via - # the Manage Lineups UI. Treat as idle rather than error. - if lineups_response.status_code == 400: - sd_data = lineups_response.json() - if sd_data.get('code') == 4102: - msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - lineups_response.raise_for_status() - lineups_data = lineups_response.json() - lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] - if not lineups: - msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no active lineups found.") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - logger.info(f"Found {len(lineups)} lineup(s) in SD account.") - - # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) - sd_lineup_country = None - for l in lineups: - lid = l.get('lineupID') or l.get('lineup') or '' - if '-' in lid: - sd_lineup_country = lid.split('-')[0] - break - logger.debug(f"SD lineup country: {sd_lineup_country}") - except requests.exceptions.RequestException as e: - msg = f"Failed to fetch Schedules Direct lineups: {e}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # Build station metadata map: stationID -> {name, callsign, logo_url} - station_map = {} - _sd_send_ws_sync(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") - for lineup in lineups: - lineup_id = lineup.get('lineupID') or lineup.get('lineup') - if not lineup_id: - continue - try: - detail_response = requests.get( - f"{SD_BASE_URL}/lineups/{lineup_id}", - headers=_sd_headers(token), - timeout=30, - ) - detail_response.raise_for_status() - detail_data = detail_response.json() - for station in detail_data.get('stations', []): - sid = station.get('stationID') - if not sid: - continue - logo_url = None - logos = station.get('stationLogo') or station.get('logo') or [] - if isinstance(logos, list) and logos: - # Read preferred logo style from source settings; default to 'dark' - logo_style = (source.custom_properties or {}).get('logo_style', 'dark') - preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) - logo_url = preferred.get('URL') or preferred.get('url') - elif isinstance(logos, dict): - logo_url = logos.get('URL') or logos.get('url') - station_map[sid] = { - 'name': station.get('name', sid), - 'callsign': station.get('callsign', ''), - 'logo_url': logo_url, - } - logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") - - if not station_map: - msg = "No stations found across all Schedules Direct lineups." - logger.warning(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - logger.info(f"Built station map with {len(station_map)} stations.") - - # ------------------------------------------------------------------------- - # Step 4: Persist station metadata to EPGData - # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_PARSING - source.last_message = f"Syncing {len(station_map)} stations..." - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") - - existing_epg_map = { - epg.tvg_id: epg - for epg in EPGData.objects.filter(epg_source=source) - } - - epgs_to_create = [] - epgs_to_update = [] - icon_max_length = EPGData._meta.get_field('icon_url').max_length - name_max_length = EPGData._meta.get_field('name').max_length - - for sid, info in station_map.items(): - display_name = (info['name'] or sid)[:name_max_length] - logo = info['logo_url'] - if logo and len(logo) > icon_max_length: - logo = None - - if sid in existing_epg_map: - epg_obj = existing_epg_map[sid] - needs_update = False - if epg_obj.name != display_name: - epg_obj.name = display_name - needs_update = True - if epg_obj.icon_url != logo: - epg_obj.icon_url = logo - needs_update = True - if needs_update: - epgs_to_update.append(epg_obj) - else: - epgs_to_create.append(EPGData( - tvg_id=sid, - name=display_name, - icon_url=logo, - epg_source=source, - )) - - if epgs_to_create: - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") - if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) - logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") - - gc.collect() - - # Rebuild map with fresh DB ids for all stations - epg_id_map = { - epg.tvg_id: epg.id - for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) - } - - # Station sync complete. Send progress update before continuing into programs phase. - # We deliberately do NOT send parsing_channels at 100 with status=success here - # because that would cause the frontend to mark the source as complete and - # stop rendering progress updates for the subsequent program fetch phases. - _sd_send_ws_sync(source.id, "parsing_programs", 30, - message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") - - # ------------------------------------------------------------------------- - # Stations-only mode. Used on initial source creation. - # Stop here so the user can run Auto-match EPG before the full program fetch. - # ------------------------------------------------------------------------- - if stations_only: - success_msg = ( - f"{len(station_map)} stations loaded from Schedules Direct. " - f"Run Auto-match EPG to map your channels, then use the Refresh " - f"button to populate guide data." - ) - source.status = EPGSource.STATUS_SUCCESS - source.last_message = success_msg - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - _sd_send_ws_sync(source.id, "parsing_channels", 100, status="success", - message=success_msg, channels_count=len(station_map)) - logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") - return - - # ------------------------------------------------------------------------- - # Step 5: MD5-delta schedule fetch - # First fetch MD5 hashes for all stations/dates. Compare against our - # locally cached hashes to determine which schedules have changed. - # Only download schedules that have actually changed; this minimises - # API calls against SD's rate-limited endpoints. - # ------------------------------------------------------------------------- - from apps.epg.models import SDScheduleMD5 - from django.utils.dateparse import parse_datetime - - _sd_send_ws_sync(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") - station_ids = list(station_map.keys()) - today = date.today() - date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] - - # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. - # These accumulate one row per station per day and are never useful once past. - pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] - if pruned_sched_md5_count: - logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") - - # Fetch MD5 hashes for all stations in batches of 5000 - STATION_BATCH_SIZE = 5000 - server_md5s = {} # (station_id, date) -> {md5, last_modified} - - logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") - - station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] - for batch in station_batches: - try: - md5_response = requests.post( - f"{SD_BASE_URL}/schedules/md5", - json=[{'stationID': sid, 'date': date_list} for sid in batch], - headers=_sd_headers(token), - timeout=120, - ) - md5_response.raise_for_status() - md5_data = md5_response.json() - for sid, dates in md5_data.items(): - for date_str, info in dates.items(): - if info.get('code', 0) == 0: - server_md5s[(sid, date_str)] = { - 'md5': info.get('md5', ''), - 'last_modified': info.get('lastModified', ''), - } - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch schedule MD5s: {e}") - - # Load our cached MD5s from DB - cached_md5s = { - (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 - for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) - } - - # Determine which station/date combinations need downloading - changed_by_station = {} # station_id -> [date_str, ...] - for (sid, date_str), server_info in server_md5s.items(): - if date_str not in date_list: - continue - cached = cached_md5s.get((sid, date_str)) - if cached != server_info['md5']: - changed_by_station.setdefault(sid, []).append(date_str) - - total_changed = sum(len(v) for v in changed_by_station.values()) - total_possible = len(station_ids) * len(date_list) - logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") - _sd_send_ws_sync(source.id, "parsing_programs", 38, - message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") - - # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} - schedules_by_station = {sid: [] for sid in station_ids} - program_ids_needed = set() - - if not changed_by_station: - logger.info("No schedule changes detected, skipping schedule and program downloads.") - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", - message="No schedule changes detected since last refresh. Guide data is up to date.") - source.status = EPGSource.STATUS_SUCCESS - source.last_message = "No schedule changes detected. Guide data is up to date." - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - return - - # Download only changed schedules, batched by 7-day windows per station - SCHEDULE_BATCH_DAYS = 7 - changed_station_ids = list(changed_by_station.keys()) - date_batches = [date_list[i:i + SCHEDULE_BATCH_DAYS] for i in range(0, len(date_list), SCHEDULE_BATCH_DAYS)] - new_md5_records = [] - updated_md5_records = [] - existing_md5_map = { - (r.station_id, r.date.strftime('%Y-%m-%d')): r - for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=changed_station_ids) - } - - for batch_idx, date_batch in enumerate(date_batches): - # Notify frontend at the start of each batch so progress updates immediately - pre_progress = 38 + int((batch_idx / len(date_batches)) * 22) - logger.info(f"Fetching schedule batch {batch_idx + 1} of {len(date_batches)}...") - _sd_send_ws_sync(source.id, "parsing_programs", min(59, pre_progress), - message=f"Fetching schedules: batch {batch_idx + 1} of {len(date_batches)}...") - # Yield to gevent hub so the WebSocket update is delivered before the blocking request - try: - import gevent; gevent.sleep(0) - except ImportError: - pass - # Only include stations that have changes in this date batch - request_body = [ - {'stationID': sid, 'date': [d for d in date_batch if d in changed_by_station.get(sid, [])]} - for sid in changed_station_ids - if any(d in changed_by_station.get(sid, []) for d in date_batch) - ] - if not request_body: - continue - try: - sched_response = requests.post( - f"{SD_BASE_URL}/schedules", - json=request_body, - headers=_sd_headers(token), - timeout=120, - ) - sched_response.raise_for_status() - sched_data = sched_response.json() - - for station_sched in sched_data: - sid = station_sched.get('stationID') - if not sid: - continue - programs = station_sched.get('programs', []) - schedules_by_station.setdefault(sid, []).extend(programs) - for prog in programs: - pid = prog.get('programID') - if pid: - program_ids_needed.add(pid) - - # Update MD5 cache for this station/date - meta = station_sched.get('metadata', {}) - start_date = meta.get('startDate') - md5_val = meta.get('md5', '') - last_mod_str = meta.get('modified', '') - if start_date and md5_val: - key = (sid, start_date) - last_mod = parse_datetime(last_mod_str) if last_mod_str else timezone.now() - if key in existing_md5_map: - rec = existing_md5_map[key] - rec.md5 = md5_val - rec.last_modified = last_mod - updated_md5_records.append(rec) - else: - import datetime as dt_module - try: - date_obj = dt_module.date.fromisoformat(start_date) - new_md5_records.append(SDScheduleMD5( - epg_source=source, - station_id=sid, - date=date_obj, - md5=md5_val, - last_modified=last_mod, - )) - except ValueError: - pass - - progress = 38 + int(((batch_idx + 1) / len(date_batches)) * 22) - _sd_send_ws_sync(source.id, "parsing_programs", min(60, progress), - message=f"Fetching changed schedules: batch {batch_idx + 1}/{len(date_batches)} ({len(program_ids_needed):,} programs found)") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch schedule batch {batch_idx + 1}: {e}") - - # Persist updated MD5 cache - if new_md5_records: - SDScheduleMD5.objects.bulk_create(new_md5_records, ignore_conflicts=True) - logger.info(f"Cached {len(new_md5_records)} new schedule MD5s.") - if updated_md5_records: - SDScheduleMD5.objects.bulk_update(updated_md5_records, ['md5', 'last_modified']) - logger.info(f"Updated {len(updated_md5_records)} existing schedule MD5s.") - - if not program_ids_needed: - msg = "No schedule data returned from Schedules Direct." - logger.warning(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Step 6: MD5-delta program metadata fetch - # The schedule response includes an MD5 hash per program airing. - # Compare against our cached program MD5s to only download programs - # whose metadata has changed since our last fetch. - # ------------------------------------------------------------------------- - - # Build map of programID -> md5 from schedule data - schedule_program_md5s = {} # programID -> md5 from schedule - for sid, airings in schedules_by_station.items(): - for airing in airings: - pid = airing.get('programID') - md5 = airing.get('md5') - if pid and md5: - schedule_program_md5s[pid] = md5 - - # Load cached program MD5s from SDProgramMD5 table, keyed by programID - cached_prog_md5s = { - r.program_id: r.md5 - for r in SDProgramMD5.objects.filter( - epg_source=source, - program_id__in=program_ids_needed, - ).only('program_id', 'md5') - } - - # Only fetch programs where MD5 differs from our cached value - programs_to_fetch = { - pid for pid in program_ids_needed - if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) - } + .select_related("m3u_account") + ) + + active_notification_keys = set() + + for profile in expiring_profiles: + active_key = evaluate_profile_expiration_notification(profile) + if active_key: + active_notification_keys.add(active_key) + + # Delete stale notifications for profiles whose expiration was extended + stale = SystemNotification.objects.filter( + is_active=True, + ).filter( + models.Q(notification_key__startswith="xc-exp-warning-") | + models.Q(notification_key__startswith="xc-exp-expired-") + ).exclude(notification_key__in=active_notification_keys) + stale_keys = list(stale.values_list("notification_key", flat=True)) + stale.delete() + for key in stale_keys: + send_notification_dismissed(key) logger.info( - f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " - f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") - - program_metadata = {} - program_id_list = list(programs_to_fetch) - total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) - - if program_id_list: - logger.info(f"Fetching metadata for {len(program_id_list)} programs in {total_batches} batch(es).") - for batch_idx in range(total_batches): - # Notify frontend at the start of each batch so progress updates immediately - pre_progress = 60 + int((batch_idx / total_batches) * 20) - logger.info(f"Fetching program metadata batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)...") - _sd_send_ws_sync(source.id, "parsing_programs", min(79, pre_progress), - message=f"Fetching program data: batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)") - # Yield to gevent hub so the WebSocket update is delivered before the blocking request - try: - import gevent; gevent.sleep(0) - except ImportError: - pass - batch = program_id_list[batch_idx * SD_PROGRAM_BATCH_SIZE:(batch_idx + 1) * SD_PROGRAM_BATCH_SIZE] - try: - prog_response = requests.post( - f"{SD_BASE_URL}/programs", - json=batch, - headers=_sd_headers(token), - timeout=120, - ) - prog_response.raise_for_status() - prog_data = prog_response.json() - for prog in prog_data: - pid = prog.get('programID') - if pid: - program_metadata[pid] = prog - - progress = 60 + int(((batch_idx + 1) / total_batches) * 20) - _sd_send_ws_sync(source.id, "parsing_programs", min(80, progress), - message=f"Fetching program details: batch {batch_idx + 1}/{total_batches} ({len(program_metadata):,} programs loaded)") - logger.debug(f"Fetched program metadata batch {batch_idx + 1}/{total_batches}") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch program metadata batch {batch_idx + 1}: {e}") - else: - logger.info("All program metadata unchanged - skipping program download.") - _sd_send_ws_sync(source.id, "parsing_programs", 80, message="Program metadata unchanged - using cached data.") - - gc.collect() - - # ------------------------------------------------------------------------- - # Step 7: Build ProgramData records and persist atomically - # ------------------------------------------------------------------------- - logger.info("Building program records...") - _sd_send_ws_sync(source.id, "parsing_programs", 80) - - # Only process stations that are mapped to channels to match the existing - # XMLTV flow (parse_programs_for_source only processes mapped channels). - from apps.channels.models import Channel as ChannelModel - mapped_epg_ids = set( - ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).values_list('epg_data_id', flat=True) + f"Account expiration check complete: {len(active_notification_keys)} active notifications" ) - mapped_tvg_ids = set( - EPGData.objects.filter( - id__in=mapped_epg_ids, - epg_source=source, - ).values_list('tvg_id', flat=True) - ) - - # Cache existing program data for unchanged programs BEFORE surgical delete. - # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only - # programs with changed program MD5s get metadata re-downloaded. The surgical delete - # wipes ALL ProgramData for changed dates, so unchanged programs lose their titles. - # This cache preserves their data for rebuilding. - unchanged_pids = set() - for sid, airings in schedules_by_station.items(): - if sid not in mapped_tvg_ids: - continue - for airing in airings: - pid = airing.get('programID') - if pid and pid not in program_metadata: - unchanged_pids.add(pid) - - existing_program_cache = {} - if unchanged_pids: - for pd in ProgramData.objects.filter( - epg__epg_source=source, - program_id__in=unchanged_pids, - ).only('program_id', 'title', 'description', 'sub_title', 'custom_properties'): - if pd.program_id not in existing_program_cache: - existing_program_cache[pd.program_id] = { - 'title': pd.title, - 'description': pd.description, - 'sub_title': pd.sub_title, - 'custom_properties': pd.custom_properties, - } - logger.info(f"Cached {len(existing_program_cache)} existing program records for unchanged programs.") - - all_programs_to_create = [] - total_programs = 0 - skipped_unmapped = 0 - - for sid, airings in schedules_by_station.items(): - if sid not in mapped_tvg_ids: - skipped_unmapped += len(airings) - continue - - epg_db_id = epg_id_map.get(sid) - if not epg_db_id: - continue - - for airing in airings: - pid = airing.get('programID') - air_time = airing.get('airDateTime') - duration_secs = airing.get('duration', 0) - - if not pid or not air_time or not duration_secs: - continue - - try: - start_dt = parse_schedules_direct_time(air_time) - end_dt = start_dt + timedelta(seconds=int(duration_secs)) - except Exception as e: - logger.debug(f"Could not parse air time '{air_time}': {e}") - continue - - meta = program_metadata.get(pid, {}) - cached_prog = existing_program_cache.get(pid) if not meta else None - - if cached_prog: - # Unchanged program — reuse cached data from before surgical delete - title = cached_prog['title'] or 'No Title' - desc = cached_prog['description'] or '' - episode_title = cached_prog['sub_title'] or '' - custom_props = cached_prog['custom_properties'] or {} - else: - titles = meta.get('titles', [{}]) - title = titles[0].get('title120', '') if titles else '' - if not title: - title = meta.get('episodeTitle150', '') or 'No Title' - title = title[:255] - - if not cached_prog: - descriptions = meta.get('descriptions', {}) - desc = '' - for key in ('description1000', 'description255', 'description100'): - candidates = descriptions.get(key, []) - if candidates: - desc = candidates[0].get('description', '') - if desc: - break - - episode_title = meta.get('episodeTitle150', '') - - # Build custom_properties following the same pattern as the XMLTV parser - custom_props = {} - - # Season/Episode — search all metadata entries, not just [0] - metadata_block = meta.get('metadata', []) - gracenote_meta = {} - for md_entry in metadata_block: - if 'Gracenote' in md_entry: - gracenote_meta = md_entry['Gracenote'] - break - if not gracenote_meta: - # Fall back to TVmaze if Gracenote is absent - for md_entry in metadata_block: - if 'TVmaze' in md_entry: - gracenote_meta = md_entry['TVmaze'] - break - season = gracenote_meta.get('season') - episode = gracenote_meta.get('episode') - if season: - custom_props['season'] = int(season) - if episode: - custom_props['episode'] = int(episode) - if season and episode: - custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" - - # Content rating — store full array, pick display rating by lineup country - content_rating = meta.get('contentRating', []) - if content_rating: - custom_props['content_ratings'] = content_rating - selected = None - if sd_lineup_country: - for cr in content_rating: - if cr.get('country', '') == sd_lineup_country: - selected = cr - break - if not selected: - # Fall back to USA, then first available - for cr in content_rating: - if cr.get('country', '') == 'USA': - selected = cr - break - if not selected: - selected = content_rating[0] - custom_props['rating'] = selected.get('code', '') - custom_props['rating_system'] = selected.get('body', '') - - # Content advisory — content warnings - content_advisory = meta.get('contentAdvisory', []) - if content_advisory: - custom_props['content_advisory'] = content_advisory - - # Categories — combine entityType, showType, and genres - categories = [] - entity_type = meta.get('entityType', '') - show_type = meta.get('showType', '') - if entity_type: - categories.append(entity_type) - if show_type and show_type != entity_type: - categories.append(show_type) - genres = meta.get('genres', []) - categories.extend(genres) - if categories: - custom_props['categories'] = categories - - # Cast — top-billed only, store characterName, drop role noise - cast = meta.get('cast', []) - crew = meta.get('crew', []) - credits = {} - if cast: - # Sort by billingOrder and cap at top-billed actors - sorted_cast = sorted( - [p for p in cast if p.get('name')], - key=lambda p: int(p.get('billingOrder', '999')) - ) - # Separate main cast from guest stars - main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] - # Store top-billed main cast (matching XMLTV parity) - credits['actor'] = [ - { - 'name': p.get('name', ''), - **(({'character': p['characterName']} ) if p.get('characterName') else {}), - } - for p in (main_cast[:6] if main_cast else sorted_cast[:6]) - ] - if crew: - for member in crew: - role = member.get('role', '').lower() - name = member.get('name', '') - if not name: - continue - if 'director' in role: - credits.setdefault('director', []).append(name) - elif 'writer' in role or 'screenwriter' in role: - credits.setdefault('writer', []).append(name) - elif 'producer' in role: - credits.setdefault('producer', []).append(name) - if credits: - custom_props['credits'] = credits - - # Airing flags - if airing.get('liveTapeDelay') == 'Live': - custom_props['live'] = True - if airing.get('new'): - custom_props['new'] = True - else: - custom_props['previously_shown'] = True - if airing.get('premiere'): - custom_props['premiere'] = True - - # Original air date — full date, not just year - original_air_date = meta.get('originalAirDate', '') - movie_year = meta.get('movie', {}).get('year', '') - if original_air_date: - custom_props['date'] = original_air_date - elif movie_year: - custom_props['date'] = str(movie_year) - - # Country of production - country = meta.get('country', []) - if country: - custom_props['country'] = country[0] if len(country) == 1 else ', '.join(country) - - # Runtime — program duration without commercials (seconds → store for display) - runtime_secs = meta.get('duration') or meta.get('movie', {}).get('duration') - if runtime_secs: - runtime_mins = int(runtime_secs) // 60 - custom_props['length'] = {'value': str(runtime_mins), 'units': 'minutes'} - - # Movie quality ratings → star_ratings (matches XMLTV key) - movie_data = meta.get('movie', {}) - quality_ratings = movie_data.get('qualityRating', []) - if quality_ratings: - star_ratings = [] - for qr in quality_ratings: - rating_str = qr.get('rating', '') - max_rating = qr.get('maxRating', '') - if rating_str and max_rating: - star_ratings.append({ - 'value': f"{rating_str}/{max_rating}", - 'system': qr.get('ratingsBody', ''), - }) - if star_ratings: - custom_props['star_ratings'] = star_ratings - - # Sports event details - event_details = meta.get('eventDetails', {}) - if event_details: - custom_props['event_details'] = event_details - - all_programs_to_create.append(ProgramData( - epg_id=epg_db_id, - start_time=start_dt, - end_time=end_dt, - title=title, - sub_title=episode_title or None, - description=desc or None, - tvg_id=sid, - program_id=pid, - custom_properties=custom_props or None, - )) - total_programs += 1 - - logger.info(f"Built {total_programs} program records " - f"({skipped_unmapped} skipped for unmapped stations).") - - _sd_send_ws_sync(source.id, "parsing_programs", 88) - - # Build a map of epg_db_id -> list of (day_start_utc, day_end_utc) for each changed date. - # Only programs that fall within changed station/date pairs will be deleted and replaced; - # programs for unchanged stations or unchanged dates are left intact. - import datetime as dt_module - epg_changed_date_ranges = {} - for sid, changed_date_strs in changed_by_station.items(): - epg_db_id = epg_id_map.get(sid) - if not epg_db_id or epg_db_id not in mapped_epg_ids: - continue - ranges = [] - for ds in changed_date_strs: - d = dt_module.date.fromisoformat(ds) - day_start = datetime(d.year, d.month, d.day, tzinfo=dt_timezone.utc) - ranges.append((day_start, day_start + timedelta(days=1))) - if ranges: - epg_changed_date_ranges[epg_db_id] = ranges - - # Atomic delete (surgical) + bulk insert - BATCH_SIZE = 1000 - try: - with transaction.atomic(): - with connection.cursor() as cursor: - cursor.execute("SET LOCAL statement_timeout = '10min'") - total_deleted = 0 - for epg_db_id, day_ranges in epg_changed_date_ranges.items(): - q = Q() - for day_start, day_end in day_ranges: - q |= Q(start_time__gte=day_start, start_time__lt=day_end) - cnt = ProgramData.objects.filter(epg_id=epg_db_id).filter(q).delete()[0] - total_deleted += cnt - logger.debug(f"Deleted {total_deleted} changed SD programs across {len(epg_changed_date_ranges)} stations.") - for i in range(0, len(all_programs_to_create), BATCH_SIZE): - ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) - progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) - _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) - - logger.info(f"Committed {total_programs} Schedules Direct programs to database.") - - # Upsert SDProgramMD5 records for programs we just downloaded - # This updates the cache so future fetches can skip unchanged programs - if schedule_program_md5s: - md5_records = [ - SDProgramMD5( - epg_source=source, - program_id=pid, - md5=md5, - ) - for pid, md5 in schedule_program_md5s.items() - if pid in program_metadata # Only cache programs that were actually downloaded - ] - if md5_records: - SDProgramMD5.objects.bulk_create( - md5_records, - update_conflicts=True, - unique_fields=['epg_source', 'program_id'], - update_fields=['md5'], - ) - logger.info(f"Cached {len(md5_records)} program MD5s for future delta detection.") - - except Exception as db_error: - msg = f"Database error persisting Schedules Direct programs: {db_error}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) - return - finally: - all_programs_to_create = None - gc.collect() - - # ------------------------------------------------------------------------- - # Step 8: Fetch program artwork (posters) if enabled - # ------------------------------------------------------------------------- - fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) - if fetch_posters and program_metadata: - logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") - _sd_send_ws_sync(source.id, "parsing_programs", 98, - message="Fetching program artwork...") - - try: - # Build a set of unique artwork lookup IDs. - # For episodes (EP...), use the series root (SH...0000) so we get - # series-level artwork — one poster per series instead of per-episode. - artwork_lookup_ids = set() - pid_to_artwork_key = {} # maps original programID -> the key we looked up - - for pid in program_metadata: - if pid.startswith('EP'): - sh_root = 'SH' + pid[2:10] + '0000' - artwork_lookup_ids.add(sh_root) - pid_to_artwork_key[pid] = sh_root - else: - artwork_lookup_ids.add(pid) - pid_to_artwork_key[pid] = pid - - artwork_map = {} # artwork_key -> poster_url - artwork_list = list(artwork_lookup_ids) - SD_ARTWORK_BATCH_SIZE = 500 - - total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) - logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " - f"in {total_art_batches} batch(es).") - - for batch_idx in range(total_art_batches): - batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] - try: - art_response = requests.post( - f"{SD_BASE_URL}/metadata/programs/", - json=batch, - headers=_sd_headers(token), - timeout=120, - ) - art_response.raise_for_status() - art_data = art_response.json() - - for entry in art_data: - if not isinstance(entry, dict): - continue - entry_pid = entry.get('programID') - images = entry.get('data') or [] - if not entry_pid or not images: - continue - - # Filter to only dict entries — SD sometimes returns bare strings - images = [img for img in images if isinstance(img, dict)] - if not images: - continue - - # Pick the best poster image: - # Prefer portrait orientation (2x3, 3x4) in larger sizes - # SD categories include: Iconic, Banner-L1, Banner-L2, Logo - # SD uses width/height instead of a "size" field - poster_url = None - - # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic - for min_width in [240, 135, 120]: - for pref_cat in ['Banner-L1', 'Iconic']: - match = next((img for img in images - if img.get('aspect') in ('2x3', '3x4') - and img.get('category') == pref_cat - and (img.get('width', 0) or 0) >= min_width), None) - if match: - poster_url = match.get('uri') - break - if poster_url: - break - - # Fallback: any portrait image at any size - if not poster_url: - portrait = next((img for img in images - if img.get('aspect') in ('2x3', '3x4')), None) - if portrait: - poster_url = portrait.get('uri') - - if poster_url: - # Complete the URL if it's relative - if not poster_url.startswith('http'): - poster_url = f"{SD_BASE_URL}/image/{poster_url}" - artwork_map[entry_pid] = poster_url - - logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " - f"{len(artwork_map)} posters found so far.") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") - - # Bulk-update ProgramData records with poster URLs - if artwork_map: - programs_to_update = [] - for prog in ProgramData.objects.filter( - epg_id__in=mapped_epg_ids, - program_id__isnull=False, - ).only('id', 'program_id', 'custom_properties'): - art_key = pid_to_artwork_key.get(prog.program_id) - poster = artwork_map.get(art_key) if art_key else None - if poster: - cp = prog.custom_properties or {} - cp['poster_url'] = poster - prog.custom_properties = cp - programs_to_update.append(prog) - - if programs_to_update: - ProgramData.objects.bulk_update( - programs_to_update, ['custom_properties'], batch_size=1000 - ) - logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") - else: - logger.info("No poster artwork matched committed programs.") - else: - logger.info("No poster artwork found from Schedules Direct.") - - except Exception as art_error: - logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) - - elif fetch_posters: - logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") - - # ------------------------------------------------------------------------- - # Step 9: Apply SD station logos to matched channels if enabled - # ------------------------------------------------------------------------- - use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) - if use_sd_logos: - try: - from apps.channels.models import Channel as ChannelModel, Logo - - channels_to_update = [] - logos_created = 0 - - for channel in ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).select_related('epg_data', 'logo'): - icon_url = (channel.epg_data.icon_url or '').strip() - if not icon_url: - continue - - # Skip if channel already has this logo URL - if channel.logo and channel.logo.url == icon_url: - continue - - # Find or create a Logo object for this URL - try: - logo = Logo.objects.get(url=icon_url) - except Logo.DoesNotExist: - logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" - logo = Logo.objects.create(name=logo_name, url=icon_url) - logos_created += 1 - - channel.logo = logo - channels_to_update.append(channel) - - if channels_to_update: - ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) - logger.info(f"Applied SD logos to {len(channels_to_update)} channels " - f"({logos_created} new logos created).") - else: - logger.info("All matched channels already have current SD logos.") - - except Exception as logo_error: - logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) - - # Prune ProgramData whose end_time has passed. With surgical per-date deletes, - # programs from dates that have rolled off the window are never explicitly removed. - today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) - try: - expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] - if expired_count: - logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") - except Exception as prune_err: - logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") - - # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. - try: - live_program_ids = set( - ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) - .values_list('program_id', flat=True) - ) - pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( - program_id__in=live_program_ids - ).delete()[0] - if pruned_prog_md5_count: - logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") - except Exception as prune_err: - logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") - - # ------------------------------------------------------------------------- - # Prune stale poster cache files (>30 days old or orphaned) - # ------------------------------------------------------------------------- - try: - cache_dir = '/data/cache/posters' - if os.path.exists(cache_dir): - import time as time_module - # Collect all poster hashes currently referenced by ProgramData - active_hashes = set() - for url in ProgramData.objects.filter( - epg__epg_source=source, - custom_properties__has_key='poster_url', - ).values_list('custom_properties__poster_url', flat=True): - if url: - active_hashes.add(url.rsplit('/', 1)[-1]) - - pruned_posters = 0 - for fname in os.listdir(cache_dir): - fpath = os.path.join(cache_dir, fname) - if not os.path.isfile(fpath): - continue - file_age = time_module.time() - os.path.getmtime(fpath) - # Remove if older than 30 days OR not referenced by any current program - if file_age > 30 * 24 * 3600 or fname not in active_hashes: - os.remove(fpath) - pruned_posters += 1 - if pruned_posters: - logger.info(f"Pruned {pruned_posters} stale poster cache files.") - except Exception as poster_prune_err: - logger.warning(f"Failed to prune poster cache: {poster_prune_err}") - - # ------------------------------------------------------------------------- - # Done - # ------------------------------------------------------------------------- - success_msg = ( - f"Successfully fetched {total_programs:,} programs for " - f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " - f"({skipped_unmapped:,} programs skipped for unmapped stations)." - ) - source.status = EPGSource.STATUS_SUCCESS - source.last_message = success_msg - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message=success_msg) - log_system_event( - event_type='epg_refresh', - source_name=source.name, - programs=total_programs, - channels=len(mapped_tvg_ids), - skipped_programs=skipped_unmapped, - ) - logger.info(f"Schedules Direct fetch complete for source: {source.name}") - - -# ------------------------------- -# Helper parse functions -# ------------------------------- -def parse_xmltv_time(time_str): - try: - # Basic format validation - if len(time_str) < 14: - logger.warning(f"XMLTV timestamp too short: '{time_str}', using as-is") - dt_obj = datetime.strptime(time_str, '%Y%m%d%H%M%S') - return timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - - # Parse base datetime - dt_obj = datetime.strptime(time_str[:14], '%Y%m%d%H%M%S') - - # Handle timezone if present - if len(time_str) >= 20: # Has timezone info - tz_sign = time_str[15] - tz_hours = int(time_str[16:18]) - tz_minutes = int(time_str[18:20]) - - # Create a timezone object - if tz_sign == '+': - tz_offset = dt_timezone(timedelta(hours=tz_hours, minutes=tz_minutes)) - elif tz_sign == '-': - tz_offset = dt_timezone(timedelta(hours=-tz_hours, minutes=-tz_minutes)) - else: - tz_offset = dt_timezone.utc - - # Make datetime aware with correct timezone - aware_dt = datetime.replace(dt_obj, tzinfo=tz_offset) - # Convert to UTC - aware_dt = aware_dt.astimezone(dt_timezone.utc) - - logger.trace(f"Parsed XMLTV time '{time_str}' to {aware_dt}") - return aware_dt - else: - # No timezone info, assume UTC - aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - logger.trace(f"Parsed XMLTV time without timezone '{time_str}' as UTC: {aware_dt}") - return aware_dt - - except Exception as e: - logger.error(f"Error parsing XMLTV time '{time_str}': {e}", exc_info=True) - raise - - -def parse_schedules_direct_time(time_str): - try: - dt_obj = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%SZ') - aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - logger.debug(f"Parsed Schedules Direct time '{time_str}' to {aware_dt}") - return aware_dt - except Exception as e: - logger.error(f"Error parsing Schedules Direct time '{time_str}': {e}", exc_info=True) - raise - - -# Re-export from utils to preserve backward compatibility for any callers -from apps.epg.utils import extract_season_episode_from_description, _ONSCREEN_RE # noqa: F401 - - -# Helper function to extract custom properties - moved to a separate function to clean up the code -def extract_custom_properties(prog): - # Create a new dictionary for each call - custom_props = {} - - # Extract categories with a single comprehension to reduce intermediate objects - categories = [cat.text.strip() for cat in prog.findall('category') if cat.text and cat.text.strip()] - if categories: - custom_props['categories'] = categories - - # Extract keywords (new) - keywords = [kw.text.strip() for kw in prog.findall('keyword') if kw.text and kw.text.strip()] - if keywords: - custom_props['keywords'] = keywords - - # Extract episode numbers - for ep_num in prog.findall('episode-num'): - system = ep_num.get('system', '') - if system == 'xmltv_ns' and ep_num.text: - # Parse XMLTV episode-num format (season.episode.part) - parts = ep_num.text.split('.') - if len(parts) >= 2: - if parts[0].strip() != '': - try: - season = int(parts[0]) + 1 # XMLTV format is zero-based - custom_props['season'] = season - except ValueError: - pass - if parts[1].strip() != '': - try: - episode = int(parts[1]) + 1 # XMLTV format is zero-based - custom_props['episode'] = episode - except ValueError: - pass - elif system == 'onscreen' and ep_num.text: - onscreen_text = ep_num.text.strip() - custom_props['onscreen_episode'] = onscreen_text - # Extract season/episode from onscreen format if not already set by xmltv_ns - if 'season' not in custom_props or 'episode' not in custom_props: - match = _ONSCREEN_RE.search(onscreen_text) - if match: - if 'season' not in custom_props: - custom_props['season'] = int(match.group(1)) - if 'episode' not in custom_props: - custom_props['episode'] = int(match.group(2)) - elif system == 'dd_progid' and ep_num.text: - # Store the dd_progid format - custom_props['dd_progid'] = ep_num.text.strip() - # Add support for other systems like thetvdb.com, themoviedb.org, imdb.com - elif system in ['thetvdb.com', 'themoviedb.org', 'imdb.com'] and ep_num.text: - custom_props[f'{system}_id'] = ep_num.text.strip() - - # Extract ratings more efficiently - rating_elem = prog.find('rating') - if rating_elem is not None: - value_elem = rating_elem.find('value') - if value_elem is not None and value_elem.text: - custom_props['rating'] = value_elem.text.strip() - if rating_elem.get('system'): - custom_props['rating_system'] = rating_elem.get('system') - - # Extract star ratings (new) - star_ratings = [] - for star_rating in prog.findall('star-rating'): - value_elem = star_rating.find('value') - if value_elem is not None and value_elem.text: - rating_data = {'value': value_elem.text.strip()} - if star_rating.get('system'): - rating_data['system'] = star_rating.get('system') - star_ratings.append(rating_data) - if star_ratings: - custom_props['star_ratings'] = star_ratings - - # Extract credits more efficiently - credits_elem = prog.find('credits') - if credits_elem is not None: - credits = {} - for credit_type in ['director', 'actor', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if credit_type == 'actor': - # Handle actors with roles and guest status - actors = [] - for actor_elem in credits_elem.findall('actor'): - if actor_elem.text and actor_elem.text.strip(): - actor_data = {'name': actor_elem.text.strip()} - if actor_elem.get('role'): - actor_data['role'] = actor_elem.get('role') - if actor_elem.get('guest') == 'yes': - actor_data['guest'] = True - actors.append(actor_data) - if actors: - credits['actor'] = actors - else: - names = [e.text.strip() for e in credits_elem.findall(credit_type) if e.text and e.text.strip()] - if names: - credits[credit_type] = names - if credits: - custom_props['credits'] = credits - - # Extract other common program metadata - date_elem = prog.find('date') - if date_elem is not None and date_elem.text: - custom_props['date'] = date_elem.text.strip() - - country_elem = prog.find('country') - if country_elem is not None and country_elem.text: - custom_props['country'] = country_elem.text.strip() - - # Extract language information (new) - language_elem = prog.find('language') - if language_elem is not None and language_elem.text: - custom_props['language'] = language_elem.text.strip() - - orig_language_elem = prog.find('orig-language') - if orig_language_elem is not None and orig_language_elem.text: - custom_props['original_language'] = orig_language_elem.text.strip() - - # Extract length (new) - length_elem = prog.find('length') - if length_elem is not None and length_elem.text: - try: - length_value = int(length_elem.text.strip()) - length_units = length_elem.get('units', 'minutes') - custom_props['length'] = {'value': length_value, 'units': length_units} - except ValueError: - pass - - # Extract video information (new) - video_elem = prog.find('video') - if video_elem is not None: - video_info = {} - for video_attr in ['present', 'colour', 'aspect', 'quality']: - attr_elem = video_elem.find(video_attr) - if attr_elem is not None and attr_elem.text: - video_info[video_attr] = attr_elem.text.strip() - if video_info: - custom_props['video'] = video_info - - # Extract audio information (new) - audio_elem = prog.find('audio') - if audio_elem is not None: - audio_info = {} - for audio_attr in ['present', 'stereo']: - attr_elem = audio_elem.find(audio_attr) - if attr_elem is not None and attr_elem.text: - audio_info[audio_attr] = attr_elem.text.strip() - if audio_info: - custom_props['audio'] = audio_info - - # Extract subtitles information (new) - subtitles = [] - for subtitle_elem in prog.findall('subtitles'): - subtitle_data = {} - if subtitle_elem.get('type'): - subtitle_data['type'] = subtitle_elem.get('type') - lang_elem = subtitle_elem.find('language') - if lang_elem is not None and lang_elem.text: - subtitle_data['language'] = lang_elem.text.strip() - if subtitle_data: - subtitles.append(subtitle_data) - - if subtitles: - custom_props['subtitles'] = subtitles - - # Extract reviews (new) - reviews = [] - for review_elem in prog.findall('review'): - if review_elem.text and review_elem.text.strip(): - review_data = {'content': review_elem.text.strip()} - if review_elem.get('type'): - review_data['type'] = review_elem.get('type') - if review_elem.get('source'): - review_data['source'] = review_elem.get('source') - if review_elem.get('reviewer'): - review_data['reviewer'] = review_elem.get('reviewer') - reviews.append(review_data) - if reviews: - custom_props['reviews'] = reviews - - # Extract images (new) - images = [] - for image_elem in prog.findall('image'): - if image_elem.text and image_elem.text.strip(): - image_data = {'url': image_elem.text.strip()} - for attr in ['type', 'size', 'orient', 'system']: - if image_elem.get(attr): - image_data[attr] = image_elem.get(attr) - images.append(image_data) - if images: - custom_props['images'] = images - - icon_elem = prog.find('icon') - if icon_elem is not None and icon_elem.get('src'): - custom_props['icon'] = icon_elem.get('src') - - # Simpler approach for boolean flags - expanded list - for kw in ['previously-shown', 'premiere', 'new', 'live', 'last-chance']: - if prog.find(kw) is not None: - custom_props[kw.replace('-', '_')] = True - - # Extract premiere and last-chance text content if available - premiere_elem = prog.find('premiere') - if premiere_elem is not None: - custom_props['premiere'] = True - if premiere_elem.text and premiere_elem.text.strip(): - custom_props['premiere_text'] = premiere_elem.text.strip() - - last_chance_elem = prog.find('last-chance') - if last_chance_elem is not None: - custom_props['last_chance'] = True - if last_chance_elem.text and last_chance_elem.text.strip(): - custom_props['last_chance_text'] = last_chance_elem.text.strip() - - # Extract previously-shown details - prev_shown_elem = prog.find('previously-shown') - if prev_shown_elem is not None: - custom_props['previously_shown'] = True - prev_shown_data = {} - if prev_shown_elem.get('start'): - prev_shown_data['start'] = prev_shown_elem.get('start') - if prev_shown_elem.get('channel'): - prev_shown_data['channel'] = prev_shown_elem.get('channel') - if prev_shown_data: - custom_props['previously_shown_details'] = prev_shown_data - - return custom_props - - -def clear_element(elem): - """Clear an XML element and its parent to free memory.""" - try: - elem.clear() - parent = elem.getparent() - if parent is not None: - while elem.getprevious() is not None: - del parent[0] - parent.remove(elem) - except Exception as e: - logger.warning(f"Error clearing XML element: {e}", exc_info=True) - - -def detect_file_format(file_path=None, content=None): - """ - Detect file format by examining content or file path. - - Args: - file_path: Path to file (optional) - content: Raw file content bytes (optional) - - Returns: - tuple: (format_type, is_compressed, file_extension) - format_type: 'gzip', 'zip', 'xml', or 'unknown' - is_compressed: Boolean indicating if the file is compressed - file_extension: Appropriate file extension including dot (.gz, .zip, .xml) - """ - # Default return values - format_type = 'unknown' - is_compressed = False - file_extension = '.tmp' - - # First priority: check content magic numbers as they're most reliable - if content: - # We only need the first few bytes for magic number detection - header = content[:20] if len(content) >= 20 else content - - # Check for gzip magic number (1f 8b) - if len(header) >= 2 and header[:2] == b'\x1f\x8b': - return 'gzip', True, '.gz' - - # Check for zip magic number (PK..) - if len(header) >= 2 and header[:2] == b'PK': - return 'zip', True, '.zip' - - # Check for XML - either standard XML header or XMLTV-specific tag - if len(header) >= 5 and (b'' in header): - return 'xml', False, '.xml' - - # Second priority: check file extension - focus on the final extension for compression - if file_path: - logger.debug(f"Detecting file format for: {file_path}") - - # Handle compound extensions like .xml.gz - prioritize compression extensions - lower_path = file_path.lower() - - # Check for compression extensions explicitly - if lower_path.endswith('.gz') or lower_path.endswith('.gzip'): - return 'gzip', True, '.gz' - elif lower_path.endswith('.zip'): - return 'zip', True, '.zip' - elif lower_path.endswith('.xml'): - return 'xml', False, '.xml' - - # Fallback to mimetypes only if direct extension check doesn't work - import mimetypes - mime_type, _ = mimetypes.guess_type(file_path) - logger.debug(f"Guessed MIME type: {mime_type}") - if mime_type: - if mime_type == 'application/gzip' or mime_type == 'application/x-gzip': - return 'gzip', True, '.gz' - elif mime_type == 'application/zip': - return 'zip', True, '.zip' - elif mime_type == 'application/xml' or mime_type == 'text/xml': - return 'xml', False, '.xml' - - # If we reach here, we couldn't reliably determine the format - return format_type, is_compressed, file_extension - - -def generate_dummy_epg(source): - """ - DEPRECATED: This function is no longer used. - - Dummy EPG programs are now generated on-demand when they are requested - (during XMLTV export or EPG grid display), rather than being pre-generated - and stored in the database. - - See: apps/output/views.py - generate_custom_dummy_programs() - - This function remains for backward compatibility but should not be called. - """ - logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " - f"Dummy EPG programs are now generated on-demand.") - return True From 8f060e8b3af6a7c2841166390c59dc16d5adfbd4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 4 Jun 2026 15:17:58 -0500 Subject: [PATCH 34/76] feat: Output SD program poster images during xml output and enhance caching logic --- apps/epg/api_views.py | 102 ++++++++++++++++++++++++++++- apps/epg/serializers.py | 6 ++ apps/epg/tasks.py | 35 +--------- apps/output/views.py | 3 + docker/init/03-init-dispatcharr.sh | 2 +- docker/nginx.conf | 10 ++- 6 files changed, 122 insertions(+), 36 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 96fc69e0..fd83f2de 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -1,6 +1,11 @@ -import logging, os, re +import hashlib +import logging +import os +import re +import time from rest_framework import viewsets, status, serializers from rest_framework.pagination import PageNumberPagination +from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.decorators import action @@ -461,6 +466,10 @@ class ProgramViewSet(viewsets.ModelViewSet): queryset = ProgramData.objects.select_related("epg").all() serializer_class = ProgramDataSerializer + # Per-source in-memory caches (token and error state) + _sd_poster_token_cache: dict = {} + _sd_poster_error_cache: dict = {} + def get_permissions(self): try: return [perm() for perm in permission_classes_by_action[self.action]] @@ -477,6 +486,97 @@ class ProgramViewSet(viewsets.ModelViewSet): serializer = self.get_serializer(instance) return Response(serializer.data) + @action(detail=True, methods=['get'], url_path='poster', permission_classes=[AllowAny]) + def poster(self, request, pk=None): + """Proxy endpoint for SD program poster images. Nginx caches the response.""" + import requests as http_requests + from apps.epg.tasks import SD_BASE_URL + + program = self.get_object() + poster_sd_url = (program.custom_properties or {}).get('sd_icon') + if not poster_sd_url: + return Response(status=status.HTTP_404_NOT_FOUND) + + source = program.epg.epg_source if program.epg else None + if not source or source.source_type != 'schedules_direct': + return Response(status=status.HTTP_404_NOT_FOUND) + + error_cache = ProgramViewSet._sd_poster_error_cache.get(source.id) + if error_cache and time.time() < error_cache['until']: + return Response( + {'error': f"SD temporarily unavailable: {error_cache['reason']}"}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + + cached = ProgramViewSet._sd_poster_token_cache.get(source.id) + token = cached['token'] if cached and time.time() < cached['expires'] else None + + if not token: + sha1_password = hashlib.sha1(source.password.encode('utf-8')).hexdigest() + try: + auth_resp = http_requests.post( + f"{SD_BASE_URL}/token", + json={'username': source.username, 'password': sha1_password}, + headers={'Content-Type': 'application/json'}, + timeout=10, + ) + auth_data = auth_resp.json() + token = auth_data.get('token') + if not token: + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 3600, + 'reason': auth_data.get('message', 'Authentication failed'), + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + token_expires = auth_data.get('tokenExpires', time.time() + 86400) + ProgramViewSet._sd_poster_token_cache[source.id] = { + 'token': token, + 'expires': token_expires, + } + except http_requests.exceptions.RequestException: + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 300, + 'reason': 'Network error reaching Schedules Direct', + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + + try: + img_resp = http_requests.get( + poster_sd_url, + headers={'token': token}, + timeout=15, + allow_redirects=True, + ) + if img_resp.status_code in (401, 403): + ProgramViewSet._sd_poster_token_cache.pop(source.id, None) + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 3600, + 'reason': f'SD returned {img_resp.status_code}', + } + return Response(status=status.HTTP_502_BAD_GATEWAY) + if img_resp.status_code == 400: + try: + err_code = img_resp.json().get('code') + except Exception: + err_code = None + if err_code == 5002: + ProgramViewSet._sd_poster_error_cache[source.id] = { + 'until': time.time() + 3600, + 'reason': 'Daily image download limit reached (SD error 5002)', + } + return Response(status=status.HTTP_429_TOO_MANY_REQUESTS) + if img_resp.status_code != 200: + return Response(status=status.HTTP_502_BAD_GATEWAY) + + from django.http import HttpResponse + content_type = img_resp.headers.get('Content-Type', 'image/jpeg') + response = HttpResponse(img_resp.content, content_type=content_type) + response['Cache-Control'] = 'public, max-age=86400' + return response + + except http_requests.exceptions.RequestException: + return Response(status=status.HTTP_502_BAD_GATEWAY) + def list(self, request, *args, **kwargs): logger.debug("Listing all EPG programs.") return super().list(request, *args, **kwargs) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 608636a9..7da93c13 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -155,6 +155,12 @@ class ProgramDetailSerializer(ProgramDataSerializer): data['icon'] = cp.get('icon') data['images'] = cp.get('images') or [] + # SD poster: expose as proxy URL so frontend never needs SD auth + if cp.get('sd_icon'): + data['poster_url'] = f"/api/epg/programs/{obj.id}/poster/" + else: + data['poster_url'] = None + return data diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index fc43c36d..78900db9 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -3048,7 +3048,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # Complete the URL if it's relative if not poster_url.startswith('http'): poster_url = f"{SD_BASE_URL}/image/{poster_url}" - artwork_map[entry_pid] = poster_url + artwork_map[entry_pid] = poster_url # raw SD endpoint URL logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " f"{len(artwork_map)} posters found so far.") @@ -3067,7 +3067,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): poster = artwork_map.get(art_key) if art_key else None if poster: cp = prog.custom_properties or {} - cp['poster_url'] = poster + cp['sd_icon'] = poster prog.custom_properties = cp programs_to_update.append(prog) @@ -3155,37 +3155,6 @@ def fetch_schedules_direct(source, stations_only=False, force=False): except Exception as prune_err: logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") - # ------------------------------------------------------------------------- - # Prune stale poster cache files (>30 days old or orphaned) - # ------------------------------------------------------------------------- - try: - cache_dir = '/data/cache/posters' - if os.path.exists(cache_dir): - import time as time_module - # Collect all poster hashes currently referenced by ProgramData - active_hashes = set() - for url in ProgramData.objects.filter( - epg__epg_source=source, - custom_properties__has_key='poster_url', - ).values_list('custom_properties__poster_url', flat=True): - if url: - active_hashes.add(url.rsplit('/', 1)[-1]) - - pruned_posters = 0 - for fname in os.listdir(cache_dir): - fpath = os.path.join(cache_dir, fname) - if not os.path.isfile(fpath): - continue - file_age = time_module.time() - os.path.getmtime(fpath) - # Remove if older than 30 days OR not referenced by any current program - if file_age > 30 * 24 * 3600 or fname not in active_hashes: - os.remove(fpath) - pruned_posters += 1 - if pruned_posters: - logger.info(f"Pruned {pruned_posters} stale poster cache files.") - except Exception as poster_prune_err: - logger.warning(f"Failed to prune poster cache: {poster_prune_err}") - # ------------------------------------------------------------------------- # Done # ------------------------------------------------------------------------- diff --git a/apps/output/views.py b/apps/output/views.py index 4d314857..854d3054 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -1646,6 +1646,7 @@ def generate_epg(request, profile_name=None, user=None): # to avoid skipping/duplicating rows if the table changes mid-stream. last_epg_id = 0 last_id = 0 + _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") while True: program_chunk = list( @@ -1849,6 +1850,8 @@ def generate_epg(request, profile_name=None, user=None): if "icon" in custom_data: program_xml.append(f' ') + elif "sd_icon" in custom_data: + program_xml.append(f' ') # Add special flags as proper tags with enhanced handling if custom_data.get("previously_shown", False): diff --git a/docker/init/03-init-dispatcharr.sh b/docker/init/03-init-dispatcharr.sh index 05bcb192..10e3e874 100644 --- a/docker/init/03-init-dispatcharr.sh +++ b/docker/init/03-init-dispatcharr.sh @@ -7,6 +7,7 @@ DATA_DIRS=( "/data/backups" "/data/logos" + "/data/logo_cache" "/data/recordings" "/data/uploads/m3us" "/data/uploads/epgs" @@ -19,7 +20,6 @@ DATA_DIRS=( # APP_DIRS live on the image layer and are always locally writable. APP_DIRS=( - "/app/logo_cache" "/app/media" "/app/static" ) diff --git a/docker/nginx.conf b/docker/nginx.conf index c5cb429e..ba75f3fe 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -1,4 +1,4 @@ -proxy_cache_path /app/logo_cache levels=1:2 keys_zone=logo_cache:10m +proxy_cache_path /data/logo_cache levels=1:2 keys_zone=logo_cache:10m inactive=24h use_temp_path=off; server { @@ -58,6 +58,14 @@ server { proxy_cache_use_stale error timeout updating; # Serve stale if Django is slow } + location ~ ^/api/epg/programs/(?\d+)/poster/ { + proxy_pass http://127.0.0.1:5656; + proxy_cache logo_cache; + proxy_cache_key "$scheme$request_uri"; + proxy_cache_valid 200 24h; + proxy_cache_use_stale error timeout updating; + } + # admin disabled when not in dev mode location ~ ^/admin/?$ { return 301 /login; From 30843c0c4c41b8b6f04eb1d8408ed05bd7934887 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 4 Jun 2026 15:20:14 -0500 Subject: [PATCH 35/76] fix: Fix migrations --- ...024_remove_epgsource_api_key_epgsource_password_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename apps/epg/migrations/{0023_remove_epgsource_api_key_epgsource_password_and_more.py => 0024_remove_epgsource_api_key_epgsource_password_and_more.py} (98%) diff --git a/apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py b/apps/epg/migrations/0024_remove_epgsource_api_key_epgsource_password_and_more.py similarity index 98% rename from apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py rename to apps/epg/migrations/0024_remove_epgsource_api_key_epgsource_password_and_more.py index 55c9e58e..3dab2c50 100644 --- a/apps/epg/migrations/0023_remove_epgsource_api_key_epgsource_password_and_more.py +++ b/apps/epg/migrations/0024_remove_epgsource_api_key_epgsource_password_and_more.py @@ -7,7 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('epg', '0022_alter_epgdata_name'), + ('epg', '0023_epgsource_programme_index'), ] operations = [ From ac3a83a1aa4d4766e53fd4168432fe9f7d215101 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 4 Jun 2026 15:37:27 -0500 Subject: [PATCH 36/76] fix: Fix removed code from bad merge. --- ProgramDetailModal.jsx | 458 ----- api_views.py | 1285 -------------- apps/epg/tasks.py | 439 +++++ docker-compose.yml | 43 - serializers.py | 866 ---------- tasks.py | 3594 ---------------------------------------- 6 files changed, 439 insertions(+), 6246 deletions(-) delete mode 100644 ProgramDetailModal.jsx delete mode 100644 api_views.py delete mode 100644 docker-compose.yml delete mode 100644 serializers.py delete mode 100644 tasks.py diff --git a/ProgramDetailModal.jsx b/ProgramDetailModal.jsx deleted file mode 100644 index 54289ba4..00000000 --- a/ProgramDetailModal.jsx +++ /dev/null @@ -1,458 +0,0 @@ -import React, { useState, useEffect, useCallback } from 'react'; -import { - Badge, - Button, - Divider, - Flex, - Group, - Image, - Modal, - Stack, - Text, - Title, -} from '@mantine/core'; -import { Calendar, Video } from 'lucide-react'; -import API from '../api'; -import useVideoStore from '../store/useVideoStore'; -import useSettingsStore from '../store/settings'; -import { getShowVideoUrl } from '../utils/cards/RecordingCardUtils'; -import { formatSeasonEpisode } from '../utils/guideUtils'; -import { - format, - initializeTime, - diff, - useDateTimeFormat, -} from '../utils/dateTimeUtils'; -import { imdbUrl, tmdbUrl } from '../utils/externalUrls'; - -const overlayProps = { color: '#000', backgroundOpacity: 0.55, blur: 0 }; - -function formatDurationMinutes(startTime, endTime) { - if (!startTime || !endTime) return null; - const start = initializeTime(startTime); - const end = initializeTime(endTime); - const minutes = diff(end, start, 'minute'); - if (minutes >= 60) { - const hours = Math.floor(minutes / 60); - const mins = minutes % 60; - return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`; - } - return `${minutes}m`; -} - -function resolveImageUrl(detail) { - if (detail?.tmdb_poster_url) return detail.tmdb_poster_url; - if (detail?.poster_url) return detail.poster_url; - if (detail?.images?.length > 0) return detail.images[0].url; - if (detail?.icon) return detail.icon; - return null; -} - -function formatCredits(actors) { - if (!actors?.length) return null; - return actors - .map((a) => (a.role ? `${a.name} (${a.role})` : a.name)) - .join(', '); -} - -export default function ProgramDetailModal({ - program, - channel, - opened, - onClose, - onRecord, -}) { - const [detailData, setDetailData] = useState(null); - - const showVideo = useVideoStore((s) => s.showVideo); - const env_mode = useSettingsStore((s) => s.environment.env_mode); - const { timeFormat } = useDateTimeFormat(); - - useEffect(() => { - if (!opened || !program) { - setDetailData(null); - return; - } - - // Dummy programs may use UUID-style IDs that aren't real DB PKs - const programId = program.id; - if (!programId || typeof programId === 'string') { - setDetailData(null); - return; - } - - let cancelled = false; - - API.getProgramDetail(programId) - .then((data) => { - if (!cancelled) setDetailData(data); - }) - .catch(() => { - if (!cancelled) setDetailData(null); - }); - - return () => { - cancelled = true; - }; - }, [opened, program?.id]); - - const handleWatchLive = useCallback(() => { - if (!channel) return; - showVideo(getShowVideoUrl(channel, env_mode), 'live', { - name: channel.name, - }); - onClose(); - }, [channel, env_mode, showVideo, onClose]); - - const handleRecord = useCallback(() => { - if (onRecord) onRecord(program); - }, [onRecord, program]); - - if (!program) return null; - - // Merge detail data with grid data (detail enriches, grid is baseline) - const d = detailData || {}; - const seasonEpisodeLabel = formatSeasonEpisode( - d.season ?? program.season, - d.episode ?? program.episode - ); - const hasBadges = - seasonEpisodeLabel || - program.is_live || - program.is_new || - d.is_previously_shown || - program.is_premiere || - program.is_finale || - d.video_quality || - d.rating; - - const categories = d.categories || []; - const credits = d.credits || {}; - const hasCredits = - credits.actors?.length > 0 || - credits.directors?.length > 0 || - credits.writers?.length > 0; - const starRatings = d.star_ratings || []; - const description = d.description || program.description; - const subtitle = d.sub_title ?? program.sub_title; - const posterUrl = - resolveImageUrl(d) || program?.custom_properties?.icon || null; - const duration = formatDurationMinutes(program.start_time, program.end_time); - const programStart = initializeTime(program.start_time || program.startMs); - const programEnd = initializeTime(program.end_time || program.endMs); - - return ( - - {channel.channel_number ? `${channel.channel_number} - ` : ''} - {channel.name} - - ) : null - } - size="lg" - centered - overlayProps={overlayProps} - zIndex={9999} - > - - - {posterUrl && ( - { - e.currentTarget.style.display = 'none'; - }} - /> - )} - - - - - {program.title} - - - {subtitle && ( - - {subtitle} - - )} - - {hasBadges && ( - - {program.is_live && ( - - LIVE - - )} - {program.is_new && ( - - NEW - - )} - {d.is_previously_shown && ( - - RERUN - - )} - {program.is_premiere && ( - - PREMIERE - - )} - {program.is_finale && ( - - FINALE - - )} - {seasonEpisodeLabel && ( - - {seasonEpisodeLabel} - - )} - {d.rating && ( - - {d.rating} - - )} - {d.video_quality && ( - - {d.video_quality} - - )} - - )} - - - - {format(programStart, timeFormat)} –{' '} - {format(programEnd, timeFormat)} - - {duration && ( - <> - - · - - - {duration} - - - )} - {categories.length > 0 && ( - <> - - · - - - {categories.join(', ')} - - - )} - - - - - {program.isLive && channel && ( - - )} - {!program.isPast && ( - - )} - - - - - {description && ( - <> - - - {description} - - - )} - - {d.content_advisory?.length > 0 && ( - - {d.content_advisory.join(', ')} - - )} - - {d.event_details && ( - <> - - - {d.event_details.venue100 && ( - - Venue: - {d.event_details.venue100} - - )} - {d.event_details.teams?.length > 0 && ( - - Teams: - {d.event_details.teams.map((t, i) => ( - - {i > 0 ? ' vs ' : ''} - {t.name}{t.isHome ? ' (Home)' : ''} - - ))} - - )} - - - )} - - {hasCredits && ( - <> - - - {credits.actors?.length > 0 && ( - - - Cast:{' '} - - {formatCredits(credits.actors)} - - )} - {credits.directors?.length > 0 && ( - - - Director:{' '} - - {credits.directors.join(', ')} - - )} - {credits.writers?.length > 0 && ( - - - Writer:{' '} - - {credits.writers.join(', ')} - - )} - - - )} - - {(d.country || - d.language || - d.original_air_date || - d.runtime || - (d.production_date && d.is_previously_shown) || - starRatings.length > 0) && ( - <> - - - {d.country && ( - - - Country:{' '} - - {d.country} - - )} - {d.language && ( - - - Language:{' '} - - {d.language} - - )} - {d.production_date && d.is_previously_shown && ( - - - First aired:{' '} - - {d.production_date} - - )} - {d.runtime && ( - - - Runtime:{' '} - - {d.runtime} {d.runtime_units || 'min'} - - )} - {d.original_air_date && ( - - - Original Air:{' '} - - {d.original_air_date} - - )} - {starRatings.map((sr, i) => ( - - ★ {sr.value} - {sr.system ? ` (${sr.system})` : ''} - - ))} - - - )} - - {(d.imdb_id || d.tmdb_id) && ( - - {d.imdb_id && ( - - IMDb ↗ - - )} - {d.tmdb_id && ( - - TMDB ↗ - - )} - - )} - - - ); -} diff --git a/api_views.py b/api_views.py deleted file mode 100644 index b7fffb05..00000000 --- a/api_views.py +++ /dev/null @@ -1,1285 +0,0 @@ -import logging, os, re -from rest_framework import viewsets, status, serializers -from rest_framework.permissions import AllowAny -from rest_framework.pagination import PageNumberPagination -from rest_framework.response import Response -from rest_framework.views import APIView -from rest_framework.decorators import action -from drf_spectacular.utils import extend_schema, OpenApiParameter, inline_serializer -from drf_spectacular.types import OpenApiTypes -from django.db.models import Q -from django.utils import timezone -from django.utils.dateparse import parse_datetime -from datetime import timedelta -from .models import EPGSource, ProgramData, EPGData -from .serializers import ( - ProgramDataSerializer, - ProgramDetailSerializer, - EPGSourceSerializer, - EPGDataSerializer, - ProgramSearchResultSerializer, -) -from .tasks import refresh_epg_data -from .query_utils import parse_text_query -from apps.accounts.permissions import ( - Authenticated, - IsAdmin, - IsStandardUser, - permission_classes_by_action, - permission_classes_by_method, -) - -logger = logging.getLogger(__name__) - - -# ───────────────────────────── -# 1) EPG Source API (CRUD) -# ───────────────────────────── -class EPGSourceViewSet(viewsets.ModelViewSet): - """ - API endpoint that allows EPG sources to be viewed or edited. - """ - - queryset = EPGSource.objects.select_related( - "refresh_task__crontab", "refresh_task__interval" - ).all() - serializer_class = EPGSourceSerializer - - def get_permissions(self): - try: - return [perm() for perm in permission_classes_by_action[self.action]] - except KeyError: - if self.action in ('sd_lineups', 'sd_lineups_search'): - if self.request.method == 'GET': - return [IsStandardUser()] - return [IsAdmin()] - return [Authenticated()] - - def get_queryset(self): - from django.db.models import Exists, OuterRef - from apps.channels.models import Channel - return EPGSource.objects.select_related( - "refresh_task__crontab", "refresh_task__interval" - ).annotate( - has_channels=Exists( - Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk')) - ) - ) - - def list(self, request, *args, **kwargs): - logger.debug("Listing all EPG sources.") - return super().list(request, *args, **kwargs) - - @action(detail=False, methods=["post"]) - def upload(self, request): - if "file" not in request.FILES: - return Response( - {"error": "No file uploaded"}, status=status.HTTP_400_BAD_REQUEST - ) - - file = request.FILES["file"] - file_name = file.name - file_path = os.path.join("/data/uploads/epgs", file_name) - - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, "wb+") as destination: - for chunk in file.chunks(): - destination.write(chunk) - - new_obj_data = request.data.copy() - new_obj_data["file_path"] = file_path - - serializer = self.get_serializer(data=new_obj_data) - serializer.is_valid(raise_exception=True) - self.perform_create(serializer) - - return Response(serializer.data, status=status.HTTP_201_CREATED) - - def partial_update(self, request, *args, **kwargs): - """Handle partial updates with special logic for is_active field""" - instance = self.get_object() - - # Check if we're toggling is_active - if ( - "is_active" in request.data - and instance.is_active != request.data["is_active"] - ): - # Set appropriate status based on new is_active value - if request.data["is_active"]: - request.data["status"] = "idle" - else: - request.data["status"] = "disabled" - - # Continue with regular partial update - return super().partial_update(request, *args, **kwargs) - - def create(self, request, *args, **kwargs): - """ - Override create to validate Schedules Direct credentials before saving. - If source_type is 'schedules_direct', authenticates against SD's /token - endpoint. Returns 400 if credentials are invalid — no source is created. - """ - source_type = request.data.get('source_type', '') - if source_type == 'schedules_direct': - import hashlib - import requests as http_requests - from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version - - username = (request.data.get('username') or '').strip() - password = (request.data.get('password') or '').strip() - - if not username or not password: - return Response( - {"error": "Username and password are required for Schedules Direct."}, - status=status.HTTP_400_BAD_REQUEST - ) - - sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() - try: - auth_response = http_requests.post( - f"{SD_BASE_URL}/token", - json={'username': username, 'password': sha1_password}, - headers={ - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - }, - timeout=15, - ) - auth_data = auth_response.json() - if not auth_data.get('token'): - error_msg = auth_data.get('message', 'Authentication failed. Check your credentials.') - return Response( - {"error": error_msg}, - status=status.HTTP_400_BAD_REQUEST - ) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Could not reach Schedules Direct: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) - - return super().create(request, *args, **kwargs) - - - def _sd_authenticate(self, source): - """ - Authenticate with Schedules Direct using stored credentials. - Returns (token, None) on success or (None, Response) on failure. - """ - import hashlib - import requests as http_requests - from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version - - username = (source.username or '').strip() - password = (source.password or '').strip() - if not username or not password: - return None, Response( - {"error": "Username and password are required."}, - status=status.HTTP_400_BAD_REQUEST - ) - - sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() - try: - auth_response = http_requests.post( - f"{SD_BASE_URL}/token", - json={'username': username, 'password': sha1_password}, - headers={'Content-Type': 'application/json', 'User-Agent': f'Dispatcharr/{dispatcharr_version}'}, - timeout=15, - ) - auth_response.raise_for_status() - token = auth_response.json().get('token') - if not token: - return None, Response( - {"error": "Authentication failed. Check your credentials."}, - status=status.HTTP_401_UNAUTHORIZED - ) - return token, None - except http_requests.exceptions.RequestException as e: - return None, Response( - {"error": f"Authentication failed: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) - - def _get_sd_reset_at(self, source): - """Retrieve stored reset timestamp from EPGSource model field.""" - reset_at_str = (source.custom_properties or {}).get('sd_changes_reset_at') - return reset_at_str - - def _get_sd_changes_remaining(self, source): - """ - Retrieve stored changesRemaining from EPGSource model field. - If a reset timestamp exists and has passed (midnight UTC), clears the - lockout automatically so the user can make adds again. - """ - from django.utils import timezone - - cp = source.custom_properties or {} - changes_remaining = cp.get('sd_changes_remaining') - reset_at_str = cp.get('sd_changes_reset_at') - from django.utils.dateparse import parse_datetime - reset_at = parse_datetime(reset_at_str) if reset_at_str else None - - # If we have a reset timestamp and it has passed, clear the lockout - if changes_remaining == 0 and reset_at: - if timezone.now() >= reset_at: - cp = source.custom_properties or {} - cp.pop('sd_changes_remaining', None) - cp.pop('sd_changes_reset_at', None) - source.custom_properties = cp - source.save(update_fields=['custom_properties']) - return None - - return changes_remaining - - def _save_sd_changes_remaining(self, source, changes_remaining): - """Persist changesRemaining to EPGSource model field.""" - cp = source.custom_properties or {} - cp['sd_changes_remaining'] = changes_remaining - source.custom_properties = cp - source.save(update_fields=['custom_properties']) - - def _save_sd_lockout(self, source): - """ - Persist a hard lockout to EPGSource custom_properties when SD returns - 4100 MAX_LINEUP_CHANGES_REACHED. Per SD API documentation, lineup adds - are limited to 6 per rolling 24-hour period (not midnight UTC reset). - Lockout clears automatically after 24 hours from when the limit was hit. - """ - from django.utils import timezone - from datetime import timedelta - - now = timezone.now() - # Rolling 24-hour window per SD docs: "6 adds in a 24 hour period" - reset_at = now + timedelta(hours=24) - - cp = source.custom_properties or {} - cp['sd_changes_remaining'] = 0 - cp['sd_changes_reset_at'] = reset_at.isoformat() - source.custom_properties = cp - source.save(update_fields=['custom_properties']) - logger.warning( - f"SD source {source.id}: daily add limit reached (4100). " - f"Lockout set until {reset_at.isoformat()} (24h rolling window)." - ) - - @action(detail=True, methods=["get", "post", "delete"], url_path="sd-lineups") - def sd_lineups(self, request, pk=None): - """ - GET — list lineups currently on the SD account - POST — add a lineup (body: {"lineup": "USA-NJ29486-X"}) - DELETE — remove a lineup (body: {"lineup": "USA-NJ29486-X"}) - """ - import requests as http_requests - from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version - - source = self.get_object() - if source.source_type != 'schedules_direct': - return Response( - {"error": "This action is only available for Schedules Direct sources."}, - status=status.HTTP_400_BAD_REQUEST - ) - - token, error = self._sd_authenticate(source) - if error: - return error - - headers = { - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - 'token': token, - } - - if request.method == "GET": - try: - resp = http_requests.get( - f"{SD_BASE_URL}/lineups", - headers=headers, - timeout=15, - ) - if resp.status_code == 400: - sd_data = resp.json() - sd_code = sd_data.get('code') - if sd_code == 4102: - return Response({ - "lineups": [], - "max_lineups": 4, - "changes_remaining": self._get_sd_changes_remaining(source), - "changes_reset_at": self._get_sd_reset_at(source), - "notice": "No lineups are currently configured on this Schedules Direct account. Use the search below to add one.", - }) - resp.raise_for_status() - data = resp.json() - lineups = [l for l in data.get('lineups', []) if not l.get('isDeleted', False)] - return Response({ - "lineups": lineups, - "max_lineups": 4, - "changes_remaining": self._get_sd_changes_remaining(source), - "changes_reset_at": self._get_sd_reset_at(source), - }) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Failed to fetch lineups: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) - - elif request.method == "POST": - lineup_id = request.data.get('lineup') - if not lineup_id: - return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) - try: - resp = http_requests.put( - f"{SD_BASE_URL}/lineups/{lineup_id}", - headers=headers, - timeout=15, - ) - sd_data = resp.json() - sd_code = sd_data.get('code') - - if resp.status_code == 400 or resp.status_code == 403: - if sd_code == 4100: - self._save_sd_lockout(source) - return Response({ - "error": "daily_limit_reached", - "message": "You have reached your daily Schedules Direct lineup addition limit. SD allows 6 adds per 24-hour period. Resets at midnight UTC.", - "changes_remaining": 0, - "docs_url": "https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform", - }, status=status.HTTP_200_OK) - if sd_code == 4101: - return Response({ - "error": "max_lineups_reached", - "message": "Your Schedules Direct account has reached the maximum of 4 lineups. Remove one before adding another.", - "changes_remaining": self._get_sd_changes_remaining(source), - }, status=status.HTTP_200_OK) - if sd_code == 2100: - return Response({ - "error": "duplicate_lineup", - "message": "This lineup is already on your Schedules Direct account.", - "changes_remaining": self._get_sd_changes_remaining(source), - }, status=status.HTTP_200_OK) - return Response({ - "error": sd_data.get('message', 'Failed to add lineup.'), - "changes_remaining": self._get_sd_changes_remaining(source), - }, status=status.HTTP_200_OK) - - resp.raise_for_status() - - # Persist changesRemaining to custom_properties - changes_remaining = sd_data.get('changesRemaining') - if changes_remaining is not None: - self._save_sd_changes_remaining(source, changes_remaining) - - logger.info( - f"SD lineup added for source {source.id}: {lineup_id}. " - f"changesRemaining: {changes_remaining}" - ) - - # Re-fetch stations so the new lineup's stations are available for matching - from apps.epg.tasks import fetch_schedules_direct_stations - fetch_schedules_direct_stations.delay(source.id) - - return Response({ - **sd_data, - "changes_remaining": changes_remaining, - }) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Failed to add lineup: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) - - elif request.method == "DELETE": - lineup_id = request.data.get('lineup') - if not lineup_id: - return Response({"error": "lineup field is required."}, status=status.HTTP_400_BAD_REQUEST) - try: - resp = http_requests.delete( - f"{SD_BASE_URL}/lineups/{lineup_id}", - headers=headers, - timeout=15, - ) - if resp.status_code == 400: - sd_data = resp.json() - sd_code = sd_data.get('code') - if sd_code == 2103: - return Response({ - "response": "OK", - "code": 0, - "message": "Lineup not found on account — already removed.", - "changes_remaining": self._get_sd_changes_remaining(source), - }) - resp.raise_for_status() - sd_data = resp.json() - # SD returns changesRemaining on deletes — persist it - changes_remaining = sd_data.get('changesRemaining') - if changes_remaining is not None: - self._save_sd_changes_remaining(source, changes_remaining) - logger.info(f"SD lineup deleted for source {source.id}: {lineup_id}") - return Response({ - **sd_data, - "changes_remaining": self._get_sd_changes_remaining(source), - }) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Failed to remove lineup: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) - - @action(detail=True, methods=["post"], url_path="sd-lineups/search") - def sd_lineups_search(self, request, pk=None): - """ - Search available headends/lineups by country and postal code. - Body: {"country": "USA", "postalcode": "07030"} - Returns a flat list of lineups across all matching headends. - """ - import requests as http_requests - from apps.epg.tasks import SD_BASE_URL - from version import __version__ as dispatcharr_version - - source = self.get_object() - if source.source_type != 'schedules_direct': - return Response( - {"error": "This action is only available for Schedules Direct sources."}, - status=status.HTTP_400_BAD_REQUEST - ) - - country = request.data.get('country', '').strip() - postalcode = request.data.get('postalcode', '').strip() - if not country or not postalcode: - return Response( - {"error": "country and postalcode are required."}, - status=status.HTTP_400_BAD_REQUEST - ) - - token, error = self._sd_authenticate(source) - if error: - return error - - headers = { - 'Content-Type': 'application/json', - 'User-Agent': f'Dispatcharr/{dispatcharr_version}', - 'token': token, - } - - try: - resp = http_requests.get( - f"{SD_BASE_URL}/headends", - params={'country': country, 'postalcode': postalcode}, - headers=headers, - timeout=15, - ) - resp.raise_for_status() - headends = resp.json() - lineups = [] - for headend in headends: - for lineup in headend.get('lineups', []): - lineups.append({ - 'lineup': lineup.get('lineup'), - 'name': lineup.get('name'), - 'transport': headend.get('transport'), - 'location': headend.get('location'), - 'headend': headend.get('headend'), - }) - return Response({"lineups": lineups}) - except http_requests.exceptions.RequestException as e: - return Response( - {"error": f"Failed to search headends: {str(e)}"}, - status=status.HTTP_502_BAD_GATEWAY - ) -# ───────────────────────────── -# 2) Program API (CRUD) -# ───────────────────────────── -class ProgramSearchPagination(PageNumberPagination): - page_size = 50 - page_size_query_param = 'page_size' - max_page_size = 500 - - -class ProgramViewSet(viewsets.ModelViewSet): - """Handles CRUD operations for EPG programs""" - - queryset = ProgramData.objects.select_related("epg").all() - serializer_class = ProgramDataSerializer - - def get_permissions(self): - if self.action == 'poster': - return [AllowAny()] - try: - return [perm() for perm in permission_classes_by_action[self.action]] - except KeyError: - return [Authenticated()] - - def get_serializer_class(self): - if self.action == 'retrieve': - return ProgramDetailSerializer - return ProgramDataSerializer - - def retrieve(self, request, *args, **kwargs): - instance = self.get_object() - serializer = self.get_serializer(instance) - return Response(serializer.data) - - # Cached SD token for poster proxy (module-level, shared across requests) - _sd_poster_token_cache = {} # {source_id: {'token': str, 'expires': float}} - # Cache SD errors to prevent hammering a blocked account - _sd_poster_error_cache = {} # {source_id: {'until': float, 'reason': str}} - - @action(detail=True, methods=['get'], url_path='poster', permission_classes=[AllowAny]) - def poster(self, request, pk=None): - """ - Proxy endpoint for SD program poster images. - Fetches from SD with auth on first request, caches to disk. - Subsequent requests served from disk cache. - """ - import time - import hashlib - import requests as http_requests - - program = self.get_object() - poster_url = (program.custom_properties or {}).get('poster_url') - if not poster_url: - return Response(status=status.HTTP_404_NOT_FOUND) - - # Extract filename hash for cache key - cache_hash = poster_url.rsplit('/', 1)[-1] - cache_dir = '/data/cache/posters' - os.makedirs(cache_dir, exist_ok=True) - cache_path = os.path.join(cache_dir, cache_hash) - - # Serve from disk cache if fresh (< 30 days) - CACHE_MAX_AGE_SECS = 30 * 24 * 3600 - if os.path.exists(cache_path): - file_age = time.time() - os.path.getmtime(cache_path) - if file_age < CACHE_MAX_AGE_SECS: - from django.http import HttpResponse - with open(cache_path, 'rb') as f: - response = HttpResponse(f.read(), content_type='image/jpeg') - response['Cache-Control'] = 'public, max-age=86400' - return response - - # Check if SD is in error state — don't hammer a blocked account - source = program.epg.epg_source if program.epg else None - if not source or source.source_type != 'schedules_direct': - return Response(status=status.HTTP_404_NOT_FOUND) - - error_cache = ProgramViewSet._sd_poster_error_cache.get(source.id) - if error_cache and time.time() < error_cache['until']: - return Response( - {'error': f"SD temporarily unavailable: {error_cache['reason']}"}, - status=status.HTTP_503_SERVICE_UNAVAILABLE - ) - - from apps.epg.tasks import SD_BASE_URL - - # Reuse cached token if still valid (using tokenExpires from SD response) - cached = ProgramViewSet._sd_poster_token_cache.get(source.id) - token = None - if cached and time.time() < cached['expires']: - token = cached['token'] - - if not token: - sha1_password = hashlib.sha1(source.password.encode('utf-8')).hexdigest() - try: - auth_resp = http_requests.post( - f"{SD_BASE_URL}/token", - json={'username': source.username, 'password': sha1_password}, - headers={'Content-Type': 'application/json'}, - timeout=10, - ) - auth_data = auth_resp.json() - token = auth_data.get('token') - if not token: - # Auth failed — cache error for 1 hour to prevent hammering - error_msg = auth_data.get('message', 'Authentication failed') - ProgramViewSet._sd_poster_error_cache[source.id] = { - 'until': time.time() + 3600, - 'reason': error_msg, - } - return Response(status=status.HTTP_502_BAD_GATEWAY) - - # Cache token using SD's tokenExpires (UNIX epoch) - token_expires = auth_data.get('tokenExpires', time.time() + 86400) - ProgramViewSet._sd_poster_token_cache[source.id] = { - 'token': token, - 'expires': token_expires, - } - except http_requests.exceptions.RequestException: - # Network error — cache for 5 minutes - ProgramViewSet._sd_poster_error_cache[source.id] = { - 'until': time.time() + 300, - 'reason': 'Network error reaching Schedules Direct', - } - return Response(status=status.HTTP_502_BAD_GATEWAY) - - # Fetch the poster image - try: - img_resp = http_requests.get( - poster_url, - headers={'token': token}, - timeout=15, - ) - if img_resp.status_code == 403 or img_resp.status_code == 401: - # Token expired or account blocked — invalidate token cache - ProgramViewSet._sd_poster_token_cache.pop(source.id, None) - ProgramViewSet._sd_poster_error_cache[source.id] = { - 'until': time.time() + 3600, - 'reason': f'SD returned {img_resp.status_code} — possible rate limit or token expiry', - } - return Response(status=status.HTTP_502_BAD_GATEWAY) - - if img_resp.status_code != 200: - return Response(status=status.HTTP_502_BAD_GATEWAY) - - # Save to disk cache - with open(cache_path, 'wb') as f: - f.write(img_resp.content) - - from django.http import HttpResponse - response = HttpResponse(img_resp.content, content_type='image/jpeg') - response['Cache-Control'] = 'public, max-age=86400' - return response - - except http_requests.exceptions.RequestException: - return Response(status=status.HTTP_502_BAD_GATEWAY) - - def list(self, request, *args, **kwargs): - logger.debug("Listing all EPG programs.") - return super().list(request, *args, **kwargs) - - @extend_schema( - summary="Search EPG programs", - description=""" -**Advanced EPG program search with multiple filter types and complex query support.** - -### Text Search Features - -**Title and Description Search**: -- Supports AND/OR logical operators (case-insensitive: `and`/`AND` both work) -- Wrap phrases in double quotes to match them literally: `"Law and Order"` -- Parenthetical grouping for complex queries: `(Newcastle OR NEW) AND (Villa OR AST)` -- Regex pattern matching with `title_regex=true` (evaluated by the database engine) -- Whole word matching with `title_whole_words=true` to avoid partial matches - -**Examples**: -- Simple: `title=football` -- AND operator: `title=premier AND league` -- OR operator: `title=Newcastle OR Villa` -- Quoted phrase: `title="Law and Order"` (matches the exact phrase; 'and' is literal) -- Mixed: `title="Law and Order" AND crime` -- Nested groups: `title=(Newcastle OR NEW) AND (Villa OR AST)` -- Regex: `title=^Premier&title_regex=true` (programs starting with "Premier") -- Whole words: `title=NEW&title_whole_words=true` (matches "NEW" but not "News") - -### Time Filtering - -**airing_at**: Find programs airing at a specific moment (start_time ≤ airing_at < end_time) - -**Time ranges**: Use combinations of start_after, start_before, end_after, end_before - -### Response Customization - -**fields**: Comma-separated list to include only specific fields in response -- Available: id, title, sub_title, description, start_time, end_time, tvg_id, custom_properties, epg_source, epg_name, epg_icon_url, channels, streams - -### Pagination - -- Default: 50 results per page -- Maximum: 500 results per page -- Use `page` and `page_size` parameters to navigate results - """, - parameters=[ - OpenApiParameter( - 'title', - OpenApiTypes.STR, - description='Title search query. Supports AND/OR operators (case-insensitive), quoted phrases, and parentheses. Double-quote a phrase to match it literally: `"Law and Order"`. Unquoted space-separated terms are matched as a phrase; use AND/OR to combine separate terms.', - ), - OpenApiParameter('title_regex', OpenApiTypes.BOOL, description='Enable regex matching for title (case-insensitive, default: false). e.g. `^The` matches titles starting with "The".'), - OpenApiParameter('title_whole_words', OpenApiTypes.BOOL, description='Match whole words only in title (default: false). e.g. `new` matches "Newcastle" normally but not with whole words enabled.'), - OpenApiParameter( - 'description', - OpenApiTypes.STR, - description='Description search query. Same syntax and features as title search.' - ), - OpenApiParameter('description_regex', OpenApiTypes.BOOL, description='Enable regex matching for description (case-insensitive, default: false).'), - OpenApiParameter('description_whole_words', OpenApiTypes.BOOL, description='Match whole words only in description (default: false). Same behaviour as title_whole_words.'), - OpenApiParameter('start_after', OpenApiTypes.DATETIME, description='Filter programs starting at or after this time. ISO 8601 format, e.g. `2026-02-14T18:00:00Z`.'), - OpenApiParameter('start_before', OpenApiTypes.DATETIME, description='Filter programs starting at or before this time. ISO 8601 format.'), - OpenApiParameter('end_after', OpenApiTypes.DATETIME, description='Filter programs ending at or after this time. ISO 8601 format.'), - OpenApiParameter('end_before', OpenApiTypes.DATETIME, description='Filter programs ending at or before this time. ISO 8601 format.'), - OpenApiParameter('airing_at', OpenApiTypes.DATETIME, description='Find programs airing at this exact moment (start_time ≤ airing_at < end_time). ISO 8601 format, e.g. `2026-02-14T20:00:00Z`.'), - OpenApiParameter('channel', OpenApiTypes.STR, description='Filter by channel name (case-insensitive substring match). e.g. `BBC One`, `Sky Sports`.'), - OpenApiParameter('channel_id', OpenApiTypes.INT, description='Filter by exact channel ID.'), - OpenApiParameter('tvg_id', OpenApiTypes.STR, description='Filter by EPG tvg_id (exact match). e.g. `bbcone.uk`.'), - OpenApiParameter('stream', OpenApiTypes.STR, description='Filter by stream name (case-insensitive substring match).'), - OpenApiParameter('group', OpenApiTypes.STR, description='Filter by channel group or stream group name (case-insensitive substring match). e.g. `Sports`, `UK Channels`.'), - OpenApiParameter('epg_source', OpenApiTypes.INT, description='Filter by EPG source ID.'), - OpenApiParameter('fields', OpenApiTypes.STR, description='Comma-separated list of fields to include. Omit to return all fields. e.g. `title,start_time,end_time`.'), - OpenApiParameter('page', OpenApiTypes.INT, description='Page number for pagination (default: 1).'), - OpenApiParameter('page_size', OpenApiTypes.INT, description='Results per page (default: 50, max: 500).'), - ], - responses={200: ProgramSearchResultSerializer(many=True)}, - tags=['EPG'], - ) - @action(detail=False, methods=['get'], url_path='search', permission_classes=[IsStandardUser]) - def search(self, request): - params = request.query_params - - # Build base queryset with prefetching - queryset = ProgramData.objects.select_related( - 'epg', 'epg__epg_source' - ).prefetch_related( - 'epg__channels', 'epg__channels__channel_group', - 'epg__channels__streams', 'epg__channels__streams__channel_group', - 'epg__channels__streams__m3u_account', - ) - - filters = Q() - - # Text filters - title = params.get('title') - if title: - title_regex = params.get('title_regex', '').lower() in ('true', '1', 'yes') - title_whole_words = params.get('title_whole_words', '').lower() in ('true', '1', 'yes') - filters &= parse_text_query('title', title, use_regex=title_regex, whole_words=title_whole_words) - - description = params.get('description') - if description: - desc_regex = params.get('description_regex', '').lower() in ('true', '1', 'yes') - desc_whole_words = params.get('description_whole_words', '').lower() in ('true', '1', 'yes') - filters &= parse_text_query('description', description, use_regex=desc_regex, whole_words=desc_whole_words) - - # Time filters with validation - start_after = params.get('start_after') - if start_after: - dt = parse_datetime(start_after) - if dt is None: - return Response( - {"error": f"Invalid datetime format for start_after: {start_after}. Use ISO 8601 format (e.g., 2026-02-14T18:00:00Z)"}, - status=status.HTTP_400_BAD_REQUEST - ) - filters &= Q(start_time__gte=dt) - - start_before = params.get('start_before') - if start_before: - dt = parse_datetime(start_before) - if dt is None: - return Response( - {"error": f"Invalid datetime format for start_before: {start_before}. Use ISO 8601 format."}, - status=status.HTTP_400_BAD_REQUEST - ) - filters &= Q(start_time__lte=dt) - - end_after = params.get('end_after') - if end_after: - dt = parse_datetime(end_after) - if dt is None: - return Response( - {"error": f"Invalid datetime format for end_after: {end_after}. Use ISO 8601 format."}, - status=status.HTTP_400_BAD_REQUEST - ) - filters &= Q(end_time__gte=dt) - - end_before = params.get('end_before') - if end_before: - dt = parse_datetime(end_before) - if dt is None: - return Response( - {"error": f"Invalid datetime format for end_before: {end_before}. Use ISO 8601 format."}, - status=status.HTTP_400_BAD_REQUEST - ) - filters &= Q(end_time__lte=dt) - - airing_at = params.get('airing_at') - if airing_at: - dt = parse_datetime(airing_at) - if dt is None: - return Response( - {"error": f"Invalid datetime format for airing_at: {airing_at}. Use ISO 8601 format."}, - status=status.HTTP_400_BAD_REQUEST - ) - filters &= Q(start_time__lte=dt, end_time__gt=dt) - - # Channel/stream filters - channel = params.get('channel') - if channel: - filters &= Q(epg__channels__name__icontains=channel) - - channel_id = params.get('channel_id') - if channel_id: - try: - filters &= Q(epg__channels__id=int(channel_id)) - except (ValueError, TypeError): - pass - - tvg_id = params.get('tvg_id') - if tvg_id: - filters &= Q(epg__tvg_id=tvg_id) - - stream = params.get('stream') - if stream: - filters &= Q(epg__channels__streams__name__icontains=stream) - - group = params.get('group') - if group: - filters &= ( - Q(epg__channels__channel_group__name__icontains=group) - | Q(epg__channels__streams__channel_group__name__icontains=group) - ) - - epg_source = params.get('epg_source') - if epg_source: - try: - filters &= Q(epg__epg_source__id=int(epg_source)) - except (ValueError, TypeError): - pass - - queryset = queryset.filter(filters).distinct().order_by('start_time') - - # Restrict results to programs on channels the user can access - user = request.user - if user.user_level < 10: - access_filter = Q(epg__channels__user_level__lte=user.user_level) - custom_props = user.custom_properties or {} - if custom_props.get('hide_adult_content', False): - access_filter &= Q(epg__channels__is_adult=False) - queryset = queryset.filter(access_filter).distinct() - - # Resolve field selection before serialization so expensive methods can short-circuit - requested_fields = params.get('fields') - allowed = set(f.strip() for f in requested_fields.split(',')) if requested_fields else None - - # Paginate - paginator = ProgramSearchPagination() - page = paginator.paginate_queryset(queryset, request) - serializer = ProgramSearchResultSerializer(page, many=True, context={'fields': allowed, 'user': request.user}) - data = serializer.data - - if allowed: - data = [{k: v for k, v in item.items() if k in allowed} for item in data] - - return paginator.get_paginated_response(data) - - -# ───────────────────────────── -# 3) EPG Grid View -# ───────────────────────────── -class EPGGridAPIView(APIView): - """Returns all programs airing in the next 24 hours including currently running ones and recent ones""" - - def get_permissions(self): - try: - return [ - perm() for perm in permission_classes_by_method[self.request.method] - ] - except KeyError: - return [Authenticated()] - - @extend_schema( - description="Retrieve programs from the previous hour, currently running and upcoming for the next 24 hours", - responses={200: ProgramDataSerializer(many=True)}, - ) - def get(self, request, format=None): - # Use current time instead of midnight - now = timezone.now() - one_hour_ago = now - timedelta(hours=1) - twenty_four_hours_later = now + timedelta(hours=24) - logger.debug( - f"EPGGridAPIView: Querying programs between {one_hour_ago} and {twenty_four_hours_later}." - ) - - programs = ProgramData.objects.filter( - end_time__gt=one_hour_ago, - start_time__lt=twenty_four_hours_later, - ) - - # Generate dummy programs for channels that have no EPG data OR dummy EPG sources - from apps.channels.models import Channel - from apps.epg.models import EPGSource - from django.db.models import Q - - # Get channels with no EPG data at all (standard dummy) - channels_without_epg = Channel.objects.filter(Q(epg_data__isnull=True)) - - # Get channels with custom dummy EPG sources (generate on-demand with patterns) - channels_with_custom_dummy = Channel.objects.filter( - epg_data__epg_source__source_type='dummy' - ).select_related('epg_data__epg_source').distinct() - - # Log what we found - without_count = channels_without_epg.count() - custom_count = channels_with_custom_dummy.count() - - if without_count > 0: - channel_names = [f"{ch.name} (ID: {ch.id})" for ch in channels_without_epg] - logger.debug( - f"EPGGridAPIView: Channels needing standard dummy EPG: {', '.join(channel_names)}" - ) - - if custom_count > 0: - channel_names = [f"{ch.name} (ID: {ch.id})" for ch in channels_with_custom_dummy] - logger.debug( - f"EPGGridAPIView: Channels needing custom dummy EPG: {', '.join(channel_names)}" - ) - - logger.debug( - f"EPGGridAPIView: Found {without_count} channels needing standard dummy, {custom_count} needing custom dummy EPG." - ) - - # Serialize the regular programs using .values() to bypass DRF overhead - programs_qs = programs.values( - 'id', 'start_time', 'end_time', 'title', 'sub_title', - 'description', 'tvg_id', 'custom_properties', - ) - serialized_programs = [] - for p in programs_qs: - cp = p['custom_properties'] or {} - premiere_text = cp.get('premiere_text', '') - serialized_programs.append({ - 'id': p['id'], - 'start_time': p['start_time'], - 'end_time': p['end_time'], - 'title': p['title'], - 'sub_title': p['sub_title'], - 'description': p['description'], - 'tvg_id': p['tvg_id'], - 'season': cp.get('season'), - 'episode': cp.get('episode'), - 'is_new': bool(cp.get('new')), - 'is_live': bool(cp.get('live')), - 'is_premiere': bool(cp.get('premiere')), - 'is_finale': bool(premiere_text and 'finale' in premiere_text.lower()), - }) - logger.debug( - f"EPGGridAPIView: Found {len(serialized_programs)} program(s), including recently ended, currently running, and upcoming shows." - ) - - # Humorous program descriptions based on time of day - same as in output/views.py - time_descriptions = { - (0, 4): [ - "Late Night with {channel} - Where insomniacs unite!", - "The 'Why Am I Still Awake?' Show on {channel}", - "Counting Sheep - A {channel} production for the sleepless", - ], - (4, 8): [ - "Dawn Patrol - Rise and shine with {channel}!", - "Early Bird Special - Coffee not included", - "Morning Zombies - Before coffee viewing on {channel}", - ], - (8, 12): [ - "Mid-Morning Meetings - Pretend you're paying attention while watching {channel}", - "The 'I Should Be Working' Hour on {channel}", - "Productivity Killer - {channel}'s daytime programming", - ], - (12, 16): [ - "Lunchtime Laziness with {channel}", - "The Afternoon Slump - Brought to you by {channel}", - "Post-Lunch Food Coma Theater on {channel}", - ], - (16, 20): [ - "Rush Hour - {channel}'s alternative to traffic", - "The 'What's For Dinner?' Debate on {channel}", - "Evening Escapism - {channel}'s remedy for reality", - ], - (20, 24): [ - "Prime Time Placeholder - {channel}'s finest not-programming", - "The 'Netflix Was Too Complicated' Show on {channel}", - "Family Argument Avoider - Courtesy of {channel}", - ], - } - - # Generate and append dummy programs - dummy_programs = [] - - # Import the function from output.views - from apps.output.views import generate_dummy_programs as gen_dummy_progs - - # Handle channels with CUSTOM dummy EPG sources (with patterns) - for channel in channels_with_custom_dummy: - # For dummy EPGs, ALWAYS use channel UUID to ensure unique programs per channel - # This prevents multiple channels assigned to the same dummy EPG from showing identical data - # Each channel gets its own unique program data even if they share the same EPG source - dummy_tvg_id = str(channel.uuid) - - try: - # Get the custom dummy EPG source - epg_source = channel.epg_data.epg_source if channel.epg_data else None - - logger.debug(f"Generating custom dummy programs for channel: {channel.name} (ID: {channel.id})") - - # Determine which name to parse based on custom properties - name_to_parse = channel.name - if epg_source and epg_source.custom_properties: - custom_props = epg_source.custom_properties - name_source = custom_props.get('name_source') - - if name_source == 'stream': - # Get the stream index (1-based from user, convert to 0-based) - stream_index = custom_props.get('stream_index', 1) - 1 - - # Get streams ordered by channelstream order - channel_streams = channel.streams.all().order_by('channelstream__order') - - if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): - stream = list(channel_streams)[stream_index] - name_to_parse = stream.name - logger.debug(f"Using stream name for parsing: {name_to_parse} (stream index: {stream_index})") - else: - logger.warning(f"Stream index {stream_index} not found for channel {channel.name}, falling back to channel name") - elif name_source == 'channel': - logger.debug(f"Using channel name for parsing: {name_to_parse}") - - # Generate programs using custom patterns from the dummy EPG source - # Use the same tvg_id that will be set in the program data - generated = gen_dummy_progs( - channel_id=dummy_tvg_id, - channel_name=name_to_parse, - num_days=1, - program_length_hours=4, - epg_source=epg_source - ) - - # Custom dummy should always return data (either from patterns or fallback) - if generated: - logger.debug(f"Generated {len(generated)} custom dummy programs for {channel.name}") - # Convert generated programs to API format - for program in generated: - prog_custom = program.get('custom_properties') or {} - dummy_program = { - "id": f"dummy-custom-{channel.id}-{program['start_time'].hour}", - "epg": {"tvg_id": dummy_tvg_id, "name": channel.name}, - "start_time": program['start_time'].isoformat(), - "end_time": program['end_time'].isoformat(), - "title": program['title'], - "description": program['description'], - "tvg_id": dummy_tvg_id, - "sub_title": program.get('sub_title'), - "custom_properties": prog_custom if prog_custom else None, - "season": None, - "episode": None, - "is_new": prog_custom.get('new', False), - "is_live": bool(prog_custom.get('live')), - "is_premiere": False, - "is_finale": False, - } - dummy_programs.append(dummy_program) - else: - logger.warning(f"No programs generated for custom dummy EPG channel: {channel.name}") - - except Exception as e: - logger.error( - f"Error creating custom dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" - ) - - # Handle channels with NO EPG data (standard dummy with humorous descriptions) - for channel in channels_without_epg: - # For channels with no EPG, use UUID to ensure uniqueness (matches frontend logic) - # The frontend uses: tvgRecord?.tvg_id ?? channel.uuid - # Since there's no EPG data, it will fall back to UUID - dummy_tvg_id = str(channel.uuid) - - try: - logger.debug(f"Generating standard dummy programs for channel: {channel.name} (ID: {channel.id})") - - # Create programs every 4 hours for the next 24 hours with humorous descriptions - for hour_offset in range(0, 24, 4): - # Use timedelta for time arithmetic instead of replace() to avoid hour overflow - start_time = now + timedelta(hours=hour_offset) - # Set minutes/seconds to zero for clean time blocks - start_time = start_time.replace(minute=0, second=0, microsecond=0) - end_time = start_time + timedelta(hours=4) - - # Get the hour for selecting a description - hour = start_time.hour - day = 0 # Use 0 as we're only doing 1 day - - # Find the appropriate time slot for description - for time_range, descriptions in time_descriptions.items(): - start_range, end_range = time_range - if start_range <= hour < end_range: - # Pick a description using the sum of the hour and day as seed - # This makes it somewhat random but consistent for the same timeslot - description = descriptions[ - (hour + day) % len(descriptions) - ].format(channel=channel.name) - break - else: - # Fallback description if somehow no range matches - description = f"Placeholder program for {channel.name} - EPG data went on vacation" - - # Create a dummy program in the same format as regular programs - dummy_program = { - "id": f"dummy-standard-{channel.id}-{hour_offset}", - "epg": {"tvg_id": dummy_tvg_id, "name": channel.name}, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "title": f"{channel.name}", - "description": description, - "tvg_id": dummy_tvg_id, - "sub_title": None, - "custom_properties": None, - "season": None, - "episode": None, - "is_new": False, - "is_live": False, - "is_premiere": False, - "is_finale": False, - } - dummy_programs.append(dummy_program) - - except Exception as e: - logger.error( - f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" - ) - - # Combine regular and dummy programs - all_programs = list(serialized_programs) + dummy_programs - logger.debug( - f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)." - ) - - return Response({"data": all_programs}, status=status.HTTP_200_OK) - - -# ───────────────────────────── -# 4) EPG Import View -# ───────────────────────────── -class EPGImportAPIView(APIView): - """Triggers an EPG data refresh""" - - def get_permissions(self): - try: - return [ - perm() for perm in permission_classes_by_method[self.request.method] - ] - except KeyError: - return [Authenticated()] - - @extend_schema( - description="Triggers an EPG data refresh for the given source.", - request=inline_serializer( - name="EPGImportRequest", - fields={ - "id": serializers.IntegerField(help_text="ID of the EPG source to refresh."), - }, - ), - ) - def post(self, request, format=None): - logger.info("EPGImportAPIView: Received request to import EPG data.") - epg_id = request.data.get("id", None) - force = bool(request.data.get("force", False)) - - # Check if this is a dummy EPG source - try: - from .models import EPGSource - epg_source = EPGSource.objects.get(id=epg_id) - if epg_source.source_type == 'dummy': - logger.info(f"EPGImportAPIView: Skipping refresh for dummy EPG source {epg_id}") - return Response( - {"success": False, "message": "Dummy EPG sources do not require refreshing."}, - status=status.HTTP_400_BAD_REQUEST, - ) - except EPGSource.DoesNotExist: - pass # Let the task handle the missing source - - refresh_epg_data.delay(epg_id, force=force) # Trigger Celery task - logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") - return Response( - {"success": True, "message": "EPG data refresh initiated."}, - status=status.HTTP_202_ACCEPTED, - ) - - -# ───────────────────────────── -# 5) EPG Data View -# ───────────────────────────── -class EPGDataViewSet(viewsets.ReadOnlyModelViewSet): - """ - API endpoint that allows EPGData objects to be viewed. - """ - - queryset = EPGData.objects.all() - serializer_class = EPGDataSerializer - - def get_permissions(self): - try: - return [perm() for perm in permission_classes_by_action[self.action]] - except KeyError: - return [Authenticated()] - - -# ───────────────────────────── -# 6) Current Programs API -# ───────────────────────────── -class CurrentProgramsAPIView(APIView): - """ - Lightweight endpoint that returns currently playing programs for specified channel IDs. - Accepts POST with JSON body containing channel_ids array, or null/empty to fetch all channels. - """ - - def get_permissions(self): - try: - return [ - perm() for perm in permission_classes_by_method[self.request.method] - ] - except KeyError: - return [Authenticated()] - - @extend_schema( - description="Get currently playing programs for specified channels or all channels", - request=inline_serializer( - name="CurrentProgramsRequest", - fields={ - "channel_uuids": serializers.ListField( - child=serializers.CharField(), - required=False, - allow_null=True, - help_text="Array of channel UUIDs. If null or omitted, returns all channels with current programs.", - ), - }, - ), - responses={200: ProgramDataSerializer(many=True)}, - ) - def post(self, request, format=None): - # Import Channel model - from apps.channels.models import Channel - - # Build query for channels with EPG data - query = Channel.objects.filter(epg_data__isnull=False) - - channel_uuids = request.data.get('channel_uuids', None) - - if channel_uuids is not None: - if not isinstance(channel_uuids, list): - return Response( - {"error": "channel_uuids must be an array of strings or null"}, - status=status.HTTP_400_BAD_REQUEST - ) - query = query.filter(uuid__in=channel_uuids) - - # Get channels with EPG data - channels = query.select_related('epg_data') - - # Get current time - now = timezone.now() - - # Build list of current programs - current_programs = [] - - for channel in channels: - # Query for current program - program = ProgramData.objects.select_related("epg").filter( - epg=channel.epg_data, - start_time__lte=now, - end_time__gt=now - ).first() - - if program: - program_data = ProgramDataSerializer(program).data - program_data['channel_uuid'] = str(channel.uuid) - current_programs.append(program_data) - - - return Response(current_programs, status=status.HTTP_200_OK) - diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 78900db9..442c0474 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -10,6 +10,7 @@ import time # Add import for tracking download progress from datetime import datetime, timedelta, timezone as dt_timezone import gc # Add garbage collection module import json +import re from lxml import etree # Using lxml exclusively import psutil # Add import for memory tracking import zipfile @@ -56,6 +57,13 @@ def _build_html_entity_doctype() -> bytes: _HTML_ENTITY_DOCTYPE = _build_html_entity_doctype() +def _parse_programme_element(element_bytes): + """Parse a single element, prepending the HTML-entity DOCTYPE + so references like é in the text resolve instead of failing.""" + parser = etree.XMLParser(resolve_entities=True, load_dtd=True, no_network=True) + return etree.fromstring(_HTML_ENTITY_DOCTYPE + element_bytes, parser) + + class _PrependStream: """Wraps an open binary file and prepends a bytes prefix to its content. @@ -358,6 +366,9 @@ def refresh_epg_data(source_id, force=False): # Continue with the normal processing... logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") if source.source_type == 'xmltv': + # Invalidate the byte-offset index before downloading the new file + # so stale offsets are never used during the refresh window. + EPGSource.objects.filter(id=source.id).update(programme_index=None) fetch_success = fetch_xmltv(source) if not fetch_success: logger.error(f"Failed to fetch XMLTV for source {source.name}") @@ -376,6 +387,8 @@ def refresh_epg_data(source_id, force=False): gc.collect() return + # Build byte-offset index for preview lookups in the background so refresh isn't blocked by it + build_programme_index_task.delay(source.id) parse_programs_for_source(source) elif source.source_type == 'schedules_direct': @@ -3561,3 +3574,429 @@ def generate_dummy_epg(source): logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " f"Dummy EPG programs are now generated on-demand.") return True + + +# --------------------------------------------------------------------------- +# Byte-offset programme index (ported from dev branch) +# These functions support fast current-program lookup for the CurrentPrograms +# API without doing a full DB query for every channel on every poll. +# --------------------------------------------------------------------------- + + +def _resolve_source_file(epg_source): + """Resolve the XML file path for an EPG source.""" + file_path = epg_source.extracted_file_path or epg_source.file_path + if not file_path: + file_path = epg_source.get_cache_file() + return file_path + + +_CHANNEL_ATTR_RE = re.compile(rb"""channel\s*=\s*(?:"([^"]+)"|'([^']+)')""") +_PROGRAMME_TAG = b'/' +_MAX_START_TAG = 4096 # generous upper bound for a start tag with namespaces/extra attrs +_OFFSET_CAP = 10 # max block-starts recorded per channel; exceeding this flags the channel as interleaved + + +def _decode_channel_id(raw): + """Match how EPGData.tvg_id is stored: resolve XML entities and strip, so byte-level index keys equal the lxml-parsed channel ids.""" + s = raw.decode('utf-8', errors='replace') + if '&' in s: + s = html.unescape(s) + return s.strip() + + +def _find_programme_tag(buf, start): + """ + Find the next ' + follow = idx + _PROGRAMME_TAG_LEN + if follow >= len(buf): + return idx, -1 # need more data + if buf[follow: follow + 1] not in _TAG_FOLLOW: + pos = follow # false match (e.g. ' that closes the opening tag (scan up to _MAX_START_TAG bytes) + tag_end = buf.find(b'>', follow, idx + _MAX_START_TAG) + if tag_end == -1: + if len(buf) >= idx + _MAX_START_TAG: + logger.warning( + f'[_find_programme_tag] start tag exceeds {_MAX_START_TAG} bytes at offset {idx}, skipping' + ) + return -1, -1 + return idx, -1 # need more data + return idx, tag_end + + +def _programme_to_dict(elem, start_time, end_time): + """Convert a lxml element to a serializable dict.""" + title_el = elem.find('title') + desc_el = elem.find('desc') + sub_el = elem.find('sub-title') + return { + 'title': title_el.text if title_el is not None and title_el.text else '', + 'description': desc_el.text if desc_el is not None and desc_el.text else '', + 'sub_title': sub_el.text if sub_el is not None and sub_el.text else '', + 'start_time': start_time.isoformat(), + 'end_time': end_time.isoformat(), + } + + +def build_programme_index(source_id): + """ + Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map. + Persists the result to EPGSource.programme_index. Most XMLTV files group programmes + by channel, but some split a channel across multiple non-contiguous blocks, so we + record block starts up to _OFFSET_CAP and mark only channels that exceed the cap + as interleaved. + """ + try: + source = EPGSource.objects.get(id=source_id) + except EPGSource.DoesNotExist: + logger.error(f'[build_programme_index] EPGSource {source_id} not found') + return + + file_path = _resolve_source_file(source) + if not file_path or not os.path.exists(file_path): + logger.warning( + f'[build_programme_index] File not found for source {source_id}: {file_path}' + ) + return + + logger.debug( + f'[build_programme_index] Building byte-offset index for source {source_id} from {file_path}' + ) + start = time.monotonic() + index = {} + prev_channel = None + interleaved_channels = set() + + CHUNK = 8 * 1024 * 1024 # 8MB + + with open(file_path, 'rb') as f: + buf = bytearray() + buf_offset = 0 # absolute file offset of buf[0] + + while True: + chunk = f.read(CHUNK) + if not chunk and not buf: + break + buf.extend(chunk) + search_from = 0 + + while True: + idx, tag_end = _find_programme_tag(buf, search_from) + if idx == -1: + break + if tag_end == -1 and chunk: + break # incomplete tag at buffer edge, need more data + + abs_pos = buf_offset + idx + m = _CHANNEL_ATTR_RE.search( + buf, idx, tag_end + 1 if tag_end != -1 else idx + _MAX_START_TAG + ) + if m: + channel_id = _decode_channel_id(m.group(1) or m.group(2)) + if channel_id not in index: + index[channel_id] = [abs_pos] + elif channel_id != prev_channel: + if len(index[channel_id]) < _OFFSET_CAP: + index[channel_id].append(abs_pos) + else: + interleaved_channels.add(channel_id) + prev_channel = channel_id + + search_from = ( + (tag_end + 1) if tag_end != -1 else (idx + _PROGRAMME_TAG_LEN) + ) + + if not chunk: + break + + # Keep unprocessed tail for next iteration + keep_from = ( + max(search_from, len(buf) - _MAX_START_TAG) if chunk else len(buf) + ) + del buf[:keep_from] + buf_offset += keep_from + + elapsed = time.monotonic() - start + logger.info( + f'[build_programme_index] Indexed {len(index)} channels in {elapsed:.1f}s for source {source_id}' + + ( + f' ({len(interleaved_channels)} interleaved)' + if interleaved_channels + else '' + ) + ) + + result = { + 'channels': index, + 'interleaved_channels': sorted(interleaved_channels), + } + EPGSource.objects.filter(id=source_id).update(programme_index=result) + + +@shared_task +def build_programme_index_task(source_id): + """Celery wrapper. Locks so refresh and preview don't both build the same source. Releases on finish rather than waiting out the TTL.""" + from core.utils import RedisClient + + redis_client = RedisClient.get_client() + lock_key = f'building_programme_index_{source_id}' + if not redis_client.set(lock_key, '1', nx=True, ex=300): + return + try: + build_programme_index(source_id) + finally: + redis_client.delete(lock_key) + + +def find_current_program_for_tvg_id(epg_or_id): + """ + Look up the currently-airing program for an EPGData instance (or id) using + the byte-offset index. If no index exists yet, queue an async build and let + the caller retry rather than doing a blocking scan. + + Returns dict, None, or "timeout". + """ + if isinstance(epg_or_id, EPGData): + epg = epg_or_id + else: + try: + epg = EPGData.objects.select_related('epg_source').get(id=epg_or_id) + except EPGData.DoesNotExist: + return None + + source = epg.epg_source + if not source or source.source_type in ('dummy', 'schedules_direct'): + return None + + tvg_id = epg.tvg_id + if not tvg_id: + return None + + file_path = _resolve_source_file(source) + if not file_path or not os.path.exists(file_path): + return None + + now = timezone.now() + # Force a fresh read of the DB-backed index to avoid using stale related-object + # state when an EPG refresh invalidates/rebuilds the index concurrently. + source.refresh_from_db(fields=['programme_index']) + index = source.programme_index + + if index is not None: + channels = index.get('channels', {}) + if tvg_id not in channels: + # Channel has no programmes in the file + return None + offsets = channels[tvg_id] + if tvg_id in (index.get('interleaved_channels') or ()): + # Check all stored offsets first (cheap: one seek + one element parse each) + result = _read_programs_at_offsets(file_path, tvg_id, offsets, now) + if result is not None: + return result + # Current programme is beyond the stored offsets; scan forward from the + # last known position to avoid re-reading the already-checked portion + result = _scan_from_offset_for_tvg_id(file_path, tvg_id, offsets[-1], now) + if result == 'timeout': + logger.warning( + f'[find_current_program_for_tvg_id] Interleaved scan timed out for ' + f'tvg_id={tvg_id} source={source.id}; index has {len(offsets)} offsets' + ) + return None + return result + return _read_programs_at_offsets(file_path, tvg_id, offsets, now) + + # No index yet: dispatch a background build and let the frontend retry. + # A sync scan can block a worker for ~10s on SMB-hosted EPGs. + build_programme_index_task.delay(source.id) + return 'timeout' + + +def _read_programs_at_offsets(file_path, tvg_id, offsets, now): + """ + Seek to each offset, extract elements for *tvg_id*, return the + first one currently airing. Chunk-based so it works on minified XML. + """ + PROG_CLOSE = b'' + CLOSE_LEN = len(PROG_CLOSE) + READ_SIZE = 2 * 1024 * 1024 # 2MB per read + + with open(file_path, 'rb') as f: + for offset in offsets: + f.seek(offset) + buf = bytearray() + done = False + + while not done: + chunk = f.read(READ_SIZE) + if not chunk and not buf: + break + buf.extend(chunk) + search_from = 0 + + while True: + tag_start, tag_end = _find_programme_tag(buf, search_from) + if tag_start == -1: + break + if tag_end == -1 and chunk: + break # incomplete tag, need more data + + # Check channel before searching for close tag + m = _CHANNEL_ATTR_RE.search( + buf, + tag_start, + tag_end + 1 if tag_end != -1 else tag_start + _MAX_START_TAG, + ) + if not m: + search_from = ( + (tag_end + 1) + if tag_end != -1 + else (tag_start + _PROGRAMME_TAG_LEN) + ) + continue + + ch = _decode_channel_id(m.group(1) or m.group(2)) + if ch != tvg_id: + done = True # different channel, end of block + break + + # Find the closing tag + close_pos = buf.find( + PROG_CLOSE, tag_end + 1 if tag_end != -1 else m.end() + ) + if close_pos == -1: + if not chunk: + done = True # EOF with no close tag + break # need more data + close_end = close_pos + CLOSE_LEN + + element_bytes = bytes(buf[tag_start:close_end]) + search_from = close_end + + try: + prog = _parse_programme_element(element_bytes) + except etree.XMLSyntaxError: + continue + + start_str = prog.get('start') + stop_str = prog.get('stop') + if not start_str or not stop_str: + continue + start_time = parse_xmltv_time(start_str) + end_time = parse_xmltv_time(stop_str) + if start_time is None or end_time is None: + continue + if start_time <= now < end_time: + return _programme_to_dict(prog, start_time, end_time) + + # Trim processed bytes + if search_from > 0: + del buf[:search_from] + search_from = 0 + + if not chunk: + break + + return None + + +def _scan_from_offset_for_tvg_id(file_path, tvg_id, start_offset, now, timeout_sec=10): + """ + Scan forward from start_offset for tvg_id, skipping other channels rather than + stopping at a channel boundary. Used for interleaved/time-sorted XMLTV files where + a channel exceeded the stored offset cap. + Returns dict, None, or 'timeout'. + """ + PROG_CLOSE = b'' + CLOSE_LEN = len(PROG_CLOSE) + READ_SIZE = 2 * 1024 * 1024 + deadline = time.monotonic() + timeout_sec + + with open(file_path, 'rb') as f: + f.seek(start_offset) + buf = bytearray() + + while True: + if time.monotonic() > deadline: + return 'timeout' + + chunk = f.read(READ_SIZE) + if not chunk and not buf: + break + buf.extend(chunk) + search_from = 0 + + trim_to = 0 + + while True: + tag_start, tag_end = _find_programme_tag(buf, search_from) + if tag_start == -1: + trim_to = search_from + break + if tag_end == -1 and chunk: + trim_to = tag_start # keep incomplete tag for next read + break + + m = _CHANNEL_ATTR_RE.search( + buf, + tag_start, + tag_end + 1 if tag_end != -1 else tag_start + _MAX_START_TAG, + ) + if not m: + search_from = ( + tag_end + 1 if tag_end != -1 else tag_start + _PROGRAMME_TAG_LEN + ) + continue + + ch = _decode_channel_id(m.group(1) or m.group(2)) + if ch != tvg_id: + search_from = ( + tag_end + 1 if tag_end != -1 else tag_start + _PROGRAMME_TAG_LEN + ) + continue + + close_pos = buf.find( + PROG_CLOSE, tag_end + 1 if tag_end != -1 else m.end() + ) + if close_pos == -1: + trim_to = tag_start # keep incomplete element for next read + break + close_end = close_pos + CLOSE_LEN + + element_bytes = bytes(buf[tag_start:close_end]) + search_from = close_end + + try: + prog = _parse_programme_element(element_bytes) + except etree.XMLSyntaxError: + continue + + start_str = prog.get('start') + stop_str = prog.get('stop') + if not start_str or not stop_str: + continue + start_time = parse_xmltv_time(start_str) + end_time = parse_xmltv_time(stop_str) + if start_time is None or end_time is None: + continue + if start_time <= now < end_time: + return _programme_to_dict(prog, start_time, end_time) + + if trim_to > 0: + del buf[:trim_to] + + if not chunk: + break + + return None diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 582d6a1b..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,43 +0,0 @@ -services: - dispatcharr: - build: - context: . - dockerfile: docker/Dockerfile - container_name: dispatcharr_dev - restart: unless-stopped - ports: - - 9191:9191 - - 8001:8001 - volumes: - - dispatcharr_data:/data - environment: - - POSTGRES_HOST=db - - POSTGRES_PORT=5432 - - POSTGRES_DB=dispatcharr - - POSTGRES_USER=dispatch - - POSTGRES_PASSWORD=secret - - DJANGO_SECRET_KEY=dev-secret-key-not-for-production - - DISPATCHARR_LOG_LEVEL=info - depends_on: - db: - condition: service_healthy - - db: - image: postgres:17 - container_name: dispatcharr_dev_db - restart: unless-stopped - environment: - - POSTGRES_DB=dispatcharr - - POSTGRES_USER=dispatch - - POSTGRES_PASSWORD=secret - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U dispatch -d dispatcharr"] - interval: 5s - timeout: 5s - retries: 5 - -volumes: - dispatcharr_data: - postgres_data: diff --git a/serializers.py b/serializers.py deleted file mode 100644 index e9ac8581..00000000 --- a/serializers.py +++ /dev/null @@ -1,866 +0,0 @@ -import json -from datetime import datetime - -from rest_framework import serializers -from .models import ( - Stream, - Channel, - ChannelGroup, - ChannelOverride, - ChannelStream, - ChannelGroupM3UAccount, - Logo, - ChannelProfile, - ChannelProfileMembership, - Recording, - RecurringRecordingRule, -) -from apps.epg.serializers import EPGDataSerializer -from core.models import StreamProfile -from apps.epg.models import EPGData -from django.db import connection, transaction -from django.urls import reverse -from rest_framework import serializers -from django.utils import timezone -from core.utils import validate_flexible_url - - -class LogoSerializer(serializers.ModelSerializer): - cache_url = serializers.SerializerMethodField() - channel_count = serializers.SerializerMethodField() - is_used = serializers.SerializerMethodField() - channel_names = serializers.SerializerMethodField() - - class Meta: - model = Logo - fields = ["id", "name", "url", "cache_url", "channel_count", "is_used", "channel_names"] - - def validate_url(self, value): - """Validate that the URL is unique for creation or update""" - if self.instance and self.instance.url == value: - return value - - if Logo.objects.filter(url=value).exists(): - raise serializers.ValidationError("A logo with this URL already exists.") - - return value - - def create(self, validated_data): - """Handle logo creation with proper URL validation""" - return Logo.objects.create(**validated_data) - - def update(self, instance, validated_data): - """Handle logo updates""" - for attr, value in validated_data.items(): - setattr(instance, attr, value) - instance.save() - return instance - - def get_cache_url(self, obj): - # Cache-busting: append a short hash of the logo's source URL so the browser - # fetches fresh when the logo changes (e.g., M3U logo replaced by SD logo). - # The backend ignores the 'v' parameter — it's purely for browser cache invalidation. - # See SD integration PR notes for context on why this was added. - import hashlib - url_hash = hashlib.md5((obj.url or '').encode()).hexdigest()[:8] - base_path = reverse("api:channels:logo-cache", args=[obj.id]) - cache_url = f"{base_path}?v={url_hash}" - request = self.context.get("request") - if request: - return request.build_absolute_uri(cache_url) - return cache_url - - def get_channel_count(self, obj): - """Get the number of channels using this logo""" - # `channel_count` is provided as an annotation in LogoViewSet.get_queryset(). - # Fall back to a query only when serializing a single un-annotated Logo - # (e.g. nested inside ChannelSerializer.get_logo()). - annotated = getattr(obj, "channel_count", None) - if annotated is not None: - return annotated - return obj.channels.count() - - def get_is_used(self, obj): - """Check if this logo is used by any channels""" - return self.get_channel_count(obj) > 0 - - def get_channel_names(self, obj): - """Get the names of channels using this logo (limited to first 5)""" - names = [] - - # When LogoViewSet.get_queryset() prefetches `channels`, iterating - # obj.channels.all() reuses the cached set; slicing happens in Python. - channels = list(obj.channels.all()[:5]) - for channel in channels: - names.append(f"Channel: {channel.name}") - - total_count = self.get_channel_count(obj) - if total_count > 5: - names.append(f"...and {total_count - 5} more") - - return names - - -# -# Stream -# -class StreamSerializer(serializers.ModelSerializer): - url = serializers.CharField( - required=False, - allow_blank=True, - allow_null=True, - validators=[validate_flexible_url] - ) - stream_profile_id = serializers.PrimaryKeyRelatedField( - queryset=StreamProfile.objects.all(), - source="stream_profile", - allow_null=True, - required=False, - ) - read_only_fields = ["is_custom", "m3u_account", "stream_hash", "stream_id", "stream_chno"] - - class Meta: - model = Stream - fields = [ - "id", - "name", - "url", - "m3u_account", # Uncomment if using M3U fields - "logo_url", - "tvg_id", - "local_file", - "current_viewers", - "updated_at", - "last_seen", - "is_stale", - "is_adult", - "stream_profile_id", - "is_custom", - "channel_group", - "stream_hash", - "stream_stats", - "stream_stats_updated_at", - "stream_id", - "stream_chno", - ] - - def get_fields(self): - fields = super().get_fields() - - # Unable to edit specific properties if this stream was created from an M3U account - if ( - self.instance - and getattr(self.instance, "m3u_account", None) - and not self.instance.is_custom - ): - fields["id"].read_only = True - fields["name"].read_only = True - fields["url"].read_only = True - fields["m3u_account"].read_only = True - fields["tvg_id"].read_only = True - fields["channel_group"].read_only = True - - return fields - - -class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): - m3u_accounts = serializers.IntegerField(source="m3u_accounts.id", read_only=True) - enabled = serializers.BooleanField() - auto_channel_sync = serializers.BooleanField(default=False) - auto_sync_channel_start = serializers.FloatField( - allow_null=True, required=False, min_value=1 - ) - auto_sync_channel_end = serializers.FloatField( - allow_null=True, required=False, min_value=1 - ) - custom_properties = serializers.JSONField(required=False) - # Provider stream count for this group+account. Lets users size an - # optional end-range without first running a blind sync. - stream_count = serializers.SerializerMethodField() - - class Meta: - model = ChannelGroupM3UAccount - fields = [ - "m3u_accounts", - "channel_group", - "enabled", - "auto_channel_sync", - "auto_sync_channel_start", - "auto_sync_channel_end", - "custom_properties", - "is_stale", - "last_seen", - "stream_count", - ] - - def get_stream_count(self, obj): - """ - Return the number of streams for this (m3u_account, channel_group) - pair. A parent serializer (e.g. M3UAccountSerializer) may seed - ``context["stream_counts"]`` with a pre-aggregated dict keyed by - ``(m3u_account_id, channel_group_id)`` to avoid one COUNT per row; - when present, it is used as the source of truth. The per-row - COUNT fallback is correct for stand-alone serialization (rare, - low-volume) and exists so direct ChannelGroupM3UAccount queries - do not require callers to know the seeding pattern. - """ - counts = self.context.get("stream_counts") - if counts is not None: - return counts.get((obj.m3u_account_id, obj.channel_group_id), 0) - from apps.channels.models import Stream - - return Stream.objects.filter( - m3u_account_id=obj.m3u_account_id, - channel_group_id=obj.channel_group_id, - ).count() - - def to_representation(self, instance): - data = super().to_representation(instance) - - custom_props = instance.custom_properties or {} - - return data - - def to_internal_value(self, data): - # Accept both dict and JSON string for custom_properties (for backward compatibility) - val = data.get("custom_properties") - if isinstance(val, str): - try: - data["custom_properties"] = json.loads(val) - except Exception: - pass - - return super().to_internal_value(data) - - def validate(self, attrs): - # Partial PATCHes only carry submitted fields; fill missing - # start/end from the instance so the validator catches a PATCH - # that lowers end past the existing start. - start = attrs.get("auto_sync_channel_start") - end = attrs.get("auto_sync_channel_end") - if start is None and self.instance is not None: - start = self.instance.auto_sync_channel_start - if end is None and self.instance is not None: - end = self.instance.auto_sync_channel_end - if start is not None and end is not None and end < start: - raise serializers.ValidationError( - { - "auto_sync_channel_end": ( - "End must be greater than or equal to start." - ) - } - ) - return super().validate(attrs) - -# -# Channel Group -# -class ChannelGroupSerializer(serializers.ModelSerializer): - channel_count = serializers.SerializerMethodField() - m3u_account_count = serializers.SerializerMethodField() - m3u_accounts = ChannelGroupM3UAccountSerializer( - many=True, - read_only=True - ) - - class Meta: - model = ChannelGroup - fields = ["id", "name", "channel_count", "m3u_account_count", "m3u_accounts"] - - def get_channel_count(self, obj): - # Use the queryset annotation when available (list path); fall back - # to a live query for retrieve/create/update where it isn't set. - v = getattr(obj, 'channel_count', None) - return v if v is not None else obj.channels.count() - - def get_m3u_account_count(self, obj): - v = getattr(obj, 'm3u_account_count', None) - return v if v is not None else obj.m3u_accounts.count() - - -class ChannelProfileSerializer(serializers.ModelSerializer): - channels = serializers.SerializerMethodField() - - class Meta: - model = ChannelProfile - fields = ["id", "name", "channels"] - - def get_channels(self, obj): - # Use prefetched attr when available, fall back to a direct query. - memberships = getattr(obj, 'enabled_memberships', None) - if memberships is not None: - return [m.channel_id for m in memberships] - return list( - ChannelProfileMembership.objects.filter( - channel_profile=obj, enabled=True - ).values_list('channel_id', flat=True) - ) - - -class ChannelProfileMembershipSerializer(serializers.ModelSerializer): - class Meta: - model = ChannelProfileMembership - fields = ["channel", "enabled"] - - -class ChanneProfilelMembershipUpdateSerializer(serializers.Serializer): - channel_id = serializers.IntegerField() # Ensure channel_id is an integer - enabled = serializers.BooleanField() - - -class BulkChannelProfileMembershipSerializer(serializers.Serializer): - channels = serializers.ListField( - child=ChanneProfilelMembershipUpdateSerializer(), # Use the nested serializer - allow_empty=False, - ) - - def validate_channels(self, value): - if not value: - raise serializers.ValidationError("At least one channel must be provided.") - return value - - -# -# Channel override -# -# Nullable per-field overrides resolved over the parent Channel in read -# paths. Embedded in ChannelSerializer so clients can upsert/clear in the -# same PATCH that targets direct channel fields. -class ChannelOverrideSerializer(serializers.ModelSerializer): - # HDHR clients reject negative GuideNumber and zero is not a real - # provider value, so reject both at the API boundary. - channel_number = serializers.FloatField( - allow_null=True, required=False, min_value=0.0001 - ) - channel_group_id = serializers.PrimaryKeyRelatedField( - queryset=ChannelGroup.objects.all(), - source="channel_group", - allow_null=True, - required=False, - ) - logo_id = serializers.PrimaryKeyRelatedField( - queryset=Logo.objects.all(), - source="logo", - allow_null=True, - required=False, - ) - epg_data_id = serializers.PrimaryKeyRelatedField( - queryset=EPGData.objects.all(), - source="epg_data", - allow_null=True, - required=False, - ) - stream_profile_id = serializers.PrimaryKeyRelatedField( - queryset=StreamProfile.objects.all(), - source="stream_profile", - allow_null=True, - required=False, - ) - - class Meta: - model = ChannelOverride - fields = [ - "name", - "channel_number", - "channel_group_id", - "logo_id", - "tvg_id", - "tvc_guide_stationid", - "epg_data_id", - "stream_profile_id", - ] - extra_kwargs = { - "name": {"allow_null": True, "required": False}, - "tvg_id": {"allow_null": True, "required": False}, - "tvc_guide_stationid": {"allow_null": True, "required": False}, - } - - -# -# Channel -# -class ChannelSerializer(serializers.ModelSerializer): - # Show nested group data, or ID - # Ensure channel_number is explicitly typed as FloatField and properly validated - channel_number = serializers.FloatField( - allow_null=True, - required=False, - error_messages={"invalid": "Channel number must be a valid decimal number."}, - ) - channel_group_id = serializers.PrimaryKeyRelatedField( - queryset=ChannelGroup.objects.all(), source="channel_group", required=False - ) - epg_data_id = serializers.PrimaryKeyRelatedField( - queryset=EPGData.objects.all(), - source="epg_data", - required=False, - allow_null=True, - ) - - stream_profile_id = serializers.PrimaryKeyRelatedField( - queryset=StreamProfile.objects.all(), - source="stream_profile", - allow_null=True, - required=False, - ) - - streams = serializers.PrimaryKeyRelatedField( - queryset=Stream.objects.all(), many=True, required=False - ) - - logo_id = serializers.PrimaryKeyRelatedField( - queryset=Logo.objects.all(), - source="logo", - allow_null=True, - required=False, - ) - - auto_created_by_name = serializers.SerializerMethodField() - override = ChannelOverrideSerializer( - required=False, - allow_null=True, - help_text=( - "Per-field overrides for an auto-created channel. " - 'Send {"override": {"name": "ESPN"}} to upsert the listed ' - 'fields, {"override": {"name": null}} to clear specific fields ' - 'while leaving others, or {"override": null} to delete the ' - "override row entirely. Omitting the key leaves any existing " - "override unchanged. Only valid for auto_created=True channels. " - "Duplicate channel_number values across channels are permitted; " - "downstream client behavior on duplicates varies by client." - ), - ) - source_stream = serializers.SerializerMethodField() - # Effective fields coalesce override over channel column. Consumers - # display these; raw fields remain in the response so the edit form - # can show them as "Provider: X" subtext. - effective_name = serializers.SerializerMethodField() - effective_channel_number = serializers.SerializerMethodField() - effective_channel_group_id = serializers.SerializerMethodField() - effective_logo_id = serializers.SerializerMethodField() - effective_tvg_id = serializers.SerializerMethodField() - effective_tvc_guide_stationid = serializers.SerializerMethodField() - effective_epg_data_id = serializers.SerializerMethodField() - effective_stream_profile_id = serializers.SerializerMethodField() - - class Meta: - model = Channel - fields = [ - "id", - "channel_number", - "name", - "channel_group_id", - "tvg_id", - "tvc_guide_stationid", - "epg_data_id", - "streams", - "stream_profile_id", - "uuid", - "logo_id", - "user_level", - "is_adult", - "hidden_from_output", - "auto_created", - "auto_created_by", - "auto_created_by_name", - "override", - "source_stream", - "effective_name", - "effective_channel_number", - "effective_channel_group_id", - "effective_logo_id", - "effective_tvg_id", - "effective_tvc_guide_stationid", - "effective_epg_data_id", - "effective_stream_profile_id", - ] - - def _effective_value(self, obj, field_name): - override = getattr(obj, "_channel_override_cache", None) - if override is None: - try: - override = obj.override - except ChannelOverride.DoesNotExist: - override = None - obj._channel_override_cache = override - if override is not None: - value = getattr(override, field_name, None) - if value is not None: - return value - return getattr(obj, field_name, None) - - def get_effective_name(self, obj): - return self._effective_value(obj, "name") - - def get_effective_channel_number(self, obj): - return self._effective_value(obj, "channel_number") - - def get_effective_channel_group_id(self, obj): - return self._effective_value(obj, "channel_group_id") - - def get_effective_logo_id(self, obj): - return self._effective_value(obj, "logo_id") - - def get_effective_tvg_id(self, obj): - return self._effective_value(obj, "tvg_id") - - def get_effective_tvc_guide_stationid(self, obj): - return self._effective_value(obj, "tvc_guide_stationid") - - def get_effective_epg_data_id(self, obj): - return self._effective_value(obj, "epg_data_id") - - def get_effective_stream_profile_id(self, obj): - return self._effective_value(obj, "stream_profile_id") - - def get_source_stream(self, obj): - """ - Return the originating provider stream for an auto-created channel. - - Surfaces the provider stream's name and owning M3U account so the - frontend can render "Auto-created from: / " - in the channel edit form. Returns None for manual channels. - """ - if not self.context.get("include_source_stream", False): - return None - if not obj.auto_created: - return None - # Viewset prefetches `channelstream_set` ordered by `order`, so - # `.all()[0]` reuses the cache and returns the lowest-order entry. - prefetched_list = list(obj.channelstream_set.all()) - if not prefetched_list: - return None - cs = prefetched_list[0] - if not cs.stream: - return None - stream = cs.stream - return { - "id": stream.id, - "name": stream.name, - "account_id": stream.m3u_account_id, - "account_name": getattr(stream.m3u_account, "name", None), - } - - def to_representation(self, instance): - include_streams = self.context.get("include_streams", False) - - if include_streams: - self.fields["streams"] = serializers.SerializerMethodField() - return super().to_representation(instance) - else: - # Read from the prefetched channelstream_set (ordered by the - # viewset's Prefetch); chaining .order_by() rebuilds the - # queryset and fires one SELECT per row in list responses. - representation = super().to_representation(instance) - if "streams" in representation: - representation["streams"] = [ - cs.stream_id for cs in instance.channelstream_set.all() - ] - return representation - - def get_logo(self, obj): - return LogoSerializer(obj.logo).data - - def get_streams(self, obj): - """Retrieve ordered stream IDs for GET requests.""" - return StreamSerializer( - obj.streams.all().order_by("channelstream__order"), many=True - ).data - - def create(self, validated_data): - streams = validated_data.pop("streams", []) - override_data = validated_data.pop("override", None) - channel_number = validated_data.pop( - "channel_number", Channel.get_next_available_channel_number() - ) - validated_data["channel_number"] = channel_number - - # Auto-assign Default Group if no channel_group is specified - if "channel_group" not in validated_data or validated_data.get("channel_group") is None: - from apps.channels.models import ChannelGroup - default_group, _ = ChannelGroup.objects.get_or_create(name="Default Group") - validated_data["channel_group"] = default_group - - # Atomic wrapper keeps the channel insert and its override row - # in the same transaction so a failure on either rolls both back. - with transaction.atomic(): - channel = Channel.objects.create(**validated_data) - - # Add streams in the specified order - for index, stream in enumerate(streams): - ChannelStream.objects.create( - channel=channel, stream_id=stream.id, order=index - ) - - if override_data: - # Manual channels (auto_created=False) have no provider - # value to override; reject the override payload here so a - # programmatic client can't write a semantically meaningless - # row that the frontend would then surface as "Overrides - # active". - if not channel.auto_created: - raise serializers.ValidationError( - { - "override": ( - "Cannot set override on a manual channel; " - "overrides only apply to auto-created channels." - ) - } - ) - obj = ChannelOverride.objects.create(channel=channel, **override_data) - # Drop an all-null override row; an empty override would - # falsely surface as active in the UI. - if not obj.has_any_override(): - obj.delete() - - return channel - - def update(self, instance, validated_data): - """ - PATCH handler for Channel rows. The ``override`` key carries - per-field user overrides for auto-created channels and follows - these rules: - - * key absent from payload: no change to existing overrides - * ``{"override": {"field": value}}``: upsert those fields - * ``{"override": {"field": null}}``: clear those specific fields - * ``{"override": null}``: delete the override row entirely - - Key presence is what distinguishes "no change" from "delete"; - an explicit null means delete. Override mutations are rejected - on manual channels (auto_created=False) since there is no - provider value to override. - """ - streams = validated_data.pop("streams", None) - has_override_key = "override" in self.initial_data - override_data = validated_data.pop("override", None) - - # Block override mutations on manual channels (no provider - # value to override). Clearing is a tolerated no-op. - if ( - has_override_key - and override_data is not None - and override_data != {} - and not instance.auto_created - ): - raise serializers.ValidationError( - { - "override": ( - "Cannot set override on a manual channel; " - "overrides only apply to auto-created channels." - ) - } - ) - - # Atomic so a failure on the override row rolls back the - # channel update too. - with transaction.atomic(): - # Skip save() when only override keys were submitted; a - # no-op UPDATE would bump updated_at and bust caches. - if validated_data: - for attr, value in validated_data.items(): - setattr(instance, attr, value) - instance.save() - - if has_override_key: - if override_data is None: - # Explicit null: remove the override row. - ChannelOverride.objects.filter(channel=instance).delete() - elif override_data == {}: - # Empty dict has no field intent; no-op. - pass - else: - obj, _ = ChannelOverride.objects.update_or_create( - channel=instance, defaults=override_data - ) - # Drop an all-null override; would falsely surface - # as active in the UI. - if not obj.has_any_override(): - obj.delete() - # Queryset writes leave the reverse-OneToOne cache stale; - # clear it so to_representation reads the new state. - try: - instance._state.fields_cache.pop("override", None) - except AttributeError: - pass - if hasattr(instance, "_channel_override_cache"): - delattr(instance, "_channel_override_cache") - - if streams is not None: - # Normalize stream IDs - normalized_ids = [ - stream.id if hasattr(stream, "id") else stream for stream in streams - ] - - # Get current mapping of stream_id -> ChannelStream - current_links = { - cs.stream_id: cs for cs in instance.channelstream_set.all() - } - - # Track existing stream IDs - existing_ids = set(current_links.keys()) - new_ids = set(normalized_ids) - - # Delete any links not in the new list - to_remove = existing_ids - new_ids - if to_remove: - instance.channelstream_set.filter(stream_id__in=to_remove).delete() - - # Update or create with new order - to_update = [] - for order, stream_id in enumerate(normalized_ids): - if stream_id in current_links: - cs = current_links[stream_id] - if cs.order != order: - cs.order = order - to_update.append(cs) - else: - ChannelStream.objects.create( - channel=instance, stream_id=stream_id, order=order - ) - - if to_update: - ChannelStream.objects.bulk_update(to_update, ["order"]) - - return instance - - def validate_channel_number(self, value): - """Ensure channel_number is properly processed as a float""" - if value is None: - return value - - try: - # Ensure it's processed as a float - return float(value) - except (ValueError, TypeError): - raise serializers.ValidationError( - "Channel number must be a valid decimal number." - ) - - def validate_stream_profile(self, value): - """Handle special case where empty/0 values mean 'use default' (null)""" - if value == "0" or value == 0 or value == "" or value is None: - return None - return value # PrimaryKeyRelatedField will handle the conversion to object - - def get_auto_created_by_name(self, obj): - """Get the name of the M3U account that auto-created this channel.""" - if obj.auto_created_by: - return obj.auto_created_by.name - return None - - -class RecordingSerializer(serializers.ModelSerializer): - class Meta: - model = Recording - fields = "__all__" - read_only_fields = ["task_id"] - - def validate(self, data): - from core.models import CoreSettings - start_time = data.get("start_time") - end_time = data.get("end_time") - - if start_time and timezone.is_naive(start_time): - start_time = timezone.make_aware(start_time, timezone.get_current_timezone()) - data["start_time"] = start_time - if end_time and timezone.is_naive(end_time): - end_time = timezone.make_aware(end_time, timezone.get_current_timezone()) - data["end_time"] = end_time - - # If this is an EPG-based recording (program provided), apply global pre/post offsets - try: - cp = data.get("custom_properties") or {} - is_epg_based = isinstance(cp, dict) and isinstance(cp.get("program"), (dict,)) - except Exception: - is_epg_based = False - - if is_epg_based and start_time and end_time: - try: - pre_min = int(CoreSettings.get_dvr_pre_offset_minutes()) - except Exception: - pre_min = 0 - try: - post_min = int(CoreSettings.get_dvr_post_offset_minutes()) - except Exception: - post_min = 0 - from datetime import timedelta - try: - if pre_min and pre_min > 0: - start_time = start_time - timedelta(minutes=pre_min) - except Exception: - pass - try: - if post_min and post_min > 0: - end_time = end_time + timedelta(minutes=post_min) - except Exception: - pass - # write back adjusted times so scheduling uses them - data["start_time"] = start_time - data["end_time"] = end_time - - now = timezone.now() # timezone-aware current time - - if end_time < now: - raise serializers.ValidationError("End time must be in the future.") - - if start_time < now: - # Optional: Adjust start_time if it's in the past but end_time is in the future - data["start_time"] = now # or: timezone.now() + timedelta(seconds=1) - if end_time <= data["start_time"]: - raise serializers.ValidationError("End time must be after start time.") - - return data - - -class RecurringRecordingRuleSerializer(serializers.ModelSerializer): - class Meta: - model = RecurringRecordingRule - fields = "__all__" - read_only_fields = ["created_at", "updated_at"] - - def validate_days_of_week(self, value): - if not value: - raise serializers.ValidationError("Select at least one day of the week") - cleaned = [] - for entry in value: - try: - iv = int(entry) - except (TypeError, ValueError): - raise serializers.ValidationError("Days of week must be integers 0-6") - if iv < 0 or iv > 6: - raise serializers.ValidationError("Days of week must be between 0 (Monday) and 6 (Sunday)") - cleaned.append(iv) - return sorted(set(cleaned)) - - def validate(self, attrs): - start = attrs.get("start_time") or getattr(self.instance, "start_time", None) - end = attrs.get("end_time") or getattr(self.instance, "end_time", None) - start_date = attrs.get("start_date") if "start_date" in attrs else getattr(self.instance, "start_date", None) - end_date = attrs.get("end_date") if "end_date" in attrs else getattr(self.instance, "end_date", None) - if start_date is None: - existing_start = getattr(self.instance, "start_date", None) - if existing_start is None: - raise serializers.ValidationError("Start date is required") - if start_date and end_date and end_date < start_date: - raise serializers.ValidationError("End date must be on or after start date") - if end_date is None: - existing_end = getattr(self.instance, "end_date", None) - if existing_end is None: - raise serializers.ValidationError("End date is required") - if start and end and start_date and end_date: - start_dt = datetime.combine(start_date, start) - end_dt = datetime.combine(end_date, end) - if end_dt <= start_dt: - raise serializers.ValidationError("End datetime must be after start datetime") - elif start and end and end == start: - raise serializers.ValidationError("End time must be different from start time") - # Normalize empty strings to None for dates - if attrs.get("end_date") == "": - attrs["end_date"] = None - if attrs.get("start_date") == "": - attrs["start_date"] = None - return super().validate(attrs) - - def create(self, validated_data): - return super().create(validated_data) diff --git a/tasks.py b/tasks.py deleted file mode 100644 index fc43c36d..00000000 --- a/tasks.py +++ /dev/null @@ -1,3594 +0,0 @@ -# apps/epg/tasks.py - -import logging -import gzip -import html.entities -import os -import uuid -import requests -import time # Add import for tracking download progress -from datetime import datetime, timedelta, timezone as dt_timezone -import gc # Add garbage collection module -import json -from lxml import etree # Using lxml exclusively -import psutil # Add import for memory tracking -import zipfile - -from celery import shared_task -from django.conf import settings -from django.db import connection, transaction -from django.db.models import Q -from django.utils import timezone -from apps.channels.models import Channel -from core.models import UserAgent, CoreSettings - -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer - -from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 -from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, send_websocket_update, cleanup_memory, log_system_event - -logger = logging.getLogger(__name__) - -SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' -SD_DAYS_TO_FETCH = 20 -SD_PROGRAM_BATCH_SIZE = 5000 - -# DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named -# entities so lxml/libxml2 can resolve references like é correctly -# instead of silently dropping them in recovery mode. -# The 5 XML-predefined entities (amp, lt, gt, quot, apos) are always -# recognised by the XML spec and must not be redeclared. -_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'}) - - -def _build_html_entity_doctype() -> bytes: - """Build a DOCTYPE internal subset declaring all HTML 4 named entities.""" - lines = [b'\n'.encode('ascii')) - lines.append(b']>\n') - return b''.join(lines) - - -_HTML_ENTITY_DOCTYPE = _build_html_entity_doctype() - - -class _PrependStream: - """Wraps an open binary file and prepends a bytes prefix to its content. - - Used by _open_xmltv_file to inject a DOCTYPE entity block before the - file content reaches lxml's iterparse, with zero disk I/O. - """ - - __slots__ = ('_prefix', '_prefix_pos', '_file') - - def __init__(self, prefix: bytes, file_obj): - self._prefix = prefix - self._prefix_pos = 0 - self._file = file_obj - - def read(self, size=-1): - prefix_len = len(self._prefix) - if self._prefix_pos >= prefix_len: - return self._file.read(size) - remaining = prefix_len - self._prefix_pos - if size < 0: - chunk = self._prefix[self._prefix_pos:] + self._file.read() - self._prefix_pos = prefix_len - return chunk - if size <= remaining: - chunk = self._prefix[self._prefix_pos:self._prefix_pos + size] - self._prefix_pos += size - return chunk - chunk = self._prefix[self._prefix_pos:] - self._prefix_pos = prefix_len - return chunk + self._file.read(size - remaining) - - def close(self): - self._file.close() - - def __enter__(self): - return self - - def __exit__(self, *_): - self.close() - - -def _open_xmltv_file(file_path: str): - """Open an XMLTV file for lxml iterparse, injecting an HTML entity DOCTYPE. - - Prepends a block that declares all 252 HTML 4 named - entities so lxml/libxml2 resolves references like é correctly - instead of silently dropping them in recovery mode. This involves zero - disk I/O (the DOCTYPE is streamed in-memory before the file content). - - If the file already contains a declaration the file is returned - unchanged; a second DOCTYPE would be invalid XML. - - The caller is responsible for closing the returned object. - """ - f = open(file_path, 'rb') - start = f.read(512) - - # Do not inject if the file already declares a DOCTYPE. - if b'= 0: - decl_end = start.find(b'?>', xml_pos) - if decl_end >= 0: - xml_decl = start[:decl_end + 2] - f.seek(decl_end + 2) - return _PrependStream(xml_decl + b'\n' + _HTML_ENTITY_DOCTYPE, f) - - # No XML declaration found; insert DOCTYPE at the very start of the file. - f.seek(0) - return _PrependStream(_HTML_ENTITY_DOCTYPE, f) - - -def validate_icon_url_fast(icon_url, max_length=None): - """ - Fast validation for icon URLs during parsing. - Returns None if URL is too long, original URL otherwise. - If max_length is None, gets it dynamically from the EPGData model field. - """ - if max_length is None: - # Get max_length dynamically from the model field - max_length = EPGData._meta.get_field('icon_url').max_length - - if icon_url and len(icon_url) > max_length: - logger.warning(f"Icon URL too long ({len(icon_url)} > {max_length}), skipping: {icon_url[:100]}...") - return None - return icon_url - - -MAX_EXTRACT_CHUNK_SIZE = 65536 # 64kb (base2) - - -def send_epg_update(source_id, action, progress, **kwargs): - """Send WebSocket update about EPG download/parsing progress""" - # Start with the base data dictionary - data = { - "progress": progress, - "type": "epg_refresh", - "source": source_id, - "action": action, - } - - # Add the additional key-value pairs from kwargs - data.update(kwargs) - - # Use the standardized update function with garbage collection for program parsing - # This is a high-frequency operation that needs more aggressive memory management - collect_garbage = action == "parsing_programs" and progress % 10 == 0 - send_websocket_update('updates', 'update', data, collect_garbage=collect_garbage) - - # Explicitly clear references - data = None - - # For high-frequency parsing, occasionally force additional garbage collection - # to prevent memory buildup - if action == "parsing_programs" and progress % 50 == 0: - gc.collect() - - -def _sd_send_ws_sync(source_id, action, progress, **kwargs): - """ - Sends a WebSocket progress update synchronously via Redis, bypassing gevent.spawn. - - In Celery prefork workers that inherit gevent monkey-patching from uWSGI, - gevent.spawn schedules coroutines that never execute because there is no - running gevent hub in the worker process. This function writes directly to - Redis using the channels_redis 4.x wire format, guaranteeing delivery - regardless of the execution context. - """ - try: - import msgpack - import redis as redis_lib - from django.conf import settings - - data = {"progress": progress, "type": "epg_refresh", "source": source_id, "action": action} - data.update(kwargs) - message = {"type": "update", "data": data} - - redis_url = getattr(settings, "CELERY_BROKER_URL", "redis://localhost:6379/0") - r = redis_lib.from_url(redis_url, decode_responses=False) - - prefix = "asgi" - group_name = "updates" - group_key = f"{prefix}:group:{group_name}" - group_expiry = 86400 - channel_expiry = 60 - rand_len = 12 - now = time.time() - - r.zremrangebyscore(group_key, 0, now - group_expiry) - raw = r.zrange(group_key, 0, -1) - if not raw: - return - - channels = [m.decode("utf-8") if isinstance(m, bytes) else m for m in raw] - nonlocal_map = {} - for ch in channels: - pos = ch.find("!") - nl = ch[:pos + 1] if pos >= 0 else ch - nonlocal_map.setdefault(nl, []).append(ch) - - pipe = r.pipeline(transaction=False) - for nl, chs in nonlocal_map.items(): - channel_key = prefix + nl - msg = dict(message) - msg["__asgi_channel__"] = chs - serialized = os.urandom(rand_len) + msgpack.packb(msg) - pipe.zadd(channel_key, {serialized: now}) - pipe.expire(channel_key, channel_expiry) - pipe.execute() - r.close() - except Exception as e: - logger.warning(f"SD WebSocket sync send failed: {e}") - # Fall back to standard path - send_epg_update(source_id, action, progress, **kwargs) - - -def delete_epg_refresh_task_by_id(epg_id): - """ - Delete the periodic task associated with an EPG source ID. - Can be called directly or from the post_delete signal. - Returns True if a task was found and deleted, False otherwise. - """ - try: - task = None - task_name = f"epg_source-refresh-{epg_id}" - - # Look for task by name - try: - from django_celery_beat.models import PeriodicTask, IntervalSchedule - task = PeriodicTask.objects.get(name=task_name) - logger.info(f"Found task by name: {task.id} for EPGSource {epg_id}") - except PeriodicTask.DoesNotExist: - logger.warning(f"No PeriodicTask found with name {task_name}") - return False - - # Now delete the task and its interval - if task: - # Store interval info before deleting the task - interval_id = None - if hasattr(task, 'interval') and task.interval: - interval_id = task.interval.id - - # Count how many TOTAL tasks use this interval (including this one) - tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() - logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") - - # Delete the task first - task_id = task.id - task.delete() - logger.info(f"Successfully deleted periodic task {task_id}") - - # Now check if we should delete the interval - # We only delete if it was the ONLY task using this interval - if interval_id and tasks_with_same_interval == 1: - try: - interval = IntervalSchedule.objects.get(id=interval_id) - logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") - interval.delete() - logger.info(f"Successfully deleted interval {interval_id}") - except IntervalSchedule.DoesNotExist: - logger.warning(f"Interval {interval_id} no longer exists") - elif interval_id: - logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") - - return True - return False - except Exception as e: - logger.error(f"Error deleting periodic task for EPGSource {epg_id}: {str(e)}", exc_info=True) - return False - - -@shared_task -def refresh_all_epg_data(): - logger.info("Starting refresh_epg_data task.") - # Exclude dummy EPG sources from refresh - they don't need refreshing - active_sources = EPGSource.objects.filter(is_active=True).exclude(source_type='dummy') - logger.debug(f"Found {active_sources.count()} active EPGSource(s) (excluding dummy EPGs).") - - for source in active_sources: - refresh_epg_data(source.id) - # Force garbage collection between sources - gc.collect() - - logger.info("Finished refresh_epg_data task.") - return "EPG data refreshed." - - -@shared_task(time_limit=14400) -def refresh_epg_data(source_id, force=False): - if not acquire_task_lock('refresh_epg_data', source_id): - logger.debug(f"EPG refresh for {source_id} already running") - return - - lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) - lock_renewer.start() - - source = None - try: - # Try to get the EPG source - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - # The EPG source doesn't exist, so delete the periodic task if it exists - logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") - - # Call the shared function to delete the task - if delete_epg_refresh_task_by_id(source_id): - logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") - else: - logger.info(f"No orphaned task found for EPG source {source_id}") - - # Release the lock and exit - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return f"EPG source {source_id} does not exist, task cleaned up" - - # The source exists but is not active, just skip processing - if not source.is_active: - logger.info(f"EPG source {source_id} is not active. Skipping.") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - # Skip refresh for dummy EPG sources - they don't need refreshing - if source.source_type == 'dummy': - logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - gc.collect() - return - - # Continue with the normal processing... - logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") - if source.source_type == 'xmltv': - fetch_success = fetch_xmltv(source) - if not fetch_success: - logger.error(f"Failed to fetch XMLTV for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - parse_channels_success = parse_channels_only(source) - if not parse_channels_success: - logger.error(f"Failed to parse channels for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - parse_programs_for_source(source) - - elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source, force=force) - - source.save(update_fields=['updated_at']) - # After successful EPG refresh, evaluate DVR series rules to schedule new episodes - try: - from apps.channels.tasks import evaluate_series_rules - evaluate_series_rules.delay() - except Exception: - pass - except Exception as e: - logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) - try: - if source: - source.status = 'error' - source.last_message = f"Error refreshing EPG data: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) - except Exception as inner_e: - logger.error(f"Error updating source status: {inner_e}") - finally: - # Clear references to ensure proper garbage collection - source = None - # Force garbage collection before releasing the lock - gc.collect() - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - - -def fetch_xmltv(source): - # Handle cases with local file but no URL - if not source.url and source.file_path and os.path.exists(source.file_path): - logger.info(f"Using existing local file for EPG source: {source.name} at {source.file_path}") - - # Check if the existing file is compressed and we need to extract it - if source.file_path.endswith(('.gz', '.zip')) and not source.file_path.endswith('.xml'): - try: - # Define the path for the extracted file in the cache directory - cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") - os.makedirs(cache_dir, exist_ok=True) - xml_path = os.path.join(cache_dir, f"{source.id}.xml") - - # Extract to the cache location keeping the original - extracted_path = extract_compressed_file(source.file_path, xml_path, delete_original=False) - - if extracted_path: - logger.info(f"Extracted mapped compressed file to: {extracted_path}") - # Update to use extracted_file_path instead of changing file_path - source.extracted_file_path = extracted_path - source.save(update_fields=['extracted_file_path']) - else: - logger.error(f"Failed to extract mapped compressed file. Using original file: {source.file_path}") - except Exception as e: - logger.error(f"Failed to extract existing compressed file: {e}") - # Continue with the original file if extraction fails - - # Set the status to success in the database - source.status = 'success' - source.save(update_fields=['status']) - - # Send a download complete notification - send_epg_update(source.id, "downloading", 100, status="success") - - # Return True to indicate successful fetch, processing will continue with parse_channels_only - return True - - # Handle cases where no URL is provided and no valid file path exists - if not source.url: - # Update source status for missing URL - source.status = 'error' - source.last_message = "No URL provided and no valid local file exists" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "downloading", 100, status="error", error="No URL provided and no valid local file exists") - return False - - logger.info(f"Fetching XMLTV data from source: {source.name}") - try: - # Get default user agent from settings - stream_settings = CoreSettings.get_stream_settings() - user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default - default_user_agent_id = stream_settings.get('default_user_agent') - if default_user_agent_id: - try: - user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_id)).first() - if user_agent_obj and user_agent_obj.user_agent: - user_agent = user_agent_obj.user_agent - logger.debug(f"Using default user agent: {user_agent}") - except (ValueError, Exception) as e: - logger.warning(f"Error retrieving default user agent, using fallback: {e}") - - headers = { - 'User-Agent': user_agent - } - - # Update status to fetching before starting download - source.status = 'fetching' - source.save(update_fields=['status']) - - # Send initial download notification - send_epg_update(source.id, "downloading", 0) - - # Use streaming response to track download progress - with requests.get(source.url, headers=headers, stream=True, timeout=60) as response: - # Handle 404 specifically - if response.status_code == 404: - logger.error(f"EPG URL not found (404): {source.url}") - # Update status to error in the database - source.status = 'error' - source.last_message = f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket about the EPG fetch failure - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": 404, - "message": f"EPG source '{source.name}' returned 404 error - will retry on next scheduled run" - } - } - ) - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error="URL not found (404)") - return False - - # For all other error status codes - if response.status_code >= 400: - error_message = f"HTTP error {response.status_code}" - user_message = f"EPG source '{source.name}' encountered HTTP error {response.status_code}" - - # Update status to error in the database - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": response.status_code, - "message": user_message - } - } - ) - # Update download progress - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - - response.raise_for_status() - logger.debug("XMLTV data fetched successfully.") - - # Define base paths for consistent file naming - cache_dir = os.path.join(settings.MEDIA_ROOT, "cached_epg") - os.makedirs(cache_dir, exist_ok=True) - - # Create temporary download file with .tmp extension - temp_download_path = os.path.join(cache_dir, f"{source.id}.tmp") - - # Check if we have content length for progress tracking - total_size = int(response.headers.get('content-length', 0)) - downloaded = 0 - start_time = time.time() - last_update_time = start_time - update_interval = 0.5 # Only update every 0.5 seconds - - # Download to temporary file - with open(temp_download_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=16384): - f.write(chunk) - - downloaded += len(chunk) - elapsed_time = time.time() - start_time - - # Calculate download speed in KB/s - speed = downloaded / elapsed_time / 1024 if elapsed_time > 0 else 0 - - # Calculate progress percentage - if total_size and total_size > 0: - progress = min(100, int((downloaded / total_size) * 100)) - else: - # If no content length header, estimate progress - progress = min(95, int((downloaded / (10 * 1024 * 1024)) * 100)) # Assume 10MB if unknown - - # Time remaining (in seconds) - time_remaining = (total_size - downloaded) / (speed * 1024) if speed > 0 and total_size > 0 else 0 - - # Only send updates at specified intervals to avoid flooding - current_time = time.time() - if current_time - last_update_time >= update_interval and progress > 0: - last_update_time = current_time - send_epg_update( - source.id, - "downloading", - progress, - speed=round(speed, 2), - elapsed_time=round(elapsed_time, 1), - time_remaining=round(time_remaining, 1), - downloaded=f"{downloaded / (1024 * 1024):.2f} MB" - ) - - # Explicitly delete the chunk to free memory immediately - del chunk - - # Send completion notification - send_epg_update(source.id, "downloading", 100) - - # Determine the appropriate file extension based on content detection - with open(temp_download_path, 'rb') as f: - content_sample = f.read(1024) # Just need the first 1KB to detect format - - # Use our helper function to detect the format - format_type, is_compressed, file_extension = detect_file_format( - file_path=source.url, # Original URL as a hint - content=content_sample # Actual file content for detection - ) - - logger.debug(f"File format detection results: type={format_type}, compressed={is_compressed}, extension={file_extension}") - - # Ensure consistent final paths - compressed_path = os.path.join(cache_dir, f"{source.id}{file_extension}" if is_compressed else f"{source.id}.compressed") - xml_path = os.path.join(cache_dir, f"{source.id}.xml") - - # Clean up old files before saving new ones - if os.path.exists(compressed_path): - try: - os.remove(compressed_path) - logger.debug(f"Removed old compressed file: {compressed_path}") - except OSError as e: - logger.warning(f"Failed to remove old compressed file: {e}") - - if os.path.exists(xml_path): - try: - os.remove(xml_path) - logger.debug(f"Removed old XML file: {xml_path}") - except OSError as e: - logger.warning(f"Failed to remove old XML file: {e}") - - # Rename the temp file to appropriate final path - if is_compressed: - try: - os.rename(temp_download_path, compressed_path) - logger.debug(f"Renamed temp file to compressed file: {compressed_path}") - current_file_path = compressed_path - except OSError as e: - logger.error(f"Failed to rename temp file to compressed file: {e}") - current_file_path = temp_download_path # Fall back to using temp file - else: - try: - os.rename(temp_download_path, xml_path) - logger.debug(f"Renamed temp file to XML file: {xml_path}") - current_file_path = xml_path - except OSError as e: - logger.error(f"Failed to rename temp file to XML file: {e}") - current_file_path = temp_download_path # Fall back to using temp file - - # Now extract the file if it's compressed - if is_compressed: - try: - logger.info(f"Extracting compressed file {current_file_path}") - send_epg_update(source.id, "extracting", 0, message="Extracting downloaded file") - - # Always extract to the standard XML path - set delete_original to True to clean up - extracted = extract_compressed_file(current_file_path, xml_path, delete_original=True) - - if extracted: - logger.info(f"Successfully extracted to {xml_path}, compressed file deleted") - send_epg_update(source.id, "extracting", 100, message=f"File extracted successfully, temporary file removed") - # Update to store only the extracted file path since the compressed file is now gone - source.file_path = xml_path - source.extracted_file_path = None - else: - logger.error("Extraction failed, using compressed file") - send_epg_update(source.id, "extracting", 100, status="error", message="Extraction failed, using compressed file") - # Use the compressed file - source.file_path = current_file_path - source.extracted_file_path = None - except Exception as e: - logger.error(f"Error extracting file: {str(e)}", exc_info=True) - send_epg_update(source.id, "extracting", 100, status="error", message=f"Error during extraction: {str(e)}") - # Use the compressed file if extraction fails - source.file_path = current_file_path - source.extracted_file_path = None - else: - # It's already an XML file - source.file_path = current_file_path - source.extracted_file_path = None - - # Update the source's file paths - source.save(update_fields=['file_path', 'status', 'extracted_file_path']) - - # Update status to parsing - source.status = 'parsing' - source.save(update_fields=['status']) - - logger.info(f"Cached EPG file saved to {source.file_path}") - - return True - - except requests.exceptions.HTTPError as e: - logger.error(f"HTTP Error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Get error details - status_code = e.response.status_code if hasattr(e, 'response') and e.response else 'unknown' - error_message = str(e) - - # Create a user-friendly message - user_message = f"EPG source '{source.name}' encountered HTTP error {status_code}" - - # Add specific handling for common HTTP errors - if status_code == 404: - user_message = f"EPG source '{source.name}' URL not found (404) - will retry on next scheduled run" - elif status_code == 401 or status_code == 403: - user_message = f"EPG source '{source.name}' access denied (HTTP {status_code}) - check credentials" - elif status_code == 429: - user_message = f"EPG source '{source.name}' rate limited (429) - try again later" - elif status_code >= 500: - user_message = f"EPG source '{source.name}' server error (HTTP {status_code}) - will retry later" - - # Update source status to error with the error message - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Notify users through the WebSocket about the EPG fetch failure - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": status_code, - "message": user_message, - "details": error_message - } - } - ) - - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - except requests.exceptions.ConnectionError as e: - # Handle connection errors separately - error_message = str(e) - user_message = f"Connection error: Unable to connect to EPG source '{source.name}'" - logger.error(f"Connection error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Update source status - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Send notifications - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - 'updates', - { - 'type': 'update', - 'data': { - "success": False, - "type": "epg_fetch_error", - "source_id": source.id, - "source_name": source.name, - "error_code": "connection_error", - "message": user_message - } - } - ) - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - except requests.exceptions.Timeout as e: - # Handle timeout errors specifically - error_message = str(e) - user_message = f"Timeout error: EPG source '{source.name}' took too long to respond" - logger.error(f"Timeout error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Update source status - source.status = 'error' - source.last_message = user_message - source.save(update_fields=['status', 'last_message']) - - # Send notifications - send_epg_update(source.id, "downloading", 100, status="error", error=user_message) - return False - except Exception as e: - error_message = str(e) - logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True) - - # Update source status for general exceptions too - source.status = 'error' - source.last_message = f"Error: {error_message}" - source.save(update_fields=['status', 'last_message']) - - # Ensure we update the download progress to 100 with error status - send_epg_update(source.id, "downloading", 100, status="error", error=f"Error: {error_message}") - return False - - -def extract_compressed_file(file_path, output_path=None, delete_original=False): - """ - Extracts a compressed file (.gz or .zip) to an XML file. - - Args: - file_path: Path to the compressed file - output_path: Specific path where the file should be extracted (optional) - delete_original: Whether to delete the original compressed file after successful extraction - - Returns: - Path to the extracted XML file, or None if extraction failed - """ - try: - if output_path is None: - base_path = os.path.splitext(file_path)[0] - extracted_path = f"{base_path}.xml" - else: - extracted_path = output_path - - # Make sure the output path doesn't already exist - if os.path.exists(extracted_path): - try: - os.remove(extracted_path) - logger.info(f"Removed existing extracted file: {extracted_path}") - except Exception as e: - logger.warning(f"Failed to remove existing extracted file: {e}") - # If we can't delete the existing file and no specific output was requested, - # create a unique filename instead - if output_path is None: - base_path = os.path.splitext(file_path)[0] - extracted_path = f"{base_path}_{uuid.uuid4().hex[:8]}.xml" - - # Use our detection helper to determine the file format instead of relying on extension - with open(file_path, 'rb') as f: - content_sample = f.read(4096) # Read a larger sample to ensure accurate detection - - format_type, is_compressed, _ = detect_file_format(file_path=file_path, content=content_sample) - - if format_type == 'gzip': - logger.debug(f"Extracting gzip file: {file_path}") - try: - # First check if the content is XML by reading a sample - with gzip.open(file_path, 'rb') as gz_file: - content_sample = gz_file.read(4096) # Read first 4KB for detection - detected_format, _, _ = detect_file_format(content=content_sample) - - if detected_format != 'xml': - logger.warning(f"GZIP file does not appear to contain XML content: {file_path} (detected as: {detected_format})") - # Continue anyway since GZIP only contains one file - - # Reset file pointer and extract the content - gz_file.seek(0) - with open(extracted_path, 'wb') as out_file: - while True: - chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk or len(chunk) == 0: - break - out_file.write(chunk) - except Exception as e: - logger.error(f"Error extracting GZIP file: {e}", exc_info=True) - return None - - logger.info(f"Successfully extracted gzip file to: {extracted_path}") - - # Delete original compressed file if requested - if delete_original: - try: - os.remove(file_path) - logger.info(f"Deleted original compressed file: {file_path}") - except Exception as e: - logger.warning(f"Failed to delete original compressed file {file_path}: {e}") - - return extracted_path - - elif format_type == 'zip': - logger.debug(f"Extracting zip file: {file_path}") - with zipfile.ZipFile(file_path, 'r') as zip_file: - # Find the first XML file in the ZIP archive - xml_files = [f for f in zip_file.namelist() if f.lower().endswith('.xml')] - - if not xml_files: - logger.info("No files with .xml extension found in ZIP archive, checking content of all files") - # Check content of each file to see if any are XML without proper extension - for filename in zip_file.namelist(): - if not filename.endswith('/'): # Skip directories - try: - # Read a sample of the file content - content_sample = zip_file.read(filename, 4096) # Read up to 4KB for detection - format_type, _, _ = detect_file_format(content=content_sample) - if format_type == 'xml': - logger.info(f"Found XML content in file without .xml extension: {filename}") - xml_files = [filename] - break - except Exception as e: - logger.warning(f"Error reading file {filename} from ZIP: {e}") - - if not xml_files: - logger.error("No XML file found in ZIP archive") - return None - - # Extract the first XML file - with open(extracted_path, 'wb') as out_file: - with zip_file.open(xml_files[0], "r") as xml_file: - while True: - chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk or len(chunk) == 0: - break - out_file.write(chunk) - - logger.info(f"Successfully extracted zip file to: {extracted_path}") - - # Delete original compressed file if requested - if delete_original: - try: - os.remove(file_path) - logger.info(f"Deleted original compressed file: {file_path}") - except Exception as e: - logger.warning(f"Failed to delete original compressed file {file_path}: {e}") - - return extracted_path - - else: - logger.error(f"Unsupported or unrecognized compressed file format: {file_path} (detected as: {format_type})") - return None - - except Exception as e: - logger.error(f"Error extracting {file_path}: {str(e)}", exc_info=True) - return None - - -def parse_channels_only(source): - # Use extracted file if available, otherwise use the original file path - file_path = source.extracted_file_path if source.extracted_file_path else source.file_path - if not file_path: - file_path = source.get_cache_file() - - # Send initial parsing notification - send_epg_update(source.id, "parsing_channels", 0) - - process = None - should_log_memory = False - - try: - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file does not exist at path: {file_path}") - - # Update the source's file_path to the default cache location - new_path = source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - source.file_path = new_path - source.save(update_fields=['file_path']) - - # If the source has a URL, fetch the data before continuing - if source.url: - logger.info(f"Fetching new EPG data from URL: {source.url}") - fetch_success = fetch_xmltv(source) # Store the result - - # Only proceed if fetch was successful AND file exists - if not fetch_success: - logger.error(f"Failed to fetch EPG data from URL: {source.url}") - # Update status to error - source.status = 'error' - source.last_message = f"Failed to fetch EPG data from URL" - source.save(update_fields=['status', 'last_message']) - # Send error notification - send_epg_update(source.id, "parsing_channels", 100, status="error", error="Failed to fetch EPG data") - return False - - # Verify the file was downloaded successfully - if not os.path.exists(source.file_path): - logger.error(f"Failed to fetch EPG data, file still missing at: {source.file_path}") - # Update status to error - source.status = 'error' - source.last_message = f"Failed to fetch EPG data, file missing after download" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found after download") - return False - - # Update file_path with the new location - file_path = source.file_path - else: - logger.error(f"No URL provided for EPG source {source.name}, cannot fetch new data") - # Update status to error - source.status = 'error' - source.last_message = f"No URL provided, cannot fetch EPG data" - source.save(update_fields=['updated_at']) - - # Initialize process variable for memory tracking only in debug mode - try: - process = None - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is DEBUG (10) or more verbose - # This is more future-proof than string comparisons - should_log_memory = current_log_level <= logging.DEBUG or settings.DEBUG - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_channels_only] Initial memory usage: {initial_memory:.2f} MB") - except (ImportError, NameError): - process = None - should_log_memory = False - logger.warning("psutil not available for memory tracking") - - # Replace full dictionary load with more efficient lookup set - existing_tvg_ids = set() - existing_epgs = {} - scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup - last_id = 0 - chunk_size = 5000 - - while True: - tvg_id_chunk = set(EPGData.objects.filter( - epg_source=source, - id__gt=last_id - ).order_by('id').values_list('tvg_id', flat=True)[:chunk_size]) - - if not tvg_id_chunk: - break - - existing_tvg_ids.update(tvg_id_chunk) - last_id = EPGData.objects.filter(tvg_id__in=tvg_id_chunk).order_by('-id')[0].id - # Update progress to show file read starting - send_epg_update(source.id, "parsing_channels", 10) - - # Stream parsing instead of loading entire file at once - # This can be simplified since we now always have XML files - epgs_to_create = [] - epgs_to_update = [] - total_channels = 0 - processed_channels = 0 - batch_size = 500 # Process in batches to limit memory usage - progress = 0 # Initialize progress variable here - icon_url_max_length = EPGData._meta.get_field('icon_url').max_length # Get max length for icon_url field - name_max_length = EPGData._meta.get_field('name').max_length # Get max length for name field - - # Track memory at key points - if process: - logger.debug(f"[parse_channels_only] Memory before opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - try: - # Attempt to count existing channels in the database - try: - total_channels = EPGData.objects.filter(epg_source=source).count() - logger.info(f"Found {total_channels} existing channels for this source") - except Exception as e: - logger.error(f"Error counting channels: {e}") - total_channels = 500 # Default estimate - if process: - logger.debug(f"[parse_channels_only] Memory after closing initial file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Update progress after counting - send_epg_update(source.id, "parsing_channels", 25, total_channels=total_channels) - - # Open the file - no need to check file type since it's always XML now - logger.debug(f"Opening file for channel parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - if process: - logger.debug(f"[parse_channels_only] Memory after opening file: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Change iterparse to look for both channel and programme elements - logger.debug(f"Creating iterparse context for channels and programmes") - channel_parser = etree.iterparse(source_file, events=('end',), tag=('channel', 'programme'), remove_blank_text=True, recover=True) - if process: - logger.debug(f"[parse_channels_only] Memory after creating iterparse: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - channel_count = 0 - total_elements_processed = 0 # Track total elements processed, not just channels - for _, elem in channel_parser: - total_elements_processed += 1 - # Only process channel elements - if elem.tag == 'channel': - channel_count += 1 - tvg_id = elem.get('id', '').strip() - if tvg_id: - scanned_tvg_ids.add(tvg_id) - display_name = None - icon_url = None - for child in elem: - if display_name is None and child.tag == 'display-name' and child.text: - display_name = child.text.strip() - elif child.tag == 'icon': - raw_icon_url = child.get('src', '').strip() - icon_url = validate_icon_url_fast(raw_icon_url, icon_url_max_length) - if display_name and icon_url: - break # No need to continue if we have both - - if not display_name: - display_name = tvg_id - - if display_name and len(display_name) > name_max_length: - logger.warning(f"EPG display name too long ({len(display_name)} > {name_max_length}), truncating: {display_name[:80]}...") - display_name = display_name[:name_max_length] - - # Use lazy loading approach to reduce memory usage - if tvg_id in existing_tvg_ids: - # Only fetch the object if we need to update it and it hasn't been loaded yet - if tvg_id not in existing_epgs: - try: - # This loads the full EPG object from the database and caches it - existing_epgs[tvg_id] = EPGData.objects.get(tvg_id=tvg_id, epg_source=source) - except EPGData.DoesNotExist: - # Handle race condition where record was deleted - existing_tvg_ids.remove(tvg_id) - epgs_to_create.append(EPGData( - tvg_id=tvg_id, - name=display_name, - icon_url=icon_url, - epg_source=source, - )) - logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 1: {tvg_id} - {display_name}") - processed_channels += 1 - continue - - # We use the cached object to check if the name or icon_url has changed - epg_obj = existing_epgs[tvg_id] - needs_update = False - if epg_obj.name != display_name: - epg_obj.name = display_name - needs_update = True - if epg_obj.icon_url != icon_url: - epg_obj.icon_url = icon_url - needs_update = True - - if needs_update: - epgs_to_update.append(epg_obj) - logger.debug(f"[parse_channels_only] Added channel to update to epgs_to_update: {tvg_id} - {display_name}") - else: - # No changes needed, just clear the element - logger.debug(f"[parse_channels_only] No changes needed for channel {tvg_id} - {display_name}") - else: - # This is a new channel that doesn't exist in our database - epgs_to_create.append(EPGData( - tvg_id=tvg_id, - name=display_name, - icon_url=icon_url, - epg_source=source, - )) - logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 2: {tvg_id} - {display_name}") - - processed_channels += 1 - - # Batch processing - if len(epgs_to_create) >= batch_size: - logger.info(f"[parse_channels_only] Bulk creating {len(epgs_to_create)} EPG entries") - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - if process: - logger.info(f"[parse_channels_only] Memory after bulk_create: {process.memory_info().rss / 1024 / 1024:.2f} MB") - del epgs_to_create # Explicit deletion - epgs_to_create = [] - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - logger.info(f"[parse_channels_only] Memory after gc.collect(): {process.memory_info().rss / 1024 / 1024:.2f} MB") - - if len(epgs_to_update) >= batch_size: - logger.info(f"[parse_channels_only] Bulk updating {len(epgs_to_update)} EPG entries") - if process: - logger.info(f"[parse_channels_only] Memory before bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") - EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) - if process: - logger.info(f"[parse_channels_only] Memory after bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") - epgs_to_update = [] - # Force garbage collection - cleanup_memory(log_usage=should_log_memory, force_collection=True) - - # Periodically clear the existing_epgs cache to prevent memory buildup - if processed_channels % 1000 == 0: - logger.info(f"[parse_channels_only] Clearing existing_epgs cache at {processed_channels} channels") - existing_epgs.clear() - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - logger.info(f"[parse_channels_only] Memory after clearing cache: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Send progress updates - if processed_channels % 100 == 0 or processed_channels == total_channels: - progress = 25 + int((processed_channels / total_channels) * 65) if total_channels > 0 else 90 - send_epg_update( - source.id, - "parsing_channels", - progress, - processed=processed_channels, - total=total_channels - ) - if processed_channels > total_channels: - logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels - total_channels} additional channels") - else: - logger.debug(f"[parse_channels_only] Processed channel {tvg_id} - processed {processed_channels}/{total_channels}") - if process: - logger.debug(f"[parse_channels_only] Memory before elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - # Clear memory - try: - # First clear the element's content - clear_element(elem) - - except Exception as e: - # Just log the error and continue - don't let cleanup errors stop processing - logger.debug(f"[parse_channels_only] Non-critical error during XML element cleanup: {e}") - if process: - logger.debug(f"[parse_channels_only] Memory after elem cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - logger.debug(f"[parse_channels_only] Total elements processed: {total_elements_processed}") - - else: - logger.trace(f"[parse_channels_only] Skipping non-channel element: {elem.get('channel', 'unknown')} - {elem.get('start', 'unknown')} {elem.tag}") - clear_element(elem) - continue - - except (etree.XMLSyntaxError, Exception) as xml_error: - logger.error(f"[parse_channels_only] XML parsing failed: {xml_error}") - # Update status to error - source.status = 'error' - source.last_message = f"Error parsing XML file: {str(xml_error)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(xml_error)) - return False - if process: - logger.info(f"[parse_channels_only] Processed {processed_channels} channels current memory: {process.memory_info().rss / 1024 / 1024:.2f} MB") - else: - logger.info(f"[parse_channels_only] Processed {processed_channels} channels") - # Process any remaining items - if epgs_to_create: - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - logger.debug(f"[parse_channels_only] Created final batch of {len(epgs_to_create)} EPG entries") - - if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) - logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") - - # Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel. - # Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list. - potentially_stale = existing_tvg_ids - scanned_tvg_ids - if potentially_stale: - stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True) - deleted_count, _ = stale_qs.delete() - if deleted_count: - logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel") - - if process: - logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") - - # Update source status with channel count - source.status = 'success' - source.last_message = f"Successfully parsed {processed_channels} channels" - source.save(update_fields=['status', 'last_message']) - - # Send completion notification - send_epg_update( - source.id, - "parsing_channels", - 100, - status="success", - channels_count=processed_channels - ) - - send_websocket_update('updates', 'update', {"success": True, "type": "epg_channels"}) - - logger.info(f"Finished parsing channel info. Found {processed_channels} channels.") - - return True - - except FileNotFoundError: - logger.error(f"EPG file not found at: {file_path}") - # Update status to error - source.status = 'error' - source.last_message = f"EPG file not found: {file_path}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error="File not found") - return False - except Exception as e: - logger.error(f"Error reading EPG file {file_path}: {e}", exc_info=True) - # Update status to error - source.status = 'error' - source.last_message = f"Error parsing EPG file: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source.id, "parsing_channels", 100, status="error", error=str(e)) - return False - finally: - # Cleanup memory and close file - if process: - logger.debug(f"[parse_channels_only] Memory before cleanup: {process.memory_info().rss / 1024 / 1024:.2f} MB") - try: - # Output any errors in the channel_parser error log - if 'channel_parser' in locals() and hasattr(channel_parser, 'error_log') and len(channel_parser.error_log) > 0: - logger.debug(f"XML parser errors found ({len(channel_parser.error_log)} total):") - for i, error in enumerate(channel_parser.error_log): - logger.debug(f" Error {i+1}: {error}") - if 'channel_parser' in locals(): - del channel_parser - if 'elem' in locals(): - del elem - if 'parent' in locals(): - del parent - - if 'source_file' in locals(): - source_file.close() - del source_file - # Clear remaining large data structures - existing_epgs.clear() - epgs_to_create.clear() - epgs_to_update.clear() - existing_epgs = None - epgs_to_create = None - epgs_to_update = None - if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None: - scanned_tvg_ids.clear() - scanned_tvg_ids = None - cleanup_memory(log_usage=should_log_memory, force_collection=True) - except Exception as e: - logger.warning(f"Cleanup error: {e}") - - try: - if process: - final_memory = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_channels_only] Final memory usage: {final_memory:.2f} MB") - process = None - except: - pass - - - -@shared_task(time_limit=3600, soft_time_limit=3500) -def parse_programs_for_tvg_id(epg_id): - # Skip XMLTV file parsing for Schedules Direct sources. Program data is - # fetched and persisted directly by fetch_schedules_direct(). - try: - from apps.epg.models import EPGData - epg_obj = EPGData.objects.select_related('epg_source').filter(id=epg_id).first() - if epg_obj and epg_obj.epg_source and epg_obj.epg_source.source_type == 'schedules_direct': - logger.info(f"Skipping XMLTV parse for SD EPGData id={epg_id} (source: {epg_obj.epg_source.name})") - return "Skipped (Schedules Direct source)" - except Exception as e: - logger.warning(f"Could not check EPG source type for id={epg_id}: {e}") - - if not acquire_task_lock('parse_epg_programs', epg_id): - logger.info(f"Program parse for {epg_id} already in progress, skipping duplicate task") - return "Task already running" - - lock_renewer = TaskLockRenewer('parse_epg_programs', epg_id) - lock_renewer.start() - - source_file = None - program_parser = None - programs_to_create = [] - programs_processed = 0 - try: - # Add memory tracking only in trace mode or higher - try: - process = None - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is TRACE or more verbose or if running in DEBUG mode - should_log_memory = current_log_level <= 5 or settings.DEBUG - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Initial memory usage: {initial_memory:.2f} MB") - mem_before = initial_memory - except ImportError: - process = None - should_log_memory = False - - epg = EPGData.objects.get(id=epg_id) - epg_source = epg.epg_source - - # Skip program parsing for dummy EPG sources - they don't have program data files - if epg_source.source_type == 'dummy': - logger.info(f"Skipping program parsing for dummy EPG source {epg_source.name} (ID: {epg_id})") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - if not Channel.objects.filter(epg_data=epg).exists(): - logger.info(f"No channels matched to EPG {epg.tvg_id}") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") - - # Optimize deletion with a single delete query instead of chunking - # This is faster for most database engines - ProgramData.objects.filter(epg=epg).delete() - - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - if not file_path: - file_path = epg_source.get_cache_file() - - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file not found at: {file_path}") - - if epg_source.url: - # Update the file path in the database - new_path = epg_source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - epg_source.file_path = new_path - epg_source.save(update_fields=['file_path']) - logger.info(f"Fetching new EPG data from URL: {epg_source.url}") - else: - logger.info(f"EPG source does not have a URL, using existing file path: {file_path} to rebuild cache") - - # Fetch new data before continuing - if epg_source: - - # Properly check the return value from fetch_xmltv - fetch_success = fetch_xmltv(epg_source) - - # If fetch was not successful or the file still doesn't exist, abort - if not fetch_success: - logger.error(f"Failed to fetch EPG data, cannot parse programs for tvg_id: {epg.tvg_id}") - # Update status to error if not already set - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data, cannot parse programs" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Also check if the file exists after download - if not os.path.exists(epg_source.file_path): - logger.error(f"Failed to fetch EPG data, file still missing at: {epg_source.file_path}") - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data, file missing after download" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="File not found after download") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Update file_path with the new location - if epg_source.extracted_file_path: - file_path = epg_source.extracted_file_path - else: - file_path = epg_source.file_path - else: - logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") - # Update status to error - epg_source.status = 'error' - epg_source.last_message = f"No URL provided, cannot fetch EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - return - - # Use streaming parsing to reduce memory usage - # No need to check file type anymore since it's always XML - logger.debug(f"Parsing programs for tvg_id={epg.tvg_id} from {file_path}") - - # Memory usage tracking - if process: - try: - mem_before = process.memory_info().rss / 1024 / 1024 - logger.debug(f"[parse_programs_for_tvg_id] Memory before parsing {epg.tvg_id} - {mem_before:.2f} MB") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - mem_before = 0 - - programs_to_create = [] - batch_size = 1000 # Process in batches to limit memory usage - - try: - # Open the file directly - no need to check compression - logger.debug(f"Opening file for parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - # Stream parse the file using lxml's iterparse - program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) - - for _, elem in program_parser: - if elem.get('channel') == epg.tvg_id: - try: - start_time = parse_xmltv_time(elem.get('start')) - end_time = parse_xmltv_time(elem.get('stop')) - title = None - desc = None - sub_title = None - - # Efficiently process child elements - for child in elem: - if child.tag == 'title': - title = child.text or 'No Title' - elif child.tag == 'desc': - desc = child.text or '' - elif child.tag == 'sub-title': - sub_title = child.text or '' - - if not title: - title = 'No Title' - - # Extract custom properties - custom_props = extract_custom_properties(elem) - custom_properties_json = None - - if custom_props: - logger.trace(f"Number of custom properties: {len(custom_props)}") - custom_properties_json = custom_props - - # Fallback: extract S/E from description when episode-num - # elements didn't provide them - if desc: - has_season = (custom_properties_json or {}).get('season') is not None - has_episode = (custom_properties_json or {}).get('episode') is not None - if not has_season or not has_episode: - d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) - if d_season is not None and d_episode is not None: - if custom_properties_json is None: - custom_properties_json = {} - if not has_season: - custom_properties_json['season'] = d_season - if not has_episode: - custom_properties_json['episode'] = d_episode - custom_properties_json['season_episode_source'] = 'description' - desc = cleaned_desc - - programs_to_create.append(ProgramData( - epg=epg, - start_time=start_time, - end_time=end_time, - title=title[:255], - description=desc, - sub_title=sub_title, - tvg_id=epg.tvg_id, - custom_properties=custom_properties_json - )) - programs_processed += 1 - # Clear the element to free memory - clear_element(elem) - # Batch processing - if len(programs_to_create) >= batch_size: - ProgramData.objects.bulk_create(programs_to_create) - logger.debug(f"Saved batch of {len(programs_to_create)} programs for {epg.tvg_id}") - programs_to_create = [] - # Only call gc.collect() every few batches - if programs_processed % (batch_size * 5) == 0: - gc.collect() - - except Exception as e: - logger.error(f"Error processing program for {epg.tvg_id}: {e}", exc_info=True) - else: - # Immediately clean up non-matching elements to reduce memory pressure - if elem is not None: - clear_element(elem) - continue - - # Make sure to close the file and release parser resources - if source_file: - source_file.close() - source_file = None - - if program_parser: - program_parser = None - - gc.collect() - - except zipfile.BadZipFile as zip_error: - logger.error(f"Bad ZIP file: {zip_error}") - raise - except etree.XMLSyntaxError as xml_error: - logger.error(f"XML syntax error parsing program data: {xml_error}") - raise - except Exception as e: - logger.error(f"Error parsing XML for programs: {e}", exc_info=True) - raise - finally: - # Ensure file is closed even if an exception occurs - if source_file: - source_file.close() - source_file = None - # Memory tracking after processing - if process: - try: - mem_after = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Memory after parsing 1 {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - - # Process any remaining items - if programs_to_create: - ProgramData.objects.bulk_create(programs_to_create) - logger.debug(f"Saved final batch of {len(programs_to_create)} programs for {epg.tvg_id}") - programs_to_create = None - custom_props = None - custom_properties_json = None - - - logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.") - finally: - # Reset internal caches and pools that lxml might be keeping - try: - etree.clear_error_log() - except: - pass - # Explicit cleanup of all potentially large objects - if source_file: - try: - source_file.close() - except: - pass - source_file = None - program_parser = None - programs_to_create = None - - epg_source = None - # Add comprehensive cleanup before releasing lock - cleanup_memory(log_usage=should_log_memory, force_collection=True) - # Memory tracking after processing - if process: - try: - mem_after = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_tvg_id] Final memory usage {epg.tvg_id} - {programs_processed} programs: {mem_after:.2f} MB (change: {mem_after-mem_before:.2f} MB)") - except Exception as e: - logger.warning(f"Error tracking memory: {e}") - process = None - epg = None - programs_processed = None - lock_renewer.stop() - release_task_lock('parse_epg_programs', epg_id) - - - -def parse_programs_for_source(epg_source, tvg_id=None): - """ - Parse programs for all MAPPED channels from an EPG source in a single pass. - - This is an optimized version that: - 1. Only processes EPG entries that are actually mapped to channels - 2. Parses the XML file ONCE instead of once per channel - 3. Skips programmes for unmapped channels entirely during parsing - - This dramatically improves performance when an EPG source has many channels - but only a fraction are mapped. - """ - # Send initial programs parsing notification - send_epg_update(epg_source.id, "parsing_programs", 0) - should_log_memory = False - process = None - initial_memory = 0 - source_file = None - - # Add memory tracking only in trace mode or higher - try: - # Get current log level as a number - current_log_level = logger.getEffectiveLevel() - - # Only track memory usage when log level is TRACE or more verbose - should_log_memory = current_log_level <= 5 or settings.DEBUG # Assuming TRACE is level 5 or lower - - if should_log_memory: - process = psutil.Process() - initial_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_source] Initial memory usage: {initial_memory:.2f} MB") - except ImportError: - logger.warning("psutil not available for memory tracking") - process = None - should_log_memory = False - - try: - # Only get EPG entries that are actually mapped to channels - mapped_epg_ids = set( - Channel.objects.filter( - epg_data__epg_source=epg_source, - epg_data__isnull=False - ).values_list('epg_data_id', flat=True) - ) - - if not mapped_epg_ids: - total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() - logger.info(f"No channels mapped to any EPG entries from source: {epg_source.name} " - f"(source has {total_epg_count} EPG entries, 0 mapped)") - # Update status - this is not an error, just no mapped entries - epg_source.status = 'success' - epg_source.last_message = f"No channels mapped to this EPG source ({total_epg_count} entries available)" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="success") - return True - - # Get the mapped EPG entries with their tvg_ids - mapped_epgs = EPGData.objects.filter(id__in=mapped_epg_ids).values('id', 'tvg_id') - tvg_id_to_epg_id = {epg['tvg_id']: epg['id'] for epg in mapped_epgs if epg['tvg_id']} - mapped_tvg_ids = set(tvg_id_to_epg_id.keys()) - - total_epg_count = EPGData.objects.filter(epg_source=epg_source).count() - mapped_count = len(mapped_tvg_ids) - - logger.info(f"Parsing programs for {mapped_count} MAPPED channels from source: {epg_source.name} " - f"(skipping {total_epg_count - mapped_count} unmapped EPG entries)") - - # Get the file path - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - if not file_path: - file_path = epg_source.get_cache_file() - - # Check if the file exists - if not os.path.exists(file_path): - logger.error(f"EPG file not found at: {file_path}") - - if epg_source.url: - # Update the file path in the database - new_path = epg_source.get_cache_file() - logger.info(f"Updating file_path from '{file_path}' to '{new_path}'") - epg_source.file_path = new_path - epg_source.save(update_fields=['file_path']) - logger.info(f"Fetching new EPG data from URL: {epg_source.url}") - - # Fetch new data before continuing - fetch_success = fetch_xmltv(epg_source) - - if not fetch_success: - logger.error(f"Failed to fetch EPG data for source: {epg_source.name}") - epg_source.status = 'error' - epg_source.last_message = f"Failed to download EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="Failed to download EPG file") - return False - - # Update file_path with the new location - file_path = epg_source.extracted_file_path if epg_source.extracted_file_path else epg_source.file_path - else: - logger.error(f"No URL provided for EPG source {epg_source.name}, cannot fetch new data") - epg_source.status = 'error' - epg_source.last_message = f"No URL provided, cannot fetch EPG data" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", error="No URL provided") - return False - - # SINGLE PASS PARSING: Parse the XML file once and collect all programs in memory - # We parse FIRST, then do an atomic delete+insert to avoid race conditions - # where clients might see empty/partial EPG data during the transition - all_programs_to_create = [] - programs_by_channel = {tvg_id: 0 for tvg_id in mapped_tvg_ids} # Track count per channel - total_programs = 0 - skipped_programs = 0 - last_progress_update = 0 - - try: - logger.debug(f"Opening file for single-pass parsing: {file_path}") - source_file = _open_xmltv_file(file_path) - - # Stream parse the file using lxml's iterparse - program_parser = etree.iterparse(source_file, events=('end',), tag='programme', remove_blank_text=True, recover=True) - - for _, elem in program_parser: - channel_id = elem.get('channel') - - # Skip programmes for unmapped channels immediately - if channel_id not in mapped_tvg_ids: - skipped_programs += 1 - # Clear element to free memory - clear_element(elem) - continue - - # This programme is for a mapped channel - process it - try: - start_time = parse_xmltv_time(elem.get('start')) - end_time = parse_xmltv_time(elem.get('stop')) - title = None - desc = None - sub_title = None - - # Efficiently process child elements - for child in elem: - if child.tag == 'title': - title = child.text or 'No Title' - elif child.tag == 'desc': - desc = child.text or '' - elif child.tag == 'sub-title': - sub_title = child.text or '' - - if not title: - title = 'No Title' - - # Extract custom properties - custom_props = extract_custom_properties(elem) - custom_properties_json = custom_props if custom_props else None - - # Fallback: extract S/E from description when episode-num - # elements didn't provide them - if desc: - has_season = (custom_properties_json or {}).get('season') is not None - has_episode = (custom_properties_json or {}).get('episode') is not None - if not has_season or not has_episode: - d_season, d_episode, cleaned_desc = extract_season_episode_from_description(desc) - if d_season is not None and d_episode is not None: - if custom_properties_json is None: - custom_properties_json = {} - if not has_season: - custom_properties_json['season'] = d_season - if not has_episode: - custom_properties_json['episode'] = d_episode - custom_properties_json['season_episode_source'] = 'description' - desc = cleaned_desc - - epg_id = tvg_id_to_epg_id[channel_id] - all_programs_to_create.append(ProgramData( - epg_id=epg_id, - start_time=start_time, - end_time=end_time, - title=title[:255], - description=desc, - sub_title=sub_title, - tvg_id=channel_id, - custom_properties=custom_properties_json - )) - total_programs += 1 - programs_by_channel[channel_id] += 1 - - # Clear the element to free memory - clear_element(elem) - - # Send progress update (estimate based on programs processed) - if total_programs - last_progress_update >= 5000: - last_progress_update = total_programs - # Cap at 70% during parsing phase (save 30% for DB operations) - progress = min(70, 10 + int((total_programs / max(total_programs + 10000, 1)) * 60)) - send_epg_update(epg_source.id, "parsing_programs", progress, - processed=total_programs, channels=mapped_count) - - # Periodic garbage collection during parsing - if total_programs % 5000 == 0: - gc.collect() - - except Exception as e: - logger.error(f"Error processing program for {channel_id}: {e}", exc_info=True) - clear_element(elem) - continue - - except etree.XMLSyntaxError as xml_error: - logger.error(f"XML syntax error parsing program data: {xml_error}") - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"XML parsing error: {str(xml_error)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(xml_error)) - return False - except Exception as e: - logger.error(f"Error parsing XML for programs: {e}", exc_info=True) - raise - finally: - if source_file: - source_file.close() - source_file = None - - # Now perform atomic delete + bulk insert - # This ensures clients never see empty/partial EPG data - logger.info(f"Parsed {total_programs} programs, performing atomic database update...") - send_epg_update(epg_source.id, "parsing_programs", 75, message="Updating database...") - - batch_size = 1000 - try: - with transaction.atomic(): - # Kill any individual statement that hangs longer than 10 minutes. - # SET LOCAL automatically resets when this transaction ends (commit or rollback). - with connection.cursor() as cursor: - cursor.execute("SET LOCAL statement_timeout = '10min'") - # Delete existing programs for mapped EPGs - deleted_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids).delete()[0] - logger.debug(f"Deleted {deleted_count} existing programs") - - # Clean up orphaned programs for unmapped EPG entries - unmapped_epg_ids = list(EPGData.objects.filter( - epg_source=epg_source - ).exclude(id__in=mapped_epg_ids).values_list('id', flat=True)) - - if unmapped_epg_ids: - orphaned_count = ProgramData.objects.filter(epg_id__in=unmapped_epg_ids).delete()[0] - if orphaned_count > 0: - logger.info(f"Cleaned up {orphaned_count} orphaned programs for {len(unmapped_epg_ids)} unmapped EPG entries") - - # Bulk insert all new programs in batches within the same transaction - for i in range(0, len(all_programs_to_create), batch_size): - batch = all_programs_to_create[i:i + batch_size] - ProgramData.objects.bulk_create(batch) - - # Update progress during insertion - progress = 75 + int((i / len(all_programs_to_create)) * 20) if all_programs_to_create else 95 - if i % (batch_size * 5) == 0: - send_epg_update(epg_source.id, "parsing_programs", min(95, progress), - message=f"Inserting programs... {i}/{len(all_programs_to_create)}") - - logger.info(f"Atomic update complete: deleted {deleted_count}, inserted {total_programs} programs") - - except Exception as db_error: - logger.error(f"Database error during atomic update: {db_error}", exc_info=True) - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"Database error: {str(db_error)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, status="error", message=str(db_error)) - return False - finally: - # Clear the large list to free memory - all_programs_to_create = None - gc.collect() - - # Count channels that actually got programs - channels_with_programs = sum(1 for count in programs_by_channel.values() if count > 0) - - # Success message - epg_source.status = EPGSource.STATUS_SUCCESS - epg_source.last_message = ( - f"Parsed {total_programs:,} programs for {channels_with_programs} channels " - f"(skipped {skipped_programs:,} programs for {total_epg_count - mapped_count} unmapped channels)" - ) - epg_source.updated_at = timezone.now() - epg_source.save(update_fields=['status', 'last_message', 'updated_at']) - - # Log system event for EPG refresh - log_system_event( - event_type='epg_refresh', - source_name=epg_source.name, - programs=total_programs, - channels=channels_with_programs, - skipped_programs=skipped_programs, - unmapped_channels=total_epg_count - mapped_count, - ) - - # Send completion notification with status - send_epg_update(epg_source.id, "parsing_programs", 100, - status="success", - message=epg_source.last_message, - updated_at=epg_source.updated_at.isoformat()) - - logger.info(f"Completed parsing programs for source: {epg_source.name} - " - f"{total_programs:,} programs for {channels_with_programs} channels, " - f"skipped {skipped_programs:,} programs for unmapped channels") - return True - - except Exception as e: - logger.error(f"Error in parse_programs_for_source: {e}", exc_info=True) - # Update status to error - epg_source.status = EPGSource.STATUS_ERROR - epg_source.last_message = f"Error parsing programs: {str(e)}" - epg_source.save(update_fields=['status', 'last_message']) - send_epg_update(epg_source.id, "parsing_programs", 100, - status="error", - message=epg_source.last_message) - return False - finally: - # Final memory cleanup and tracking - if source_file: - try: - source_file.close() - except: - pass - source_file = None - - # Explicitly release any remaining large data structures - programs_to_create = None - programs_by_channel = None - mapped_epg_ids = None - mapped_tvg_ids = None - tvg_id_to_epg_id = None - gc.collect() - - # Add comprehensive memory cleanup at the end - cleanup_memory(log_usage=should_log_memory, force_collection=True) - if process: - final_memory = process.memory_info().rss / 1024 / 1024 - logger.info(f"[parse_programs_for_source] Final memory usage: {final_memory:.2f} MB difference: {final_memory - initial_memory:.2f} MB") - # Explicitly clear the process object to prevent potential memory leaks - process = None -@shared_task(bind=True) -def fetch_schedules_direct_stations(self, source_id): - """ - Lightweight Celery task that runs a stations-only Schedules Direct fetch. - Called on initial source creation so EPGData entries exist for auto-matching - before the user commits to a full schedule/program fetch. - """ - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - logger.error(f"EPGSource {source_id} not found for SD stations fetch") - return - fetch_schedules_direct(source, stations_only=True) - - -def fetch_schedules_direct(source, stations_only=False, force=False): - """ - Fetch EPG data from the Schedules Direct JSON API and persist it to the - EPGData / ProgramData models. - - Authentication flow (as required by the SD API specification): - 1. POST credentials to the token endpoint (password must be SHA1-hashed - as required by the Schedules Direct API specification. - 2. Use the returned token for all subsequent requests via the 'token' header. - 3. Tokens are valid for 24 hours; SD returns the current valid token if one - already exists for the account. - - Data flow: - 1. Fetch subscribed lineups for the account. - 2. Fetch station metadata for each lineup. - 3. Persist station metadata to EPGData. - 4. If stations_only=True, stop here. Used on initial source creation so - the user can run Auto-match EPG before the full program fetch. - 5. Fetch schedule grids in 14-day date-batched requests per station. - 6. Fetch program metadata in batched requests (up to 5000 programIDs per request). - 7. Persist channels to EPGData and programs to ProgramData. - - Args: - source: EPGSource instance - stations_only: If True, only fetch and persist station metadata (no schedules/programs). - Used on initial source creation to populate EPGData for auto-matching - before channels are assigned. - """ - import hashlib - from datetime import date - - - logger.info(f"Fetching Schedules Direct data for source: {source.name}") - - # ------------------------------------------------------------------------- - # Validate credentials - # ------------------------------------------------------------------------- - username = (source.username or '').strip() - password = (source.password or '').strip() - - if not username or not password: - msg = "Schedules Direct source requires both a username and password." - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Enforce 2-hour minimum interval between full fetches (not stations-only). - # Schedules Direct enforces rate limits of ~200 requests per 2-hour window. - # This prevents automated abuse regardless of how the refresh was triggered. - # - # Exception: if no SDScheduleMD5 records exist yet, this is the first full - # refresh after initial source creation (stations-only runs first and updates - # updated_at, which would otherwise incorrectly trigger this guard). Always - # allow the first full refresh through so guide data is immediately available. - # ------------------------------------------------------------------------- - if not stations_only and not force and source.updated_at: - from apps.epg.models import SDScheduleMD5 as _SDScheduleMD5 - has_prior_full_refresh = _SDScheduleMD5.objects.filter(epg_source=source).exists() - if has_prior_full_refresh: - elapsed = (timezone.now() - source.updated_at).total_seconds() - min_interval_seconds = 2 * 3600 # 2 hours - if elapsed < min_interval_seconds: - remaining_minutes = int((min_interval_seconds - elapsed) / 60) - msg = ( - f"Schedules Direct refresh skipped. Minimum 2-hour interval not reached. " - f"Last refreshed {int(elapsed / 60)} minutes ago. " - f"Please wait {remaining_minutes} more minute(s)." - ) - logger.warning(f"SD source {source.id}: {msg}") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - else: - logger.info(f"SD source {source.id}: No prior full refresh detected, skipping 2-hour guard for first full fetch.") - elif force and not stations_only: - logger.info(f"SD source {source.id}: Force flag set, bypassing 2-hour refresh guard.") - - # ------------------------------------------------------------------------- - # Build SD-specific headers - # SD API spec requires the User-Agent to identify the application and version. - # SergeantPanda confirmed Dispatcharr should identify itself properly. - # ------------------------------------------------------------------------- - from version import __version__ as dispatcharr_version - sd_user_agent = f"Dispatcharr/{dispatcharr_version}" - - def _sd_headers(token=None): - h = { - 'Content-Type': 'application/json', - 'User-Agent': sd_user_agent, - } - if token: - h['token'] = token - return h - - # ------------------------------------------------------------------------- - # Step 1: Authenticate and obtain session token - # The SD API requires the password to be SHA1-hashed before transmission. - # This is a requirement of the Schedules Direct API specification, not an - # architectural choice. - # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_FETCHING - source.last_message = "Authenticating with Schedules Direct..." - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 2, message="Authenticating with Schedules Direct...") - - try: - sha1_password = hashlib.sha1(password.encode('utf-8')).hexdigest() - token_response = requests.post( - f"{SD_BASE_URL}/token", - json={'username': username, 'password': sha1_password}, - headers=_sd_headers(), - timeout=30, - ) - token_response.raise_for_status() - token_data = token_response.json() - - auth_code = token_data.get('code', 0) - if auth_code != 0: - if auth_code == 4007: - msg = "Schedules Direct: this application is not authorized. Please contact the Dispatcharr maintainers." - elif auth_code == 4004: - msg = "Schedules Direct: account locked due to too many failed login attempts. Try again in 15 minutes." - elif auth_code == 4009: - msg = "Schedules Direct: too many login attempts in 24 hours. Token is valid for 24 hours. Check for misconfiguration." - elif auth_code == 4001: - msg = "Schedules Direct: account has expired. Please renew your subscription at schedulesdirect.org." - elif auth_code == 4008: - msg = "Schedules Direct: account is inactive. Please log in to schedulesdirect.org to reactivate." - else: - msg = f"Schedules Direct authentication failed (code {auth_code}): {token_data.get('message', 'Unknown error')}" - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - token = token_data.get('token') - if not token: - msg = "Schedules Direct returned no token." - logger.error(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - logger.info("Schedules Direct authentication successful.") - - except requests.exceptions.RequestException as e: - msg = f"Network error authenticating with Schedules Direct: {e}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Step 2: Check account status (respect OFFLINE system status) - # ------------------------------------------------------------------------- - try: - status_response = requests.get( - f"{SD_BASE_URL}/status", - headers=_sd_headers(token), - timeout=30, - ) - status_response.raise_for_status() - status_data = status_response.json() - system_status = status_data.get('systemStatus', [{}])[0].get('status', 'Online') - if system_status == 'Offline': - # Per SD API spec: if system is offline, disconnect and do not - # retry for 1 hour. We set idle status rather than error since - # this is a temporary SD-side condition. - msg = "Schedules Direct system is currently offline. Per SD guidelines, retrying in 1 hour." - logger.warning(msg) - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - logger.debug(f"Schedules Direct system status: {system_status}") - except requests.exceptions.RequestException as e: - logger.warning(f"Could not fetch SD system status, proceeding anyway: {e}") - - # ------------------------------------------------------------------------- - # Step 3: Fetch subscribed lineups and build station map - # ------------------------------------------------------------------------- - _sd_send_ws_sync(source.id, "parsing_programs", 10, message="Fetching subscribed lineups...") - try: - lineups_response = requests.get( - f"{SD_BASE_URL}/lineups", - headers=_sd_headers(token), - timeout=30, - ) - # SD returns 400 with code 4102 when no lineups are configured. - # This is a valid account state. The user needs to add lineups via - # the Manage Lineups UI. Treat as idle rather than error. - if lineups_response.status_code == 400: - sd_data = lineups_response.json() - if sd_data.get('code') == 4102: - msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no lineups configured on account (4102).") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - lineups_response.raise_for_status() - lineups_data = lineups_response.json() - lineups = [l for l in lineups_data.get('lineups', []) if not l.get('isDeleted', False)] - if not lineups: - msg = "No lineups configured. Use the Manage Lineups option in the EPG source settings to add a lineup." - logger.warning(f"SD source {source.id}: no active lineups found.") - source.status = EPGSource.STATUS_IDLE - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="idle", message=msg) - return - logger.info(f"Found {len(lineups)} lineup(s) in SD account.") - - # Extract country from lineup IDs (format: "USA-NJ29486-X", "GBR-...", etc.) - sd_lineup_country = None - for l in lineups: - lid = l.get('lineupID') or l.get('lineup') or '' - if '-' in lid: - sd_lineup_country = lid.split('-')[0] - break - logger.debug(f"SD lineup country: {sd_lineup_country}") - except requests.exceptions.RequestException as e: - msg = f"Failed to fetch Schedules Direct lineups: {e}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - # Build station metadata map: stationID -> {name, callsign, logo_url} - station_map = {} - _sd_send_ws_sync(source.id, "parsing_programs", 18, message=f"Fetching station metadata for {len(lineups)} lineup(s)...") - for lineup in lineups: - lineup_id = lineup.get('lineupID') or lineup.get('lineup') - if not lineup_id: - continue - try: - detail_response = requests.get( - f"{SD_BASE_URL}/lineups/{lineup_id}", - headers=_sd_headers(token), - timeout=30, - ) - detail_response.raise_for_status() - detail_data = detail_response.json() - for station in detail_data.get('stations', []): - sid = station.get('stationID') - if not sid: - continue - logo_url = None - logos = station.get('stationLogo') or station.get('logo') or [] - if isinstance(logos, list) and logos: - # Read preferred logo style from source settings; default to 'dark' - logo_style = (source.custom_properties or {}).get('logo_style', 'dark') - preferred = next((l for l in logos if l.get('category') == logo_style), logos[0]) - logo_url = preferred.get('URL') or preferred.get('url') - elif isinstance(logos, dict): - logo_url = logos.get('URL') or logos.get('url') - station_map[sid] = { - 'name': station.get('name', sid), - 'callsign': station.get('callsign', ''), - 'logo_url': logo_url, - } - logger.debug(f"Fetched {len(detail_data.get('stations', []))} stations from lineup {lineup_id}") - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch lineup details for {lineup_id}: {e}") - - if not station_map: - msg = "No stations found across all Schedules Direct lineups." - logger.warning(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "refresh", 100, status="error", error=msg) - return - - logger.info(f"Built station map with {len(station_map)} stations.") - - # ------------------------------------------------------------------------- - # Step 4: Persist station metadata to EPGData - # ------------------------------------------------------------------------- - source.status = EPGSource.STATUS_PARSING - source.last_message = f"Syncing {len(station_map)} stations..." - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 28, message=f"Syncing {len(station_map)} stations to database...") - - existing_epg_map = { - epg.tvg_id: epg - for epg in EPGData.objects.filter(epg_source=source) - } - - epgs_to_create = [] - epgs_to_update = [] - icon_max_length = EPGData._meta.get_field('icon_url').max_length - name_max_length = EPGData._meta.get_field('name').max_length - - for sid, info in station_map.items(): - display_name = (info['name'] or sid)[:name_max_length] - logo = info['logo_url'] - if logo and len(logo) > icon_max_length: - logo = None - - if sid in existing_epg_map: - epg_obj = existing_epg_map[sid] - needs_update = False - if epg_obj.name != display_name: - epg_obj.name = display_name - needs_update = True - if epg_obj.icon_url != logo: - epg_obj.icon_url = logo - needs_update = True - if needs_update: - epgs_to_update.append(epg_obj) - else: - epgs_to_create.append(EPGData( - tvg_id=sid, - name=display_name, - icon_url=logo, - epg_source=source, - )) - - if epgs_to_create: - EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) - logger.info(f"Created {len(epgs_to_create)} new EPGData entries.") - if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ['name', 'icon_url']) - logger.info(f"Updated {len(epgs_to_update)} existing EPGData entries.") - - gc.collect() - - # Rebuild map with fresh DB ids for all stations - epg_id_map = { - epg.tvg_id: epg.id - for epg in EPGData.objects.filter(epg_source=source, tvg_id__in=list(station_map.keys())) - } - - # Station sync complete. Send progress update before continuing into programs phase. - # We deliberately do NOT send parsing_channels at 100 with status=success here - # because that would cause the frontend to mark the source as complete and - # stop rendering progress updates for the subsequent program fetch phases. - _sd_send_ws_sync(source.id, "parsing_programs", 30, - message=f"Stations synced ({len(station_map)} stations). Preparing schedule fetch...") - - # ------------------------------------------------------------------------- - # Stations-only mode. Used on initial source creation. - # Stop here so the user can run Auto-match EPG before the full program fetch. - # ------------------------------------------------------------------------- - if stations_only: - success_msg = ( - f"{len(station_map)} stations loaded from Schedules Direct. " - f"Run Auto-match EPG to map your channels, then use the Refresh " - f"button to populate guide data." - ) - source.status = EPGSource.STATUS_SUCCESS - source.last_message = success_msg - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - _sd_send_ws_sync(source.id, "parsing_channels", 100, status="success", - message=success_msg, channels_count=len(station_map)) - logger.info(f"Stations-only fetch complete for source: {source.name} ({len(station_map)} stations)") - return - - # ------------------------------------------------------------------------- - # Step 5: MD5-delta schedule fetch - # First fetch MD5 hashes for all stations/dates. Compare against our - # locally cached hashes to determine which schedules have changed. - # Only download schedules that have actually changed; this minimises - # API calls against SD's rate-limited endpoints. - # ------------------------------------------------------------------------- - from apps.epg.models import SDScheduleMD5 - from django.utils.dateparse import parse_datetime - - _sd_send_ws_sync(source.id, "parsing_programs", 33, message=f"Checking schedule MD5s for {len(station_map)} stations over {SD_DAYS_TO_FETCH} days...") - station_ids = list(station_map.keys()) - today = date.today() - date_list = [(today + timedelta(days=i)).strftime('%Y-%m-%d') for i in range(SD_DAYS_TO_FETCH)] - - # Prune SDScheduleMD5 records whose dates have rolled off the fetch window. - # These accumulate one row per station per day and are never useful once past. - pruned_sched_md5_count = SDScheduleMD5.objects.filter(epg_source=source, date__lt=today).delete()[0] - if pruned_sched_md5_count: - logger.info(f"Pruned {pruned_sched_md5_count} expired SDScheduleMD5 records (before {today}).") - - # Fetch MD5 hashes for all stations in batches of 5000 - STATION_BATCH_SIZE = 5000 - server_md5s = {} # (station_id, date) -> {md5, last_modified} - - logger.info(f"Fetching schedule MD5s for {len(station_ids)} stations over {SD_DAYS_TO_FETCH} days.") - - station_batches = [station_ids[i:i + STATION_BATCH_SIZE] for i in range(0, len(station_ids), STATION_BATCH_SIZE)] - for batch in station_batches: - try: - md5_response = requests.post( - f"{SD_BASE_URL}/schedules/md5", - json=[{'stationID': sid, 'date': date_list} for sid in batch], - headers=_sd_headers(token), - timeout=120, - ) - md5_response.raise_for_status() - md5_data = md5_response.json() - for sid, dates in md5_data.items(): - for date_str, info in dates.items(): - if info.get('code', 0) == 0: - server_md5s[(sid, date_str)] = { - 'md5': info.get('md5', ''), - 'last_modified': info.get('lastModified', ''), - } - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch schedule MD5s: {e}") - - # Load our cached MD5s from DB - cached_md5s = { - (r.station_id, r.date.strftime('%Y-%m-%d')): r.md5 - for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=station_ids) - } - - # Determine which station/date combinations need downloading - changed_by_station = {} # station_id -> [date_str, ...] - for (sid, date_str), server_info in server_md5s.items(): - if date_str not in date_list: - continue - cached = cached_md5s.get((sid, date_str)) - if cached != server_info['md5']: - changed_by_station.setdefault(sid, []).append(date_str) - - total_changed = sum(len(v) for v in changed_by_station.values()) - total_possible = len(station_ids) * len(date_list) - logger.info(f"Schedule MD5 check: {len(server_md5s)} hashes checked, {total_changed} station/date combinations changed (of {total_possible} possible).") - _sd_send_ws_sync(source.id, "parsing_programs", 38, - message=f"MD5 check complete: {len(changed_by_station)} stations have schedule updates.") - - # schedules_by_station: stationID -> list of {programID, airDateTime, duration, ...} - schedules_by_station = {sid: [] for sid in station_ids} - program_ids_needed = set() - - if not changed_by_station: - logger.info("No schedule changes detected, skipping schedule and program downloads.") - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", - message="No schedule changes detected since last refresh. Guide data is up to date.") - source.status = EPGSource.STATUS_SUCCESS - source.last_message = "No schedule changes detected. Guide data is up to date." - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - return - - # Download only changed schedules, batched by 7-day windows per station - SCHEDULE_BATCH_DAYS = 7 - changed_station_ids = list(changed_by_station.keys()) - date_batches = [date_list[i:i + SCHEDULE_BATCH_DAYS] for i in range(0, len(date_list), SCHEDULE_BATCH_DAYS)] - new_md5_records = [] - updated_md5_records = [] - existing_md5_map = { - (r.station_id, r.date.strftime('%Y-%m-%d')): r - for r in SDScheduleMD5.objects.filter(epg_source=source, station_id__in=changed_station_ids) - } - - for batch_idx, date_batch in enumerate(date_batches): - # Notify frontend at the start of each batch so progress updates immediately - pre_progress = 38 + int((batch_idx / len(date_batches)) * 22) - logger.info(f"Fetching schedule batch {batch_idx + 1} of {len(date_batches)}...") - _sd_send_ws_sync(source.id, "parsing_programs", min(59, pre_progress), - message=f"Fetching schedules: batch {batch_idx + 1} of {len(date_batches)}...") - # Yield to gevent hub so the WebSocket update is delivered before the blocking request - try: - import gevent; gevent.sleep(0) - except ImportError: - pass - # Only include stations that have changes in this date batch - request_body = [ - {'stationID': sid, 'date': [d for d in date_batch if d in changed_by_station.get(sid, [])]} - for sid in changed_station_ids - if any(d in changed_by_station.get(sid, []) for d in date_batch) - ] - if not request_body: - continue - try: - sched_response = requests.post( - f"{SD_BASE_URL}/schedules", - json=request_body, - headers=_sd_headers(token), - timeout=120, - ) - sched_response.raise_for_status() - sched_data = sched_response.json() - - for station_sched in sched_data: - sid = station_sched.get('stationID') - if not sid: - continue - programs = station_sched.get('programs', []) - schedules_by_station.setdefault(sid, []).extend(programs) - for prog in programs: - pid = prog.get('programID') - if pid: - program_ids_needed.add(pid) - - # Update MD5 cache for this station/date - meta = station_sched.get('metadata', {}) - start_date = meta.get('startDate') - md5_val = meta.get('md5', '') - last_mod_str = meta.get('modified', '') - if start_date and md5_val: - key = (sid, start_date) - last_mod = parse_datetime(last_mod_str) if last_mod_str else timezone.now() - if key in existing_md5_map: - rec = existing_md5_map[key] - rec.md5 = md5_val - rec.last_modified = last_mod - updated_md5_records.append(rec) - else: - import datetime as dt_module - try: - date_obj = dt_module.date.fromisoformat(start_date) - new_md5_records.append(SDScheduleMD5( - epg_source=source, - station_id=sid, - date=date_obj, - md5=md5_val, - last_modified=last_mod, - )) - except ValueError: - pass - - progress = 38 + int(((batch_idx + 1) / len(date_batches)) * 22) - _sd_send_ws_sync(source.id, "parsing_programs", min(60, progress), - message=f"Fetching changed schedules: batch {batch_idx + 1}/{len(date_batches)} ({len(program_ids_needed):,} programs found)") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch schedule batch {batch_idx + 1}: {e}") - - # Persist updated MD5 cache - if new_md5_records: - SDScheduleMD5.objects.bulk_create(new_md5_records, ignore_conflicts=True) - logger.info(f"Cached {len(new_md5_records)} new schedule MD5s.") - if updated_md5_records: - SDScheduleMD5.objects.bulk_update(updated_md5_records, ['md5', 'last_modified']) - logger.info(f"Updated {len(updated_md5_records)} existing schedule MD5s.") - - if not program_ids_needed: - msg = "No schedule data returned from Schedules Direct." - logger.warning(msg) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) - return - - # ------------------------------------------------------------------------- - # Step 6: MD5-delta program metadata fetch - # The schedule response includes an MD5 hash per program airing. - # Compare against our cached program MD5s to only download programs - # whose metadata has changed since our last fetch. - # ------------------------------------------------------------------------- - - # Build map of programID -> md5 from schedule data - schedule_program_md5s = {} # programID -> md5 from schedule - for sid, airings in schedules_by_station.items(): - for airing in airings: - pid = airing.get('programID') - md5 = airing.get('md5') - if pid and md5: - schedule_program_md5s[pid] = md5 - - # Load cached program MD5s from SDProgramMD5 table, keyed by programID - cached_prog_md5s = { - r.program_id: r.md5 - for r in SDProgramMD5.objects.filter( - epg_source=source, - program_id__in=program_ids_needed, - ).only('program_id', 'md5') - } - - # Only fetch programs where MD5 differs from our cached value - programs_to_fetch = { - pid for pid in program_ids_needed - if schedule_program_md5s.get(pid) != cached_prog_md5s.get(pid) - } - - logger.info( - f"Program MD5 delta: {len(program_ids_needed)} programs in schedules, " - f"{len(programs_to_fetch)} need downloading ({len(program_ids_needed) - len(programs_to_fetch)} unchanged).") - - program_metadata = {} - program_id_list = list(programs_to_fetch) - total_batches = max(1, (len(program_id_list) + SD_PROGRAM_BATCH_SIZE - 1) // SD_PROGRAM_BATCH_SIZE) - - if program_id_list: - logger.info(f"Fetching metadata for {len(program_id_list)} programs in {total_batches} batch(es).") - for batch_idx in range(total_batches): - # Notify frontend at the start of each batch so progress updates immediately - pre_progress = 60 + int((batch_idx / total_batches) * 20) - logger.info(f"Fetching program metadata batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)...") - _sd_send_ws_sync(source.id, "parsing_programs", min(79, pre_progress), - message=f"Fetching program data: batch {batch_idx + 1} of {total_batches} ({batch_idx * SD_PROGRAM_BATCH_SIZE:,} of {len(program_id_list):,} programs)") - # Yield to gevent hub so the WebSocket update is delivered before the blocking request - try: - import gevent; gevent.sleep(0) - except ImportError: - pass - batch = program_id_list[batch_idx * SD_PROGRAM_BATCH_SIZE:(batch_idx + 1) * SD_PROGRAM_BATCH_SIZE] - try: - prog_response = requests.post( - f"{SD_BASE_URL}/programs", - json=batch, - headers=_sd_headers(token), - timeout=120, - ) - prog_response.raise_for_status() - prog_data = prog_response.json() - for prog in prog_data: - pid = prog.get('programID') - if pid: - program_metadata[pid] = prog - - progress = 60 + int(((batch_idx + 1) / total_batches) * 20) - _sd_send_ws_sync(source.id, "parsing_programs", min(80, progress), - message=f"Fetching program details: batch {batch_idx + 1}/{total_batches} ({len(program_metadata):,} programs loaded)") - logger.debug(f"Fetched program metadata batch {batch_idx + 1}/{total_batches}") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch program metadata batch {batch_idx + 1}: {e}") - else: - logger.info("All program metadata unchanged - skipping program download.") - _sd_send_ws_sync(source.id, "parsing_programs", 80, message="Program metadata unchanged - using cached data.") - - gc.collect() - - # ------------------------------------------------------------------------- - # Step 7: Build ProgramData records and persist atomically - # ------------------------------------------------------------------------- - logger.info("Building program records...") - _sd_send_ws_sync(source.id, "parsing_programs", 80) - - # Only process stations that are mapped to channels to match the existing - # XMLTV flow (parse_programs_for_source only processes mapped channels). - from apps.channels.models import Channel as ChannelModel - mapped_epg_ids = set( - ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).values_list('epg_data_id', flat=True) - ) - mapped_tvg_ids = set( - EPGData.objects.filter( - id__in=mapped_epg_ids, - epg_source=source, - ).values_list('tvg_id', flat=True) - ) - - # Cache existing program data for unchanged programs BEFORE surgical delete. - # When a station/date schedule MD5 changes, ALL airings are re-fetched, but only - # programs with changed program MD5s get metadata re-downloaded. The surgical delete - # wipes ALL ProgramData for changed dates, so unchanged programs lose their titles. - # This cache preserves their data for rebuilding. - unchanged_pids = set() - for sid, airings in schedules_by_station.items(): - if sid not in mapped_tvg_ids: - continue - for airing in airings: - pid = airing.get('programID') - if pid and pid not in program_metadata: - unchanged_pids.add(pid) - - existing_program_cache = {} - if unchanged_pids: - for pd in ProgramData.objects.filter( - epg__epg_source=source, - program_id__in=unchanged_pids, - ).only('program_id', 'title', 'description', 'sub_title', 'custom_properties'): - if pd.program_id not in existing_program_cache: - existing_program_cache[pd.program_id] = { - 'title': pd.title, - 'description': pd.description, - 'sub_title': pd.sub_title, - 'custom_properties': pd.custom_properties, - } - logger.info(f"Cached {len(existing_program_cache)} existing program records for unchanged programs.") - - all_programs_to_create = [] - total_programs = 0 - skipped_unmapped = 0 - - for sid, airings in schedules_by_station.items(): - if sid not in mapped_tvg_ids: - skipped_unmapped += len(airings) - continue - - epg_db_id = epg_id_map.get(sid) - if not epg_db_id: - continue - - for airing in airings: - pid = airing.get('programID') - air_time = airing.get('airDateTime') - duration_secs = airing.get('duration', 0) - - if not pid or not air_time or not duration_secs: - continue - - try: - start_dt = parse_schedules_direct_time(air_time) - end_dt = start_dt + timedelta(seconds=int(duration_secs)) - except Exception as e: - logger.debug(f"Could not parse air time '{air_time}': {e}") - continue - - meta = program_metadata.get(pid, {}) - cached_prog = existing_program_cache.get(pid) if not meta else None - - if cached_prog: - # Unchanged program — reuse cached data from before surgical delete - title = cached_prog['title'] or 'No Title' - desc = cached_prog['description'] or '' - episode_title = cached_prog['sub_title'] or '' - custom_props = cached_prog['custom_properties'] or {} - else: - titles = meta.get('titles', [{}]) - title = titles[0].get('title120', '') if titles else '' - if not title: - title = meta.get('episodeTitle150', '') or 'No Title' - title = title[:255] - - if not cached_prog: - descriptions = meta.get('descriptions', {}) - desc = '' - for key in ('description1000', 'description255', 'description100'): - candidates = descriptions.get(key, []) - if candidates: - desc = candidates[0].get('description', '') - if desc: - break - - episode_title = meta.get('episodeTitle150', '') - - # Build custom_properties following the same pattern as the XMLTV parser - custom_props = {} - - # Season/Episode — search all metadata entries, not just [0] - metadata_block = meta.get('metadata', []) - gracenote_meta = {} - for md_entry in metadata_block: - if 'Gracenote' in md_entry: - gracenote_meta = md_entry['Gracenote'] - break - if not gracenote_meta: - # Fall back to TVmaze if Gracenote is absent - for md_entry in metadata_block: - if 'TVmaze' in md_entry: - gracenote_meta = md_entry['TVmaze'] - break - season = gracenote_meta.get('season') - episode = gracenote_meta.get('episode') - if season: - custom_props['season'] = int(season) - if episode: - custom_props['episode'] = int(episode) - if season and episode: - custom_props['onscreen_episode'] = f"S{int(season)} E{int(episode)}" - - # Content rating — store full array, pick display rating by lineup country - content_rating = meta.get('contentRating', []) - if content_rating: - custom_props['content_ratings'] = content_rating - selected = None - if sd_lineup_country: - for cr in content_rating: - if cr.get('country', '') == sd_lineup_country: - selected = cr - break - if not selected: - # Fall back to USA, then first available - for cr in content_rating: - if cr.get('country', '') == 'USA': - selected = cr - break - if not selected: - selected = content_rating[0] - custom_props['rating'] = selected.get('code', '') - custom_props['rating_system'] = selected.get('body', '') - - # Content advisory — content warnings - content_advisory = meta.get('contentAdvisory', []) - if content_advisory: - custom_props['content_advisory'] = content_advisory - - # Categories — combine entityType, showType, and genres - categories = [] - entity_type = meta.get('entityType', '') - show_type = meta.get('showType', '') - if entity_type: - categories.append(entity_type) - if show_type and show_type != entity_type: - categories.append(show_type) - genres = meta.get('genres', []) - categories.extend(genres) - if categories: - custom_props['categories'] = categories - - # Cast — top-billed only, store characterName, drop role noise - cast = meta.get('cast', []) - crew = meta.get('crew', []) - credits = {} - if cast: - # Sort by billingOrder and cap at top-billed actors - sorted_cast = sorted( - [p for p in cast if p.get('name')], - key=lambda p: int(p.get('billingOrder', '999')) - ) - # Separate main cast from guest stars - main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] - # Store top-billed main cast (matching XMLTV parity) - credits['actor'] = [ - { - 'name': p.get('name', ''), - **(({'character': p['characterName']} ) if p.get('characterName') else {}), - } - for p in (main_cast[:6] if main_cast else sorted_cast[:6]) - ] - if crew: - for member in crew: - role = member.get('role', '').lower() - name = member.get('name', '') - if not name: - continue - if 'director' in role: - credits.setdefault('director', []).append(name) - elif 'writer' in role or 'screenwriter' in role: - credits.setdefault('writer', []).append(name) - elif 'producer' in role: - credits.setdefault('producer', []).append(name) - if credits: - custom_props['credits'] = credits - - # Airing flags - if airing.get('liveTapeDelay') == 'Live': - custom_props['live'] = True - if airing.get('new'): - custom_props['new'] = True - else: - custom_props['previously_shown'] = True - if airing.get('premiere'): - custom_props['premiere'] = True - - # Original air date — full date, not just year - original_air_date = meta.get('originalAirDate', '') - movie_year = meta.get('movie', {}).get('year', '') - if original_air_date: - custom_props['date'] = original_air_date - elif movie_year: - custom_props['date'] = str(movie_year) - - # Country of production - country = meta.get('country', []) - if country: - custom_props['country'] = country[0] if len(country) == 1 else ', '.join(country) - - # Runtime — program duration without commercials (seconds → store for display) - runtime_secs = meta.get('duration') or meta.get('movie', {}).get('duration') - if runtime_secs: - runtime_mins = int(runtime_secs) // 60 - custom_props['length'] = {'value': str(runtime_mins), 'units': 'minutes'} - - # Movie quality ratings → star_ratings (matches XMLTV key) - movie_data = meta.get('movie', {}) - quality_ratings = movie_data.get('qualityRating', []) - if quality_ratings: - star_ratings = [] - for qr in quality_ratings: - rating_str = qr.get('rating', '') - max_rating = qr.get('maxRating', '') - if rating_str and max_rating: - star_ratings.append({ - 'value': f"{rating_str}/{max_rating}", - 'system': qr.get('ratingsBody', ''), - }) - if star_ratings: - custom_props['star_ratings'] = star_ratings - - # Sports event details - event_details = meta.get('eventDetails', {}) - if event_details: - custom_props['event_details'] = event_details - - all_programs_to_create.append(ProgramData( - epg_id=epg_db_id, - start_time=start_dt, - end_time=end_dt, - title=title, - sub_title=episode_title or None, - description=desc or None, - tvg_id=sid, - program_id=pid, - custom_properties=custom_props or None, - )) - total_programs += 1 - - logger.info(f"Built {total_programs} program records " - f"({skipped_unmapped} skipped for unmapped stations).") - - _sd_send_ws_sync(source.id, "parsing_programs", 88) - - # Build a map of epg_db_id -> list of (day_start_utc, day_end_utc) for each changed date. - # Only programs that fall within changed station/date pairs will be deleted and replaced; - # programs for unchanged stations or unchanged dates are left intact. - import datetime as dt_module - epg_changed_date_ranges = {} - for sid, changed_date_strs in changed_by_station.items(): - epg_db_id = epg_id_map.get(sid) - if not epg_db_id or epg_db_id not in mapped_epg_ids: - continue - ranges = [] - for ds in changed_date_strs: - d = dt_module.date.fromisoformat(ds) - day_start = datetime(d.year, d.month, d.day, tzinfo=dt_timezone.utc) - ranges.append((day_start, day_start + timedelta(days=1))) - if ranges: - epg_changed_date_ranges[epg_db_id] = ranges - - # Atomic delete (surgical) + bulk insert - BATCH_SIZE = 1000 - try: - with transaction.atomic(): - with connection.cursor() as cursor: - cursor.execute("SET LOCAL statement_timeout = '10min'") - total_deleted = 0 - for epg_db_id, day_ranges in epg_changed_date_ranges.items(): - q = Q() - for day_start, day_end in day_ranges: - q |= Q(start_time__gte=day_start, start_time__lt=day_end) - cnt = ProgramData.objects.filter(epg_id=epg_db_id).filter(q).delete()[0] - total_deleted += cnt - logger.debug(f"Deleted {total_deleted} changed SD programs across {len(epg_changed_date_ranges)} stations.") - for i in range(0, len(all_programs_to_create), BATCH_SIZE): - ProgramData.objects.bulk_create(all_programs_to_create[i:i + BATCH_SIZE]) - progress = 88 + int(((i + BATCH_SIZE) / max(len(all_programs_to_create), 1)) * 10) - _sd_send_ws_sync(source.id, "parsing_programs", min(98, progress)) - - logger.info(f"Committed {total_programs} Schedules Direct programs to database.") - - # Upsert SDProgramMD5 records for programs we just downloaded - # This updates the cache so future fetches can skip unchanged programs - if schedule_program_md5s: - md5_records = [ - SDProgramMD5( - epg_source=source, - program_id=pid, - md5=md5, - ) - for pid, md5 in schedule_program_md5s.items() - if pid in program_metadata # Only cache programs that were actually downloaded - ] - if md5_records: - SDProgramMD5.objects.bulk_create( - md5_records, - update_conflicts=True, - unique_fields=['epg_source', 'program_id'], - update_fields=['md5'], - ) - logger.info(f"Cached {len(md5_records)} program MD5s for future delta detection.") - - except Exception as db_error: - msg = f"Database error persisting Schedules Direct programs: {db_error}" - logger.error(msg, exc_info=True) - source.status = EPGSource.STATUS_ERROR - source.last_message = msg - source.save(update_fields=['status', 'last_message']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="error", error=msg) - return - finally: - all_programs_to_create = None - gc.collect() - - # ------------------------------------------------------------------------- - # Step 8: Fetch program artwork (posters) if enabled - # ------------------------------------------------------------------------- - fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) - if fetch_posters and program_metadata: - logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") - _sd_send_ws_sync(source.id, "parsing_programs", 98, - message="Fetching program artwork...") - - try: - # Build a set of unique artwork lookup IDs. - # For episodes (EP...), use the series root (SH...0000) so we get - # series-level artwork — one poster per series instead of per-episode. - artwork_lookup_ids = set() - pid_to_artwork_key = {} # maps original programID -> the key we looked up - - for pid in program_metadata: - if pid.startswith('EP'): - sh_root = 'SH' + pid[2:10] + '0000' - artwork_lookup_ids.add(sh_root) - pid_to_artwork_key[pid] = sh_root - else: - artwork_lookup_ids.add(pid) - pid_to_artwork_key[pid] = pid - - artwork_map = {} # artwork_key -> poster_url - artwork_list = list(artwork_lookup_ids) - SD_ARTWORK_BATCH_SIZE = 500 - - total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) - logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " - f"in {total_art_batches} batch(es).") - - for batch_idx in range(total_art_batches): - batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] - try: - art_response = requests.post( - f"{SD_BASE_URL}/metadata/programs/", - json=batch, - headers=_sd_headers(token), - timeout=120, - ) - art_response.raise_for_status() - art_data = art_response.json() - - for entry in art_data: - if not isinstance(entry, dict): - continue - entry_pid = entry.get('programID') - images = entry.get('data') or [] - if not entry_pid or not images: - continue - - # Filter to only dict entries — SD sometimes returns bare strings - images = [img for img in images if isinstance(img, dict)] - if not images: - continue - - # Pick the best poster image: - # Prefer portrait orientation (2x3, 3x4) in larger sizes - # SD categories include: Iconic, Banner-L1, Banner-L2, Logo - # SD uses width/height instead of a "size" field - poster_url = None - - # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic - for min_width in [240, 135, 120]: - for pref_cat in ['Banner-L1', 'Iconic']: - match = next((img for img in images - if img.get('aspect') in ('2x3', '3x4') - and img.get('category') == pref_cat - and (img.get('width', 0) or 0) >= min_width), None) - if match: - poster_url = match.get('uri') - break - if poster_url: - break - - # Fallback: any portrait image at any size - if not poster_url: - portrait = next((img for img in images - if img.get('aspect') in ('2x3', '3x4')), None) - if portrait: - poster_url = portrait.get('uri') - - if poster_url: - # Complete the URL if it's relative - if not poster_url.startswith('http'): - poster_url = f"{SD_BASE_URL}/image/{poster_url}" - artwork_map[entry_pid] = poster_url - - logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " - f"{len(artwork_map)} posters found so far.") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") - - # Bulk-update ProgramData records with poster URLs - if artwork_map: - programs_to_update = [] - for prog in ProgramData.objects.filter( - epg_id__in=mapped_epg_ids, - program_id__isnull=False, - ).only('id', 'program_id', 'custom_properties'): - art_key = pid_to_artwork_key.get(prog.program_id) - poster = artwork_map.get(art_key) if art_key else None - if poster: - cp = prog.custom_properties or {} - cp['poster_url'] = poster - prog.custom_properties = cp - programs_to_update.append(prog) - - if programs_to_update: - ProgramData.objects.bulk_update( - programs_to_update, ['custom_properties'], batch_size=1000 - ) - logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") - else: - logger.info("No poster artwork matched committed programs.") - else: - logger.info("No poster artwork found from Schedules Direct.") - - except Exception as art_error: - logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) - - elif fetch_posters: - logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") - - # ------------------------------------------------------------------------- - # Step 9: Apply SD station logos to matched channels if enabled - # ------------------------------------------------------------------------- - use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) - if use_sd_logos: - try: - from apps.channels.models import Channel as ChannelModel, Logo - - channels_to_update = [] - logos_created = 0 - - for channel in ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).select_related('epg_data', 'logo'): - icon_url = (channel.epg_data.icon_url or '').strip() - if not icon_url: - continue - - # Skip if channel already has this logo URL - if channel.logo and channel.logo.url == icon_url: - continue - - # Find or create a Logo object for this URL - try: - logo = Logo.objects.get(url=icon_url) - except Logo.DoesNotExist: - logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" - logo = Logo.objects.create(name=logo_name, url=icon_url) - logos_created += 1 - - channel.logo = logo - channels_to_update.append(channel) - - if channels_to_update: - ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) - logger.info(f"Applied SD logos to {len(channels_to_update)} channels " - f"({logos_created} new logos created).") - else: - logger.info("All matched channels already have current SD logos.") - - except Exception as logo_error: - logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) - - # Prune ProgramData whose end_time has passed. With surgical per-date deletes, - # programs from dates that have rolled off the window are never explicitly removed. - today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) - try: - expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] - if expired_count: - logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") - except Exception as prune_err: - logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") - - # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. - try: - live_program_ids = set( - ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) - .values_list('program_id', flat=True) - ) - pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( - program_id__in=live_program_ids - ).delete()[0] - if pruned_prog_md5_count: - logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") - except Exception as prune_err: - logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") - - # ------------------------------------------------------------------------- - # Prune stale poster cache files (>30 days old or orphaned) - # ------------------------------------------------------------------------- - try: - cache_dir = '/data/cache/posters' - if os.path.exists(cache_dir): - import time as time_module - # Collect all poster hashes currently referenced by ProgramData - active_hashes = set() - for url in ProgramData.objects.filter( - epg__epg_source=source, - custom_properties__has_key='poster_url', - ).values_list('custom_properties__poster_url', flat=True): - if url: - active_hashes.add(url.rsplit('/', 1)[-1]) - - pruned_posters = 0 - for fname in os.listdir(cache_dir): - fpath = os.path.join(cache_dir, fname) - if not os.path.isfile(fpath): - continue - file_age = time_module.time() - os.path.getmtime(fpath) - # Remove if older than 30 days OR not referenced by any current program - if file_age > 30 * 24 * 3600 or fname not in active_hashes: - os.remove(fpath) - pruned_posters += 1 - if pruned_posters: - logger.info(f"Pruned {pruned_posters} stale poster cache files.") - except Exception as poster_prune_err: - logger.warning(f"Failed to prune poster cache: {poster_prune_err}") - - # ------------------------------------------------------------------------- - # Done - # ------------------------------------------------------------------------- - success_msg = ( - f"Successfully fetched {total_programs:,} programs for " - f"{len(mapped_tvg_ids)} mapped stations from Schedules Direct " - f"({skipped_unmapped:,} programs skipped for unmapped stations)." - ) - source.status = EPGSource.STATUS_SUCCESS - source.last_message = success_msg - source.updated_at = timezone.now() - source.save(update_fields=['status', 'last_message', 'updated_at']) - _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message=success_msg) - log_system_event( - event_type='epg_refresh', - source_name=source.name, - programs=total_programs, - channels=len(mapped_tvg_ids), - skipped_programs=skipped_unmapped, - ) - logger.info(f"Schedules Direct fetch complete for source: {source.name}") - - -# ------------------------------- -# Helper parse functions -# ------------------------------- -def parse_xmltv_time(time_str): - try: - # Basic format validation - if len(time_str) < 14: - logger.warning(f"XMLTV timestamp too short: '{time_str}', using as-is") - dt_obj = datetime.strptime(time_str, '%Y%m%d%H%M%S') - return timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - - # Parse base datetime - dt_obj = datetime.strptime(time_str[:14], '%Y%m%d%H%M%S') - - # Handle timezone if present - if len(time_str) >= 20: # Has timezone info - tz_sign = time_str[15] - tz_hours = int(time_str[16:18]) - tz_minutes = int(time_str[18:20]) - - # Create a timezone object - if tz_sign == '+': - tz_offset = dt_timezone(timedelta(hours=tz_hours, minutes=tz_minutes)) - elif tz_sign == '-': - tz_offset = dt_timezone(timedelta(hours=-tz_hours, minutes=-tz_minutes)) - else: - tz_offset = dt_timezone.utc - - # Make datetime aware with correct timezone - aware_dt = datetime.replace(dt_obj, tzinfo=tz_offset) - # Convert to UTC - aware_dt = aware_dt.astimezone(dt_timezone.utc) - - logger.trace(f"Parsed XMLTV time '{time_str}' to {aware_dt}") - return aware_dt - else: - # No timezone info, assume UTC - aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - logger.trace(f"Parsed XMLTV time without timezone '{time_str}' as UTC: {aware_dt}") - return aware_dt - - except Exception as e: - logger.error(f"Error parsing XMLTV time '{time_str}': {e}", exc_info=True) - raise - - -def parse_schedules_direct_time(time_str): - try: - dt_obj = datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%SZ') - aware_dt = timezone.make_aware(dt_obj, timezone=dt_timezone.utc) - logger.debug(f"Parsed Schedules Direct time '{time_str}' to {aware_dt}") - return aware_dt - except Exception as e: - logger.error(f"Error parsing Schedules Direct time '{time_str}': {e}", exc_info=True) - raise - - -# Re-export from utils to preserve backward compatibility for any callers -from apps.epg.utils import extract_season_episode_from_description, _ONSCREEN_RE # noqa: F401 - - -# Helper function to extract custom properties - moved to a separate function to clean up the code -def extract_custom_properties(prog): - # Create a new dictionary for each call - custom_props = {} - - # Extract categories with a single comprehension to reduce intermediate objects - categories = [cat.text.strip() for cat in prog.findall('category') if cat.text and cat.text.strip()] - if categories: - custom_props['categories'] = categories - - # Extract keywords (new) - keywords = [kw.text.strip() for kw in prog.findall('keyword') if kw.text and kw.text.strip()] - if keywords: - custom_props['keywords'] = keywords - - # Extract episode numbers - for ep_num in prog.findall('episode-num'): - system = ep_num.get('system', '') - if system == 'xmltv_ns' and ep_num.text: - # Parse XMLTV episode-num format (season.episode.part) - parts = ep_num.text.split('.') - if len(parts) >= 2: - if parts[0].strip() != '': - try: - season = int(parts[0]) + 1 # XMLTV format is zero-based - custom_props['season'] = season - except ValueError: - pass - if parts[1].strip() != '': - try: - episode = int(parts[1]) + 1 # XMLTV format is zero-based - custom_props['episode'] = episode - except ValueError: - pass - elif system == 'onscreen' and ep_num.text: - onscreen_text = ep_num.text.strip() - custom_props['onscreen_episode'] = onscreen_text - # Extract season/episode from onscreen format if not already set by xmltv_ns - if 'season' not in custom_props or 'episode' not in custom_props: - match = _ONSCREEN_RE.search(onscreen_text) - if match: - if 'season' not in custom_props: - custom_props['season'] = int(match.group(1)) - if 'episode' not in custom_props: - custom_props['episode'] = int(match.group(2)) - elif system == 'dd_progid' and ep_num.text: - # Store the dd_progid format - custom_props['dd_progid'] = ep_num.text.strip() - # Add support for other systems like thetvdb.com, themoviedb.org, imdb.com - elif system in ['thetvdb.com', 'themoviedb.org', 'imdb.com'] and ep_num.text: - custom_props[f'{system}_id'] = ep_num.text.strip() - - # Extract ratings more efficiently - rating_elem = prog.find('rating') - if rating_elem is not None: - value_elem = rating_elem.find('value') - if value_elem is not None and value_elem.text: - custom_props['rating'] = value_elem.text.strip() - if rating_elem.get('system'): - custom_props['rating_system'] = rating_elem.get('system') - - # Extract star ratings (new) - star_ratings = [] - for star_rating in prog.findall('star-rating'): - value_elem = star_rating.find('value') - if value_elem is not None and value_elem.text: - rating_data = {'value': value_elem.text.strip()} - if star_rating.get('system'): - rating_data['system'] = star_rating.get('system') - star_ratings.append(rating_data) - if star_ratings: - custom_props['star_ratings'] = star_ratings - - # Extract credits more efficiently - credits_elem = prog.find('credits') - if credits_elem is not None: - credits = {} - for credit_type in ['director', 'actor', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if credit_type == 'actor': - # Handle actors with roles and guest status - actors = [] - for actor_elem in credits_elem.findall('actor'): - if actor_elem.text and actor_elem.text.strip(): - actor_data = {'name': actor_elem.text.strip()} - if actor_elem.get('role'): - actor_data['role'] = actor_elem.get('role') - if actor_elem.get('guest') == 'yes': - actor_data['guest'] = True - actors.append(actor_data) - if actors: - credits['actor'] = actors - else: - names = [e.text.strip() for e in credits_elem.findall(credit_type) if e.text and e.text.strip()] - if names: - credits[credit_type] = names - if credits: - custom_props['credits'] = credits - - # Extract other common program metadata - date_elem = prog.find('date') - if date_elem is not None and date_elem.text: - custom_props['date'] = date_elem.text.strip() - - country_elem = prog.find('country') - if country_elem is not None and country_elem.text: - custom_props['country'] = country_elem.text.strip() - - # Extract language information (new) - language_elem = prog.find('language') - if language_elem is not None and language_elem.text: - custom_props['language'] = language_elem.text.strip() - - orig_language_elem = prog.find('orig-language') - if orig_language_elem is not None and orig_language_elem.text: - custom_props['original_language'] = orig_language_elem.text.strip() - - # Extract length (new) - length_elem = prog.find('length') - if length_elem is not None and length_elem.text: - try: - length_value = int(length_elem.text.strip()) - length_units = length_elem.get('units', 'minutes') - custom_props['length'] = {'value': length_value, 'units': length_units} - except ValueError: - pass - - # Extract video information (new) - video_elem = prog.find('video') - if video_elem is not None: - video_info = {} - for video_attr in ['present', 'colour', 'aspect', 'quality']: - attr_elem = video_elem.find(video_attr) - if attr_elem is not None and attr_elem.text: - video_info[video_attr] = attr_elem.text.strip() - if video_info: - custom_props['video'] = video_info - - # Extract audio information (new) - audio_elem = prog.find('audio') - if audio_elem is not None: - audio_info = {} - for audio_attr in ['present', 'stereo']: - attr_elem = audio_elem.find(audio_attr) - if attr_elem is not None and attr_elem.text: - audio_info[audio_attr] = attr_elem.text.strip() - if audio_info: - custom_props['audio'] = audio_info - - # Extract subtitles information (new) - subtitles = [] - for subtitle_elem in prog.findall('subtitles'): - subtitle_data = {} - if subtitle_elem.get('type'): - subtitle_data['type'] = subtitle_elem.get('type') - lang_elem = subtitle_elem.find('language') - if lang_elem is not None and lang_elem.text: - subtitle_data['language'] = lang_elem.text.strip() - if subtitle_data: - subtitles.append(subtitle_data) - - if subtitles: - custom_props['subtitles'] = subtitles - - # Extract reviews (new) - reviews = [] - for review_elem in prog.findall('review'): - if review_elem.text and review_elem.text.strip(): - review_data = {'content': review_elem.text.strip()} - if review_elem.get('type'): - review_data['type'] = review_elem.get('type') - if review_elem.get('source'): - review_data['source'] = review_elem.get('source') - if review_elem.get('reviewer'): - review_data['reviewer'] = review_elem.get('reviewer') - reviews.append(review_data) - if reviews: - custom_props['reviews'] = reviews - - # Extract images (new) - images = [] - for image_elem in prog.findall('image'): - if image_elem.text and image_elem.text.strip(): - image_data = {'url': image_elem.text.strip()} - for attr in ['type', 'size', 'orient', 'system']: - if image_elem.get(attr): - image_data[attr] = image_elem.get(attr) - images.append(image_data) - if images: - custom_props['images'] = images - - icon_elem = prog.find('icon') - if icon_elem is not None and icon_elem.get('src'): - custom_props['icon'] = icon_elem.get('src') - - # Simpler approach for boolean flags - expanded list - for kw in ['previously-shown', 'premiere', 'new', 'live', 'last-chance']: - if prog.find(kw) is not None: - custom_props[kw.replace('-', '_')] = True - - # Extract premiere and last-chance text content if available - premiere_elem = prog.find('premiere') - if premiere_elem is not None: - custom_props['premiere'] = True - if premiere_elem.text and premiere_elem.text.strip(): - custom_props['premiere_text'] = premiere_elem.text.strip() - - last_chance_elem = prog.find('last-chance') - if last_chance_elem is not None: - custom_props['last_chance'] = True - if last_chance_elem.text and last_chance_elem.text.strip(): - custom_props['last_chance_text'] = last_chance_elem.text.strip() - - # Extract previously-shown details - prev_shown_elem = prog.find('previously-shown') - if prev_shown_elem is not None: - custom_props['previously_shown'] = True - prev_shown_data = {} - if prev_shown_elem.get('start'): - prev_shown_data['start'] = prev_shown_elem.get('start') - if prev_shown_elem.get('channel'): - prev_shown_data['channel'] = prev_shown_elem.get('channel') - if prev_shown_data: - custom_props['previously_shown_details'] = prev_shown_data - - return custom_props - - -def clear_element(elem): - """Clear an XML element and its parent to free memory.""" - try: - elem.clear() - parent = elem.getparent() - if parent is not None: - while elem.getprevious() is not None: - del parent[0] - parent.remove(elem) - except Exception as e: - logger.warning(f"Error clearing XML element: {e}", exc_info=True) - - -def detect_file_format(file_path=None, content=None): - """ - Detect file format by examining content or file path. - - Args: - file_path: Path to file (optional) - content: Raw file content bytes (optional) - - Returns: - tuple: (format_type, is_compressed, file_extension) - format_type: 'gzip', 'zip', 'xml', or 'unknown' - is_compressed: Boolean indicating if the file is compressed - file_extension: Appropriate file extension including dot (.gz, .zip, .xml) - """ - # Default return values - format_type = 'unknown' - is_compressed = False - file_extension = '.tmp' - - # First priority: check content magic numbers as they're most reliable - if content: - # We only need the first few bytes for magic number detection - header = content[:20] if len(content) >= 20 else content - - # Check for gzip magic number (1f 8b) - if len(header) >= 2 and header[:2] == b'\x1f\x8b': - return 'gzip', True, '.gz' - - # Check for zip magic number (PK..) - if len(header) >= 2 and header[:2] == b'PK': - return 'zip', True, '.zip' - - # Check for XML - either standard XML header or XMLTV-specific tag - if len(header) >= 5 and (b'' in header): - return 'xml', False, '.xml' - - # Second priority: check file extension - focus on the final extension for compression - if file_path: - logger.debug(f"Detecting file format for: {file_path}") - - # Handle compound extensions like .xml.gz - prioritize compression extensions - lower_path = file_path.lower() - - # Check for compression extensions explicitly - if lower_path.endswith('.gz') or lower_path.endswith('.gzip'): - return 'gzip', True, '.gz' - elif lower_path.endswith('.zip'): - return 'zip', True, '.zip' - elif lower_path.endswith('.xml'): - return 'xml', False, '.xml' - - # Fallback to mimetypes only if direct extension check doesn't work - import mimetypes - mime_type, _ = mimetypes.guess_type(file_path) - logger.debug(f"Guessed MIME type: {mime_type}") - if mime_type: - if mime_type == 'application/gzip' or mime_type == 'application/x-gzip': - return 'gzip', True, '.gz' - elif mime_type == 'application/zip': - return 'zip', True, '.zip' - elif mime_type == 'application/xml' or mime_type == 'text/xml': - return 'xml', False, '.xml' - - # If we reach here, we couldn't reliably determine the format - return format_type, is_compressed, file_extension - - -def generate_dummy_epg(source): - """ - DEPRECATED: This function is no longer used. - - Dummy EPG programs are now generated on-demand when they are requested - (during XMLTV export or EPG grid display), rather than being pre-generated - and stored in the database. - - See: apps/output/views.py - generate_custom_dummy_programs() - - This function remains for backward compatibility but should not be called. - """ - logger.warning(f"generate_dummy_epg() called for {source.name} but this function is deprecated. " - f"Dummy EPG programs are now generated on-demand.") - return True From 661d1690d252cd6fce48b202a272fd15b1d78a3e Mon Sep 17 00:00:00 2001 From: mwhit Date: Thu, 4 Jun 2026 22:28:18 +0000 Subject: [PATCH 37/76] feat: expose SD enrichment fields in serializer and frontend Add content_advisory, content_ratings, event_details, runtime to ProgramDetailSerializer. Display content advisory, sports venue/teams, and first-aired date in ProgramDetailModal. --- apps/epg/serializers.py | 14 +++++ .../src/components/ProgramDetailModal.jsx | 51 +++++++++++++++---- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 7da93c13..c196c4db 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -146,6 +146,20 @@ class ProgramDetailSerializer(ProgramDataSerializer): previously_shown = cp.get('previously_shown_details') or {} data['original_air_date'] = previously_shown.get('start') + # Content advisory (SD) + data['content_advisory'] = cp.get('content_advisory') or [] + + # Full content ratings array (SD — all regional ratings) + data['content_ratings'] = cp.get('content_ratings') or [] + + # Sports event details (SD) + data['event_details'] = cp.get('event_details') + + # Runtime (duration without commercials) + length = cp.get('length') or {} + data['runtime'] = length.get('value') if length else None + data['runtime_units'] = length.get('units') if length else None + # External IDs data['imdb_id'] = cp.get('imdb.com_id') data['tmdb_id'] = cp.get('themoviedb.org_id') diff --git a/frontend/src/components/ProgramDetailModal.jsx b/frontend/src/components/ProgramDetailModal.jsx index 89388cb4..d3d9c5e5 100644 --- a/frontend/src/components/ProgramDetailModal.jsx +++ b/frontend/src/components/ProgramDetailModal.jsx @@ -298,6 +298,37 @@ export default function ProgramDetailModal({ )} + {d.content_advisory?.length > 0 && ( + + {d.content_advisory.join(', ')} + + )} + + {d.event_details && ( + <> + + + {d.event_details.venue100 && ( + + Venue: + {d.event_details.venue100} + + )} + {d.event_details.teams?.length > 0 && ( + + Teams: + {d.event_details.teams.map((t, i) => ( + + {i > 0 ? ' vs ' : ''} + {t.name}{t.isHome ? ' (Home)' : ''} + + ))} + + )} + + + )} + {hasCredits && ( <> @@ -330,21 +361,13 @@ export default function ProgramDetailModal({ )} - {(d.country || - d.language || + {(d.language || d.original_air_date || + (d.production_date && d.is_previously_shown) || starRatings.length > 0) && ( <> - {d.country && ( - - - Country:{' '} - - {d.country} - - )} {d.language && ( @@ -353,6 +376,14 @@ export default function ProgramDetailModal({ {d.language} )} + {d.production_date && d.is_previously_shown && ( + + + First aired:{' '} + + {d.production_date} + + )} {d.original_air_date && ( From 7c91508f3c2a40184acb650c886555f563bddd03 Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Thu, 4 Jun 2026 23:46:40 +0000 Subject: [PATCH 38/76] fix: add missing updateSDSettings API method and EPG refresh handlers Add updateSDSettings method to API client for SD logo style and settings management. Add fetchEPGData and requeryChannels calls after EPG match completion to ensure UI refreshes immediately. Add SD logo style picker and cache-busting for channel logos. --- apps/channels/serializers.py | 19 ++- frontend/src/WebSocket.jsx | 11 +- frontend/src/api.js | 19 +++ frontend/src/components/forms/EPG.jsx | 195 +++++++++++++++++++++++--- 4 files changed, 209 insertions(+), 35 deletions(-) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 4598f8b2..e9ac8581 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -19,9 +19,10 @@ from apps.epg.serializers import EPGDataSerializer from core.models import StreamProfile from apps.epg.models import EPGData from django.db import connection, transaction +from django.urls import reverse from rest_framework import serializers from django.utils import timezone -from core.utils import validate_flexible_url, build_absolute_uri_with_port +from core.utils import validate_flexible_url class LogoSerializer(serializers.ModelSerializer): @@ -56,12 +57,18 @@ class LogoSerializer(serializers.ModelSerializer): return instance def get_cache_url(self, obj): + # Cache-busting: append a short hash of the logo's source URL so the browser + # fetches fresh when the logo changes (e.g., M3U logo replaced by SD logo). + # The backend ignores the 'v' parameter — it's purely for browser cache invalidation. + # See SD integration PR notes for context on why this was added. + import hashlib + url_hash = hashlib.md5((obj.url or '').encode()).hexdigest()[:8] + base_path = reverse("api:channels:logo-cache", args=[obj.id]) + cache_url = f"{base_path}?v={url_hash}" request = self.context.get("request") - if not request: - return f"/api/channels/logos/{obj.id}/cache/" - if not hasattr(self, "_cache_url_prefix"): - self._cache_url_prefix = build_absolute_uri_with_port(request, "") - return f"{self._cache_url_prefix}/api/channels/logos/{obj.id}/cache/" + if request: + return request.build_absolute_uri(cache_url) + return cache_url def get_channel_count(self, obj): """Get the number of channels using this logo""" diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index 1223a6a0..d8912f0d 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -381,6 +381,11 @@ export const WebsocketProvider = ({ children }) => { ) { API.batchSetEPG(parsedEvent.data.associations); } + + // Refresh EPG store first, then requery channels so the table + // cross-references updated epg_data_id assignments immediately + fetchEPGData(); + API.requeryChannels(); break; case 'epg_matching_progress': { @@ -965,12 +970,6 @@ export const WebsocketProvider = ({ children }) => { break; } - case 'ip_lookup_complete': { - const { type: _t, ...ipData } = parsedEvent.data; - useSettingsStore.getState().setEnvironmentFields(ipData); - break; - } - default: console.error( `Unknown websocket event type: ${parsedEvent.data?.type}` diff --git a/frontend/src/api.js b/frontend/src/api.js index 30d69c85..43f53040 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -3879,6 +3879,25 @@ export default class API { } } + static async updateSDSettings(sourceId, settings) { + try { + // Read current custom_properties from the store to merge, not replace + const epgs = useEPGsStore.getState().epgs; + const source = epgs[sourceId]; + const cp = { ...(source?.custom_properties || {}), ...settings }; + + const response = await request(`${host}/api/epg/sources/${sourceId}/`, { + method: 'PATCH', + body: { custom_properties: cp }, + }); + + useEPGsStore.getState().updateEPG(response); + return response; + } catch (e) { + errorNotification('Failed to update Schedules Direct settings', e); + } + } + static async searchSDLineups(sourceId, country, postalcode) { try { const response = await request( diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 369011f3..6215e40d 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -18,6 +18,8 @@ import { Badge, ScrollArea, Table, + Switch, + UnstyledButton, } from '@mantine/core'; import { TriangleAlert, Trash2, Plus, Search } from 'lucide-react'; import { isNotEmpty, useForm } from '@mantine/form'; @@ -45,6 +47,138 @@ const SD_COUNTRIES_FALLBACK = [ { value: 'NZL', label: 'New Zealand' }, ]; +// ESPN HD logo previews — packaged as static URLs so no API call is needed. +// These are publicly accessible S3 URLs that don't require authentication. +const SD_LOGO_PREVIEW_BASE = 'https://schedulesdirect-api20141201-logos.s3.dualstack.us-east-1.amazonaws.com/stationLogos/s32645'; +const SD_LOGO_STYLES = [ + { value: 'dark', label: 'Dark', url: `${SD_LOGO_PREVIEW_BASE}_dark_360w_270h.png` }, + { value: 'white', label: 'White', url: `${SD_LOGO_PREVIEW_BASE}_white_360w_270h.png` }, + { value: 'gray', label: 'Gray', url: `${SD_LOGO_PREVIEW_BASE}_gray_360w_270h.png` }, + { value: 'light', label: 'Light', url: `${SD_LOGO_PREVIEW_BASE}_light_360w_270h.png` }, +]; + +// ─── SD Settings: Logo toggle + style selector + Poster toggle ────────────── +const SDSettings = ({ sourceId, customProperties }) => { + const cp = customProperties || {}; + const [useSDLogos, setUseSDLogos] = useState(cp.use_sd_logos || false); + const [logoStyle, setLogoStyle] = useState(cp.logo_style || 'dark'); + const [fetchPosters, setFetchPosters] = useState(cp.fetch_posters || false); + const [saving, setSaving] = useState(false); + + // Sync from parent when customProperties changes + useEffect(() => { + const newCp = customProperties || {}; + setUseSDLogos(newCp.use_sd_logos || false); + setLogoStyle(newCp.logo_style || 'dark'); + setFetchPosters(newCp.fetch_posters || false); + + // Persist the default logo_style if not already set + if (sourceId && !newCp.logo_style) { + API.updateSDSettings(sourceId, { logo_style: 'dark' }); + } + }, [customProperties, sourceId]); + + const saveSetting = async (key, value) => { + setSaving(true); + try { + await API.updateSDSettings(sourceId, { [key]: value }); + } finally { + setSaving(false); + } + }; + + const handleLogoToggle = (checked) => { + setUseSDLogos(checked); + saveSetting('use_sd_logos', checked); + }; + + const handleLogoChange = (style) => { + if (!useSDLogos) return; + setLogoStyle(style); + saveSetting('logo_style', style); + }; + + const handlePosterToggle = (checked) => { + setFetchPosters(checked); + saveSetting('fetch_posters', checked); + }; + + const logosDisabled = !useSDLogos; + + return ( + + handleLogoToggle(e.currentTarget.checked)} + disabled={saving} + size="sm" + mb="sm" + /> + + + Station Logo Style + + + Choose which logo variant to use for SD stations. + + + {SD_LOGO_STYLES.map((style) => ( + handleLogoChange(style.value)} + style={{ + border: !logosDisabled && logoStyle === style.value + ? '2px solid var(--mantine-color-blue-5)' + : '2px solid var(--mantine-color-default-border)', + borderRadius: 'var(--mantine-radius-sm)', + padding: 3, + opacity: logosDisabled ? 0.3 : (saving ? 0.6 : 1), + cursor: logosDisabled ? 'not-allowed' : (saving ? 'wait' : 'pointer'), + flex: 1, + textAlign: 'center', + pointerEvents: logosDisabled ? 'none' : 'auto', + }} + > + {style.label} + + {style.label} + + + ))} + + + + + handlePosterToggle(e.currentTarget.checked)} + disabled={saving} + size="sm" + /> + + ); +}; + +// ─── SD Lineup Manager ───────────────────────────────────────────────────── const SDLineupManager = ({ sourceId }) => { const [countries, setCountries] = useState(SD_COUNTRIES_FALLBACK); const [activeLineups, setActiveLineups] = useState([]); @@ -175,7 +309,7 @@ const SDLineupManager = ({ sourceId }) => { const atMax = activeLineups.length >= maxLineups; return ( - + @@ -335,10 +469,12 @@ const SDLineupManager = ({ sourceId }) => { ); }; +// ─── Main EPG Form ────────────────────────────────────────────────────────── const EPG = ({ epg = null, isOpen, onClose }) => { const [sourceType, setSourceType] = useState('xmltv'); const [scheduleType, setScheduleType] = useState('interval'); const [savedEpgId, setSavedEpgId] = useState(null); + const [sdCustomProps, setSdCustomProps] = useState(null); const form = useForm({ mode: 'uncontrolled', @@ -372,22 +508,18 @@ const EPG = ({ epg = null, isOpen, onClose }) => { values.cron_expression = ''; } - if (epg?.id) { - if (!epg || typeof epg !== 'object' || !epg.id) { - showNotification({ - title: 'Error', - message: 'Invalid EPG data. Please close and reopen this form.', - color: 'red', - }); - return; - } + const existingId = epg?.id || savedEpgId; - await updateEPG(values, epg); + if (existingId) { + const epgObj = epg || { id: existingId }; + await updateEPG(values, epgObj); onClose(); } else { const result = await addEPG(values); if (result?.id) { setSavedEpgId(result.id); + // Load custom_properties for the new source + setSdCustomProps(result.custom_properties || {}); } else { form.reset(); onClose(); @@ -411,6 +543,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { form.setValues(values); setSourceType(epg.source_type); setSavedEpgId(epg.id); + setSdCustomProps(epg.custom_properties || {}); setScheduleType( epg.cron_expression && epg.cron_expression.trim() !== '' ? 'cron' @@ -421,6 +554,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { setSourceType('xmltv'); setScheduleType('interval'); setSavedEpgId(null); + setSdCustomProps(null); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [epg]); @@ -433,6 +567,9 @@ const EPG = ({ epg = null, isOpen, onClose }) => { const handleClose = () => { form.reset(); setSavedEpgId(null); + setSdCustomProps(null); + setSourceType('xmltv'); + setScheduleType('interval'); onClose(); }; @@ -441,14 +578,15 @@ const EPG = ({ epg = null, isOpen, onClose }) => { } const isSD = sourceType === 'schedules_direct'; + const hasSDPanel = isSD && savedEpgId; return ( <> - +
- + {/* Left Column */} - + { - {/* Right Column */} - + {/* Middle Column */} + {!isSD && ( { - - {/* Lineup Manager — only shown for saved SD sources */} - {isSD && savedEpgId && ( - - )} + {/* Right Column — SD settings + Lineup Manager */} + {hasSDPanel && ( + <> + + + + + + + + + )} + {/* Full Width Section */} @@ -623,7 +772,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { {isSD && !savedEpgId && ( - Save this source first to manage Schedules Direct lineups. + Save this source first to manage Schedules Direct settings and lineups. )} @@ -632,7 +781,7 @@ const EPG = ({ epg = null, isOpen, onClose }) => { Cancel From abdf2d7864d29a4e834c5c521522ad0144e129f1 Mon Sep 17 00:00:00 2001 From: Goldenfreddy0703 <62456796+Goldenfreddy0703@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:51:20 -0400 Subject: [PATCH 39/76] Fix credential release on deleted profile and round 3 review items. Store credential Redis keys at reserve so release works when the profile row is deleted. Return reserve failure reasons to avoid fingerprint DB queries on logging paths. Document unlimited profile bypass in pool logic and Server Group UI. Co-authored-by: Cursor --- apps/channels/models.py | 19 +++-- apps/m3u/connection_pool.py | 78 +++++++++++++++---- apps/m3u/models.py | 6 +- apps/m3u/tests/test_connection_pool.py | 47 +++++++++-- .../multi_worker_connection_manager.py | 4 +- frontend/src/components/forms/M3U.jsx | 2 +- frontend/src/components/forms/ServerGroup.jsx | 2 +- 7 files changed, 123 insertions(+), 35 deletions(-) diff --git a/apps/channels/models.py b/apps/channels/models.py index 154f6fa2..9742626b 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -234,7 +234,9 @@ class Stream(models.Model): continue # Atomic slot reservation via shared connection pool helper - reserved, _count = reserve_profile_slot(profile, redis_client) + reserved, _count, _failure_reason = reserve_profile_slot( + profile, redis_client + ) if reserved: redis_client.set(f"channel_stream:{self.id}", self.id) @@ -694,7 +696,9 @@ class Channel(models.Model): has_active_profiles = True # Atomically check and reserve a slot (INCR-first pattern) - reserved, current_count = reserve_profile_slot(profile, redis_client) + reserved, current_count, failure_reason = reserve_profile_slot( + profile, redis_client + ) if reserved: # Slot reserved — assign stream to this channel @@ -712,11 +716,6 @@ class Channel(models.Model): True, ) # Return newly assigned stream and matched profile else: - from apps.m3u.connection_pool import ( - group_has_capacity_for_profile, - profile_has_capacity_for_selection, - ) - # At capacity: try to preempt a lower-impact channel on this profile victim_channel_id = self._pick_channel_to_preempt( profile_id=profile.id, @@ -729,14 +728,14 @@ class Channel(models.Model): # return self.id, profile.id, victim_channel_id has_streams_but_maxed_out = True - if not profile_has_capacity_for_selection(profile, redis_client): + if failure_reason == "profile_full": logger.info( f"Profile {profile.id} at max connections: " f"{current_count}/{profile.max_streams}, trying next profile" ) - elif not group_has_capacity_for_profile(profile, redis_client): + elif failure_reason == "credential_full": logger.info( - f"Profile {profile.id} login or server group pool full, trying next profile" + f"Profile {profile.id} shared login pool full, trying next profile" ) else: logger.debug( diff --git a/apps/m3u/connection_pool.py b/apps/m3u/connection_pool.py index 35c35b0e..36a74d17 100644 --- a/apps/m3u/connection_pool.py +++ b/apps/m3u/connection_pool.py @@ -5,7 +5,8 @@ Profile selection rotates across M3UAccountProfile rows using each profile's own Redis counter (the pre-pool behavior). When an account belongs to a ServerGroup with max_streams > 0, a credential-scoped counter is checked on reserve/release so accounts sharing the same provider login share one limit without blocking -unrelated logins on the same group. +unrelated logins on the same group. Account profiles with max_streams=0 skip +credential enforcement for that profile. """ from __future__ import annotations @@ -13,11 +14,14 @@ from __future__ import annotations import hashlib import logging import re -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple logger = logging.getLogger(__name__) +ReserveFailureReason = Literal["profile_full", "credential_full"] + PROFILE_CONNECTIONS_KEY = "profile_connections:{profile_id}" +PROFILE_CREDENTIAL_RELEASE_KEY = "profile_credential_release:{profile_id}" SERVER_GROUP_CONNECTIONS_KEY = "server_group_connections:{group_id}:{fingerprint}" _XC_URL_CREDENTIALS_RE = re.compile( @@ -30,6 +34,11 @@ def profile_connections_key(profile_id: int) -> str: return PROFILE_CONNECTIONS_KEY.format(profile_id=profile_id) +def profile_credential_release_key(profile_id: int) -> str: + """Redis key storing the credential counter to release when the profile row is gone.""" + return PROFILE_CREDENTIAL_RELEASE_KEY.format(profile_id=profile_id) + + def server_group_connections_key(group_id: int, fingerprint: str) -> str: """Redis key for per-credential usage within a ServerGroup.""" return SERVER_GROUP_CONNECTIONS_KEY.format( @@ -161,6 +170,8 @@ def profile_has_capacity_for_selection(profile, redis_client) -> bool: def group_has_capacity_for_profile(profile, redis_client) -> bool: + # Profiles with max_streams=0 skip credential enforcement entirely. An unlimited + # profile in a pooled group can still stream while other accounts share the login. group = get_enforced_server_group_for_profile(profile) if not group or profile.max_streams == 0: return True @@ -186,21 +197,43 @@ def _safe_decr(redis_client, key: str) -> None: redis_client.set(key, 0) -def _reserve_server_group_slot_for_profile(profile, redis_client) -> bool: +def _remember_credential_release_key( + profile_id: int, cred_key: str, redis_client +) -> None: + redis_client.set(profile_credential_release_key(profile_id), cred_key) + + +def _release_credential_slot_by_profile_id(profile_id: int, redis_client) -> bool: + """Release a reserved credential counter using the key stored at reserve time.""" + release_key = profile_credential_release_key(profile_id) + cred_key = redis_client.get(release_key) + if not cred_key: + return False + + if isinstance(cred_key, bytes): + cred_key = cred_key.decode() + _safe_decr(redis_client, cred_key) + redis_client.delete(release_key) + return True + + +def _reserve_server_group_slot_for_profile( + profile, redis_client +) -> Tuple[bool, Optional[str]]: group = get_enforced_server_group_for_profile(profile) if not group or profile.max_streams == 0: - return True + return True, None cred_key = _credential_counter_key(profile, group) if not cred_key: - return True + return True, None cred_count = redis_client.incr(cred_key) if cred_count <= profile.max_streams: - return True + return True, cred_key redis_client.decr(cred_key) - return False + return False, None def _release_server_group_slot_for_profile(profile, redis_client) -> None: @@ -212,11 +245,14 @@ def _release_server_group_slot_for_profile(profile, redis_client) -> None: _safe_decr(redis_client, cred_key) -def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: +def reserve_profile_slot( + profile, redis_client +) -> Tuple[bool, int, Optional[ReserveFailureReason]]: """ Atomically reserve profile + optional credential slots (INCR-first). - Returns (reserved, profile_count_after_attempt). + Returns (reserved, profile_count_after_attempt, failure_reason). + failure_reason is set when reserved is False. """ profile_key = profile_connections_key(profile.id) profile_count = 0 @@ -225,20 +261,34 @@ def reserve_profile_slot(profile, redis_client) -> Tuple[bool, int]: profile_count = redis_client.incr(profile_key) if profile_count > profile.max_streams: redis_client.decr(profile_key) - return False, profile_count - 1 + return False, profile_count - 1, "profile_full" - if not _reserve_server_group_slot_for_profile(profile, redis_client): + cred_reserved, cred_key = _reserve_server_group_slot_for_profile( + profile, redis_client + ) + if not cred_reserved: if profile.max_streams > 0: redis_client.decr(profile_key) - return False, profile_count - 1 if profile.max_streams > 0 else 0 + return ( + False, + profile_count - 1 if profile.max_streams > 0 else 0, + "credential_full", + ) - return True, profile_count + if cred_key: + _remember_credential_release_key(profile.id, cred_key, redis_client) + + return True, profile_count, None def release_profile_slot(profile_id: int, redis_client) -> None: """Release profile and shared credential slots after a stream end.""" from apps.m3u.models import M3UAccountProfile + released_via_stored_key = _release_credential_slot_by_profile_id( + profile_id, redis_client + ) + try: profile = M3UAccountProfile.objects.get(id=profile_id) except M3UAccountProfile.DoesNotExist: @@ -250,5 +300,5 @@ def release_profile_slot(profile_id: int, redis_client) -> None: if current > 0: redis_client.decr(profile_key) - if profile: + if profile and not released_via_stored_key: _release_server_group_slot_for_profile(profile, redis_client) diff --git a/apps/m3u/models.py b/apps/m3u/models.py index e6ccaadb..46d1a068 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -201,7 +201,11 @@ class ServerGroup(models.Model): ) max_streams = models.PositiveIntegerField( default=0, - help_text="Maximum number of concurrent streams shared across all accounts in this group (0 for unlimited)", + help_text=( + "Set above 0 to enable shared credential pooling for accounts in this group. " + "Per-login limits come from each account profile's max_streams. " + "Profiles with max_streams=0 (unlimited) skip cross-account credential enforcement." + ), ) def __str__(self): diff --git a/apps/m3u/tests/test_connection_pool.py b/apps/m3u/tests/test_connection_pool.py index 1dbfc2a9..e5de6c10 100644 --- a/apps/m3u/tests/test_connection_pool.py +++ b/apps/m3u/tests/test_connection_pool.py @@ -14,6 +14,7 @@ from apps.m3u.connection_pool import ( pool_has_capacity_for_profile, profile_has_capacity_for_selection, profile_connections_key, + profile_credential_release_key, release_profile_slot, reserve_profile_slot, server_group_connections_key, @@ -31,10 +32,15 @@ class FakeRedis: val = self._data.get(key) if val is None: return None + if isinstance(val, str): + return val.encode() return str(val).encode() def set(self, key, value, ex=None): - self._data[key] = int(value) + try: + self._data[key] = int(value) + except (ValueError, TypeError): + self._data[key] = value def incr(self, key): self._data[key] = self._data.get(key, 0) + 1 @@ -44,6 +50,9 @@ class FakeRedis: self._data[key] = self._data.get(key, 0) - 1 return self._data[key] + def delete(self, key): + self._data.pop(key, None) + def pipeline(self): return FakeRedisPipeline(self) @@ -125,10 +134,10 @@ class ManualServerGroupTests(TestCase): profile2.save() redis = FakeRedis() - reserved1, _ = reserve_profile_slot(profile1, redis) + reserved1, _, _ = reserve_profile_slot(profile1, redis) self.assertTrue(reserved1) - reserved2, _ = reserve_profile_slot(profile2, redis) + reserved2, _, _ = reserve_profile_slot(profile2, redis) self.assertFalse(reserved2) self.assertFalse(group_has_capacity_for_profile(profile2, redis)) @@ -153,7 +162,7 @@ class ManualServerGroupTests(TestCase): ) redis = FakeRedis() - reserved, _ = reserve_profile_slot(default, redis) + reserved, _, _ = reserve_profile_slot(default, redis) self.assertTrue(reserved) self.assertFalse(profile_has_capacity_for_selection(default, redis)) self.assertTrue(profile_has_capacity_for_selection(alt, redis)) @@ -182,7 +191,7 @@ class PoolEnforcementTests(TestCase): self.profile.save() def test_reserve_and_release_both_counters(self): - reserved, count = reserve_profile_slot(self.profile, self.redis) + reserved, count, _ = reserve_profile_slot(self.profile, self.redis) self.assertTrue(reserved) self.assertEqual(count, 1) @@ -286,13 +295,37 @@ class PoolEnforcementTests(TestCase): fp = get_profile_credential_fingerprint(self.profile) cred_key = server_group_connections_key(self.group.id, fp) - self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + reserved, _, failure_reason = reserve_profile_slot( + self.profile, self.redis + ) + self.assertTrue(reserved) + self.assertIsNone(failure_reason) self.profile.delete() release_profile_slot(profile_id, self.redis) self.assertEqual(self.redis._data[profile_connections_key(profile_id)], 0) - self.assertEqual(self.redis._data[cred_key], 1) + self.assertEqual(self.redis._data[cred_key], 0) + self.assertNotIn(profile_credential_release_key(profile_id), self.redis._data) + + def test_reserve_returns_failure_reason_without_extra_checks(self): + account2 = M3UAccount.objects.create( + name="Reason Account", + account_type="XC", + username="user", + password="pass", + server_url="http://xc.example.com", + server_group=self.group, + max_streams=5, + ) + profile2 = M3UAccountProfile.objects.get(m3u_account=account2, is_default=True) + profile2.max_streams = 1 + profile2.save() + + self.assertTrue(reserve_profile_slot(self.profile, self.redis)[0]) + reserved, _count, reason = reserve_profile_slot(profile2, self.redis) + self.assertFalse(reserved) + self.assertEqual(reason, "credential_full") class UpdateStreamProfileTests(TestCase): diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index 77db3b1b..8d803dcf 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -745,7 +745,9 @@ class MultiWorkerVODConnectionManager: from apps.m3u.connection_pool import reserve_profile_slot try: - reserved, new_count = reserve_profile_slot(m3u_profile, self.redis_client) + reserved, new_count, _failure_reason = reserve_profile_slot( + m3u_profile, self.redis_client + ) if reserved: logger.info( f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: " diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 3a4161cb..6e19c93a 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -365,7 +365,7 @@ const M3U = ({ id="server_group" name="server_group" label="Server Group" - description="Share a connection limit across multiple accounts (e.g. XC + M3U on the same subscription)" + description="Share login limits across accounts in a server group. Set max streams on each profile (unlimited profiles skip group enforcement)." {...form.getInputProps('server_group')} key={form.key('server_group')} data={[{ value: '0', label: '(None)' }].concat( diff --git a/frontend/src/components/forms/ServerGroup.jsx b/frontend/src/components/forms/ServerGroup.jsx index 229479ea..a01a0a76 100644 --- a/frontend/src/components/forms/ServerGroup.jsx +++ b/frontend/src/components/forms/ServerGroup.jsx @@ -70,7 +70,7 @@ const ServerGroupForm = ({ serverGroup = null, isOpen, onClose }) => { setValue('max_streams', value ?? 0)} From aa9e78f85834e35b9df4c12edc23f53ef67036de Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 00:12:17 +0000 Subject: [PATCH 40/76] fix: allow unauthenticated access to SD poster proxy endpoint ProgramViewSet.get_permissions() was falling through to Authenticated() for the poster action since it wasn't in permission_classes_by_action. Add explicit AllowAny check for the poster action. --- apps/epg/api_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index fd83f2de..6961b79e 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -471,6 +471,8 @@ class ProgramViewSet(viewsets.ModelViewSet): _sd_poster_error_cache: dict = {} def get_permissions(self): + if self.action == 'poster': + return [AllowAny()] try: return [perm() for perm in permission_classes_by_action[self.action]] except KeyError: From 4ae0796b3d8b1bfc003fcf3471e240841cfb5f0a Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 00:28:50 +0000 Subject: [PATCH 41/76] fix: add Switch mock to EPG test for SD logo toggle Frontend test was failing because the Mantine core mock didn't include the Switch component used by the SD logo style picker. --- frontend/src/components/forms/__tests__/EPG.test.jsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index bd8ad1cd..1121f865 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -223,6 +223,18 @@ vi.mock('@mantine/core', async () => ({ ScrollArea: ({ children }) =>
{children}
, Table: ({ children }) => {children}
, Tooltip: ({ children }) =>
{children}
, + Switch: ({ label, checked, onChange, disabled, description }) => ( + + ), })); // ── Imports after mocks ──────────────────────────────────────────────────────── From 6b5c7459622dead1b8ddbcade0ea60a6669d55ac Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 00:57:19 +0000 Subject: [PATCH 42/76] fix: add remaining Mantine component mocks to EPG test Add UnstyledButton, Alert, Stack, Text, TextInput mocks to prevent test failures from unmocked Mantine components in SD settings UI. --- frontend/src/components/forms/__tests__/EPG.test.jsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index 1121f865..b798bb4a 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -235,6 +235,17 @@ vi.mock('@mantine/core', async () => ({ {label} ), + UnstyledButton: ({ children, onClick, ...props }) => ( + + ), + Alert: ({ children, title, color, icon }) => ( +
{title}{children}
+ ), + Stack: ({ children, gap }) =>
{children}
, + Text: ({ children, ...props }) => {children}, + TextInput: ({ label, value, onChange, placeholder, ...props }) => ( + + ), })); // ── Imports after mocks ──────────────────────────────────────────────────────── From c0b7cc5fe83febb92986f2c3db2cc8f8bcfabba2 Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 01:19:53 +0000 Subject: [PATCH 43/76] fix: add updateSDSettings to API mock in EPG test --- frontend/src/components/forms/__tests__/EPG.test.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index b798bb4a..0ad578ef 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -12,6 +12,7 @@ vi.mock('../../../api.js', () => ({ addSDLineup: vi.fn().mockResolvedValue({ success: true }), deleteSDLineup: vi.fn().mockResolvedValue({ success: true }), searchSDLineups: vi.fn().mockResolvedValue([]), + updateSDSettings: vi.fn().mockResolvedValue({}), }, })); From e731527104278b3d3b3fa6429718d5f094fa9271 Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 01:26:51 +0000 Subject: [PATCH 44/76] fix: add data-testid to TextInput mock in EPG test --- frontend/src/components/forms/__tests__/EPG.test.jsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/forms/__tests__/EPG.test.jsx b/frontend/src/components/forms/__tests__/EPG.test.jsx index 0ad578ef..1e64a563 100644 --- a/frontend/src/components/forms/__tests__/EPG.test.jsx +++ b/frontend/src/components/forms/__tests__/EPG.test.jsx @@ -245,7 +245,14 @@ vi.mock('@mantine/core', async () => ({ Stack: ({ children, gap }) =>
{children}
, Text: ({ children, ...props }) => {children}, TextInput: ({ label, value, onChange, placeholder, ...props }) => ( - + ), })); From 38359540241b9c0927c9f5e82384a69d30e1c5d7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 4 Jun 2026 21:30:05 -0500 Subject: [PATCH 45/76] fix: enhance API request headers and improve cast handling in schedule fetching Add User-Agent header to API requests in ProgramViewSet for better tracking. Update cast processing logic in fetch_schedules_direct to align with XMLTV standards, ensuring proper categorization of main cast and guest stars. --- apps/epg/api_views.py | 6 +++++- apps/epg/tasks.py | 30 ++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 6961b79e..547be152 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -516,10 +516,14 @@ class ProgramViewSet(viewsets.ModelViewSet): if not token: sha1_password = hashlib.sha1(source.password.encode('utf-8')).hexdigest() try: + from version import __version__ as dispatcharr_version auth_resp = http_requests.post( f"{SD_BASE_URL}/token", json={'username': source.username, 'password': sha1_password}, - headers={'Content-Type': 'application/json'}, + headers={ + 'Content-Type': 'application/json', + 'User-Agent': f'Dispatcharr/{dispatcharr_version}', + }, timeout=10, ) auth_data = auth_resp.json() diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 442c0474..f9230e6b 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2799,7 +2799,12 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if categories: custom_props['categories'] = categories - # Cast — top-billed only, store characterName, drop role noise + # Cast — top-billed only. SD's 'role' field = job type (Actor/Guest Star); + # SD's 'characterName' = the character played. We store characterName under + # the key 'role' to match the XMLTV parser convention + # + # Guest stars are stored with guest=True so the XMLTV generator emits + # per the XMLTV DTD standard. cast = meta.get('cast', []) crew = meta.get('crew', []) credits = {} @@ -2809,16 +2814,29 @@ def fetch_schedules_direct(source, stations_only=False, force=False): [p for p in cast if p.get('name')], key=lambda p: int(p.get('billingOrder', '999')) ) - # Separate main cast from guest stars + # Separate regular cast from guest stars (SD 'role' = job type here) main_cast = [p for p in sorted_cast if p.get('role', '').lower() != 'guest star'] - # Store top-billed main cast (matching XMLTV parity) - credits['actor'] = [ + guest_stars = [p for p in sorted_cast if p.get('role', '').lower() == 'guest star'] + # Use main cast if available, otherwise fall back to full sorted list + primary = main_cast[:6] if main_cast else sorted_cast[:6] + actors = [ { 'name': p.get('name', ''), - **(({'character': p['characterName']} ) if p.get('characterName') else {}), + **(({'role': p['characterName']}) if p.get('characterName') else {}), } - for p in (main_cast[:6] if main_cast else sorted_cast[:6]) + for p in primary ] + # Append notable guest stars with XMLTV guest="yes" marker (cap at 3) + actors += [ + { + 'name': p.get('name', ''), + **(({'role': p['characterName']}) if p.get('characterName') else {}), + 'guest': True, + } + for p in guest_stars[:3] + ] + if actors: + credits['actor'] = actors if crew: for member in crew: role = member.get('role', '').lower() From c4d73eae3983e767a5d5f696aea67a62554a73dc Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 4 Jun 2026 22:00:15 -0500 Subject: [PATCH 46/76] fix(epg): repair SD poster display and backfill missing artwork Build absolute poster proxy URLs via build_absolute_uri_with_port so program detail images resolve correctly in dev and behind reverse proxies. Backfill sd_icon for programs when poster fetch is enabled but a delta refresh skips program metadata. Add a dev-mode frontend fallback for relative image paths and correct the poster caching description in the SD settings UI. --- apps/epg/serializers.py | 11 +++++++--- apps/epg/tasks.py | 22 ++++++++++++++++--- .../src/components/ProgramDetailModal.jsx | 14 +++++++++--- frontend/src/components/forms/EPG.jsx | 2 +- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index c196c4db..ed99efb4 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -1,4 +1,4 @@ -from core.utils import validate_flexible_url +from core.utils import validate_flexible_url, build_absolute_uri_with_port from rest_framework import serializers from .models import EPGSource, EPGData, ProgramData from apps.channels.models import Channel, Stream @@ -169,9 +169,14 @@ class ProgramDetailSerializer(ProgramDataSerializer): data['icon'] = cp.get('icon') data['images'] = cp.get('images') or [] - # SD poster: expose as proxy URL so frontend never needs SD auth + # SD poster: expose as absolute proxy URL so frontend/img tags never need SD auth if cp.get('sd_icon'): - data['poster_url'] = f"/api/epg/programs/{obj.id}/poster/" + poster_path = f"/api/epg/programs/{obj.id}/poster/" + request = self.context.get('request') + if request: + data['poster_url'] = build_absolute_uri_with_port(request, poster_path) + else: + data['poster_url'] = poster_path else: data['poster_url'] = None diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index f9230e6b..fe1b1e7c 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2995,7 +2995,23 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # Step 8: Fetch program artwork (posters) if enabled # ------------------------------------------------------------------------- fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) - if fetch_posters and program_metadata: + poster_program_ids = set(program_metadata.keys()) if program_metadata else set() + if fetch_posters and not poster_program_ids: + # Backfill when posters were enabled after initial fetch, or a delta + # refresh skipped program metadata but programs still lack sd_icon. + poster_program_ids = set( + ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).exclude(custom_properties__has_key='sd_icon') + .values_list('program_id', flat=True) + ) + if poster_program_ids: + logger.info( + f"Poster backfill: {len(poster_program_ids)} programs missing sd_icon." + ) + + if fetch_posters and poster_program_ids: logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") _sd_send_ws_sync(source.id, "parsing_programs", 98, message="Fetching program artwork...") @@ -3007,7 +3023,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): artwork_lookup_ids = set() pid_to_artwork_key = {} # maps original programID -> the key we looked up - for pid in program_metadata: + for pid in poster_program_ids: if pid.startswith('EP'): sh_root = 'SH' + pid[2:10] + '0000' artwork_lookup_ids.add(sh_root) @@ -3116,7 +3132,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) elif fetch_posters: - logger.info("Poster fetch enabled but no new program metadata downloaded — skipping artwork.") + logger.info("Poster fetch enabled but all mapped programs already have artwork.") # ------------------------------------------------------------------------- # Step 9: Apply SD station logos to matched channels if enabled diff --git a/frontend/src/components/ProgramDetailModal.jsx b/frontend/src/components/ProgramDetailModal.jsx index d3d9c5e5..fb715269 100644 --- a/frontend/src/components/ProgramDetailModal.jsx +++ b/frontend/src/components/ProgramDetailModal.jsx @@ -40,11 +40,19 @@ function formatDurationMinutes(startTime, endTime) { return `${minutes}m`; } +function resolveApiUrl(url) { + if (!url || url.startsWith('http')) return url; + const apiHost = import.meta.env.DEV + ? `http://${window.location.hostname}:5656` + : ''; + return `${apiHost}${url}`; +} + function resolveImageUrl(detail) { if (detail?.tmdb_poster_url) return detail.tmdb_poster_url; - if (detail?.poster_url) return detail.poster_url; - if (detail?.images?.length > 0) return detail.images[0].url; - if (detail?.icon) return detail.icon; + if (detail?.poster_url) return resolveApiUrl(detail.poster_url); + if (detail?.images?.length > 0) return resolveApiUrl(detail.images[0].url); + if (detail?.icon) return resolveApiUrl(detail.icon); return null; } diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 6215e40d..0273c43c 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -168,7 +168,7 @@ const SDSettings = ({ sourceId, customProperties }) => { handlePosterToggle(e.currentTarget.checked)} disabled={saving} From 8ed003022356a4288eec3cd44af169565ac24f2c Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 03:37:50 +0000 Subject: [PATCH 47/76] refactor: always enable logo style picker per reviewer feedback Logo style selection should always be available so users can set their preference before enabling SD logos. --- frontend/src/components/forms/EPG.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 0273c43c..b2ea9e1e 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -103,7 +103,7 @@ const SDSettings = ({ sourceId, customProperties }) => { saveSetting('fetch_posters', checked); }; - const logosDisabled = !useSDLogos; + const logosDisabled = false; return ( From cae040d45d61cef653ebfeed1821dd471e240369 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 5 Jun 2026 00:01:22 -0500 Subject: [PATCH 48/76] fix(channels): make compact numbering repack idempotent The compact repack read its channels with no ORDER BY, so the pack followed PostgreSQL's physical row order. That order drifts after the UPDATEs each repack issues, so successive syncs packed the same channels into different numbers within the configured range. Auto-synced channel numbers reshuffled on every sync even when the provider had not changed. - Add .order_by("id") to the _repack_inner channel query so the pack is deterministic. id order is creation order, which tracks the provider stream order used by the default "provider" sort. - Add c.id as a secondary key to the name / tvg_id / updated_at sorts so equal values (e.g. blank tvg_id) break ties deterministically instead of churning. - Add a deterministic regression test that forces a physical heap reorder (CLUSTER) and asserts two consecutive repacks produce identical channel numbers. --- apps/channels/compact_numbering.py | 18 ++-- apps/m3u/tests/test_sync_correctness.py | 107 +++++++++++++++++++++++- 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/apps/channels/compact_numbering.py b/apps/channels/compact_numbering.py index 8cb62f53..fd29a153 100644 --- a/apps/channels/compact_numbering.py +++ b/apps/channels/compact_numbering.py @@ -324,12 +324,17 @@ def _repack_inner(group_relation): # into the SAME target group, their channels are indistinguishable # here (channels carry no source-group back-reference), so each repack # renumbers the shared target's channels into its own range. + # order_by("id") makes the pack deterministic. Without it the query + # returns rows in unspecified physical order, which shifts after the + # renumber's own UPDATEs and autovacuum, so the default "provider" sort + # below would repack channels into different numbers on every sync. + # id order is creation order, which tracks the provider stream order. channels = list( Channel.objects.filter( auto_created=True, auto_created_by_id=account_id, channel_group_id__in=group_ids, - ).select_related("override") + ).select_related("override").order_by("id") ) visible = [] @@ -344,21 +349,22 @@ def _repack_inner(group_relation): visible.append(ch) # Sort the visible set by the group's configured channel_sort_order. - # Provider order (the default) preserves DB-iteration order which is - # roughly creation order; treat unrecognized values the same way. + # Provider order (the default) keeps the id order from the query above. + # Each explicit sort carries c.id as a secondary key so equal values + # (e.g. blank tvg_id) break ties deterministically instead of churning. if sort_order == "name": visible.sort( - key=lambda c: natural_sort_key(c.name or ""), + key=lambda c: (natural_sort_key(c.name or ""), c.id), reverse=sort_reverse, ) elif sort_order == "tvg_id": visible.sort( - key=lambda c: c.tvg_id or "", + key=lambda c: (c.tvg_id or "", c.id), reverse=sort_reverse, ) elif sort_order == "updated_at": visible.sort( - key=lambda c: c.updated_at, + key=lambda c: (c.updated_at, c.id), reverse=sort_reverse, ) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 509171c9..9f8b87a2 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -6,7 +6,10 @@ first (fails on HEAD prior to the Tier 2 patch), then is flipped to assert the correct post-fix behavior. Comments call out the failure mode and the fix location. """ -from django.test import TestCase +from unittest import skipUnless + +from django.db import connection +from django.test import TestCase, TransactionTestCase from django.utils import timezone from apps.channels.models import ( @@ -2195,3 +2198,105 @@ class CompactNumberingWithGroupOverrideTests(TestCase): large, f"repack query count scaled with channel count: {small} -> {large}", ) + + +@skipUnless( + connection.vendor == "postgresql", + "Idempotency repro forces a physical heap reorder via CLUSTER, which is " + "PostgreSQL-specific (the suite's target DB).", +) +class CompactNumberingIdempotencyTests(TransactionTestCase): + """ + A compact repack must be idempotent: with no change to hide state or + overrides, repacking again must leave every channel on the same number. + + The unpatched _repack_inner read its channels with no ORDER BY, so the + pack followed PostgreSQL's physical row order. That order drifts after + the UPDATEs each repack issues (and after autovacuum), so successive + syncs packed the same channels into different numbers. That is the daily + channel-number churn users reported. + + This test forces the divergence deterministically. After the first pack + it rewrites every channel_number to the reverse of id order, then + physically clusters the table on that column so the heap order becomes + the reverse of id order. An unordered SELECT then returns the rows in the + opposite order from the first pass. Unpatched, the second pack assigns + numbers in that reversed order and the channel->number mapping flips; + patched, .order_by("id") keeps both packs identical. + + Fail signature: channel->number mapping differs between the two repacks + = _repack_inner is following physical row order instead of id order. + + Fix location: apps/channels/compact_numbering.py (_repack_inner channel + query .order_by("id")). + """ + + # TransactionTestCase commits its rows (TestCase's savepoint rollback + # would hide them from CLUSTER, which also cannot run inside the + # transaction block TestCase wraps each test in). + + def _mapping(self, account): + return { + c.id: c.channel_number + for c in Channel.objects.filter( + auto_created=True, auto_created_by=account + ) + } + + def test_repack_is_idempotent_under_physical_reorder(self): + from apps.channels.compact_numbering import repack_group + + account = _make_account() + group = _make_group(name="Sports") + rel = _attach_group_to_account( + account, group, custom_properties={"compact_numbering": True} + ) + rel.auto_sync_channel_start = 8000 + rel.auto_sync_channel_end = 8099 + rel.save() + + # Eight visible auto channels; ascending id is creation order. + channels = [ + Channel.objects.create( + name=f"C{i}", + channel_group=group, + auto_created=True, + auto_created_by=account, + ) + for i in range(8) + ] + + repack_group(rel) + first = self._mapping(account) + # Provider-order pack (the default) assigns by id, so the lowest id + # takes the range start. + lowest_id = min(c.id for c in channels) + self.assertEqual(first[lowest_id], 8000) + + # Set channel_number to the reverse of id order, then cluster the + # heap on that column so physical order becomes reverse-id order. + # Values sit above the range so they cannot collide with the pack. + table = Channel._meta.db_table + with connection.cursor() as cur: + for pos, ch in enumerate(channels): + cur.execute( + f"UPDATE {table} SET channel_number = %s WHERE id = %s", + [9000 - pos, ch.id], + ) + cur.execute( + f"CREATE INDEX IF NOT EXISTS churn_cn_idx " + f"ON {table} (channel_number)" + ) + cur.execute(f"CLUSTER {table} USING churn_cn_idx") + cur.execute("DROP INDEX IF EXISTS churn_cn_idx") + + repack_group(rel) + second = self._mapping(account) + + self.assertEqual( + first, + second, + "Repack is not idempotent: channel numbers changed on a second " + "pass with no hide or override change. _repack_inner is following " + "physical row order instead of id order.", + ) From b6a5738f7d17c682e66a4aec65f722349ea790a2 Mon Sep 17 00:00:00 2001 From: Shokkstokk Date: Fri, 5 Jun 2026 07:33:03 +0000 Subject: [PATCH 49/76] refactor: remove useSDLogos guard from logo style handler Allow logo style selection to save regardless of toggle state. --- frontend/src/components/forms/EPG.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index b2ea9e1e..b94fadcd 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -93,7 +93,6 @@ const SDSettings = ({ sourceId, customProperties }) => { }; const handleLogoChange = (style) => { - if (!useSDLogos) return; setLogoStyle(style); saveSetting('logo_style', style); }; From 4eab95793e6bf23e6d7f6870bc3e3ffb1205352e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 5 Jun 2026 09:08:26 -0500 Subject: [PATCH 50/76] refactor: rename useSDLogos to autoApplySDLogos for clarity Update the variable name in the EPG settings to better reflect its purpose. Adjust related logic in the UI to ensure consistent handling of the auto-apply feature for SD logos. --- apps/epg/tasks.py | 4 +- frontend/src/components/forms/EPG.jsx | 352 +++++++++++++------ frontend/src/components/tables/EPGsTable.jsx | 3 +- 3 files changed, 241 insertions(+), 118 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index fe1b1e7c..8d2bb96e 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -3137,8 +3137,8 @@ def fetch_schedules_direct(source, stations_only=False, force=False): # ------------------------------------------------------------------------- # Step 9: Apply SD station logos to matched channels if enabled # ------------------------------------------------------------------------- - use_sd_logos = (source.custom_properties or {}).get('use_sd_logos', False) - if use_sd_logos: + auto_apply_sd_logos = (source.custom_properties or {}).get('auto_apply_sd_logos', False) + if auto_apply_sd_logos: try: from apps.channels.models import Channel as ChannelModel, Logo diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index b94fadcd..409307e9 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -27,6 +27,7 @@ import ScheduleInput from './ScheduleInput'; import { addEPG, updateEPG } from '../../utils/forms/DummyEpgUtils.js'; import { showNotification } from '../../utils/notificationUtils.js'; import API from '../../api.js'; +import useEPGsStore from '../../store/epgs'; // Countries are fetched dynamically from the SD API on component mount. // Fallback list used if the API call fails. @@ -49,34 +50,52 @@ const SD_COUNTRIES_FALLBACK = [ // ESPN HD logo previews — packaged as static URLs so no API call is needed. // These are publicly accessible S3 URLs that don't require authentication. -const SD_LOGO_PREVIEW_BASE = 'https://schedulesdirect-api20141201-logos.s3.dualstack.us-east-1.amazonaws.com/stationLogos/s32645'; +const SD_LOGO_PREVIEW_BASE = + 'https://schedulesdirect-api20141201-logos.s3.dualstack.us-east-1.amazonaws.com/stationLogos/s32645'; const SD_LOGO_STYLES = [ - { value: 'dark', label: 'Dark', url: `${SD_LOGO_PREVIEW_BASE}_dark_360w_270h.png` }, - { value: 'white', label: 'White', url: `${SD_LOGO_PREVIEW_BASE}_white_360w_270h.png` }, - { value: 'gray', label: 'Gray', url: `${SD_LOGO_PREVIEW_BASE}_gray_360w_270h.png` }, - { value: 'light', label: 'Light', url: `${SD_LOGO_PREVIEW_BASE}_light_360w_270h.png` }, + { + value: 'dark', + label: 'Dark', + url: `${SD_LOGO_PREVIEW_BASE}_dark_360w_270h.png`, + }, + { + value: 'white', + label: 'White', + url: `${SD_LOGO_PREVIEW_BASE}_white_360w_270h.png`, + }, + { + value: 'gray', + label: 'Gray', + url: `${SD_LOGO_PREVIEW_BASE}_gray_360w_270h.png`, + }, + { + value: 'light', + label: 'Light', + url: `${SD_LOGO_PREVIEW_BASE}_light_360w_270h.png`, + }, ]; // ─── SD Settings: Logo toggle + style selector + Poster toggle ────────────── const SDSettings = ({ sourceId, customProperties }) => { - const cp = customProperties || {}; - const [useSDLogos, setUseSDLogos] = useState(cp.use_sd_logos || false); - const [logoStyle, setLogoStyle] = useState(cp.logo_style || 'dark'); - const [fetchPosters, setFetchPosters] = useState(cp.fetch_posters || false); + const storeCustomProps = useEPGsStore((s) => + sourceId ? s.epgs[sourceId]?.custom_properties : null + ); + const resolvedCp = storeCustomProps || customProperties || {}; + + const [autoApplySDLogos, setAutoApplySDLogos] = useState( + !!resolvedCp.auto_apply_sd_logos + ); + const [logoStyle, setLogoStyle] = useState(resolvedCp.logo_style || 'dark'); + const [fetchPosters, setFetchPosters] = useState(!!resolvedCp.fetch_posters); const [saving, setSaving] = useState(false); - // Sync from parent when customProperties changes + // Sync from store (preferred) or parent props when the form opens or settings save useEffect(() => { - const newCp = customProperties || {}; - setUseSDLogos(newCp.use_sd_logos || false); + const newCp = storeCustomProps || customProperties || {}; + setAutoApplySDLogos(!!newCp.auto_apply_sd_logos); setLogoStyle(newCp.logo_style || 'dark'); - setFetchPosters(newCp.fetch_posters || false); - - // Persist the default logo_style if not already set - if (sourceId && !newCp.logo_style) { - API.updateSDSettings(sourceId, { logo_style: 'dark' }); - } - }, [customProperties, sourceId]); + setFetchPosters(!!newCp.fetch_posters); + }, [storeCustomProps, customProperties]); const saveSetting = async (key, value) => { setSaving(true); @@ -87,9 +106,9 @@ const SDSettings = ({ sourceId, customProperties }) => { } }; - const handleLogoToggle = (checked) => { - setUseSDLogos(checked); - saveSetting('use_sd_logos', checked); + const handleAutoApplyToggle = (checked) => { + setAutoApplySDLogos(checked); + saveSetting('auto_apply_sd_logos', checked); }; const handleLogoChange = (style) => { @@ -102,25 +121,14 @@ const SDSettings = ({ sourceId, customProperties }) => { saveSetting('fetch_posters', checked); }; - const logosDisabled = false; - return ( - handleLogoToggle(e.currentTarget.checked)} - disabled={saving} - size="sm" - mb="sm" - /> - - + Station Logo Style - Choose which logo variant to use for SD stations. + Choose which logo variant to store in EPG data. Available for Set Logo + from EPG even when auto-apply is disabled. {SD_LOGO_STYLES.map((style) => ( @@ -128,16 +136,17 @@ const SDSettings = ({ sourceId, customProperties }) => { key={style.value} onClick={() => handleLogoChange(style.value)} style={{ - border: !logosDisabled && logoStyle === style.value - ? '2px solid var(--mantine-color-blue-5)' - : '2px solid var(--mantine-color-default-border)', + border: + logoStyle === style.value + ? '2px solid var(--mantine-color-blue-5)' + : '2px solid var(--mantine-color-default-border)', borderRadius: 'var(--mantine-radius-sm)', padding: 3, - opacity: logosDisabled ? 0.3 : (saving ? 0.6 : 1), - cursor: logosDisabled ? 'not-allowed' : (saving ? 'wait' : 'pointer'), + opacity: saving ? 0.6 : 1, + cursor: saving ? 'wait' : 'pointer', flex: 1, textAlign: 'center', - pointerEvents: logosDisabled ? 'none' : 'auto', + pointerEvents: saving ? 'none' : 'auto', }} > { height: 50, objectFit: 'contain', display: 'block', - backgroundColor: style.value === 'white' || style.value === 'light' - ? '#333' - : 'transparent', + backgroundColor: + style.value === 'white' || style.value === 'light' + ? '#333' + : 'transparent', borderRadius: 2, - filter: logosDisabled ? 'grayscale(100%)' : 'none', }} /> - + {style.label} ))} + handleAutoApplyToggle(e.currentTarget.checked)} + disabled={saving} + size="sm" + mb="sm" + /> + { const [addingLineup, setAddingLineup] = useState(null); const [removingLineup, setRemovingLineup] = useState(null); const maxLineups = 4; - const SD_DOCS_URL = 'https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform'; + const SD_DOCS_URL = + 'https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#tasks-your-client-must-perform'; const fetchActiveLineups = useCallback(async () => { setLoadingLineups(true); @@ -219,12 +243,12 @@ const SDLineupManager = ({ sourceId }) => { fetchActiveLineups(); // Fetch country list from SD API per their recommendation to not hardcode fetch('https://json.schedulesdirect.org/20141201/available/countries') - .then(r => r.json()) - .then(data => { + .then((r) => r.json()) + .then((data) => { const all = Object.values(data).flat(); const mapped = all - .filter(c => c.shortName && c.fullName) - .map(c => ({ value: c.shortName, label: c.fullName })) + .filter((c) => c.shortName && c.fullName) + .map((c) => ({ value: c.shortName, label: c.fullName })) .sort((a, b) => a.label.localeCompare(b.label)); if (mapped.length > 0) setCountries(mapped); }) @@ -237,7 +261,11 @@ const SDLineupManager = ({ sourceId }) => { setSearching(true); setSearchResults([]); try { - const data = await API.searchSDLineups(sourceId, country, postalCode.trim()); + const data = await API.searchSDLineups( + sourceId, + country, + postalCode.trim() + ); if (data) { setSearchResults(data.lineups || []); } @@ -253,7 +281,10 @@ const SDLineupManager = ({ sourceId }) => { if (!result) return; // Update changesRemaining from response - if (result.changes_remaining !== undefined && result.changes_remaining !== null) { + if ( + result.changes_remaining !== undefined && + result.changes_remaining !== null + ) { setChangesRemaining(result.changes_remaining); } @@ -264,22 +295,38 @@ const SDLineupManager = ({ sourceId }) => { } if (result.error === 'duplicate_lineup') { - showNotification({ title: 'Already added', message: result.message, color: 'yellow' }); + showNotification({ + title: 'Already added', + message: result.message, + color: 'yellow', + }); return; } if (result.error === 'max_lineups_reached') { - showNotification({ title: 'Maximum lineups reached', message: result.message, color: 'orange' }); + showNotification({ + title: 'Maximum lineups reached', + message: result.message, + color: 'orange', + }); return; } if (result.error) { - showNotification({ title: 'Unable to add lineup', message: result.message, color: 'red' }); + showNotification({ + title: 'Unable to add lineup', + message: result.message, + color: 'red', + }); return; } if (result.code === 0) { - showNotification({ title: 'Lineup added', message: lineup.name, color: 'green' }); + showNotification({ + title: 'Lineup added', + message: lineup.name, + color: 'green', + }); await fetchActiveLineups(); } } finally { @@ -292,8 +339,15 @@ const SDLineupManager = ({ sourceId }) => { try { const result = await API.deleteSDLineup(sourceId, lineup.lineup); if (result && result.code === 0) { - showNotification({ title: 'Lineup removed', message: lineup.name, color: 'blue' }); - if (result.changes_remaining !== undefined && result.changes_remaining !== null) { + showNotification({ + title: 'Lineup removed', + message: lineup.name, + color: 'blue', + }); + if ( + result.changes_remaining !== undefined && + result.changes_remaining !== null + ) { setChangesRemaining(result.changes_remaining); } await fetchActiveLineups(); @@ -344,7 +398,9 @@ const SDLineupManager = ({ sourceId }) => { }} > - {lineup.name} + + {lineup.name} + {lineup.transport} · {lineup.location} · {lineup.lineup} @@ -370,33 +426,67 @@ const SDLineupManager = ({ sourceId }) => { {atMax && ( - }> + } + > Maximum of {maxLineups} lineups reached. Remove one to add another. )} {changesRemaining === 0 && ( - }> - You have reached your daily Schedules Direct lineup addition limit. SD allows 6 adds per 24-hour period.{' '} + } + > + You have reached your daily Schedules Direct lineup addition limit. SD + allows 6 adds per 24-hour period.{' '} {changesResetAt && ( - Limit resets at {new Date(changesResetAt).toUTCString()}. + + Limit resets at{' '} + {new Date(changesResetAt).toUTCString()}.{' '} + )} - {!changesResetAt && Limit resets 24 hours after the first add of the day. } - Learn more + {!changesResetAt && ( + Limit resets 24 hours after the first add of the day. + )} + + Learn more + )} {changesRemaining === 1 && ( - }> - You have 1 lineup addition remaining today. Use it carefully — Schedules Direct limits adds to 6 per 24-hour period.{' '} - Learn more + } + > + You have 1 lineup addition remaining today. Use it + carefully — Schedules Direct limits adds to 6 per 24-hour period.{' '} + + Learn more + )} {changesRemaining === 2 && ( - }> - You have 2 lineup additions remaining today. Schedules Direct limits adds to 6 per 24-hour period.{' '} - Learn more + } + > + You have 2 lineup additions remaining today. + Schedules Direct limits adds to 6 per 24-hour period.{' '} + + Learn more + )} @@ -430,7 +520,13 @@ const SDLineupManager = ({ sourceId }) => { {searchResults.length > 0 && ( - + {searchResults.map((lineup) => { @@ -438,12 +534,20 @@ const SDLineupManager = ({ sourceId }) => { return ( - {lineup.name} - {lineup.transport} · {lineup.location} + + {lineup.name} + + + {lineup.transport} · {lineup.location} + - + {isActive ? ( - Active + + Active + ) : ( diff --git a/frontend/src/components/tables/EPGsTable.jsx b/frontend/src/components/tables/EPGsTable.jsx index 8b464ee2..c1dc9ef6 100644 --- a/frontend/src/components/tables/EPGsTable.jsx +++ b/frontend/src/components/tables/EPGsTable.jsx @@ -399,7 +399,8 @@ const EPGsTable = () => { const [sorting, setSorting] = useState([]); const editEPG = async (epg = null) => { - setEPG(epg); + const freshEpg = epg?.id ? (epgs[epg.id] || epg) : epg; + setEPG(freshEpg); // Open the appropriate modal based on source type if (epg?.source_type === 'dummy') { setDummyEpgModalOpen(true); From ff081d7dc8fb32b1f53981786a3cb72b340b00b5 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 5 Jun 2026 09:28:55 -0500 Subject: [PATCH 51/76] feat(epg): implement poster artwork fetching and logo auto-apply functionality Add a new task to fetch program artwork from Schedules Direct and auto-apply logos to channels based on EPG data. Enhance the existing logic to handle poster backfill and update program records with fetched artwork. Readd the WebSocket handler to manage IP lookup events and adjust the EPGsTable component to display relevant credentials based on the source type. --- apps/channels/serializers.py | 4 +- apps/epg/tasks.py | 392 +++++++++---------- frontend/src/WebSocket.jsx | 6 + frontend/src/components/tables/EPGsTable.jsx | 12 +- 4 files changed, 199 insertions(+), 215 deletions(-) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index e9ac8581..9b7d3a2a 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -22,7 +22,7 @@ from django.db import connection, transaction from django.urls import reverse from rest_framework import serializers from django.utils import timezone -from core.utils import validate_flexible_url +from core.utils import validate_flexible_url, build_absolute_uri_with_port class LogoSerializer(serializers.ModelSerializer): @@ -67,7 +67,7 @@ class LogoSerializer(serializers.ModelSerializer): cache_url = f"{base_path}?v={url_hash}" request = self.context.get("request") if request: - return request.build_absolute_uri(cache_url) + return build_absolute_uri_with_port(request, cache_url) return cache_url def get_channel_count(self, obj): diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 8d2bb96e..0b173ef2 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -2087,6 +2087,180 @@ def fetch_schedules_direct(source, stations_only=False, force=False): h['token'] = token return h + def _sd_post_refresh_tasks(mapped_epg_ids, program_metadata, today): + """Poster fetch, logo auto-apply, and pruning — runs even when schedules are unchanged.""" + from apps.epg.models import SDProgramMD5 + + fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) + poster_program_ids = set(program_metadata.keys()) if program_metadata else set() + if fetch_posters and not poster_program_ids: + poster_program_ids = set( + ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).exclude(custom_properties__has_key='sd_icon') + .values_list('program_id', flat=True) + ) + if poster_program_ids: + logger.info( + f"Poster backfill: {len(poster_program_ids)} programs missing sd_icon." + ) + + if fetch_posters and poster_program_ids: + logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") + _sd_send_ws_sync(source.id, "parsing_programs", 98, + message="Fetching program artwork...") + try: + artwork_lookup_ids = set() + pid_to_artwork_key = {} + for pid in poster_program_ids: + if pid.startswith('EP'): + sh_root = 'SH' + pid[2:10] + '0000' + artwork_lookup_ids.add(sh_root) + pid_to_artwork_key[pid] = sh_root + else: + artwork_lookup_ids.add(pid) + pid_to_artwork_key[pid] = pid + + artwork_map = {} + artwork_list = list(artwork_lookup_ids) + SD_ARTWORK_BATCH_SIZE = 500 + total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) + logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " + f"in {total_art_batches} batch(es).") + + for batch_idx in range(total_art_batches): + batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] + try: + art_response = requests.post( + f"{SD_BASE_URL}/metadata/programs/", + json=batch, + headers=_sd_headers(token), + timeout=120, + ) + art_response.raise_for_status() + art_data = art_response.json() + + for entry in art_data: + if not isinstance(entry, dict): + continue + entry_pid = entry.get('programID') + images = entry.get('data') or [] + if not entry_pid or not images: + continue + images = [img for img in images if isinstance(img, dict)] + if not images: + continue + + poster_url = None + for min_width in [240, 135, 120]: + for pref_cat in ['Banner-L1', 'Iconic']: + match = next((img for img in images + if img.get('aspect') in ('2x3', '3x4') + and img.get('category') == pref_cat + and (img.get('width', 0) or 0) >= min_width), None) + if match: + poster_url = match.get('uri') + break + if poster_url: + break + if not poster_url: + portrait = next((img for img in images + if img.get('aspect') in ('2x3', '3x4')), None) + if portrait: + poster_url = portrait.get('uri') + if poster_url: + if not poster_url.startswith('http'): + poster_url = f"{SD_BASE_URL}/image/{poster_url}" + artwork_map[entry_pid] = poster_url + + logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " + f"{len(artwork_map)} posters found so far.") + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") + + if artwork_map: + programs_to_update = [] + for prog in ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).only('id', 'program_id', 'custom_properties'): + art_key = pid_to_artwork_key.get(prog.program_id) + poster = artwork_map.get(art_key) if art_key else None + if poster: + cp = prog.custom_properties or {} + cp['sd_icon'] = poster + prog.custom_properties = cp + programs_to_update.append(prog) + if programs_to_update: + ProgramData.objects.bulk_update( + programs_to_update, ['custom_properties'], batch_size=1000 + ) + logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") + else: + logger.info("No poster artwork matched committed programs.") + else: + logger.info("No poster artwork found from Schedules Direct.") + except Exception as art_error: + logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) + elif fetch_posters: + logger.info("Poster fetch enabled but all mapped programs already have artwork.") + + auto_apply_sd_logos = (source.custom_properties or {}).get('auto_apply_sd_logos', False) + if auto_apply_sd_logos: + try: + from apps.channels.models import Channel as ChannelModel, Logo + + channels_to_update = [] + logos_created = 0 + for channel in ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).select_related('epg_data', 'logo'): + icon_url = (channel.epg_data.icon_url or '').strip() + if not icon_url: + continue + if channel.logo and channel.logo.url == icon_url: + continue + try: + logo = Logo.objects.get(url=icon_url) + except Logo.DoesNotExist: + logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" + logo = Logo.objects.create(name=logo_name, url=icon_url) + logos_created += 1 + channel.logo = logo + channels_to_update.append(channel) + + if channels_to_update: + ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) + logger.info(f"Applied SD logos to {len(channels_to_update)} channels " + f"({logos_created} new logos created).") + else: + logger.info("All matched channels already have current SD logos.") + except Exception as logo_error: + logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) + + today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) + try: + expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] + if expired_count: + logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") + except Exception as prune_err: + logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") + + try: + live_program_ids = set( + ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) + .values_list('program_id', flat=True) + ) + pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( + program_id__in=live_program_ids + ).delete()[0] + if pruned_prog_md5_count: + logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") + except Exception as prune_err: + logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + # ------------------------------------------------------------------------- # Step 1: Authenticate and obtain session token # The SD API requires the password to be SHA1-hashed before transmission. @@ -2438,6 +2612,14 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if not changed_by_station: logger.info("No schedule changes detected, skipping schedule and program downloads.") + from apps.channels.models import Channel as ChannelModel + mapped_epg_ids_no_change = set( + ChannelModel.objects.filter( + epg_data__epg_source=source, + epg_data__isnull=False, + ).values_list('epg_data_id', flat=True) + ) + _sd_post_refresh_tasks(mapped_epg_ids_no_change, {}, today) _sd_send_ws_sync(source.id, "parsing_programs", 100, status="success", message="No schedule changes detected since last refresh. Guide data is up to date.") source.status = EPGSource.STATUS_SUCCESS @@ -2992,215 +3174,9 @@ def fetch_schedules_direct(source, stations_only=False, force=False): gc.collect() # ------------------------------------------------------------------------- - # Step 8: Fetch program artwork (posters) if enabled + # Step 8–9: Posters, logo auto-apply, and pruning # ------------------------------------------------------------------------- - fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) - poster_program_ids = set(program_metadata.keys()) if program_metadata else set() - if fetch_posters and not poster_program_ids: - # Backfill when posters were enabled after initial fetch, or a delta - # refresh skipped program metadata but programs still lack sd_icon. - poster_program_ids = set( - ProgramData.objects.filter( - epg_id__in=mapped_epg_ids, - program_id__isnull=False, - ).exclude(custom_properties__has_key='sd_icon') - .values_list('program_id', flat=True) - ) - if poster_program_ids: - logger.info( - f"Poster backfill: {len(poster_program_ids)} programs missing sd_icon." - ) - - if fetch_posters and poster_program_ids: - logger.info("Poster fetch enabled — retrieving program artwork from Schedules Direct.") - _sd_send_ws_sync(source.id, "parsing_programs", 98, - message="Fetching program artwork...") - - try: - # Build a set of unique artwork lookup IDs. - # For episodes (EP...), use the series root (SH...0000) so we get - # series-level artwork — one poster per series instead of per-episode. - artwork_lookup_ids = set() - pid_to_artwork_key = {} # maps original programID -> the key we looked up - - for pid in poster_program_ids: - if pid.startswith('EP'): - sh_root = 'SH' + pid[2:10] + '0000' - artwork_lookup_ids.add(sh_root) - pid_to_artwork_key[pid] = sh_root - else: - artwork_lookup_ids.add(pid) - pid_to_artwork_key[pid] = pid - - artwork_map = {} # artwork_key -> poster_url - artwork_list = list(artwork_lookup_ids) - SD_ARTWORK_BATCH_SIZE = 500 - - total_art_batches = max(1, (len(artwork_list) + SD_ARTWORK_BATCH_SIZE - 1) // SD_ARTWORK_BATCH_SIZE) - logger.info(f"Fetching artwork index for {len(artwork_list)} unique program/series IDs " - f"in {total_art_batches} batch(es).") - - for batch_idx in range(total_art_batches): - batch = artwork_list[batch_idx * SD_ARTWORK_BATCH_SIZE:(batch_idx + 1) * SD_ARTWORK_BATCH_SIZE] - try: - art_response = requests.post( - f"{SD_BASE_URL}/metadata/programs/", - json=batch, - headers=_sd_headers(token), - timeout=120, - ) - art_response.raise_for_status() - art_data = art_response.json() - - for entry in art_data: - if not isinstance(entry, dict): - continue - entry_pid = entry.get('programID') - images = entry.get('data') or [] - if not entry_pid or not images: - continue - - # Filter to only dict entries — SD sometimes returns bare strings - images = [img for img in images if isinstance(img, dict)] - if not images: - continue - - # Pick the best poster image: - # Prefer portrait orientation (2x3, 3x4) in larger sizes - # SD categories include: Iconic, Banner-L1, Banner-L2, Logo - # SD uses width/height instead of a "size" field - poster_url = None - - # First pass: portrait images (2x3 or 3x4) at decent size, prefer Iconic - for min_width in [240, 135, 120]: - for pref_cat in ['Banner-L1', 'Iconic']: - match = next((img for img in images - if img.get('aspect') in ('2x3', '3x4') - and img.get('category') == pref_cat - and (img.get('width', 0) or 0) >= min_width), None) - if match: - poster_url = match.get('uri') - break - if poster_url: - break - - # Fallback: any portrait image at any size - if not poster_url: - portrait = next((img for img in images - if img.get('aspect') in ('2x3', '3x4')), None) - if portrait: - poster_url = portrait.get('uri') - - if poster_url: - # Complete the URL if it's relative - if not poster_url.startswith('http'): - poster_url = f"{SD_BASE_URL}/image/{poster_url}" - artwork_map[entry_pid] = poster_url # raw SD endpoint URL - - logger.info(f"Artwork batch {batch_idx + 1}/{total_art_batches}: " - f"{len(artwork_map)} posters found so far.") - - except requests.exceptions.RequestException as e: - logger.warning(f"Failed to fetch artwork batch {batch_idx + 1}: {e}") - - # Bulk-update ProgramData records with poster URLs - if artwork_map: - programs_to_update = [] - for prog in ProgramData.objects.filter( - epg_id__in=mapped_epg_ids, - program_id__isnull=False, - ).only('id', 'program_id', 'custom_properties'): - art_key = pid_to_artwork_key.get(prog.program_id) - poster = artwork_map.get(art_key) if art_key else None - if poster: - cp = prog.custom_properties or {} - cp['sd_icon'] = poster - prog.custom_properties = cp - programs_to_update.append(prog) - - if programs_to_update: - ProgramData.objects.bulk_update( - programs_to_update, ['custom_properties'], batch_size=1000 - ) - logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") - else: - logger.info("No poster artwork matched committed programs.") - else: - logger.info("No poster artwork found from Schedules Direct.") - - except Exception as art_error: - logger.warning(f"Poster artwork fetch failed (non-fatal): {art_error}", exc_info=True) - - elif fetch_posters: - logger.info("Poster fetch enabled but all mapped programs already have artwork.") - - # ------------------------------------------------------------------------- - # Step 9: Apply SD station logos to matched channels if enabled - # ------------------------------------------------------------------------- - auto_apply_sd_logos = (source.custom_properties or {}).get('auto_apply_sd_logos', False) - if auto_apply_sd_logos: - try: - from apps.channels.models import Channel as ChannelModel, Logo - - channels_to_update = [] - logos_created = 0 - - for channel in ChannelModel.objects.filter( - epg_data__epg_source=source, - epg_data__isnull=False, - ).select_related('epg_data', 'logo'): - icon_url = (channel.epg_data.icon_url or '').strip() - if not icon_url: - continue - - # Skip if channel already has this logo URL - if channel.logo and channel.logo.url == icon_url: - continue - - # Find or create a Logo object for this URL - try: - logo = Logo.objects.get(url=icon_url) - except Logo.DoesNotExist: - logo_name = channel.epg_data.name or f"SD Logo {channel.epg_data.tvg_id}" - logo = Logo.objects.create(name=logo_name, url=icon_url) - logos_created += 1 - - channel.logo = logo - channels_to_update.append(channel) - - if channels_to_update: - ChannelModel.objects.bulk_update(channels_to_update, ['logo'], batch_size=100) - logger.info(f"Applied SD logos to {len(channels_to_update)} channels " - f"({logos_created} new logos created).") - else: - logger.info("All matched channels already have current SD logos.") - - except Exception as logo_error: - logger.warning(f"SD logo application failed (non-fatal): {logo_error}", exc_info=True) - - # Prune ProgramData whose end_time has passed. With surgical per-date deletes, - # programs from dates that have rolled off the window are never explicitly removed. - today_utc = datetime(today.year, today.month, today.day, tzinfo=dt_timezone.utc) - try: - expired_count = ProgramData.objects.filter(epg_id__in=mapped_epg_ids, end_time__lt=today_utc).delete()[0] - if expired_count: - logger.info(f"Pruned {expired_count} expired SD ProgramData records (end_time before {today}).") - except Exception as prune_err: - logger.warning(f"Failed to prune expired SD ProgramData: {prune_err}") - - # Prune SDProgramMD5 rows no longer referenced by any live ProgramData for this source. - try: - live_program_ids = set( - ProgramData.objects.filter(epg_id__in=mapped_epg_ids, program_id__isnull=False) - .values_list('program_id', flat=True) - ) - pruned_prog_md5_count = SDProgramMD5.objects.filter(epg_source=source).exclude( - program_id__in=live_program_ids - ).delete()[0] - if pruned_prog_md5_count: - logger.info(f"Pruned {pruned_prog_md5_count} stale SDProgramMD5 records no longer referenced by live ProgramData.") - except Exception as prune_err: - logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + _sd_post_refresh_tasks(mapped_epg_ids, program_metadata, today) # ------------------------------------------------------------------------- # Done diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index d8912f0d..294e879e 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -970,6 +970,12 @@ export const WebsocketProvider = ({ children }) => { break; } + case 'ip_lookup_complete': { + const { type: _t, ...ipData } = parsedEvent.data; + useSettingsStore.getState().setEnvironmentFields(ipData); + break; + } + default: console.error( `Unknown websocket event type: ${parsedEvent.data?.type}` diff --git a/frontend/src/components/tables/EPGsTable.jsx b/frontend/src/components/tables/EPGsTable.jsx index c1dc9ef6..56a3de62 100644 --- a/frontend/src/components/tables/EPGsTable.jsx +++ b/frontend/src/components/tables/EPGsTable.jsx @@ -399,7 +399,7 @@ const EPGsTable = () => { const [sorting, setSorting] = useState([]); const editEPG = async (epg = null) => { - const freshEpg = epg?.id ? (epgs[epg.id] || epg) : epg; + const freshEpg = epg?.id ? epgs[epg.id] || epg : epg; setEPG(freshEpg); // Open the appropriate modal based on source type if (epg?.source_type === 'dummy') { @@ -740,10 +740,12 @@ const EPGsTable = () => { Name: ${epgToDelete.name} Source Type: ${epgToDelete.source_type} ${ - epgToDelete.url - ? `URL: ${epgToDelete.url}` - : epgToDelete.password - ? `API Key: ${epgToDelete.password}` + epgToDelete.source_type === 'schedules_direct' + ? epgToDelete.username + ? `Username: ${epgToDelete.username}` + : '(credentials set)' + : epgToDelete.url + ? `URL: ${epgToDelete.url}` : epgToDelete.file_path ? `File Path: ${epgToDelete.file_path}` : '' From 7e906ad78ec9ba1398398ca6a22e2e4c56f234cf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 5 Jun 2026 10:14:07 -0500 Subject: [PATCH 52/76] feat(epg): enhance poster style selection and fetching logic Introduce a new poster style selection feature in the EPG settings, allowing users to choose their preferred artwork style. Update the logic for fetching and displaying program artwork from Schedules Direct, ensuring that the selected style is honored with appropriate fallbacks. Set default poster art to 'primary'. Add comprehensive tests to validate the poster selection functionality across various scenarios. --- apps/epg/tasks.py | 191 +++++++++++++++++++++--- apps/epg/tests/test_schedules_direct.py | 171 +++++++++++++++++++++ frontend/src/components/forms/EPG.jsx | 65 +++++++- 3 files changed, 402 insertions(+), 25 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0b173ef2..45a5cf12 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -35,6 +35,161 @@ SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' SD_DAYS_TO_FETCH = 20 SD_PROGRAM_BATCH_SIZE = 5000 +SD_POSTER_CATEGORIES = ( + 'Iconic', 'Banner-L1', 'Banner-L2', 'Banner-L3', 'Banner', + 'Staple', 'Poster Art', 'Box Art', +) + +SD_POSTER_STYLE_CONFIG = { + 'portrait_iconic': { + 'aspect_groups': (('2x3', '3x4'),), + 'categories': ('Iconic',), + }, + 'portrait_banner': { + 'aspect_groups': (('2x3', '3x4'),), + 'categories': ('Banner-L1', 'Banner-L2', 'Banner-L3', 'Banner'), + }, + 'landscape_iconic': { + 'aspect_groups': (('16x9', '4x3'),), + 'categories': ('Iconic',), + }, + 'landscape_banner': { + 'aspect_groups': (('16x9', '4x3'),), + 'categories': ('Banner-L1', 'Banner-L2', 'Banner-L3', 'Banner'), + }, + 'square_iconic': { + 'aspect_groups': (('1x1',),), + 'categories': ('Iconic',), + }, +} + + +def _sd_image_width(img): + try: + return int(img.get('width') or 0) + except (TypeError, ValueError): + return 0 + + +def _sd_is_primary(img): + val = img.get('primary') + if val is True: + return True + if isinstance(val, str): + return val.lower() in ('true', '1', 'yes') + return False + + +def _sd_matching_images(images, *, categories=None, aspects=None, min_width=0, primary_only=False): + matches = [] + for img in images: + if not isinstance(img, dict): + continue + if primary_only and not _sd_is_primary(img): + continue + if categories is not None and img.get('category') not in categories: + continue + if aspects is not None and img.get('aspect') not in aspects: + continue + if _sd_image_width(img) < min_width: + continue + if img.get('uri'): + matches.append(img) + return matches + + +def _sd_best_image(matches): + if not matches: + return None + best = max(matches, key=lambda img: (_sd_is_primary(img), _sd_image_width(img))) + return best.get('uri') + + +def _sd_find_image(images, *, categories=None, aspects=None, min_width=0, primary_only=False): + return _sd_best_image(_sd_matching_images( + images, + categories=categories, + aspects=aspects, + min_width=min_width, + primary_only=primary_only, + )) + + +SD_POSTER_STYLE_DEFAULT = 'sd_recommended' +SD_POSTER_PORTRAIT_FALLBACK = 'portrait_iconic' + + +def _sd_pick_recommended_poster_url(images): + """Use Gracenote's primary flag, then fall back to portrait iconic.""" + min_widths = (240, 135, 120, 0) + for min_w in min_widths: + uri = _sd_find_image( + images, + categories=SD_POSTER_CATEGORIES, + aspects=None, + min_width=min_w, + primary_only=True, + ) + if uri: + return uri + for min_w in min_widths: + uri = _sd_find_image( + images, + categories=None, + aspects=None, + min_width=min_w, + primary_only=True, + ) + if uri: + return uri + return _sd_pick_poster_url(images, SD_POSTER_PORTRAIT_FALLBACK) + + +def _sd_pick_poster_url(images, poster_style=SD_POSTER_STYLE_DEFAULT): + """Pick the best SD poster URI for the user's style preference, with fallbacks.""" + if poster_style == 'sd_recommended': + return _sd_pick_recommended_poster_url(images) + + config = SD_POSTER_STYLE_CONFIG.get(poster_style) + if not config: + return _sd_pick_recommended_poster_url(images) + min_widths = (240, 135, 120, 0) + + for min_w in min_widths: + for cat in config['categories']: + for aspects in config['aspect_groups']: + uri = _sd_find_image(images, categories=(cat,), aspects=aspects, min_width=min_w) + if uri: + return uri + + for min_w in min_widths: + for aspects in config['aspect_groups']: + uri = _sd_find_image(images, categories=SD_POSTER_CATEGORIES, aspects=aspects, min_width=min_w) + if uri: + return uri + + for aspects in config['aspect_groups']: + uri = _sd_find_image(images, categories=None, aspects=aspects, min_width=0) + if uri: + return uri + + # Fallback: SD primary among poster categories (any aspect) + for min_w in min_widths: + uri = _sd_find_image( + images, + categories=SD_POSTER_CATEGORIES, + aspects=None, + min_width=min_w, + primary_only=True, + ) + if uri: + return uri + + if poster_style != SD_POSTER_PORTRAIT_FALLBACK: + return _sd_pick_poster_url(images, SD_POSTER_PORTRAIT_FALLBACK) + + return None + # DOCTYPE internal subset for XMLTV files. Declares all 252 HTML 4 named # entities so lxml/libxml2 can resolve references like é correctly # instead of silently dropping them in recovery mode. @@ -2092,18 +2247,24 @@ def fetch_schedules_direct(source, stations_only=False, force=False): from apps.epg.models import SDProgramMD5 fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) - poster_program_ids = set(program_metadata.keys()) if program_metadata else set() - if fetch_posters and not poster_program_ids: + poster_style = (source.custom_properties or {}).get('poster_style', SD_POSTER_STYLE_DEFAULT) + poster_program_ids = set() + if fetch_posters: + needs_poster_q = ( + Q(custom_properties__isnull=True) + | ~Q(custom_properties__has_key='sd_icon') + | ~Q(custom_properties__sd_poster_style=poster_style) + ) poster_program_ids = set( ProgramData.objects.filter( epg_id__in=mapped_epg_ids, program_id__isnull=False, - ).exclude(custom_properties__has_key='sd_icon') - .values_list('program_id', flat=True) + ).filter(needs_poster_q).values_list('program_id', flat=True) ) if poster_program_ids: logger.info( - f"Poster backfill: {len(poster_program_ids)} programs missing sd_icon." + f"Poster fetch: {len(poster_program_ids)} programs need artwork " + f"(missing, style change, or first fetch; style={poster_style})." ) if fetch_posters and poster_program_ids: @@ -2152,23 +2313,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if not images: continue - poster_url = None - for min_width in [240, 135, 120]: - for pref_cat in ['Banner-L1', 'Iconic']: - match = next((img for img in images - if img.get('aspect') in ('2x3', '3x4') - and img.get('category') == pref_cat - and (img.get('width', 0) or 0) >= min_width), None) - if match: - poster_url = match.get('uri') - break - if poster_url: - break - if not poster_url: - portrait = next((img for img in images - if img.get('aspect') in ('2x3', '3x4')), None) - if portrait: - poster_url = portrait.get('uri') + poster_url = _sd_pick_poster_url(images, poster_style) if poster_url: if not poster_url.startswith('http'): poster_url = f"{SD_BASE_URL}/image/{poster_url}" @@ -2183,6 +2328,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): programs_to_update = [] for prog in ProgramData.objects.filter( epg_id__in=mapped_epg_ids, + program_id__in=poster_program_ids, program_id__isnull=False, ).only('id', 'program_id', 'custom_properties'): art_key = pid_to_artwork_key.get(prog.program_id) @@ -2190,6 +2336,7 @@ def fetch_schedules_direct(source, stations_only=False, force=False): if poster: cp = prog.custom_properties or {} cp['sd_icon'] = poster + cp['sd_poster_style'] = poster_style prog.custom_properties = cp programs_to_update.append(prog) if programs_to_update: diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index 8b696899..16899dac 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -261,3 +261,174 @@ class SDSourceSignalTests(TestCase): ) mock_parse.delay.assert_not_called() + + +# --------------------------------------------------------------------------- +# Poster selection tests +# --------------------------------------------------------------------------- + +class SDPosterSelectionTests(TestCase): + """_sd_pick_poster_url must honour style preference with sensible fallbacks.""" + + def _images(self): + return [ + { + 'uri': 'assets/iconic_portrait.jpg', + 'width': '960', + 'aspect': '2x3', + 'category': 'Iconic', + }, + { + 'uri': 'assets/banner_portrait.jpg', + 'width': '360', + 'aspect': '2x3', + 'category': 'Banner-L1', + }, + { + 'uri': 'assets/iconic_landscape.jpg', + 'width': '1920', + 'aspect': '16x9', + 'category': 'Iconic', + }, + { + 'uri': 'assets/banner_landscape.jpg', + 'width': '1280', + 'aspect': '16x9', + 'category': 'Banner-L1', + }, + ] + + def test_portrait_iconic_prefers_iconic_over_banner(self): + from apps.epg.tasks import _sd_pick_poster_url + + self.assertEqual( + _sd_pick_poster_url(self._images(), 'portrait_iconic'), + 'assets/iconic_portrait.jpg', + ) + + def test_portrait_banner_prefers_banner(self): + from apps.epg.tasks import _sd_pick_poster_url + + self.assertEqual( + _sd_pick_poster_url(self._images(), 'portrait_banner'), + 'assets/banner_portrait.jpg', + ) + + def test_landscape_iconic_prefers_landscape_iconic(self): + from apps.epg.tasks import _sd_pick_poster_url + + self.assertEqual( + _sd_pick_poster_url(self._images(), 'landscape_iconic'), + 'assets/iconic_landscape.jpg', + ) + + def test_landscape_falls_back_to_portrait_when_unavailable(self): + from apps.epg.tasks import _sd_pick_poster_url + + images = [img for img in self._images() if img['aspect'] in ('2x3', '3x4')] + self.assertEqual( + _sd_pick_poster_url(images, 'landscape_iconic'), + 'assets/iconic_portrait.jpg', + ) + + def test_unknown_style_defaults_to_sd_recommended(self): + from apps.epg.tasks import _sd_pick_poster_url + + self.assertEqual( + _sd_pick_poster_url(self._images(), 'not_a_real_style'), + 'assets/iconic_portrait.jpg', + ) + + def test_prefers_primary_when_category_and_aspect_match(self): + from apps.epg.tasks import _sd_pick_poster_url + + images = [ + { + 'uri': 'assets/banner_small.jpg', + 'width': '120', + 'aspect': '2x3', + 'category': 'Banner-L1', + }, + { + 'uri': 'assets/banner_primary.jpg', + 'width': '360', + 'aspect': '2x3', + 'category': 'Banner-L1', + 'primary': 'true', + }, + ] + self.assertEqual( + _sd_pick_poster_url(images, 'portrait_banner'), + 'assets/banner_primary.jpg', + ) + + def test_sd_recommended_uses_primary_poster_category(self): + from apps.epg.tasks import _sd_pick_poster_url + + images = [ + { + 'uri': 'assets/cast_primary.jpg', + 'width': '500', + 'aspect': '3x4', + 'category': 'Cast in Character', + 'primary': 'true', + }, + { + 'uri': 'assets/iconic_primary.jpg', + 'width': '300', + 'aspect': '16x9', + 'category': 'Iconic', + 'primary': 'true', + }, + ] + self.assertEqual( + _sd_pick_poster_url(images, 'sd_recommended'), + 'assets/iconic_primary.jpg', + ) + + def test_sd_recommended_falls_back_to_portrait_iconic(self): + from apps.epg.tasks import _sd_pick_poster_url + + self.assertEqual( + _sd_pick_poster_url(self._images(), 'sd_recommended'), + 'assets/iconic_portrait.jpg', + ) + + def test_default_style_is_sd_recommended(self): + from apps.epg.tasks import _sd_pick_poster_url, SD_POSTER_STYLE_DEFAULT + + self.assertEqual(SD_POSTER_STYLE_DEFAULT, 'sd_recommended') + images = [ + { + 'uri': 'assets/primary.jpg', + 'width': '960', + 'aspect': '16x9', + 'category': 'Iconic', + 'primary': 'true', + }, + ] + self.assertEqual(_sd_pick_poster_url(images), 'assets/primary.jpg') + + def test_style_fallback_uses_primary_before_cross_orientation(self): + from apps.epg.tasks import _sd_pick_poster_url + + images = [ + { + 'uri': 'assets/iconic_portrait.jpg', + 'width': '960', + 'aspect': '2x3', + 'category': 'Iconic', + }, + { + 'uri': 'assets/landscape_primary.jpg', + 'width': '1920', + 'aspect': '16x9', + 'category': 'Iconic', + 'primary': 'true', + }, + ] + # square_iconic has no 1x1 images; should pick SD primary before portrait iconic fallback + self.assertEqual( + _sd_pick_poster_url(images, 'square_iconic'), + 'assets/landscape_primary.jpg', + ) diff --git a/frontend/src/components/forms/EPG.jsx b/frontend/src/components/forms/EPG.jsx index 409307e9..10699566 100644 --- a/frontend/src/components/forms/EPG.jsx +++ b/frontend/src/components/forms/EPG.jsx @@ -75,6 +75,33 @@ const SD_LOGO_STYLES = [ }, ]; +const SD_POSTER_STYLES = [ + { + value: 'sd_recommended', + label: 'SD Recommended (primary artwork)', + }, + { + value: 'portrait_banner', + label: 'Portrait, Banner with title (2×3 / 3×4)', + }, + { + value: 'portrait_iconic', + label: 'Portrait, Iconic (2×3 / 3×4)', + }, + { + value: 'landscape_iconic', + label: 'Landscape, Iconic (16×9 / 4×3)', + }, + { + value: 'landscape_banner', + label: 'Landscape, Banner with title (16×9 / 4×3)', + }, + { + value: 'square_iconic', + label: 'Square, Iconic (1×1)', + }, +]; + // ─── SD Settings: Logo toggle + style selector + Poster toggle ────────────── const SDSettings = ({ sourceId, customProperties }) => { const storeCustomProps = useEPGsStore((s) => @@ -87,6 +114,9 @@ const SDSettings = ({ sourceId, customProperties }) => { ); const [logoStyle, setLogoStyle] = useState(resolvedCp.logo_style || 'dark'); const [fetchPosters, setFetchPosters] = useState(!!resolvedCp.fetch_posters); + const [posterStyle, setPosterStyle] = useState( + resolvedCp.poster_style || 'sd_recommended' + ); const [saving, setSaving] = useState(false); // Sync from store (preferred) or parent props when the form opens or settings save @@ -95,6 +125,7 @@ const SDSettings = ({ sourceId, customProperties }) => { setAutoApplySDLogos(!!newCp.auto_apply_sd_logos); setLogoStyle(newCp.logo_style || 'dark'); setFetchPosters(!!newCp.fetch_posters); + setPosterStyle(newCp.poster_style || 'sd_recommended'); }, [storeCustomProps, customProperties]); const saveSetting = async (key, value) => { @@ -121,6 +152,12 @@ const SDSettings = ({ sourceId, customProperties }) => { saveSetting('fetch_posters', checked); }; + const handlePosterStyleChange = (style) => { + if (!style) return; + setPosterStyle(style); + saveSetting('poster_style', style); + }; + return ( @@ -195,7 +232,21 @@ const SDSettings = ({ sourceId, customProperties }) => { onChange={(e) => handlePosterToggle(e.currentTarget.checked)} disabled={saving} size="sm" + mb={fetchPosters ? 'xs' : undefined} /> + + {fetchPosters && ( + )} + + { + setServerGroupsManagerOpen(false); + setServerGroupsCreateOnOpen(false); + }} + openCreateOnMount={serverGroupsCreateOnOpen} + onGroupCreated={(group) => { + if (group?.id) { + form.setFieldValue('server_group', `${group.id}`); + } + }} + /> ); }; diff --git a/frontend/src/components/forms/ServerGroup.jsx b/frontend/src/components/forms/ServerGroup.jsx index a01a0a76..80c8c1a4 100644 --- a/frontend/src/components/forms/ServerGroup.jsx +++ b/frontend/src/components/forms/ServerGroup.jsx @@ -3,20 +3,21 @@ import { useForm } from 'react-hook-form'; import { yupResolver } from '@hookform/resolvers/yup'; import * as Yup from 'yup'; import API from '../../api'; -import { Button, Flex, Modal, NumberInput, TextInput } from '@mantine/core'; +import { Button, Flex, Modal, TextInput } from '@mantine/core'; const schema = Yup.object({ name: Yup.string().required('Name is required'), - max_streams: Yup.number() - .min(0, 'Must be 0 or greater') - .required('Max streams is required'), }); -const ServerGroupForm = ({ serverGroup = null, isOpen, onClose }) => { +const ServerGroupForm = ({ + serverGroup = null, + isOpen, + onClose, + onSaved, +}) => { const defaultValues = useMemo( () => ({ name: serverGroup?.name || '', - max_streams: serverGroup?.max_streams ?? 0, }), [serverGroup] ); @@ -26,23 +27,21 @@ const ServerGroupForm = ({ serverGroup = null, isOpen, onClose }) => { handleSubmit, formState: { errors, isSubmitting }, reset, - setValue, - watch, } = useForm({ defaultValues, resolver: yupResolver(schema), }); const onSubmit = async (values) => { - const payload = { - ...values, - max_streams: Number(values.max_streams), - }; - + let response; if (serverGroup?.id) { - await API.updateServerGroup({ id: serverGroup.id, ...payload }); + response = await API.updateServerGroup({ id: serverGroup.id, ...values }); } else { - await API.addServerGroup(payload); + response = await API.addServerGroup(values); + } + + if (response) { + onSaved?.(response); } reset(); @@ -57,26 +56,23 @@ const ServerGroupForm = ({ serverGroup = null, isOpen, onClose }) => { return null; } - const maxStreams = watch('max_streams'); - return ( - + - setValue('max_streams', value ?? 0)} - error={errors.max_streams?.message} - /> - + + + + { playlistCreated={playlistCreated} /> + setServerGroupsManagerOpen(false)} + /> + setConfirmDeleteOpen(false)} diff --git a/frontend/src/components/tables/ServerGroupsTable.jsx b/frontend/src/components/tables/ServerGroupsTable.jsx index e4280037..acad57f4 100644 --- a/frontend/src/components/tables/ServerGroupsTable.jsx +++ b/frontend/src/components/tables/ServerGroupsTable.jsx @@ -1,7 +1,10 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import API from '../../api'; import useServerGroupsStore from '../../store/serverGroups'; +import usePlaylistsStore from '../../store/playlists'; +import useWarningsStore from '../../store/warnings'; import ServerGroupForm from '../forms/ServerGroup'; +import ConfirmationDialog from '../ConfirmationDialog'; import { ActionIcon, Box, @@ -11,15 +14,15 @@ import { Paper, Stack, Text, - Tooltip, } from '@mantine/core'; import { SquareMinus, SquarePen, SquarePlus } from 'lucide-react'; import { CustomTable, useTable } from './CustomTable'; import useLocalStorage from '../../hooks/useLocalStorage'; +import './table.css'; const RowActions = ({ row, editServerGroup, deleteServerGroup }) => { return ( - <> + { > - + ); }; -const ServerGroupsTable = () => { +const ServerGroupsTable = ({ + onGroupCreated, + openCreateOnMount = false, +}) => { const [serverGroup, setServerGroup] = useState(null); const [serverGroupModalOpen, setServerGroupModalOpen] = useState(false); + const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false); + const [deleteTarget, setDeleteTarget] = useState(null); + const [groupToDelete, setGroupToDelete] = useState(null); + const [deleting, setDeleting] = useState(false); + const openedCreateOnMount = useRef(false); + + const isWarningSuppressed = useWarningsStore((s) => s.isWarningSuppressed); + const suppressWarning = useWarningsStore((s) => s.suppressWarning); const serverGroups = useServerGroupsStore((state) => state.serverGroups); const fetchServerGroups = useServerGroupsStore( (state) => state.fetchServerGroups ); + const playlists = usePlaylistsStore((state) => state.playlists); const [tableSize] = useLocalStorage('table-size', 'default'); + const tableData = useMemo( + () => + serverGroups.map((group) => ({ + ...group, + accountCount: playlists.filter( + (playlist) => playlist.server_group === group.id + ).length, + })), + [serverGroups, playlists] + ); + const columns = useMemo( () => [ { header: 'Name', accessorKey: 'name', - size: 175, + size: 150, + cell: ({ cell }) => {cell.getValue()}, }, { - header: 'Max Streams', - accessorKey: 'max_streams', - size: 100, - cell: ({ cell }) => { - const value = cell.getValue(); - return value === 0 ? 'Unlimited' : value; - }, + header: 'Accounts', + accessorKey: 'accountCount', + size: 80, + cell: ({ cell }) => {cell.getValue() ?? 0}, }, { id: 'actions', @@ -82,8 +106,26 @@ const ServerGroupsTable = () => { setServerGroupModalOpen(true); }; - const deleteServerGroup = async (id) => { - await API.deleteServerGroup(id); + const executeDeleteServerGroup = async (id) => { + setDeleting(true); + try { + await API.deleteServerGroup(id); + } finally { + setDeleting(false); + setConfirmDeleteOpen(false); + } + }; + + const deleteServerGroup = (id) => { + const group = tableData.find((item) => item.id === id); + setGroupToDelete(group); + setDeleteTarget(id); + + if (isWarningSuppressed('delete-server-group')) { + return executeDeleteServerGroup(id); + } + + setConfirmDeleteOpen(true); }; const closeServerGroupForm = () => { @@ -91,17 +133,32 @@ const ServerGroupsTable = () => { setServerGroupModalOpen(false); }; + const handleServerGroupSaved = (savedGroup) => { + if (!serverGroup?.id) { + onGroupCreated?.(savedGroup); + } + }; + useEffect(() => { fetchServerGroups().finally(() => setIsLoading(false)); }, [fetchServerGroups]); - const renderHeaderCell = (header) => { - return ( - - {header.column.columnDef.header} - - ); - }; + useEffect(() => { + if (!openCreateOnMount) { + openedCreateOnMount.current = false; + return; + } + if (!isLoading && !openedCreateOnMount.current) { + openedCreateOnMount.current = true; + editServerGroup(); + } + }, [openCreateOnMount, isLoading]); + + const renderHeaderCell = (header) => ( + + {header.column.columnDef.header} + + ); const renderBodyCell = ({ row }) => { return ( @@ -115,14 +172,14 @@ const ServerGroupsTable = () => { const table = useTable({ columns, - data: serverGroups, - allRowIds: serverGroups.map((group) => group.id), + data: tableData, + allRowIds: tableData.map((group) => group.id), bodyCellRenderFns: { actions: renderBodyCell, }, headerCellRenderFns: { name: renderHeaderCell, - max_streams: renderHeaderCell, + accountCount: renderHeaderCell, actions: renderHeaderCell, }, }); @@ -146,23 +203,22 @@ const ServerGroupsTable = () => { }} > - - - + @@ -185,7 +241,7 @@ const ServerGroupsTable = () => { borderRadius: 'var(--mantine-radius-default)', }} > -
+
@@ -195,6 +251,34 @@ const ServerGroupsTable = () => { serverGroup={serverGroup} isOpen={serverGroupModalOpen} onClose={closeServerGroupForm} + onSaved={handleServerGroupSaved} + /> + + setConfirmDeleteOpen(false)} + onConfirm={() => executeDeleteServerGroup(deleteTarget)} + loading={deleting} + title="Confirm Server Group Deletion" + message={ + groupToDelete ? ( +
+ {`Are you sure you want to delete the following server group? + +Name: ${groupToDelete.name} +Accounts: ${groupToDelete.accountCount ?? 0} + +Accounts in this group will no longer share connection limits. This action cannot be undone.`} +
+ ) : ( + 'Are you sure you want to delete this server group? This action cannot be undone.' + ) + } + confirmLabel="Delete" + cancelLabel="Cancel" + actionKey="delete-server-group" + onSuppressChange={suppressWarning} + zIndex={401} /> ); diff --git a/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx b/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx new file mode 100644 index 00000000..93f8b480 --- /dev/null +++ b/frontend/src/components/tables/__tests__/ServerGroupsTable.test.jsx @@ -0,0 +1,62 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import { MantineProvider } from '@mantine/core'; +import ServerGroupsTable from '../ServerGroupsTable'; + +const renderTable = () => + render( + + + + ); + +vi.mock('../../../api', () => ({ + default: { + deleteServerGroup: vi.fn(), + }, +})); + +vi.mock('../../../store/serverGroups', () => ({ + default: (selector) => + selector({ + serverGroups: [ + { id: 1, name: 'Pool A' }, + { id: 2, name: 'Pool B' }, + ], + fetchServerGroups: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock('../../../store/playlists', () => ({ + default: (selector) => + selector({ + playlists: [ + { id: 10, server_group: 2 }, + { id: 11, server_group: 1 }, + ], + }), +})); + +vi.mock('../../forms/ServerGroup', () => ({ + default: () => null, +})); + +vi.mock('../../../hooks/useLocalStorage', () => ({ + default: () => ['default', vi.fn()], +})); + +describe('ServerGroupsTable', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('renders server groups without tooltip errors', async () => { + expect(() => renderTable()).not.toThrow(); + + await waitFor(() => { + expect(screen.getByText('Pool A')).toBeInTheDocument(); + expect(screen.getByText('Pool B')).toBeInTheDocument(); + expect(screen.getByText('Accounts')).toBeInTheDocument(); + }); + }); +}); diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index c3481984..29d46dd8 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -14,9 +14,6 @@ import { const UserAgentsTable = React.lazy( () => import('../components/tables/UserAgentsTable.jsx') ); -const ServerGroupsTable = React.lazy( - () => import('../components/tables/ServerGroupsTable.jsx') -); const StreamProfilesTable = React.lazy( () => import('../components/tables/StreamProfilesTable.jsx') ); @@ -138,19 +135,6 @@ const SettingsPage = () => { - - Server Groups - - - }> - - - - - - User-Agents From 705041ea81c1d110830eb104421e84646b3c93f4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 14:46:26 -0500 Subject: [PATCH 71/76] refactor(channel_status, vod_proxy): remove stream URL credential extraction - Eliminated redundant extraction of provider usernames from stream URLs in both ChannelStatus and VOD stats data functions, simplifying the codebase. - Updated frontend components to reflect the removal of provider username display, streamlining the UI. - Enhanced overall readability and maintainability of the code by reducing complexity in handling stream URLs. --- apps/proxy/live_proxy/channel_status.py | 12 +--------- apps/proxy/vod_proxy/views.py | 12 ---------- .../components/cards/StreamConnectionCard.jsx | 22 +++++-------------- .../components/cards/VodConnectionCard.jsx | 7 ------ 4 files changed, 7 insertions(+), 46 deletions(-) diff --git a/apps/proxy/live_proxy/channel_status.py b/apps/proxy/live_proxy/channel_status.py index 52fc8859..f52a5fbb 100644 --- a/apps/proxy/live_proxy/channel_status.py +++ b/apps/proxy/live_proxy/channel_status.py @@ -390,12 +390,11 @@ class ChannelStatus: created_at = float(init_time_bytes) uptime = time.time() - created_at if created_at > 0 else 0 - stream_url = metadata.get(ChannelMetadataField.URL, "") or "" # Simplified info info = { 'channel_id': channel_id, 'state': metadata.get(ChannelMetadataField.STATE), - 'url': stream_url, + 'url': metadata.get(ChannelMetadataField.URL, ""), 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""), 'owner': metadata.get(ChannelMetadataField.OWNER), 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, @@ -403,15 +402,6 @@ class ChannelStatus: 'uptime': uptime, 'started_at': created_at if created_at > 0 else None, } - if stream_url: - try: - from apps.m3u.connection_pool import extract_credentials_from_stream_url - - provider_user, _ = extract_credentials_from_stream_url(stream_url) - if provider_user: - info['provider_username'] = provider_user - except Exception as e: - logger.debug(f"Could not parse provider username from stream URL: {e}") channel_name = metadata.get(ChannelMetadataField.CHANNEL_NAME) if channel_name: diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index fc825cf4..27208ec9 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -911,15 +911,9 @@ def build_vod_stats_data(redis_client): # Get M3U profile information m3u_profile_info = {} m3u_profile_id = combined_data.get('m3u_profile_id') - stream_url_for_creds = ( - combined_data.get('final_url') - or combined_data.get('stream_url') - or '' - ) if m3u_profile_id: try: from apps.m3u.models import M3UAccountProfile - from apps.m3u.connection_pool import extract_credentials_from_stream_url profile = M3UAccountProfile.objects.select_related('m3u_account').get(id=m3u_profile_id) m3u_profile_info = { @@ -929,12 +923,6 @@ def build_vod_stats_data(redis_client): 'max_streams': profile.m3u_account.max_streams, 'm3u_profile_id': int(m3u_profile_id), } - if stream_url_for_creds: - provider_user, _ = extract_credentials_from_stream_url( - stream_url_for_creds - ) - if provider_user: - m3u_profile_info['provider_username'] = provider_user except Exception as e: logger.warning(f"Could not fetch M3U profile {m3u_profile_id}: {e}") diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 6b043cb1..a1748143 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -500,7 +500,6 @@ const StreamConnectionCard = ({ channel.m3u_profile?.name || channel.m3u_profile_name || 'Unknown M3U Profile'; - const providerUsername = channel.provider_username; // Create select options for available streams const streamOptions = getStreamOptions(availableStreams, m3uAccountsMap); @@ -602,21 +601,12 @@ const StreamConnectionCard = ({ {/* M3U Profile on right - absolutely positioned */} - - - - - {m3uProfileName} - - - {providerUsername && ( - - - Login: {providerUsername} - - - )} - + + + + {m3uProfileName} + + {/* Channel Name on left */} diff --git a/frontend/src/components/cards/VodConnectionCard.jsx b/frontend/src/components/cards/VodConnectionCard.jsx index 6539426a..90b4c034 100644 --- a/frontend/src/components/cards/VodConnectionCard.jsx +++ b/frontend/src/components/cards/VodConnectionCard.jsx @@ -349,13 +349,6 @@ const VodConnectionCard = ({ vodContent, stopVODClient }) => { {connection.m3u_profile.profile_name || 'Default Profile'} - {connection.m3u_profile.provider_username && ( - - - Login: {connection.m3u_profile.provider_username} - - - )} From 2dae24a02bff15e423925a35fb0edd29cba0bed2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 16:18:09 -0500 Subject: [PATCH 72/76] refactor(ServerGroupsTable, ServerGroupsManagerModal): adjust layout and component structure - Changed the size of the ServerGroupsManagerModal from "lg" to "md" for improved UI consistency. - Refactored ServerGroupsTable to enhance the layout, including adjustments to column sizes and the addition of minimum sizes for better responsiveness. - Simplified the RowActions component for better readability and alignment. - Updated the loading and empty state handling in the table to provide clearer user feedback. - Removed unused local storage hook from the component, streamlining the codebase. --- .../components/ServerGroupsManagerModal.jsx | 2 +- .../components/tables/ServerGroupsTable.jsx | 162 +++++++++--------- .../__tests__/ServerGroupsTable.test.jsx | 4 - 3 files changed, 83 insertions(+), 85 deletions(-) diff --git a/frontend/src/components/ServerGroupsManagerModal.jsx b/frontend/src/components/ServerGroupsManagerModal.jsx index 5e3005ec..2d403aa1 100644 --- a/frontend/src/components/ServerGroupsManagerModal.jsx +++ b/frontend/src/components/ServerGroupsManagerModal.jsx @@ -12,7 +12,7 @@ const ServerGroupsManagerModal = ({ opened={isOpen} onClose={onClose} title="Server Groups" - size="lg" + size="md" centered > {isOpen ? ( diff --git a/frontend/src/components/tables/ServerGroupsTable.jsx b/frontend/src/components/tables/ServerGroupsTable.jsx index acad57f4..54500294 100644 --- a/frontend/src/components/tables/ServerGroupsTable.jsx +++ b/frontend/src/components/tables/ServerGroupsTable.jsx @@ -11,42 +11,39 @@ import { Button, Center, Flex, - Paper, Stack, Text, } from '@mantine/core'; import { SquareMinus, SquarePen, SquarePlus } from 'lucide-react'; import { CustomTable, useTable } from './CustomTable'; -import useLocalStorage from '../../hooks/useLocalStorage'; import './table.css'; -const RowActions = ({ row, editServerGroup, deleteServerGroup }) => { - return ( - - editServerGroup(row.original)} - > - - - deleteServerGroup(row.original.id)} - > - - - - ); -}; +const TABLE_WIDTH = 360; +const ACTIONS_COLUMN_SIZE = 76; +const ACCOUNTS_COLUMN_SIZE = 72; -const ServerGroupsTable = ({ - onGroupCreated, - openCreateOnMount = false, -}) => { +const RowActions = ({ row, editServerGroup, deleteServerGroup }) => ( + + editServerGroup(row.original)} + > + + + deleteServerGroup(row.original.id)} + > + + + +); + +const ServerGroupsTable = ({ onGroupCreated, openCreateOnMount = false }) => { const [serverGroup, setServerGroup] = useState(null); const [serverGroupModalOpen, setServerGroupModalOpen] = useState(false); const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false); @@ -63,7 +60,6 @@ const ServerGroupsTable = ({ (state) => state.fetchServerGroups ); const playlists = usePlaylistsStore((state) => state.playlists); - const [tableSize] = useLocalStorage('table-size', 'default'); const tableData = useMemo( () => @@ -81,22 +77,33 @@ const ServerGroupsTable = ({ { header: 'Name', accessorKey: 'name', - size: 150, - cell: ({ cell }) => {cell.getValue()}, + grow: true, + minSize: 100, + cell: ({ cell }) => ( + + {cell.getValue()} + + ), }, { header: 'Accounts', accessorKey: 'accountCount', - size: 80, - cell: ({ cell }) => {cell.getValue() ?? 0}, + size: ACCOUNTS_COLUMN_SIZE, + minSize: 65, + cell: ({ cell }) => ( +
+ {cell.getValue() ?? 0} +
+ ), }, { id: 'actions', header: 'Actions', - size: tableSize == 'compact' ? 50 : 75, + size: ACTIONS_COLUMN_SIZE, + minSize: 65, }, ], - [tableSize] + [] ); const [isLoading, setIsLoading] = useState(true); @@ -160,20 +167,19 @@ const ServerGroupsTable = ({ ); - const renderBodyCell = ({ row }) => { - return ( - - ); - }; + const renderBodyCell = ({ row }) => ( + + ); const table = useTable({ columns, data: tableData, allRowIds: tableData.map((group) => group.id), + enableColumnResizing: false, bodyCellRenderFns: { actions: renderBodyCell, }, @@ -186,23 +192,23 @@ const ServerGroupsTable = ({ if (isLoading) { return ( -
+
Loading server groups...
); } return ( - - - - + + + Group accounts that share the same provider login so their connection + limits are enforced together. Assign a group when editing an M3U + account. + + + + + - - - - -
- -
- - + + {tableData.length === 0 ? ( +
+ + No server groups yet. + +
+ ) : ( + + )} + + + ({ default: () => null, })); -vi.mock('../../../hooks/useLocalStorage', () => ({ - default: () => ['default', vi.fn()], -})); - describe('ServerGroupsTable', () => { beforeEach(() => { vi.clearAllMocks(); From 647c1316ba176cc9614bb32dcc2f3dc781ca9228 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 9 Jun 2026 16:34:27 -0500 Subject: [PATCH 73/76] refactor(M3U): enhance form layout and streamline input handling - Increased modal size for better visibility and user experience. - Adjusted form layout for improved alignment and spacing between elements. - Removed unused Checkbox component and simplified the account type selection logic. - Consolidated VOD-related input fields under the Xtream Codes account type for clarity. - Enhanced overall readability and maintainability of the component by reducing complexity in the form structure. --- frontend/src/components/forms/M3U.jsx | 221 +++++++++++++------------- 1 file changed, 111 insertions(+), 110 deletions(-) diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index bfc015de..07b430ef 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -6,7 +6,6 @@ import M3UProfiles from './M3UProfiles'; import { Box, Button, - Checkbox, Divider, FileInput, Flex, @@ -58,7 +57,8 @@ const M3U = ({ const [filterModalOpen, setFilterModalOpen] = useState(false); const [scheduleType, setScheduleType] = useState('interval'); const [serverGroupsManagerOpen, setServerGroupsManagerOpen] = useState(false); - const [serverGroupsCreateOnOpen, setServerGroupsCreateOnOpen] = useState(false); + const [serverGroupsCreateOnOpen, setServerGroupsCreateOnOpen] = + useState(false); const form = useForm({ mode: 'uncontrolled', @@ -212,7 +212,7 @@ const M3U = ({ return ( <> - - + + - Manage server groups -