From f99133837690e431b5fd8b7d5a111cddbf0bb7b2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 17 Jun 2026 17:22:28 -0500 Subject: [PATCH 01/28] fix(proxy): update for improved DB connection management - Enhanced the live proxy to release geventpool DB connections more effectively across various paths. - Implemented `close_old_connections()` in `stream_ts()`, `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, `get_connections_left()`, and the cleanup process in `StreamGenerator`, ensuring that clients do not hold onto pool slots unnecessarily during streaming operations. --- CHANGELOG.md | 4 + apps/proxy/live_proxy/output/ts/generator.py | 104 +++++++------ .../live_proxy/services/channel_service.py | 138 +++++++++-------- .../live_proxy/tests/test_live_db_cleanup.py | 142 ++++++++++++++++++ apps/proxy/live_proxy/url_utils.py | 9 ++ apps/proxy/live_proxy/views.py | 5 +- 6 files changed, 286 insertions(+), 116 deletions(-) create mode 100644 apps/proxy/live_proxy/tests/test_live_db_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1da516a1..1f1234c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. + ## [0.27.0] - 2026-06-16 ### Added diff --git a/apps/proxy/live_proxy/output/ts/generator.py b/apps/proxy/live_proxy/output/ts/generator.py index 2bd88725..3524a2a6 100644 --- a/apps/proxy/live_proxy/output/ts/generator.py +++ b/apps/proxy/live_proxy/output/ts/generator.py @@ -7,6 +7,7 @@ import time import gevent from apps.proxy.config import TSConfig as Config from apps.channels.models import Channel, Stream +from django.db import close_old_connections from core.utils import log_system_event from ...server import ProxyServer from ...utils import create_ts_packet, get_logger @@ -590,60 +591,63 @@ class StreamGenerator: def _cleanup(self): """Clean up resources and report final statistics.""" - # Client cleanup - elapsed = time.time() - self.stream_start_time - local_clients = 0 - total_clients = 0 - proxy_server = ProxyServer.get_instance() + try: + # Client cleanup + elapsed = time.time() - self.stream_start_time + local_clients = 0 + total_clients = 0 + proxy_server = ProxyServer.get_instance() - # Release M3U profile stream allocation if this is the last client - stream_released = False - if proxy_server.redis_client: - try: - if self.channel_id in proxy_server.client_managers: - client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() - # Pool slots are global; the last client on any worker must release. - # During shutdown_delay, keep the slot until coordinated stop runs. - if client_count <= 1 and ConfigHelper.channel_shutdown_delay() <= 0: - try: + # Release M3U profile stream allocation if this is the last client + stream_released = False + if proxy_server.redis_client: + try: + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Pool slots are global; the last client on any worker must release. + # During shutdown_delay, keep the slot until coordinated stop runs. + if client_count <= 1 and ConfigHelper.channel_shutdown_delay() <= 0: try: - obj = Channel.objects.get(uuid=self.channel_id) - except (Channel.DoesNotExist, Exception): - obj = Stream.objects.get(stream_hash=self.channel_id) - stream_released = obj.release_stream() - if stream_released: - logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") - else: - logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") - except Exception as e: - logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") - except Exception as e: - logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") + try: + obj = Channel.objects.get(uuid=self.channel_id) + except (Channel.DoesNotExist, Exception): + obj = Stream.objects.get(stream_hash=self.channel_id) + stream_released = obj.release_stream() + if stream_released: + logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}") + else: + logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}") + except Exception as e: + logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") - if self.channel_id in proxy_server.client_managers: - client_manager = proxy_server.client_managers[self.channel_id] - if self.client_id in client_manager.clients: - local_clients = client_manager.remove_client(self.client_id) - else: - local_clients = client_manager.get_client_count() - total_clients = client_manager.get_total_client_count() - logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") + if self.channel_id in proxy_server.client_managers: + client_manager = proxy_server.client_managers[self.channel_id] + if self.client_id in client_manager.clients: + local_clients = client_manager.remove_client(self.client_id) + else: + local_clients = client_manager.get_client_count() + total_clients = client_manager.get_total_client_count() + logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") - # Log client disconnect event - try: - log_system_event( - 'client_disconnect', - channel_id=self.channel_id, - channel_name=self.channel_name, - client_ip=self.client_ip, - client_id=self.client_id, - user_agent=self.client_user_agent[:100] if self.client_user_agent else None, - duration=round(elapsed, 2), - bytes_sent=self.bytes_sent, - username=self.user.username if self.user else None - ) - except Exception as e: - logger.error(f"Could not log client disconnect event: {e}") + # Log client disconnect event + try: + log_system_event( + 'client_disconnect', + channel_id=self.channel_id, + channel_name=self.channel_name, + client_ip=self.client_ip, + client_id=self.client_id, + user_agent=self.client_user_agent[:100] if self.client_user_agent else None, + duration=round(elapsed, 2), + bytes_sent=self.bytes_sent, + username=self.user.username if self.user else None + ) + except Exception as e: + logger.error(f"Could not log client disconnect event: {e}") + finally: + close_old_connections() def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None, buffer=None): diff --git a/apps/proxy/live_proxy/services/channel_service.py b/apps/proxy/live_proxy/services/channel_service.py index 400f55f4..7ad4e71b 100644 --- a/apps/proxy/live_proxy/services/channel_service.py +++ b/apps/proxy/live_proxy/services/channel_service.py @@ -241,34 +241,38 @@ class ChannelService: # Store additional metadata if initialization was successful if success and proxy_server.redis_client: - metadata_key = RedisKeys.channel_metadata(channel_id) - update_data = {} - if stream_profile_value: - update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value - if stream_id: - update_data[ChannelMetadataField.STREAM_ID] = str(stream_id) - if m3u_profile_id: - update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - - # Store channel name and stream name so stats workers don't need DB calls try: - if not channel_name: - from apps.channels.models import Channel - channel_name = Channel.objects.filter(uuid=channel_id).values_list('name', flat=True).first() - if channel_name: - update_data[ChannelMetadataField.CHANNEL_NAME] = channel_name - else: - # No channel name means stream preview mode, use stream name as display fallback - if stream_id and not stream_name: - from apps.channels.models import Stream - stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() - if stream_name: - update_data[ChannelMetadataField.STREAM_NAME] = stream_name - except Exception as e: - logger.warning(f"Failed to store channel/stream names in Redis for {channel_id}: {e}") + metadata_key = RedisKeys.channel_metadata(channel_id) + update_data = {} + if stream_profile_value: + update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value + if stream_id: + update_data[ChannelMetadataField.STREAM_ID] = str(stream_id) + if m3u_profile_id: + update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - if update_data: - proxy_server.redis_client.hset(metadata_key, mapping=update_data) + # Store channel name and stream name so stats workers don't need DB calls + try: + if not channel_name: + from apps.channels.models import Channel + channel_name = Channel.objects.filter(uuid=channel_id).values_list('name', flat=True).first() + if channel_name: + update_data[ChannelMetadataField.CHANNEL_NAME] = channel_name + else: + # No channel name means stream preview mode, use stream name as display fallback + if stream_id and not stream_name: + from apps.channels.models import Stream + stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() + if stream_name: + update_data[ChannelMetadataField.STREAM_NAME] = stream_name + except Exception as e: + logger.warning(f"Failed to store channel/stream names in Redis for {channel_id}: {e}") + + if update_data: + proxy_server.redis_client.hset(metadata_key, mapping=update_data) + finally: + from django.db import close_old_connections + close_old_connections() return success @@ -736,53 +740,57 @@ class ChannelService: @staticmethod def _update_channel_metadata(channel_id, url, user_agent=None, stream_id=None, m3u_profile_id=None, stream_name=None): """Update channel metadata in Redis""" - proxy_server = ProxyServer.get_instance() + try: + proxy_server = ProxyServer.get_instance() - if not proxy_server.redis_client: - return False + if not proxy_server.redis_client: + return False - metadata_key = RedisKeys.channel_metadata(channel_id) + metadata_key = RedisKeys.channel_metadata(channel_id) - # First check if the key exists and what type it is - key_type = proxy_server.redis_client.type(metadata_key) - logger.debug(f"Redis key {metadata_key} is of type: {key_type}") + # First check if the key exists and what type it is + key_type = proxy_server.redis_client.type(metadata_key) + logger.debug(f"Redis key {metadata_key} is of type: {key_type}") - # Build metadata update dict - metadata = {ChannelMetadataField.URL: url} - if user_agent: - metadata[ChannelMetadataField.USER_AGENT] = user_agent - if stream_id: - metadata[ChannelMetadataField.STREAM_ID] = str(stream_id) - if not stream_name: - try: - from apps.channels.models import Stream - stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() - except Exception as e: - logger.warning(f"Failed to update stream name in Redis for stream {stream_id}: {e}") - if stream_name: - metadata[ChannelMetadataField.STREAM_NAME] = stream_name - if m3u_profile_id: - metadata[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) + # Build metadata update dict + metadata = {ChannelMetadataField.URL: url} + if user_agent: + metadata[ChannelMetadataField.USER_AGENT] = user_agent + if stream_id: + metadata[ChannelMetadataField.STREAM_ID] = str(stream_id) + if not stream_name: + try: + from apps.channels.models import Stream + stream_name = Stream.objects.filter(id=stream_id).values_list('name', flat=True).first() + except Exception as e: + logger.warning(f"Failed to update stream name in Redis for stream {stream_id}: {e}") + if stream_name: + metadata[ChannelMetadataField.STREAM_NAME] = stream_name + if m3u_profile_id: + metadata[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) - # Also update the stream switch time field - metadata[ChannelMetadataField.STREAM_SWITCH_TIME] = str(time.time()) + # Also update the stream switch time field + metadata[ChannelMetadataField.STREAM_SWITCH_TIME] = str(time.time()) - # Use the appropriate method based on the key type - if key_type == 'hash': - proxy_server.redis_client.hset(metadata_key, mapping=metadata) - elif key_type == 'none': # Key doesn't exist yet - proxy_server.redis_client.hset(metadata_key, mapping=metadata) - else: - # If key exists with wrong type, delete it and recreate - proxy_server.redis_client.delete(metadata_key) - proxy_server.redis_client.hset(metadata_key, mapping=metadata) + # Use the appropriate method based on the key type + if key_type == 'hash': + proxy_server.redis_client.hset(metadata_key, mapping=metadata) + elif key_type == 'none': # Key doesn't exist yet + proxy_server.redis_client.hset(metadata_key, mapping=metadata) + else: + # If key exists with wrong type, delete it and recreate + proxy_server.redis_client.delete(metadata_key) + proxy_server.redis_client.hset(metadata_key, mapping=metadata) - # Set switch request flag to ensure all workers see it - switch_key = RedisKeys.switch_request(channel_id) - proxy_server.redis_client.setex(switch_key, 30, url) # 30 second TTL + # Set switch request flag to ensure all workers see it + switch_key = RedisKeys.switch_request(channel_id) + proxy_server.redis_client.setex(switch_key, 30, url) # 30 second TTL - logger.debug(f"Updated metadata for channel {channel_id} in Redis") - return True + logger.debug(f"Updated metadata for channel {channel_id} in Redis") + return True + finally: + from django.db import close_old_connections + close_old_connections() @staticmethod def _publish_stream_switch_event(channel_id, new_url, user_agent=None, stream_id=None, m3u_profile_id=None): diff --git a/apps/proxy/live_proxy/tests/test_live_db_cleanup.py b/apps/proxy/live_proxy/tests/test_live_db_cleanup.py new file mode 100644 index 00000000..f2015a41 --- /dev/null +++ b/apps/proxy/live_proxy/tests/test_live_db_cleanup.py @@ -0,0 +1,142 @@ +"""Live proxy must release geventpool checkouts after ORM on stream and URL paths.""" + +from unittest.mock import MagicMock, patch + +from django.http import StreamingHttpResponse +from django.test import RequestFactory, SimpleTestCase + + +class StreamTsDbCleanupTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + @patch("apps.proxy.live_proxy.views.close_old_connections") + @patch("apps.proxy.live_proxy.views.create_stream_generator") + @patch("apps.proxy.live_proxy.views._resolve_output_format", return_value="mpegts") + @patch("apps.proxy.live_proxy.views._resolve_output_profile", return_value=None) + @patch("apps.proxy.live_proxy.views.ChannelService.is_channel_unavailable_for_new_clients", return_value=False) + @patch("apps.proxy.live_proxy.views.get_stream_object") + @patch("apps.proxy.live_proxy.views.network_access_allowed", return_value=True) + @patch("apps.proxy.live_proxy.views.ProxyServer") + def test_stream_ts_closes_db_before_streaming_response( + self, + mock_proxy_cls, + _network_ok, + mock_get_stream_object, + _unavailable, + _output_profile, + _output_format, + mock_create_generator, + mock_close, + ): + channel = MagicMock() + channel.id = 1 + channel.uuid = "channel-uuid" + channel.name = "Test Channel" + mock_get_stream_object.return_value = channel + + client_manager = MagicMock() + proxy_server = MagicMock() + proxy_server.redis_client = MagicMock() + proxy_server.redis_client.exists.return_value = True + proxy_server.redis_client.hgetall.return_value = {"state": "active"} + proxy_server.stream_buffers = {"channel-uuid": MagicMock()} + proxy_server.client_managers = {"channel-uuid": client_manager} + proxy_server.check_if_channel_exists.return_value = True + proxy_server.get_buffer.return_value = MagicMock() + mock_proxy_cls.get_instance.return_value = proxy_server + + def _generate(): + yield b"chunk" + + mock_create_generator.return_value = lambda: _generate() + + request = self.factory.get("/proxy/live/channel-uuid/") + request.user = MagicMock(is_authenticated=False) + + from apps.proxy.live_proxy.views import stream_ts + + response = stream_ts(request, "channel-uuid") + + self.assertIsInstance(response, StreamingHttpResponse) + mock_close.assert_called_once() + + +class UrlUtilsDbCleanupTests(SimpleTestCase): + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_stream_object") + def test_generate_stream_url_closes_db(self, mock_get_object, mock_close): + channel = MagicMock() + channel.get_stream.return_value = (None, None, "no streams", False) + mock_get_object.return_value = channel + + from apps.proxy.live_proxy.url_utils import generate_stream_url + + result = generate_stream_url("channel-uuid") + + self.assertIsNone(result[0]) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_stream_object") + def test_get_alternate_streams_closes_db(self, mock_get_object, mock_close): + channel = MagicMock() + channel.streams.all.return_value.order_by.return_value.exists.return_value = False + mock_get_object.return_value = channel + + from apps.proxy.live_proxy.url_utils import get_alternate_streams + + result = get_alternate_streams("channel-uuid", current_stream_id=1) + + self.assertEqual(result, []) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.get_object_or_404") + def test_get_stream_info_for_switch_closes_db_on_error(self, mock_get_404, mock_close): + mock_get_404.side_effect = RuntimeError("db error") + + from apps.proxy.live_proxy.url_utils import get_stream_info_for_switch + + result = get_stream_info_for_switch("channel-uuid", target_stream_id=99) + + self.assertIn("error", result) + mock_close.assert_called_once() + + @patch("apps.proxy.live_proxy.url_utils.close_old_connections") + @patch("apps.proxy.live_proxy.url_utils.M3UAccountProfile.objects.get") + def test_get_connections_left_closes_db(self, mock_get, mock_close): + mock_get.side_effect = Exception("not found") + + from apps.proxy.live_proxy.url_utils import get_connections_left + + result = get_connections_left(999) + + self.assertEqual(result, 0) + mock_close.assert_called_once() + + +class TsGeneratorDbCleanupTests(SimpleTestCase): + @patch("apps.proxy.live_proxy.output.ts.generator.close_old_connections") + @patch("apps.proxy.live_proxy.output.ts.generator.ProxyServer.get_instance") + def test_ts_cleanup_closes_db(self, mock_proxy_cls, mock_close): + proxy_server = MagicMock() + proxy_server.redis_client = None + proxy_server.client_managers = {} + mock_proxy_cls.return_value = proxy_server + + from apps.proxy.live_proxy.output.ts.generator import StreamGenerator + + gen = StreamGenerator.__new__(StreamGenerator) + gen.channel_id = "channel-uuid" + gen.client_id = "client-1" + gen.stream_start_time = 0 + gen.channel_name = "Test" + gen.client_ip = "127.0.0.1" + gen.client_user_agent = "agent" + gen.bytes_sent = 0 + gen.user = None + + gen._cleanup() + + mock_close.assert_called_once() diff --git a/apps/proxy/live_proxy/url_utils.py b/apps/proxy/live_proxy/url_utils.py index 7cdc92ef..2b32f7e6 100644 --- a/apps/proxy/live_proxy/url_utils.py +++ b/apps/proxy/live_proxy/url_utils.py @@ -5,6 +5,7 @@ Utilities for handling stream URLs and transformations. import logging import regex from typing import Optional, Tuple, List +from django.db import close_old_connections from django.shortcuts import get_object_or_404 from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile @@ -156,6 +157,8 @@ def generate_stream_url( except Exception as e: logger.error(f"Error generating stream URL: {e}") return None, None, False, None, False, str(e) + finally: + close_old_connections() def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -300,6 +303,8 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] channel.release_stream() logger.error(f"Error getting stream info for switch: {e}", exc_info=True) return {'error': f'Error: {str(e)}'} + finally: + close_old_connections() def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = None) -> List[dict]: """ @@ -422,6 +427,8 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No except Exception as e: logger.error(f"Error getting alternate streams for channel {channel_id}: {e}", exc_info=True) return [] + finally: + close_old_connections() def validate_stream_url(url, user_agent=None, timeout=(5, 5)): """ @@ -592,3 +599,5 @@ def get_connections_left(m3u_profile_id: int) -> int: except Exception as e: logger.error(f"Error getting connections left for M3U profile {m3u_profile_id}: {e}") return 0 + finally: + close_old_connections() diff --git a/apps/proxy/live_proxy/views.py b/apps/proxy/live_proxy/views.py index 2119b4bd..001cfc14 100644 --- a/apps/proxy/live_proxy/views.py +++ b/apps/proxy/live_proxy/views.py @@ -3,6 +3,7 @@ import time import random import re import pathlib +from django.db import close_old_connections from django.http import StreamingHttpResponse, JsonResponse, HttpResponseRedirect, HttpResponse from django.views.decorators.csrf import csrf_exempt from django.shortcuts import get_object_or_404 @@ -617,7 +618,9 @@ def stream_ts(request, channel_id, user=None, force_output_format=None): ) content_type = "video/mp2t" - # Return the StreamingHttpResponse from the main function + # Release ORM checkout before returning a long-lived StreamingHttpResponse. + close_old_connections() + response = StreamingHttpResponse( streaming_content=generate(), content_type=content_type ) From cfe58db222ea489cbc9ebc210ab4bd5f175aa287 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 10:12:12 -0500 Subject: [PATCH 02/28] fix(xc): normalize server URLs for live playback and stream URLs (Fixes #1363) - Implemented `normalize_server_url()` to standardize account server URLs, ensuring that on-demand live URLs are built correctly without including API endpoints or query parameters. - Updated `get_transformed_credentials()` and stream URL generation in `M3UMovieRelation` and `M3UEpisodeRelation` to utilize the new normalization function, improving URL handling for Xtream Codes accounts. --- CHANGELOG.md | 1 + apps/m3u/tasks.py | 5 +- apps/m3u/tests/test_xc_live_url.py | 139 +++++++++++++++++++++++++++++ apps/vod/models.py | 21 +++-- core/xtream_codes.py | 38 ++++---- 5 files changed, 176 insertions(+), 28 deletions(-) create mode 100644 apps/m3u/tests/test_xc_live_url.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f1234c5..1712d447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. ## [0.27.0] - 2026-06-16 diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 8c69965e..2d5ac1ee 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -2816,12 +2816,13 @@ def get_transformed_credentials(account, profile=None): logger.warning(f"Could not get primary profile for account {account.name}: {e}") profile = None - base_url = account.server_url + from core.xtream_codes import normalize_server_url + + base_url = normalize_server_url(account.server_url) base_username = account.username base_password = account.password # Build a complete URL with credentials (similar to how IPTV URLs are structured) # Format: http://server.com:port/live/username/password/1234.ts if base_url and base_username and base_password: - # Remove trailing slash from server URL if present clean_server_url = base_url.rstrip('/') # Build the complete URL with embedded credentials diff --git a/apps/m3u/tests/test_xc_live_url.py b/apps/m3u/tests/test_xc_live_url.py new file mode 100644 index 00000000..8353241d --- /dev/null +++ b/apps/m3u/tests/test_xc_live_url.py @@ -0,0 +1,139 @@ +"""Tests for XC stream URL normalization and on-demand URL building.""" + +from django.test import TestCase + +from apps.channels.models import Stream +from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.m3u.tasks import get_transformed_credentials +from apps.proxy.live_proxy.url_utils import _resolve_live_stream_url +from apps.vod.models import Episode, M3UEpisodeRelation, M3UMovieRelation, Movie, Series +from core.xtream_codes import normalize_server_url + + +class NormalizeServerUrlTests(TestCase): + def test_preserves_sub_path(self): + url = "https://myserver.fun/server1" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_strips_player_api_php_and_query_params(self): + url = "https://myserver.fun/server1/player_api.php?username=foo&password=bar" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_strips_trailing_slash(self): + url = "https://myserver.fun/server1/" + self.assertEqual(normalize_server_url(url), "https://myserver.fun/server1") + + def test_nested_sub_path_with_php_endpoint(self): + url = "http://server/Pluto/gb/player_api.php" + self.assertEqual(normalize_server_url(url), "http://server/Pluto/gb") + + +class GetTransformedCredentialsTests(TestCase): + def test_returns_normalized_server_url(self): + account = M3UAccount.objects.create( + name="Sub-path XC", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + + server_url, username, password = get_transformed_credentials(account, profile) + + self.assertEqual(server_url, "https://myserver.fun/server1") + self.assertEqual(username, "alice") + self.assertEqual(password, "secret") + + +class ResolveLiveStreamUrlTests(TestCase): + def test_builds_url_from_normalized_base_not_raw_account_url(self): + account = M3UAccount.objects.create( + name="Live sub-path", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + stream = Stream.objects.create( + name="Test Channel", + m3u_account=account, + stream_id="12345", + url="https://myserver.fun/server1/live/olduser/oldpass/12345.ts", + ) + + url = _resolve_live_stream_url(stream, account, profile) + + self.assertEqual( + url, + "https://myserver.fun/server1/live/alice/secret/12345.ts", + ) + + def test_std_account_uses_stored_stream_url(self): + account = M3UAccount.objects.create( + name="STD account", + account_type="STD", + server_url="https://example.com/list.m3u", + username="alice", + password="secret", + ) + profile = M3UAccountProfile.objects.get(m3u_account=account, is_default=True) + stream = Stream.objects.create( + name="STD Stream", + m3u_account=account, + url="https://provider.example/stream/abc123", + ) + + url = _resolve_live_stream_url(stream, account, profile) + + self.assertEqual(url, "https://provider.example/stream/abc123") + + +class VodStreamUrlTests(TestCase): + def setUp(self): + self.account = M3UAccount.objects.create( + name="VOD sub-path", + account_type="XC", + server_url="https://myserver.fun/server1/player_api.php?username=foo", + username="alice", + password="secret", + ) + + def test_movie_relation_builds_normalized_url(self): + movie = Movie.objects.create(name="Test Movie") + relation = M3UMovieRelation.objects.create( + m3u_account=self.account, + movie=movie, + stream_id="999", + container_extension="mkv", + ) + + url = relation.get_stream_url() + + self.assertEqual( + url, + "https://myserver.fun/server1/movie/alice/secret/999.mkv", + ) + + def test_episode_relation_builds_normalized_url(self): + series = Series.objects.create(name="Test Series") + episode = Episode.objects.create( + series=series, + name="Pilot", + season_number=1, + episode_number=1, + ) + relation = M3UEpisodeRelation.objects.create( + m3u_account=self.account, + episode=episode, + stream_id="888", + container_extension="mp4", + ) + + url = relation.get_stream_url() + + self.assertEqual( + url, + "https://myserver.fun/server1/series/alice/secret/888.mp4", + ) diff --git a/apps/vod/models.py b/apps/vod/models.py index dd7d1753..f609be81 100644 --- a/apps/vod/models.py +++ b/apps/vod/models.py @@ -243,12 +243,12 @@ class M3UMovieRelation(models.Model): def get_stream_url(self): """Get the full stream URL for this movie from this provider""" - # Build URL dynamically for XtreamCodes accounts if self.m3u_account.account_type == 'XC': - from core.xtream_codes import Client as XCClient - # Use XC client's URL normalization to handle malformed URLs - # (e.g., URLs with /player_api.php or query parameters) - normalized_url = XCClient(self.m3u_account.server_url, '', '')._normalize_url(self.m3u_account.server_url) + from core.xtream_codes import normalize_server_url + + normalized_url = normalize_server_url(self.m3u_account.server_url) + if not normalized_url: + return None username = self.m3u_account.username password = self.m3u_account.password return f"{normalized_url}/movie/{username}/{password}/{self.stream_id}.{self.container_extension or 'mp4'}" @@ -292,13 +292,12 @@ class M3UEpisodeRelation(models.Model): def get_stream_url(self): """Get the full stream URL for this episode from this provider""" - from core.xtream_codes import Client as XtreamCodesClient - if self.m3u_account.account_type == 'XC': - # For XtreamCodes accounts, build the URL dynamically - # Use XC client's URL normalization to handle malformed URLs - # (e.g., URLs with /player_api.php or query parameters) - normalized_url = XtreamCodesClient(self.m3u_account.server_url, '', '')._normalize_url(self.m3u_account.server_url) + from core.xtream_codes import normalize_server_url + + normalized_url = normalize_server_url(self.m3u_account.server_url) + if not normalized_url: + return None username = self.m3u_account.username password = self.m3u_account.password return f"{normalized_url}/series/{username}/{password}/{self.stream_id}.{self.container_extension or 'mp4'}" diff --git a/core/xtream_codes.py b/core/xtream_codes.py index 195d4428..bcbe22e2 100644 --- a/core/xtream_codes.py +++ b/core/xtream_codes.py @@ -5,6 +5,26 @@ import json logger = logging.getLogger(__name__) + +def normalize_server_url(url): + """Normalize server URL: strip XC API endpoints and query params, preserve base path.""" + if not url: + return url + + from urllib.parse import urlparse, urlunparse + + parsed = urlparse(url.strip()) + path = parsed.path.rstrip('/') + + # XC API endpoints are always .php files; legitimate base paths never are. + # Stripping the trailing segment when it ends in .php handles any pasted API URL. + last_segment = path.rsplit('/', 1)[-1] + if last_segment.endswith('.php'): + path = path[:-(len(last_segment) + 1)] if '/' in path else '' + + return urlunparse((parsed.scheme, parsed.netloc, path, '', '', '')) + + class Client: """Xtream Codes API Client with robust error handling""" @@ -43,22 +63,10 @@ class Client: self.server_info = None def _normalize_url(self, url): - """Normalize server URL: strip XC API endpoints and query params, preserve base path.""" - if not url: + normalized = normalize_server_url(url) + if not normalized: raise ValueError("Server URL cannot be empty") - - from urllib.parse import urlparse, urlunparse - - parsed = urlparse(url.strip()) - path = parsed.path.rstrip('/') - - # XC API endpoints are always .php files; legitimate base paths never are. - # Stripping the trailing segment when it ends in .php handles any pasted API URL. - last_segment = path.rsplit('/', 1)[-1] - if last_segment.endswith('.php'): - path = path[:-(len(last_segment) + 1)] if '/' in path else '' - - return urlunparse((parsed.scheme, parsed.netloc, path, '', '', '')) + return normalized def _make_request(self, endpoint, params=None): """Make request with detailed error handling""" From 46695588f73dc27f17930b52e7d84f4ac73860d7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 14:59:09 -0500 Subject: [PATCH 03/28] fix(m3u): enhance custom properties handling and DB connection management - Implemented `ensure_custom_properties_dict()` to normalize custom properties across various models and serializers, addressing issues with legacy JSON-encoded strings. - Updated M3U account and channel group models to ensure custom properties are consistently stored as dictionaries during save operations. - Enhanced Celery task management by ensuring old DB connections are closed before and after tasks, improving reliability and preventing errors during account refresh operations. (Fixes #1338) --- CHANGELOG.md | 2 + apps/channels/compact_numbering.py | 11 +- apps/channels/models.py | 9 +- apps/channels/serializers.py | 11 - apps/m3u/api_views.py | 10 +- apps/m3u/forms.py | 8 +- apps/m3u/models.py | 11 + apps/m3u/serializers.py | 20 +- apps/m3u/tasks.py | 273 ++++++++++++++------- apps/m3u/tests/test_refresh_db_recovery.py | 74 ++++++ core/utils.py | 42 +++- dispatcharr/celery.py | 20 +- tests/test_custom_properties_as_dict.py | 74 ++++++ 13 files changed, 446 insertions(+), 119 deletions(-) create mode 100644 apps/m3u/tests/test_refresh_db_recovery.py create mode 100644 tests/test_custom_properties_as_dict.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1712d447..74a5f6a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) +- **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/channels/compact_numbering.py b/apps/channels/compact_numbering.py index fd29a153..27a26cb6 100644 --- a/apps/channels/compact_numbering.py +++ b/apps/channels/compact_numbering.py @@ -25,7 +25,8 @@ import logging from django.db import transaction from .models import Channel, ChannelOverride, ChannelGroupM3UAccount -from apps.m3u.tasks import _custom_properties_as_dict, _next_available_number +from apps.m3u.tasks import _next_available_number +from core.utils import ensure_custom_properties_dict from core.utils import ( acquire_task_lock, natural_sort_key, @@ -37,7 +38,7 @@ logger = logging.getLogger(__name__) def is_compact_group(group_relation): """Return True if the given ChannelGroupM3UAccount is in compact mode.""" - cp = _custom_properties_as_dict(group_relation.custom_properties) + cp = ensure_custom_properties_dict(group_relation.custom_properties) return bool(cp.get("compact_numbering")) @@ -72,7 +73,7 @@ def get_group_relation_for_channel(channel): for rel in ChannelGroupM3UAccount.objects.filter( m3u_account_id=channel.auto_created_by_id ): - cp = _custom_properties_as_dict(rel.custom_properties) + cp = ensure_custom_properties_dict(rel.custom_properties) if str(cp.get("group_override", "")) == target: return rel return None @@ -179,7 +180,7 @@ def assign_compact_numbers_for_channels(channel_ids): for rel in ChannelGroupM3UAccount.objects.filter( m3u_account_id__in={k[0] for k in unresolved} ): - cp = _custom_properties_as_dict(rel.custom_properties) + cp = ensure_custom_properties_dict(rel.custom_properties) target = cp.get("group_override") if not target: continue @@ -295,7 +296,7 @@ def _repack_inner(group_relation): account_id = group_relation.m3u_account_id group_id = group_relation.channel_group_id - cp = _custom_properties_as_dict(group_relation.custom_properties) + cp = ensure_custom_properties_dict(group_relation.custom_properties) sort_order = cp.get("channel_sort_order") or "" sort_reverse = bool(cp.get("channel_sort_reverse")) diff --git a/apps/channels/models.py b/apps/channels/models.py index ef1d0cff..747d0b89 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -2,7 +2,7 @@ from django.db import models from django.core.exceptions import ValidationError from django.conf import settings from core.models import StreamProfile, CoreSettings -from core.utils import RedisClient +from core.utils import RedisClient, custom_properties_as_dict from apps.proxy.live_proxy.redis_keys import RedisKeys from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState import logging @@ -1119,6 +1119,13 @@ class ChannelGroupM3UAccount(models.Model): def __str__(self): return f"{self.channel_group.name} - {self.m3u_account.name} (Enabled: {self.enabled})" + def save(self, *args, **kwargs): + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + super().save(*args, **kwargs) + class Logo(models.Model): name = models.CharField(max_length=255) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 9b7d3a2a..2e80d840 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -221,17 +221,6 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): return data - def to_internal_value(self, data): - # Accept both dict and JSON string for custom_properties (for backward compatibility) - val = data.get("custom_properties") - if isinstance(val, str): - try: - data["custom_properties"] = json.loads(val) - except Exception: - pass - - return super().to_internal_value(data) - def validate(self, attrs): # Partial PATCHes only carry submitted fields; fill missing # start/end from the instance so the validator catches a PATCH diff --git a/apps/m3u/api_views.py b/apps/m3u/api_views.py index a49ddd6f..1117a226 100644 --- a/apps/m3u/api_views.py +++ b/apps/m3u/api_views.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile from core.models import UserAgent -from core.utils import safe_upload_path +from core.utils import safe_upload_path, ensure_custom_properties_dict from apps.channels.models import ChannelGroupM3UAccount from core.serializers import UserAgentSerializer from apps.vod.models import M3UVODCategoryRelation @@ -536,7 +536,9 @@ class M3UAccountViewSet(viewsets.ModelViewSet): auto_channel_sync=setting.get("auto_channel_sync", False), auto_sync_channel_start=setting.get("auto_sync_channel_start"), auto_sync_channel_end=setting.get("auto_sync_channel_end"), - custom_properties=setting.get("custom_properties", {}), + custom_properties=ensure_custom_properties_dict( + setting.get("custom_properties") + ), ) for setting in group_settings if setting.get("channel_group") @@ -561,7 +563,9 @@ class M3UAccountViewSet(viewsets.ModelViewSet): category_id=setting["id"], m3u_account=account, enabled=setting.get("enabled", True), - custom_properties=setting.get("custom_properties", {}), + custom_properties=ensure_custom_properties_dict( + setting.get("custom_properties") + ), ) for setting in category_settings if setting.get("id") diff --git a/apps/m3u/forms.py b/apps/m3u/forms.py index cf6586c3..af4d6072 100644 --- a/apps/m3u/forms.py +++ b/apps/m3u/forms.py @@ -1,6 +1,7 @@ # apps/m3u/forms.py from django import forms from .models import M3UAccount, M3UFilter +from core.utils import ensure_custom_properties_dict import re class M3UAccountForm(forms.ModelForm): @@ -28,7 +29,9 @@ class M3UAccountForm(forms.ModelForm): # Set initial value for enable_vod from custom_properties if self.instance and self.instance.custom_properties: - custom_props = self.instance.custom_properties or {} + custom_props = self.instance.custom_properties + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) self.fields['enable_vod'].initial = custom_props.get('enable_vod', False) def save(self, commit=True): @@ -37,8 +40,9 @@ class M3UAccountForm(forms.ModelForm): # Handle enable_vod field enable_vod = self.cleaned_data.get('enable_vod', False) - # Parse existing custom_properties custom_props = instance.custom_properties or {} + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) # Update VOD preference custom_props['enable_vod'] = enable_vod diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 53c18d38..7d4fe65a 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -7,6 +7,7 @@ from django.dispatch import receiver from apps.channels.models import StreamProfile from django_celery_beat.models import PeriodicTask from core.models import CoreSettings, UserAgent +from core.utils import custom_properties_as_dict CUSTOM_M3U_ACCOUNT_NAME = "custom" @@ -124,6 +125,11 @@ class M3UAccount(models.Model): return user_agent def save(self, *args, **kwargs): + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + # Prevent auto_now behavior by handling updated_at manually if "update_fields" in kwargs and "updated_at" not in kwargs["update_fields"]: # Don't modify updated_at for regular updates @@ -263,6 +269,11 @@ class M3UAccountProfile(models.Model): def save(self, *args, **kwargs): """Auto-sync exp_date from custom_properties for XC accounts on every save. For non-XC accounts, exp_date is set directly and left untouched here.""" + if self.custom_properties is not None and not isinstance( + self.custom_properties, dict + ): + self.custom_properties = custom_properties_as_dict(self.custom_properties) + parsed = self._parse_exp_date_from_custom_properties() if parsed is not None: # XC account with exp_date in custom_properties — always sync diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 485a3687..c1dc6f92 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -1,4 +1,4 @@ -from core.utils import validate_flexible_url +from core.utils import validate_flexible_url, ensure_custom_properties_dict from rest_framework import serializers, status from rest_framework.response import Response from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile @@ -270,11 +270,15 @@ class M3UAccountSerializer(serializers.ModelSerializer): # overwrite their corresponding keys; clients should set those via # the typed top-level fields rather than the custom_properties # payload. - incoming_custom = validated_data.get("custom_properties") or {} - custom_props = { - **(instance.custom_properties or {}), - **incoming_custom, - } + incoming_custom = {} + if "custom_properties" in validated_data: + incoming_custom = validated_data["custom_properties"] or {} + if not isinstance(incoming_custom, dict): + incoming_custom = ensure_custom_properties_dict(incoming_custom) + existing_custom = instance.custom_properties or {} + if not isinstance(existing_custom, dict): + existing_custom = ensure_custom_properties_dict(existing_custom) + custom_props = {**existing_custom, **incoming_custom} if enable_vod is not None: custom_props["enable_vod"] = enable_vod @@ -346,7 +350,9 @@ class M3UAccountSerializer(serializers.ModelSerializer): auto_enable_new_groups_series = validated_data.pop("auto_enable_new_groups_series", True) # Parse existing custom_properties or create new - custom_props = validated_data.get("custom_properties", {}) + custom_props = validated_data.get("custom_properties") or {} + if not isinstance(custom_props, dict): + custom_props = ensure_custom_properties_dict(custom_props) # Set preferences (default to True for auto_enable_new_groups) custom_props["enable_vod"] = enable_vod diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 2d5ac1ee..be2a612f 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -7,29 +7,23 @@ import os import gc import gzip, zipfile from concurrent.futures import ThreadPoolExecutor, as_completed -from celery.app.control import Inspect -from celery.result import AsyncResult -from celery import shared_task, current_app, group +from celery import shared_task from django.conf import settings -from django.core.cache import cache from django.db import models, transaction from .models import M3UAccount from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer from django.utils import timezone import time import json from core.utils import ( - RedisClient, acquire_task_lock, release_task_lock, TaskLockRenewer, natural_sort_key, log_system_event, + ensure_custom_properties_dict, ) from core.models import CoreSettings, UserAgent -from asgiref.sync import async_to_sync from core.xtream_codes import Client as XCClient from core.utils import send_websocket_update from .utils import normalize_stream_url @@ -39,6 +33,104 @@ logger = logging.getLogger(__name__) BATCH_SIZE = 1500 # Optimized batch size for threading m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") +_NON_TERMINAL_REFRESH_STATUSES = frozenset({ + M3UAccount.Status.FETCHING, + M3UAccount.Status.PARSING, +}) + + +def _release_task_db_connection(): + """Return the Celery worker's DB connection to a clean state after ORM errors.""" + from django.db import close_old_connections + close_old_connections() + + +def _db_query_with_retry(fn, *, label="DB query", max_retries=2): + """ + Run an ORM read with one connection reset + retry on transient failures. + + Poisoned Celery worker connections often surface as OperationalError or as + ``IndexError: list index out of range`` inside Django's row converters. + """ + from django.db import InterfaceError, OperationalError + + transient_errors = (OperationalError, InterfaceError, IndexError) + for attempt in range(max_retries): + try: + return fn() + except transient_errors as exc: + if attempt + 1 >= max_retries: + raise + logger.warning( + "%s failed (%s), resetting DB connection (%s/%s)", + label, + exc, + attempt + 1, + max_retries, + ) + _release_task_db_connection() + + +def _get_active_m3u_account(account_id): + return _db_query_with_retry( + lambda: M3UAccount.objects.get(id=account_id, is_active=True), + label=f"load active M3U account {account_id}", + ) + + +def _set_m3u_account_status( + account_id, + status, + last_message=None, + *, + notify_error=False, + ws_action="parsing", + ws_error=None, +): + """Update account status using a fresh connection (safe after DB failures).""" + _release_task_db_connection() + update = {"status": status} + if last_message is not None: + update["last_message"] = last_message + try: + M3UAccount.objects.filter(id=account_id).update(**update) + if notify_error: + send_m3u_update( + account_id, + ws_action, + 100, + status="error", + error=ws_error or last_message, + ) + except Exception as e: + logger.error( + f"Failed to set account {account_id} status to {status}: {e}" + ) + + +def _ensure_m3u_refresh_terminal_status(account_id): + """Mark refresh as failed when the task exits while still in progress.""" + _release_task_db_connection() + try: + current_status = ( + M3UAccount.objects.filter(id=account_id) + .values_list("status", flat=True) + .first() + ) + if current_status in _NON_TERMINAL_REFRESH_STATUSES: + message = "Refresh did not complete successfully" + M3UAccount.objects.filter(id=account_id).update( + status=M3UAccount.Status.ERROR, + last_message=message, + ) + send_m3u_update( + account_id, "parsing", 100, status="error", error=message + ) + except Exception as e: + logger.debug( + f"Could not verify terminal refresh status for account {account_id}: {e}" + ) + _EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') @@ -630,7 +722,7 @@ def process_groups(account, groups, scan_start_time=None): logger.info(f"Currently {len(existing_groups)} existing groups") # Check if we should auto-enable new groups based on account settings - account_custom_props = account.custom_properties or {} + account_custom_props = ensure_custom_properties_dict(account.custom_properties) auto_enable_new_groups_live = account_custom_props.get("auto_enable_new_groups_live", True) # Separate existing groups from groups that need to be created @@ -673,7 +765,9 @@ def process_groups(account, groups, scan_start_time=None): existing_rel = existing_relationships[group.name] # Get existing custom properties (now JSONB, no need to parse) - existing_custom_props = existing_rel.custom_properties or {} + existing_custom_props = ensure_custom_properties_dict( + existing_rel.custom_properties + ) # Get the new xc_id from groups data new_xc_id = custom_props.get("xc_id") @@ -696,6 +790,8 @@ def process_groups(account, groups, scan_start_time=None): logger.debug(f"Updated xc_id for group '{group.name}' from '{existing_xc_id}' to '{new_xc_id}' - account {account.id}") else: # Update last_seen even if xc_id hasn't changed + if isinstance(existing_rel.custom_properties, str): + existing_rel.custom_properties = existing_custom_props existing_rel.last_seen = scan_start_time existing_rel.is_stale = False relations_to_update.append(existing_rel) @@ -1755,33 +1851,6 @@ def _range_exhausted_error(mode, start_number, end_number, fallback_start): return f"Channel number range {range_start}-{int(end_number)} is full" -def _custom_properties_as_dict(value): - """ - Normalize a JSONField-backed custom_properties value into a dict. - - Historical data has rows where the field holds a JSON-encoded string - instead of a dict. Django's JSONField serializes whatever it gets, so - `.get()` on one of those rows raises AttributeError and aborts the - entire sync. Treat string values as JSON to parse, and fall back to an - empty dict for anything that isn't a dict after parsing. - """ - import json - - if isinstance(value, dict): - return value - if isinstance(value, str): - try: - parsed = json.loads(value) - except (ValueError, TypeError): - logger.warning( - "custom_properties stored as non-JSON string; ignoring: %r", - value[:100], - ) - return {} - return parsed if isinstance(parsed, dict) else {} - return {} - - def _classify_sync_failure(exc): """ Map an exception raised during per-stream sync to a coarse typed @@ -1890,7 +1959,7 @@ def sync_auto_channels(account_id, scan_start_time=None): custom_epg_id = None # New option: select specific EPG source (takes priority over force_dummy_epg) channel_numbering_mode = "fixed" # Default mode channel_numbering_fallback = 1 # Default fallback for provider mode - group_custom_props = _custom_properties_as_dict( + group_custom_props = ensure_custom_properties_dict( group_relation.custom_properties ) if group_custom_props: @@ -2734,7 +2803,7 @@ def sync_auto_channels(account_id, scan_start_time=None): # "preserve_customized" keeps those with a ChannelOverride row; # "never" disables cleanup. Hidden channels are preserved across all # modes so event/PPV channels that come and go are not silently lost. - cleanup_mode = (account.custom_properties or {}).get( + cleanup_mode = ensure_custom_properties_dict(account.custom_properties).get( "orphan_channel_cleanup", "always" ) if cleanup_mode != "never": @@ -2948,12 +3017,19 @@ def refresh_account_profiles(account_id): if profile_client.authenticate(): # Get account information specific to this profile's credentials profile_account_info = profile_client.get_account_info() + if not isinstance(profile_account_info, dict): + raise TypeError( + f"Unexpected account info type: {type(profile_account_info).__name__}" + ) # Merge with existing custom_properties if they exist - existing_props = profile.custom_properties or {} - existing_props.update(profile_account_info) - profile.custom_properties = existing_props - profile.save(update_fields=['custom_properties']) + profile.custom_properties = { + **ensure_custom_properties_dict( + profile.custom_properties + ), + **profile_account_info, + } + profile.save(update_fields=['custom_properties', 'exp_date']) profiles_updated += 1 logger.info(f"Updated account information for profile '{profile.name}' ({profiles_updated}/{profiles.count()})") @@ -2964,6 +3040,7 @@ def refresh_account_profiles(account_id): except Exception as profile_error: profiles_failed += 1 logger.error(f"Failed to update account information for profile '{profile.name}': {str(profile_error)}") + _release_task_db_connection() # Continue with other profiles even if one fails result_msg = f"Profile refresh complete for account {account.name}: {profiles_updated} updated, {profiles_failed} failed" @@ -2978,6 +3055,8 @@ def refresh_account_profiles(account_id): error_msg = f"Error refreshing profiles for account {account_id}: {str(e)}" logger.error(error_msg) return error_msg + finally: + _release_task_db_connection() @shared_task @@ -3032,10 +3111,11 @@ def refresh_account_info(profile_id): account_info = client.get_account_info() # Update only this specific profile with the new account info - if not profile.custom_properties: - profile.custom_properties = {} - profile.custom_properties.update(account_info) - profile.save() + profile.custom_properties = { + **ensure_custom_properties_dict(profile.custom_properties), + **account_info, + } + profile.save(update_fields=['custom_properties', 'exp_date']) # Send success notification to frontend via websocket send_websocket_update( @@ -3098,10 +3178,26 @@ def refresh_single_m3u_account(account_id): lock_renewer = TaskLockRenewer("refresh_single_m3u_account", account_id) lock_renewer.start() + _release_task_db_connection() + try: return _refresh_single_m3u_account_impl(account_id) + except Exception as e: + logger.error( + f"refresh_single_m3u_account failed for account {account_id}: {e}", + exc_info=True, + ) + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + f"Error processing M3U: {str(e)[:500]}", + notify_error=True, + ws_error=str(e)[:500], + ) + raise finally: - # Guaranteed cleanup on all exit paths (success, exception, early return) + _ensure_m3u_refresh_terminal_status(account_id) + _release_task_db_connection() lock_renewer.stop() release_task_lock("refresh_single_m3u_account", account_id) @@ -3116,22 +3212,25 @@ def _refresh_single_m3u_account_impl(account_id): streams_deleted = 0 try: - account = M3UAccount.objects.get(id=account_id, is_active=True) + account = _get_active_m3u_account(account_id) if not account.is_active: logger.debug(f"Account {account_id} is not active, skipping.") return - # Set status to fetching - account.status = M3UAccount.Status.FETCHING - account.save(update_fields=['status']) + # Set status to fetching and replace stale completion messages. + _set_m3u_account_status( + account_id, + M3UAccount.Status.FETCHING, + "Refresh in progress...", + ) + account = _get_active_m3u_account(account_id) filters = list(account.filters.all()) # Check if VOD is enabled for this account - vod_enabled = False - if account.custom_properties: - custom_props = account.custom_properties or {} - vod_enabled = custom_props.get('enable_vod', False) + vod_enabled = ensure_custom_properties_dict(account.custom_properties).get( + 'enable_vod', False + ) except M3UAccount.DoesNotExist: # The M3U account doesn't exist, so delete the periodic task if it exists @@ -3197,6 +3296,16 @@ def _refresh_single_m3u_account_impl(account_id): logger.error( f"Failed to refresh M3U groups for account {account_id}: {result}" ) + error_msg = ( + "Failed to refresh M3U groups - download failed or other error" + ) + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, + ) return "Failed to update m3u account - download failed or other error" extinf_data, groups = result @@ -3211,23 +3320,23 @@ def _refresh_single_m3u_account_impl(account_id): # For XC accounts, empty extinf_data is normal at this stage if not extinf_data and not is_xc_account: logger.error(f"No streams found for non-XC account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No streams found in M3U source" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( - account_id, "parsing", 100, status="error", error="No streams found" + error_msg = "No streams found in M3U source" + _set_m3u_account_status( + account_id, + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) except Exception as e: logger.error(f"Exception in refresh_m3u_groups: {str(e)}", exc_info=True) - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error refreshing M3U groups: {str(e)}" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( + error_msg = f"Error refreshing M3U groups: {str(e)[:500]}" + _set_m3u_account_status( account_id, - "parsing", - 100, - status="error", - error=f"Error refreshing M3U groups: {str(e)}", + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) return "Failed to update m3u account" @@ -3241,15 +3350,13 @@ def _refresh_single_m3u_account_impl(account_id): # Modified validation logic for different account types if (not groups) or (not is_xc_account and not extinf_data): logger.error(f"No data to process for account {account_id}") - account.status = M3UAccount.Status.ERROR - account.last_message = "No data available for processing" - account.save(update_fields=["status", "last_message"]) - send_m3u_update( + error_msg = "No data available for processing" + _set_m3u_account_status( account_id, - "parsing", - 100, - status="error", - error="No data available for processing", + M3UAccount.Status.ERROR, + error_msg, + notify_error=True, + ws_error=error_msg, ) return "Failed to update m3u account, no data available" @@ -3365,7 +3472,7 @@ def _refresh_single_m3u_account_impl(account_id): group_id = rel.channel_group.id # Load the custom properties with the xc_id - custom_props = rel.custom_properties or {} + custom_props = ensure_custom_properties_dict(rel.custom_properties) if "xc_id" in custom_props: filtered_groups[group_name] = { "xc_id": custom_props["xc_id"], @@ -3590,13 +3697,7 @@ def _refresh_single_m3u_account_impl(account_id): except Exception as e: logger.error(f"Error processing M3U for account {account_id}: {str(e)}") - try: - account.status = M3UAccount.Status.ERROR - account.last_message = f"Error processing M3U: {str(e)}" - account.save(update_fields=["status", "last_message"]) - except Exception: - logger.debug(f"Failed to update account {account_id} status during error handling") - raise # Re-raise the exception for Celery to handle + raise finally: # Free large data structures regardless of success or failure if 'existing_groups' in locals(): @@ -3623,6 +3724,8 @@ def _refresh_single_m3u_account_impl(account_id): except OSError: pass + _release_task_db_connection() + return f"Dispatched jobs complete." diff --git a/apps/m3u/tests/test_refresh_db_recovery.py b/apps/m3u/tests/test_refresh_db_recovery.py new file mode 100644 index 00000000..42db8bd5 --- /dev/null +++ b/apps/m3u/tests/test_refresh_db_recovery.py @@ -0,0 +1,74 @@ +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.m3u.tasks import ( + _db_query_with_retry, + _get_active_m3u_account, + _release_task_db_connection, + refresh_single_m3u_account, +) + + +class DbQueryWithRetryTests(SimpleTestCase): + def test_retries_after_index_error_from_poisoned_connection(self): + fn = MagicMock(side_effect=[IndexError("list index out of range"), "ok"]) + + with patch( + "apps.m3u.tasks._release_task_db_connection" + ) as mock_release: + result = _db_query_with_retry(fn, label="test query") + + self.assertEqual(result, "ok") + self.assertEqual(fn.call_count, 2) + mock_release.assert_called_once() + + def test_raises_after_exhausting_retries(self): + fn = MagicMock(side_effect=IndexError("list index out of range")) + + with patch("apps.m3u.tasks._release_task_db_connection"): + with self.assertRaises(IndexError): + _db_query_with_retry(fn, label="test query", max_retries=2) + + self.assertEqual(fn.call_count, 2) + + +class RefreshTaskDbStartupTests(SimpleTestCase): + @patch("apps.m3u.tasks._ensure_m3u_refresh_terminal_status") + @patch("apps.m3u.tasks._refresh_single_m3u_account_impl") + @patch("apps.m3u.tasks.release_task_lock") + @patch("apps.m3u.tasks.acquire_task_lock", return_value=True) + @patch("apps.m3u.tasks.TaskLockRenewer") + @patch("apps.m3u.tasks._release_task_db_connection") + def test_refresh_releases_db_connection_before_impl( + self, + mock_release, + _mock_renewer, + _mock_acquire, + _mock_release_lock, + mock_impl, + _mock_ensure_terminal, + ): + call_order = [] + + def track_release(): + call_order.append("release") + + mock_release.side_effect = track_release + mock_impl.side_effect = lambda *_a, **_k: call_order.append("impl") or "done" + + result = refresh_single_m3u_account(140) + + self.assertEqual(result, "done") + self.assertEqual(call_order[:2], ["release", "impl"]) + + @patch("apps.m3u.tasks.M3UAccount") + def test_get_active_m3u_account_uses_retry_helper(self, mock_model): + mock_model.objects.get.return_value = MagicMock(is_active=True) + + with patch("apps.m3u.tasks._db_query_with_retry") as mock_retry: + mock_retry.side_effect = lambda fn, **_: fn() + account = _get_active_m3u_account(140) + + mock_retry.assert_called_once() + self.assertTrue(account.is_active) diff --git a/core/utils.py b/core/utils.py index 188d9cb2..29c613b3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,3 +1,4 @@ +import json import redis import logging import time @@ -7,7 +8,6 @@ from pathlib import Path import re from django.conf import settings from redis.exceptions import ConnectionError, TimeoutError -from django.core.cache import cache from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.core.validators import URLValidator @@ -71,6 +71,46 @@ def natural_sort_key(text): return [convert(c) for c in re.split('([0-9]+)', text)] + +def custom_properties_as_dict(value): + """ + Normalize a JSONField-backed custom_properties value into a dict. + + Historical rows (TextField era and early JSONField migration) may store a + JSON-encoded string instead of an object. API clients can also submit a + string value because JSONField accepts any JSON type. Call this before + reading or merging custom_properties. + """ + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except (ValueError, TypeError): + logger.warning( + "custom_properties stored as non-JSON string; ignoring: %r", + value[:100], + ) + return {} + return parsed if isinstance(parsed, dict) else {} + if value is None: + return {} + return {} + + +def ensure_custom_properties_dict(value): + """ + Return a dict for read/merge/bulk-write paths. Dict values pass through + without re-parsing. Use model ``save()`` (not this) as the canonical + normalizer for ORM writes that go through ``save()``. + """ + if isinstance(value, dict): + return value + if value is None: + return {} + return custom_properties_as_dict(value) + + class RedisClient: _client = None _buffer = None diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 8154b52e..18be81eb 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -2,7 +2,7 @@ import os from celery import Celery import logging -from celery.signals import task_postrun, worker_ready +from celery.signals import task_postrun, task_prerun, worker_ready logger = logging.getLogger(__name__) @@ -68,18 +68,30 @@ app.conf.task_routes = { 'apps.channels.tasks.run_recording': {'queue': 'dvr'}, } + +@task_prerun.connect +def reset_db_connection_before_task(**kwargs): + """Discard stale DB connections before each task (Celery workers are long-lived).""" + from django.db import close_old_connections + + try: + close_old_connections() + except Exception: + pass + + # Add memory cleanup after task completion @task_postrun.connect # Use the imported signal def cleanup_task_memory(**kwargs): """Clean up memory and database connections after each task completes""" - from django.db import connection + from django.db import close_old_connections # Get task name from kwargs task_name = kwargs.get('task').name if kwargs.get('task') else '' - # Close database connection for this Celery worker process + # Return all DB connections to the pool in a clean state try: - connection.close() + close_old_connections() except Exception: pass diff --git a/tests/test_custom_properties_as_dict.py b/tests/test_custom_properties_as_dict.py new file mode 100644 index 00000000..bd863fdc --- /dev/null +++ b/tests/test_custom_properties_as_dict.py @@ -0,0 +1,74 @@ +import uuid + +from django.test import SimpleTestCase, TestCase + +from core.utils import custom_properties_as_dict, ensure_custom_properties_dict +from apps.m3u.models import M3UAccount, M3UAccountProfile +from apps.channels.models import ChannelGroupM3UAccount, ChannelGroup + + +class CustomPropertiesAsDictTests(SimpleTestCase): + def test_dict_passthrough(self): + value = {"enable_vod": True} + self.assertIs(custom_properties_as_dict(value), value) + + def test_json_string_parsed(self): + self.assertEqual( + custom_properties_as_dict('{"enable_vod": true}'), + {"enable_vod": True}, + ) + + def test_non_json_string_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict("not-json"), {}) + + def test_json_array_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict("[1, 2]"), {}) + + def test_none_returns_empty_dict(self): + self.assertEqual(custom_properties_as_dict(None), {}) + + +class EnsureCustomPropertiesDictTests(SimpleTestCase): + def test_dict_passthrough_without_reparse(self): + value = {"enable_vod": True} + self.assertIs(ensure_custom_properties_dict(value), value) + + def test_none_returns_empty_dict(self): + self.assertEqual(ensure_custom_properties_dict(None), {}) + + +class CustomPropertiesSaveNormalizationTests(TestCase): + def test_m3u_account_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}", + custom_properties='{"enable_vod": true}', + ) + account.refresh_from_db() + self.assertEqual(account.custom_properties, {"enable_vod": True}) + + def test_profile_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}" + ) + profile = M3UAccountProfile.objects.get( + m3u_account=account, is_default=True + ) + profile.custom_properties = '{"notes": "hello"}' + profile.save(update_fields=["custom_properties"]) + profile.refresh_from_db() + self.assertEqual(profile.custom_properties, {"notes": "hello"}) + + def test_group_relation_save_rewrites_string_custom_properties(self): + account = M3UAccount.objects.create( + name=f"Test Account {uuid.uuid4().hex[:8]}" + ) + group = ChannelGroup.objects.create( + name=f"Sports {uuid.uuid4().hex[:8]}" + ) + rel = ChannelGroupM3UAccount.objects.create( + m3u_account=account, + channel_group=group, + custom_properties='{"force_dummy_epg": true}', + ) + rel.refresh_from_db() + self.assertEqual(rel.custom_properties, {"force_dummy_epg": True}) From 460677aeea84f6c8c9b25ea4f30c09cbeca16d0d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 15:24:23 -0500 Subject: [PATCH 04/28] refactor(channels): optimize stream retrieval in ChannelSerializer and add tests for query efficiency - Updated `get_streams` method in `ChannelSerializer` to use prefetched `channelstream_set`, improving performance by reducing database queries. - Added `ChannelListIncludeStreamsQueryTests` to ensure that the number of queries remains stable as channels grow, verifying efficient stream inclusion in API responses. --- apps/channels/serializers.py | 11 ++-- apps/channels/tests/test_channel_api.py | 69 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index 2e80d840..61bbef8d 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -551,10 +551,13 @@ class ChannelSerializer(serializers.ModelSerializer): return LogoSerializer(obj.logo).data def get_streams(self, obj): - """Retrieve ordered stream IDs for GET requests.""" - return StreamSerializer( - obj.streams.all().order_by("channelstream__order"), many=True - ).data + """Retrieve ordered streams for GET requests using prefetched channelstream_set.""" + ordered_streams = [ + cs.stream + for cs in obj.channelstream_set.all() + if cs.stream_id is not None + ] + return StreamSerializer(ordered_streams, many=True).data def create(self, validated_data): streams = validated_data.pop("streams", []) diff --git a/apps/channels/tests/test_channel_api.py b/apps/channels/tests/test_channel_api.py index 363f5925..5d60a8da 100644 --- a/apps/channels/tests/test_channel_api.py +++ b/apps/channels/tests/test_channel_api.py @@ -504,3 +504,72 @@ class SeriesRuleAPITests(TestCase): def test_bulk_remove_requires_tvg_id_or_title(self): resp = self.client.post(self.bulk_remove_url, {}, format="json") self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + + +class ChannelListIncludeStreamsQueryTests(TestCase): + """include_streams=true must not issue one stream query per channel.""" + + def setUp(self): + from apps.channels.models import ChannelStream, Stream + from apps.m3u.models import M3UAccount + + self.user = User.objects.create_user(username="list_admin", password="x") + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + + self.account = M3UAccount.objects.create( + name="list-test-account", + account_type="XC", + username="user", + password="pass", + ) + self.group = ChannelGroup.objects.create(name=f"List Group {self.id}") + + def _add_channel_with_stream(self, number): + from apps.channels.models import ChannelStream, Stream + + channel = Channel.objects.create( + channel_number=float(number), + name=f"Channel {number}", + channel_group=self.group, + ) + stream = Stream.objects.create( + name=f"Stream {number}", + url=f"http://example.com/{number}.ts", + m3u_account=self.account, + ) + ChannelStream.objects.create(channel=channel, stream=stream, order=0) + return channel + + def _query_count_for_list(self): + from django.db import connection + from django.test.utils import CaptureQueriesContext + + with CaptureQueriesContext(connection) as ctx: + response = self.client.get( + "/api/channels/channels/", + {"page": 1, "page_size": 50, "include_streams": "true"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + return len(ctx.captured_queries) + + def test_include_streams_query_count_stable_as_channels_grow(self): + self._add_channel_with_stream(1) + self._add_channel_with_stream(2) + self._add_channel_with_stream(3) + q_small = self._query_count_for_list() + + self._add_channel_with_stream(4) + self._add_channel_with_stream(5) + self._add_channel_with_stream(6) + self._add_channel_with_stream(7) + q_large = self._query_count_for_list() + + self.assertEqual( + q_small, + q_large, + "include_streams list should use prefetched channelstream_set, " + "not one streams M2M query per channel", + ) From c2ac08fdfdc13142a694f10ba9bfa4fee661cd9f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 15:53:55 -0500 Subject: [PATCH 05/28] enhancement(epg): Performance improvements and API enhancements for EPG refreshes. - Updated EPG import logic to reject dummy sources without loading the large `programme_index`, optimizing memory usage during API calls. --- CHANGELOG.md | 5 ++ apps/epg/api_views.py | 22 ++++--- apps/epg/tests/test_epg_import_api.py | 95 +++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 apps/epg/tests/test_epg_import_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a5f6a0..9c0b1729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. + ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. +- **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index b268ef65..fbc419c5 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -1119,18 +1119,24 @@ class EPGImportAPIView(APIView): epg_id = request.data.get("id", None) force = bool(request.data.get("force", False)) - # Check if this is a dummy EPG source - try: + # Reject dummy sources without loading programme_index (multi-MB JSON). + if epg_id is not None: from .models import EPGSource - epg_source = EPGSource.objects.get(id=epg_id) - if epg_source.source_type == 'dummy': - logger.info(f"EPGImportAPIView: Skipping refresh for dummy EPG source {epg_id}") + + if EPGSource.objects.filter( + id=epg_id, source_type="dummy" + ).exists(): + logger.info( + "EPGImportAPIView: Skipping refresh for dummy EPG source %s", + epg_id, + ) return Response( - {"success": False, "message": "Dummy EPG sources do not require refreshing."}, + { + "success": False, + "message": "Dummy EPG sources do not require refreshing.", + }, status=status.HTTP_400_BAD_REQUEST, ) - except EPGSource.DoesNotExist: - pass # Let the task handle the missing source refresh_epg_data.delay(epg_id, force=force) # Trigger Celery task logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") diff --git a/apps/epg/tests/test_epg_import_api.py b/apps/epg/tests/test_epg_import_api.py new file mode 100644 index 00000000..c58f2a45 --- /dev/null +++ b/apps/epg/tests/test_epg_import_api.py @@ -0,0 +1,95 @@ +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.db import connection +from django.test.utils import CaptureQueriesContext +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIClient + +from apps.epg.models import EPGSource + +User = get_user_model() + +IMPORT_URL = "/api/epg/import/" + + +class EPGImportAPITests(TestCase): + def setUp(self): + self.user = User.objects.create_user( + username="epg_import_admin", password="testpass123" + ) + self.user.user_level = 10 + self.user.save() + self.client = APIClient() + self.client.force_authenticate(user=self.user) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_dummy_source_rejected_without_dispatch(self, mock_delay): + source = EPGSource.objects.create( + name="Dummy EPG", + source_type="dummy", + ) + + response = self.client.post( + IMPORT_URL, {"id": source.id}, format="json" + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(response.data["success"]) + mock_delay.assert_not_called() + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_xmltv_dispatches_without_loading_programme_index( + self, mock_delay + ): + source = EPGSource.objects.create( + name="Large Index XMLTV", + source_type="xmltv", + url="http://example.com/epg.xml", + programme_index={ + "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)}, + "interleaved_channels": [], + }, + ) + mock_delay.reset_mock() + + with CaptureQueriesContext(connection) as ctx: + response = self.client.post( + IMPORT_URL, {"id": source.id}, format="json" + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertTrue(response.data["success"]) + mock_delay.assert_called_once_with(source.id, force=False) + for query in ctx.captured_queries: + self.assertNotIn( + "programme_index", + query["sql"].lower(), + "import trigger should not read programme_index", + ) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_missing_source_still_dispatches(self, mock_delay): + response = self.client.post(IMPORT_URL, {"id": 99999}, format="json") + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + mock_delay.assert_called_once_with(99999, force=False) + + @patch("apps.epg.api_views.refresh_epg_data.delay") + def test_import_honours_force_flag(self, mock_delay): + source = EPGSource.objects.create( + name="Force XMLTV", + source_type="xmltv", + url="http://example.com/epg.xml", + ) + mock_delay.reset_mock() + + response = self.client.post( + IMPORT_URL, + {"id": source.id, "force": True}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + mock_delay.assert_called_once_with(source.id, force=True) From 04394b7eac686db1eef98433e022a22f62800914 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 16:39:52 -0500 Subject: [PATCH 06/28] fix(epg): enhance DB connection management and error handling during EPG refreshes - Implemented robust DB connection management in `refresh_epg_data` to handle transient errors, ensuring reliable status updates for EPG sources. - Added retry logic for database queries and improved error handling to prevent task failures from unhandled exceptions. - Updated EPG source status handling to ensure accurate reporting of errors and terminal states during refresh operations. --- CHANGELOG.md | 1 + apps/epg/tasks.py | 275 ++++++++++++++------- apps/epg/tests/test_refresh_db_recovery.py | 106 ++++++++ 3 files changed, 291 insertions(+), 91 deletions(-) create mode 100644 apps/epg/tests/test_refresh_db_recovery.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c0b1729..5d8aeeb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) +- **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. - **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0e8eda42..9661c5b9 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -39,6 +39,105 @@ from core.utils import ( logger = logging.getLogger(__name__) +_NON_TERMINAL_REFRESH_STATUSES = frozenset({ + EPGSource.STATUS_FETCHING, + EPGSource.STATUS_PARSING, +}) + + +def _release_task_db_connection(): + """Return the Celery worker's DB connection to a clean state after ORM errors.""" + from django.db import close_old_connections + close_old_connections() + + +def _db_query_with_retry(fn, *, label="DB query", max_retries=2): + """ + Run an ORM read with one connection reset + retry on transient failures. + + Poisoned Celery worker connections often surface as OperationalError or as + ``IndexError: list index out of range`` inside Django's row converters. + """ + from django.db import InterfaceError, OperationalError + + transient_errors = (OperationalError, InterfaceError, IndexError) + for attempt in range(max_retries): + try: + return fn() + except transient_errors as exc: + if attempt + 1 >= max_retries: + raise + logger.warning( + "%s failed (%s), resetting DB connection (%s/%s)", + label, + exc, + attempt + 1, + max_retries, + ) + _release_task_db_connection() + + +def _get_epg_source(source_id): + return _db_query_with_retry( + lambda: EPGSource.objects.get(id=source_id), + label=f"load EPG source {source_id}", + ) + + +def _set_epg_source_status( + source_id, + status, + last_message=None, + *, + notify_error=False, + ws_action="refresh", + ws_error=None, +): + """Update source status using a fresh connection (safe after DB failures).""" + _release_task_db_connection() + update = {"status": status} + if last_message is not None: + update["last_message"] = last_message + try: + EPGSource.objects.filter(id=source_id).update(**update) + if notify_error: + send_epg_update( + source_id, + ws_action, + 100, + status="error", + error=ws_error or last_message, + ) + except Exception as e: + logger.error( + f"Failed to set EPG source {source_id} status to {status}: {e}" + ) + + +def _ensure_epg_refresh_terminal_status(source_id): + """Mark refresh as failed when the task exits while still in progress.""" + _release_task_db_connection() + try: + current_status = ( + EPGSource.objects.filter(id=source_id) + .values_list("status", flat=True) + .first() + ) + if current_status in _NON_TERMINAL_REFRESH_STATUSES: + message = "Refresh did not complete successfully" + EPGSource.objects.filter(id=source_id).update( + status=EPGSource.STATUS_ERROR, + last_message=message, + ) + send_epg_update( + source_id, "refresh", 100, status="error", error=message + ) + except Exception as e: + logger.debug( + f"Could not verify terminal refresh status for EPG source {source_id}: {e}" + ) + + SD_BASE_URL = 'https://json.schedulesdirect.org/20141201' SD_DAYS_TO_FETCH = 20 SD_PROGRAM_BATCH_SIZE = 5000 @@ -498,104 +597,90 @@ def refresh_epg_data(source_id, force=False): lock_renewer = TaskLockRenewer('refresh_epg_data', source_id) lock_renewer.start() - source = None + _release_task_db_connection() + try: - # Try to get the EPG source - try: - source = EPGSource.objects.get(id=source_id) - except EPGSource.DoesNotExist: - # The EPG source doesn't exist, so delete the periodic task if it exists - logger.warning(f"EPG source with ID {source_id} not found, but task was triggered. Cleaning up orphaned task.") - - # Call the shared function to delete the task - if delete_epg_refresh_task_by_id(source_id): - logger.info(f"Successfully cleaned up orphaned task for EPG source {source_id}") - else: - logger.info(f"No orphaned task found for EPG source {source_id}") - - # Release the lock and exit - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return f"EPG source {source_id} does not exist, task cleaned up" - - # The source exists but is not active, just skip processing - if not source.is_active: - logger.info(f"EPG source {source_id} is not active. Skipping.") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - # Skip refresh for dummy EPG sources - they don't need refreshing - if source.source_type == 'dummy': - logger.info(f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - gc.collect() - return - - # Continue with the normal processing... - logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") - if source.source_type == 'xmltv': - # Invalidate the byte-offset index before downloading the new file - # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) - fetch_success = fetch_xmltv(source) - if not fetch_success: - logger.error(f"Failed to fetch XMLTV for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - parse_channels_success = parse_channels_only(source) - if not parse_channels_success: - logger.error(f"Failed to parse channels for source {source.name}") - lock_renewer.stop() - release_task_lock('refresh_epg_data', source_id) - # Force garbage collection before exit - gc.collect() - return - - # Build byte-offset index after programme data is committed so refresh - # does not compete for memory/IO during the programme swap. - parse_programs_for_source(source) - build_programme_index_task.delay(source.id) - - elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source, force=force) - - source.save(update_fields=['updated_at']) - # After successful EPG refresh, evaluate DVR series rules to schedule new episodes - try: - from apps.channels.tasks import evaluate_series_rules - evaluate_series_rules.delay() - except Exception: - pass + return _refresh_epg_data_impl(source_id, force=force) except Exception as e: - logger.error(f"Error in refresh_epg_data for source {source_id}: {e}", exc_info=True) - try: - if source: - source.status = 'error' - source.last_message = f"Error refreshing EPG data: {str(e)}" - source.save(update_fields=['status', 'last_message']) - send_epg_update(source_id, "refresh", 100, status="error", error=str(e)) - except Exception as inner_e: - logger.error(f"Error updating source status: {inner_e}") + logger.error( + f"Error in refresh_epg_data for source {source_id}: {e}", + exc_info=True, + ) + _set_epg_source_status( + source_id, + EPGSource.STATUS_ERROR, + f"Error refreshing EPG data: {str(e)[:500]}", + notify_error=True, + ws_error=str(e)[:500], + ) finally: - # Clear references to ensure proper garbage collection - source = None - # Force garbage collection before releasing the lock + _ensure_epg_refresh_terminal_status(source_id) + _release_task_db_connection() gc.collect() - connection.close() lock_renewer.stop() release_task_lock('refresh_epg_data', source_id) +def _refresh_epg_data_impl(source_id, force=False): + try: + source = _get_epg_source(source_id) + except EPGSource.DoesNotExist: + logger.warning( + f"EPG source with ID {source_id} not found, but task was triggered. " + "Cleaning up orphaned task." + ) + + if delete_epg_refresh_task_by_id(source_id): + logger.info( + f"Successfully cleaned up orphaned task for EPG source {source_id}" + ) + else: + logger.info(f"No orphaned task found for EPG source {source_id}") + + return f"EPG source {source_id} does not exist, task cleaned up" + + if not source.is_active: + logger.info(f"EPG source {source_id} is not active. Skipping.") + return + + if source.source_type == 'dummy': + logger.info( + f"Skipping refresh for dummy EPG source {source.name} (ID: {source_id})" + ) + return + + logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") + if source.source_type == 'xmltv': + # Invalidate the byte-offset index before downloading the new file + # so stale offsets are never used during the refresh window. + EPGSource.objects.filter(id=source.id).update(programme_index=None) + if not fetch_xmltv(source): + logger.error(f"Failed to fetch XMLTV for source {source.name}") + return + + if not parse_channels_only(source): + logger.error(f"Failed to parse channels for source {source.name}") + return + + # Build byte-offset index after programme data is committed so refresh + # does not compete for memory/IO during the programme swap. + if not parse_programs_for_source(source): + logger.error(f"Failed to parse programs for source {source.name}") + return + + build_programme_index_task.delay(source.id) + + elif source.source_type == 'schedules_direct': + fetch_schedules_direct(source, force=force) + + EPGSource.objects.filter(id=source.id).update(updated_at=timezone.now()) + try: + from apps.channels.tasks import evaluate_series_rules + evaluate_series_rules.delay() + except Exception: + pass + + def fetch_xmltv(source): # Handle cases with local file but no URL if not source.url and source.file_path and os.path.exists(source.file_path): @@ -1167,7 +1252,15 @@ def parse_channels_only(source): # Update status to error source.status = 'error' source.last_message = f"No URL provided, cannot fetch EPG data" - source.save(update_fields=['updated_at']) + source.save(update_fields=['status', 'last_message']) + send_epg_update( + source.id, + "parsing_channels", + 100, + status="error", + error="No URL provided", + ) + return False # Initialize process variable for memory tracking only in debug mode try: diff --git a/apps/epg/tests/test_refresh_db_recovery.py b/apps/epg/tests/test_refresh_db_recovery.py new file mode 100644 index 00000000..11885b34 --- /dev/null +++ b/apps/epg/tests/test_refresh_db_recovery.py @@ -0,0 +1,106 @@ +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +from apps.epg.tasks import ( + _db_query_with_retry, + _ensure_epg_refresh_terminal_status, + _get_epg_source, + _release_task_db_connection, + refresh_epg_data, +) + + +class DbQueryWithRetryTests(SimpleTestCase): + def test_retries_after_index_error_from_poisoned_connection(self): + fn = MagicMock(side_effect=[IndexError("list index out of range"), "ok"]) + + with patch( + "apps.epg.tasks._release_task_db_connection" + ) as mock_release: + result = _db_query_with_retry(fn, label="test query") + + self.assertEqual(result, "ok") + self.assertEqual(fn.call_count, 2) + mock_release.assert_called_once() + + def test_raises_after_exhausting_retries(self): + fn = MagicMock(side_effect=IndexError("list index out of range")) + + with patch("apps.epg.tasks._release_task_db_connection"): + with self.assertRaises(IndexError): + _db_query_with_retry(fn, label="test query", max_retries=2) + + self.assertEqual(fn.call_count, 2) + + +class RefreshTaskDbStartupTests(SimpleTestCase): + @patch("apps.epg.tasks._ensure_epg_refresh_terminal_status") + @patch("apps.epg.tasks._refresh_epg_data_impl") + @patch("apps.epg.tasks.release_task_lock") + @patch("apps.epg.tasks.acquire_task_lock", return_value=True) + @patch("apps.epg.tasks.TaskLockRenewer") + @patch("apps.epg.tasks._release_task_db_connection") + def test_refresh_releases_db_connection_before_impl( + self, + mock_release, + _mock_renewer, + _mock_acquire, + _mock_release_lock, + mock_impl, + _mock_ensure_terminal, + ): + call_order = [] + + def track_release(): + call_order.append("release") + + mock_release.side_effect = track_release + mock_impl.side_effect = lambda *_a, **_k: call_order.append("impl") or "done" + + result = refresh_epg_data(42) + + self.assertEqual(result, "done") + self.assertEqual(call_order[:2], ["release", "impl"]) + + @patch("apps.epg.tasks.EPGSource") + def test_get_epg_source_uses_retry_helper(self, mock_model): + mock_source = MagicMock(id=42) + mock_model.objects.get.return_value = mock_source + + with patch("apps.epg.tasks._db_query_with_retry") as mock_retry: + mock_retry.side_effect = lambda fn, **_: fn() + source = _get_epg_source(42) + + mock_retry.assert_called_once() + mock_model.objects.get.assert_called_once_with(id=42) + self.assertIs(source, mock_source) + + +class EnsureEpgTerminalStatusTests(SimpleTestCase): + @patch("apps.epg.tasks.send_epg_update") + @patch("apps.epg.tasks._release_task_db_connection") + def test_marks_stuck_fetching_as_error(self, _mock_release, mock_ws): + with patch("apps.epg.tasks.EPGSource") as mock_model: + mock_model.STATUS_ERROR = "error" + qs = MagicMock() + mock_model.objects.filter.return_value = qs + qs.values_list.return_value.first.return_value = "fetching" + + _ensure_epg_refresh_terminal_status(7) + + qs.update.assert_called_once() + mock_ws.assert_called_once() + + @patch("apps.epg.tasks.send_epg_update") + @patch("apps.epg.tasks._release_task_db_connection") + def test_leaves_success_unchanged(self, _mock_release, mock_ws): + with patch("apps.epg.tasks.EPGSource") as mock_model: + qs = MagicMock() + mock_model.objects.filter.return_value = qs + qs.values_list.return_value.first.return_value = "success" + + _ensure_epg_refresh_terminal_status(7) + + qs.update.assert_not_called() + mock_ws.assert_not_called() From 7139c88c350adf25414e07decc5feadc11946777 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 18 Jun 2026 17:12:43 -0500 Subject: [PATCH 07/28] enhancement(epg): improve custom dummy EPG generation performance - Optimized `generate_epg()` to reduce database queries by prefetching ordered channel streams and implementing a new method for custom dummy epg name matching. - Enhanced handling of dummy EPG sources to minimize SQL round-trips, improving performance for large guides with numerous custom-dummy channels. --- CHANGELOG.md | 4 ++ apps/output/views.py | 87 ++++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d8aeeb6..0517493d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. +### Performance + +- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. + ### Fixed - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) diff --git a/apps/output/views.py b/apps/output/views.py index 8bcc37dc..dba35951 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -338,6 +338,29 @@ def generate_m3u(request, profile_name=None, user=None): return response +def _ordered_channel_streams(channel): + """Return a channel's streams ordered by channelstream join order.""" + prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') + if prefetched is not None: + return list(prefetched) + return list(channel.streams.all().order_by('channelstream__order')) + + +def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): + """Name used for custom dummy EPG regex matching (channel or stream title). + + Returns (name, stream_lookup_failed). stream_lookup_failed is True only when + name_source is 'stream' but the configured index is missing or out of range. + """ + if custom_props.get('name_source') != 'stream': + return effective_name, False + stream_index = custom_props.get('stream_index', 1) - 1 + streams = _ordered_channel_streams(channel) + if 0 <= stream_index < len(streams): + return streams[stream_index].name, False + return effective_name, True + + def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): """ Generate dummy programs using custom fallback templates when patterns don't match. @@ -1397,9 +1420,11 @@ def generate_epg(request, profile_name=None, user=None): with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") + .prefetch_related( + Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + ) ) - # For dummy EPG, use either the specified value or default to 3 days dummy_days = num_days if num_days > 0 else 3 @@ -1440,6 +1465,8 @@ def generate_epg(request, profile_name=None, user=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw + dummy_epg_ids_for_program_check = set() + # Process channels for the section for channel in channels: effective_name = channel.effective_name @@ -1465,23 +1492,17 @@ def generate_epg(request, profile_name=None, user=None): # Check if this is a custom dummy EPG with channel logo URL template if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + if channel.effective_epg_data_id: + dummy_epg_ids_for_program_check.add(channel.effective_epg_data_id) epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties channel_logo_url_template = custom_props.get('channel_logo_url', '') if channel_logo_url_template: - # Determine which name to use for pattern matching (same logic as program generation) - pattern_match_name = effective_name - name_source = custom_props.get('name_source') - - if name_source == 'stream': - stream_index = custom_props.get('stream_index', 1) - 1 - channel_streams = channel.streams.all().order_by('channelstream__order') - - if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): - stream = list(channel_streams)[stream_index] - pattern_match_name = stream.name + pattern_match_name, _ = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) # Try to extract groups from the channel/stream name and build the logo URL title_pattern = custom_props.get('title_pattern', '') @@ -1538,10 +1559,17 @@ def generate_epg(request, profile_name=None, user=None): yield channel_xml xml_lines = [] # Clear to save memory + dummy_epg_with_programs = set() + if dummy_epg_ids_for_program_check: + dummy_epg_with_programs = set( + ProgramData.objects.filter(epg_id__in=dummy_epg_ids_for_program_check) + .values_list('epg_id', flat=True) + .distinct() + ) + # Pre-pass: categorize channels into dummy and real EPG groups dummy_program_list = [] # (channel_id, pattern_match_name, epg_source_or_None) real_epg_map = {} # epg_data_id -> [channel_id, ...] - dummy_epg_checked = {} # epg_data_id -> bool (has stored programs) for channel in channels: effective_name = channel.effective_name @@ -1569,26 +1597,31 @@ def generate_epg(request, profile_name=None, user=None): epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties - name_source = custom_props.get('name_source') - - if name_source == 'stream': + pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + if ( + custom_props.get('name_source') == 'stream' + and not stream_lookup_failed + and pattern_match_name != effective_name + ): stream_index = custom_props.get('stream_index', 1) - 1 - channel_streams = channel.streams.all().order_by('channelstream__order') - - if channel_streams.exists() and 0 <= stream_index < channel_streams.count(): - stream = list(channel_streams)[stream_index] - pattern_match_name = stream.name - logger.debug(f"Using stream name for parsing: {pattern_match_name} (stream index: {stream_index})") - else: - logger.warning(f"Stream index {stream_index} not found for channel {effective_name}, falling back to channel name") + logger.debug( + f"Using stream name for parsing: {pattern_match_name} " + f"(stream index: {stream_index})" + ) + elif stream_lookup_failed: + stream_index = custom_props.get('stream_index', 1) - 1 + logger.warning( + f"Stream index {stream_index} not found for channel " + f"{effective_name}, falling back to channel name" + ) if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) else: if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if effective_epg_data_id not in dummy_epg_checked: - dummy_epg_checked[effective_epg_data_id] = effective_epg_data.programs.exists() - if dummy_epg_checked[effective_epg_data_id]: + if effective_epg_data_id in dummy_epg_with_programs: real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) else: dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) From a29704af55015597e4f5215b300a4b584d42f892 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:40:10 -0500 Subject: [PATCH 08/28] fix(channels): account for group_override in the auto-sync range-conflict check The range-conflict warning classified each channel in the configured range as either this config's own auto-sync output or a real conflict, comparing each occupant against the source group. When a group sets a group_override, auto-sync creates its channels in the override target group, so the config's own channels failed that comparison and were misclassified as a conflict. - Add effectiveSyncGroupId to resolve the group the sync's channels actually land in (the group_override target when set, otherwise the source group). - Compare occupants against that effective target, so the config's own output is recognized while genuine conflicts (manual channels, channels from another account, channels in a different group, user-pinned numbers) still warn. - Add Vitest coverage for the helper, the override case, and an over-suppression guard, plus a backend test asserting the numbers-in-range endpoint reports the override target group. --- apps/m3u/tests/test_sync_correctness.py | 28 +++++ .../src/components/forms/LiveGroupFilter.jsx | 13 +- .../src/utils/forms/LiveGroupFilterUtils.js | 12 ++ .../__tests__/LiveGroupFilterUtils.test.js | 111 ++++++++++++++++++ 4 files changed, 161 insertions(+), 3 deletions(-) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 3cc24942..8ae09fa8 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -644,6 +644,34 @@ class NumbersInRangeLookupTests(TestCase): ) self.assertFalse(occupant["has_channel_number_override"]) + def test_group_override_channel_reports_target_group(self): + # When auto-sync routes channels into a different group via + # group_override, the occupant's channel_group_id is the override + # target, not the source group being configured. The frontend relies + # on this to recognize override-routed channels as the config's own + # output (effectiveSyncGroupId), so the warning does not flag them. + account = _make_account() + source = _make_group(name="SourceGrp") + target = _make_group(name="TargetGrp") + Channel.objects.create( + name="Routed", + channel_number=3210, + channel_group=target, + auto_created=True, + auto_created_by=account, + ) + client = self._client() + + response = client.get( + "/api/channels/channels/numbers-in-range/?start=3210&end=3210" + ) + + occupant = response.data["occupants"][0] + self.assertEqual(occupant["channel_group_id"], target.id) + self.assertNotEqual(occupant["channel_group_id"], source.id) + self.assertTrue(occupant["auto_created"]) + self.assertEqual(occupant["auto_created_by_account_id"], account.id) + def test_manual_channel_exposed_with_auto_created_false(self): # Manual channels are always a real collision worth surfacing. # The response must flag them with auto_created=False and a null diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index 626ec55e..5ed2be94 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -34,6 +34,7 @@ import { getRegexOptions, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, isGroupVisible, rangeFor, } from '../../utils/forms/LiveGroupFilterUtils.js'; @@ -131,7 +132,12 @@ const LiveGroupFilter = ({ // (in-memory range overlap with sibling groups) sources so the sweep // can refresh form-overlap synchronously without firing HTTP for // groups that did not change. - const scheduleConflictScan = (groupId, rawStart, rawEnd) => { + const scheduleConflictScan = ( + groupId, + rawStart, + rawEnd, + expectedGroupId = groupId + ) => { if (conflictTimersRef.current[groupId]) { clearTimeout(conflictTimersRef.current[groupId]); } @@ -156,7 +162,7 @@ const LiveGroupFilter = ({ ? result.occupants : []; const unexpected = occupants.filter( - (o) => !isExpectedOccupantForGroup(o, groupId, playlist) + (o) => !isExpectedOccupantForGroup(o, expectedGroupId, playlist) ); setConflictSource(groupId, 'occupant', unexpected.length > 0); } catch (e) { @@ -221,7 +227,8 @@ const LiveGroupFilter = ({ scheduleConflictScan( g.channel_group, range.startRaw, - g.auto_sync_channel_end + g.auto_sync_channel_end, + effectiveSyncGroupId(g) ); } } diff --git a/frontend/src/utils/forms/LiveGroupFilterUtils.js b/frontend/src/utils/forms/LiveGroupFilterUtils.js index a1372887..3cb62bf5 100644 --- a/frontend/src/utils/forms/LiveGroupFilterUtils.js +++ b/frontend/src/utils/forms/LiveGroupFilterUtils.js @@ -57,6 +57,18 @@ export const isExpectedOccupantForGroup = ( ); }; +// The group the sync's own channels actually land in. A group_override +// routes auto-created channels into a different ChannelGroup, so the +// conflict check must recognize occupants of that target group as this +// config's own output rather than flagging them against the source group. +export const effectiveSyncGroupId = (group) => { + const override = group?.custom_properties?.group_override; + if (override !== undefined && override !== null && override !== '') { + return Number(override); + } + return group?.channel_group; +}; + export const rangeFor = (g) => { if (!g.enabled || !g.auto_channel_sync) return null; const mode = g.custom_properties?.channel_numbering_mode || 'fixed'; diff --git a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js index feb878e1..86874f23 100644 --- a/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js +++ b/frontend/src/utils/forms/__tests__/LiveGroupFilterUtils.test.js @@ -4,6 +4,7 @@ import { getChannelsInRange, getStreamsRegexPreview, isExpectedOccupantForGroup, + effectiveSyncGroupId, rangeFor, abortTimers, getRegexOptions, @@ -227,6 +228,116 @@ describe('LiveGroupFilterUtils', () => { }); }); + // ── effectiveSyncGroupId ─────────────────────────────────────────────────── + describe('effectiveSyncGroupId', () => { + it('returns the source channel_group when there is no override', () => { + expect(effectiveSyncGroupId(makeGroup({ channel_group: 7 }))).toBe(7); + }); + + it('returns the group_override target when set', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('coerces a string-stored group_override to a number', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '9' }, + }); + expect(effectiveSyncGroupId(group)).toBe(9); + }); + + it('falls back to the source group when group_override is blank', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: '' }, + }); + expect(effectiveSyncGroupId(group)).toBe(7); + }); + + // Regression guard for the group-override range-conflict false positive: + // the auto-sync's own channels land in the override target group, so + // comparing against the source group (pre-fix) flags them as a conflict, + // while comparing against the effective target recognizes them as this + // config's own output. + it("makes group-override occupants count as this group's own", () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const occupant = makeOccupant({ channel_group_id: 9 }); + // Pre-fix comparison (source group) treats own channels as a conflict. + expect( + isExpectedOccupantForGroup( + occupant, + group.channel_group, + makePlaylist() + ) + ).toBe(false); + // Comparing against the effective target recognizes them as expected. + expect( + isExpectedOccupantForGroup( + occupant, + effectiveSyncGroupId(group), + makePlaylist() + ) + ).toBe(true); + }); + + // Guards against over-suppression: resolving the effective target group + // must still surface genuine collisions in an override config's range. + // Only the config's own output (auto-created, this account, in the + // target group, unpinned) is excluded. + it('still flags genuine collisions in a group-override config', () => { + const group = makeGroup({ + channel_group: 7, + custom_properties: { group_override: 9 }, + }); + const target = effectiveSyncGroupId(group); + // Manual channel sitting in the range. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 9, auto_created: false }), + target, + makePlaylist() + ) + ).toBe(false); + // Auto-created by a different account. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + auto_created_by_account_id: 999, + }), + target, + makePlaylist() + ) + ).toBe(false); + // A channel in a different group than the override target. + expect( + isExpectedOccupantForGroup( + makeOccupant({ channel_group_id: 123 }), + target, + makePlaylist() + ) + ).toBe(false); + // A user-pinned channel number. + expect( + isExpectedOccupantForGroup( + makeOccupant({ + channel_group_id: 9, + has_channel_number_override: true, + }), + target, + makePlaylist() + ) + ).toBe(false); + }); + }); + // ── rangeFor ────────────────────────────────────────────────────────────── describe('rangeFor', () => { it('returns null when group is disabled', () => { From 3868f02c45e118acc9965745115bf4289eea2b24 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 10:57:06 -0500 Subject: [PATCH 09/28] enhancement(epg): optimize EPG export performance and database interactions - Streamlined `generate_epg()` to incrementally stream EPG data without loading the entire guide, improving memory efficiency. - Reduced database load by deferring the fetching of large `programme_index` blobs, enhancing response times for EPG generation. - Introduced a composite index on `(epg_id, id)` in `ProgramData` to optimize query performance during EPG exports. - Updated tests to ensure proper functionality and performance of new features. --- CHANGELOG.md | 6 + .../0025_programdata_epg_id_index.py | 40 ++ apps/epg/models.py | 5 + apps/output/epg_chunk_cache.py | 230 +++++++ apps/output/test_epg_chunk_cache.py | 187 ++++++ apps/output/tests.py | 292 +++++++-- apps/output/views.py | 573 ++++++++++-------- core/tests.py | 13 +- core/utils.py | 19 + 9 files changed, 1065 insertions(+), 300 deletions(-) create mode 100644 apps/epg/migrations/0025_programdata_epg_id_index.py create mode 100644 apps/output/epg_chunk_cache.py create mode 100644 apps/output/test_epg_chunk_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0517493d..3315456b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Performance + +- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. + ### Changed - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. diff --git a/apps/epg/migrations/0025_programdata_epg_id_index.py b/apps/epg/migrations/0025_programdata_epg_id_index.py new file mode 100644 index 00000000..1a350b20 --- /dev/null +++ b/apps/epg/migrations/0025_programdata_epg_id_index.py @@ -0,0 +1,40 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class AddIndexConcurrentlyIfPostgres(AddIndexConcurrently): + """Create the index CONCURRENTLY on PostgreSQL (no table lock on large + tables), falling back to a normal blocking AddIndex on other backends + such as the sqlite dev/test fallback.""" + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_forwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_forwards( + self, app_label, schema_editor, from_state, to_state + ) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + if schema_editor.connection.vendor == 'postgresql': + super().database_backwards(app_label, schema_editor, from_state, to_state) + else: + migrations.AddIndex.database_backwards( + self, app_label, schema_editor, from_state, to_state + ) + + +class Migration(migrations.Migration): + # CREATE INDEX CONCURRENTLY cannot run inside a transaction. + atomic = False + + dependencies = [ + ('epg', '0024_remove_epgsource_api_key_epgsource_password_and_more'), + ] + + operations = [ + AddIndexConcurrentlyIfPostgres( + model_name='programdata', + index=models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 025945db..04c6072c 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -153,6 +153,11 @@ class ProgramData(models.Model): program_id = models.CharField(max_length=64, null=True, blank=True, help_text='Schedules Direct programID (e.g. EP123456789). Null for XMLTV sources.') custom_properties = models.JSONField(default=dict, blank=True, null=True) + class Meta: + indexes = [ + models.Index(fields=['epg', 'id'], name='epg_prog_epg_id_idx'), + ] + def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" diff --git a/apps/output/epg_chunk_cache.py b/apps/output/epg_chunk_cache.py new file mode 100644 index 00000000..92ea6eeb --- /dev/null +++ b/apps/output/epg_chunk_cache.py @@ -0,0 +1,230 @@ +"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" + +import logging +import time + +from django.http import StreamingHttpResponse + +logger = logging.getLogger(__name__) + +STATUS_BUILDING = "building" +STATUS_READY = "ready" +STATUS_ERROR = "error" + +DEFAULT_CACHE_TTL = 300 +DEFAULT_LOCK_TTL = 120 +DEFAULT_POLL_INTERVAL = 0.05 +DEFAULT_MAX_FOLLOWER_WAIT = 600 + + +def _chunks_key(base_key): + return f"{base_key}:chunks" + + +def _ready_key(base_key): + return f"{base_key}:ready" + + +def _status_key(base_key): + return f"{base_key}:status" + + +def _lock_key(base_key): + return f"{base_key}:lock" + + +def _decode_chunk(chunk): + if chunk is None: + return None + if isinstance(chunk, bytes): + return chunk.decode("utf-8") + return chunk + + +def _encode_chunk(chunk): + if isinstance(chunk, bytes): + return chunk + return chunk.encode("utf-8") + + +def _poll_wait(interval): + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.sleep(interval) + return + except ImportError: + pass + time.sleep(interval) + + +def _get_redis(): + from django_redis import get_redis_connection + + return get_redis_connection("default") + + +def _get_status(redis, base_key): + raw = redis.get(_status_key(base_key)) + if raw is None: + return None + return _decode_chunk(raw) + + +def _clear_build_keys(redis, base_key): + redis.delete( + _chunks_key(base_key), + _status_key(base_key), + _ready_key(base_key), + _lock_key(base_key), + ) + + +def _try_acquire_lock(redis, base_key, lock_ttl): + return bool(redis.set(_lock_key(base_key), "1", nx=True, ex=lock_ttl)) + + +def _refresh_build_ttl(redis, base_key, lock_ttl): + redis.expire(_lock_key(base_key), lock_ttl) + redis.expire(_status_key(base_key), lock_ttl) + redis.expire(_chunks_key(base_key), lock_ttl) + + +def _stream_ready(redis, base_key): + offset = 0 + chunks_key = _chunks_key(base_key) + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is None: + break + yield _decode_chunk(chunk) + offset += 1 + + +def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): + """Leader: stream to client and append each chunk to Redis.""" + chunks_key = _chunks_key(base_key) + status_key = _status_key(base_key) + try: + from django.core.cache import cache as django_cache + + django_cache.delete(base_key) # legacy monolithic entry + redis.delete(chunks_key, _ready_key(base_key)) + redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + for chunk in source(): + redis.rpush(chunks_key, _encode_chunk(chunk)) + _refresh_build_ttl(redis, base_key, lock_ttl) + yield chunk + redis.set(status_key, STATUS_READY) + redis.set(_ready_key(base_key), "1") + redis.expire(chunks_key, cache_ttl) + redis.expire(status_key, cache_ttl) + redis.expire(_ready_key(base_key), cache_ttl) + logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + except Exception: + logger.exception("EPG cache build failed for %s", base_key) + redis.delete(chunks_key) + redis.set(status_key, STATUS_ERROR, ex=60) + raise + finally: + redis.delete(_lock_key(base_key)) + + +def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, max_follower_wait): + """Follower: read chunks as the leader writes them.""" + offset = 0 + deadline = time.monotonic() + max_follower_wait + idle_polls = 0 + chunks_key = _chunks_key(base_key) + lock_key = _lock_key(base_key) + + while True: + chunk = redis.lindex(chunks_key, offset) + if chunk is not None: + idle_polls = 0 + yield _decode_chunk(chunk) + offset += 1 + continue + + status = _get_status(redis, base_key) + if status == STATUS_READY: + break + + if status == STATUS_ERROR: + _clear_build_keys(redis, base_key) + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + raise RuntimeError("EPG cache build failed") + + if time.monotonic() >= deadline: + if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + logger.warning("EPG cache follower timed out after partial read for %s", base_key) + break + + lock_active = bool(redis.exists(lock_key)) + if status != STATUS_BUILDING and not lock_active: + idle_polls += 1 + if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): + if _try_acquire_lock(redis, base_key, lock_ttl): + logger.warning("EPG cache leader lost; rebuilding %s", base_key) + yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) + return + else: + idle_polls = 0 + + _poll_wait(poll_interval) + + +def stream_epg_response( + cache_key, + source, + *, + cache_ttl=DEFAULT_CACHE_TTL, + lock_ttl=DEFAULT_LOCK_TTL, + poll_interval=DEFAULT_POLL_INTERVAL, + max_follower_wait=DEFAULT_MAX_FOLLOWER_WAIT, + redis=None, +): + """ + Stream XMLTV EPG output with single-flight Redis chunk caching. + + ``source`` must be a callable returning a chunk iterator. Only the leader + invokes it; followers read chunks already written to Redis. + """ + if redis is None: + redis = _get_redis() + + if redis.get(_ready_key(cache_key)): + logger.debug("Serving EPG from chunk cache") + stream = _stream_ready(redis, cache_key) + else: + status = _get_status(redis, cache_key) + if status == STATUS_ERROR: + _clear_build_keys(redis, cache_key) + + if _try_acquire_lock(redis, cache_key, lock_ttl): + logger.debug("Building EPG (cache leader)") + stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) + else: + logger.debug("Following in-flight EPG build") + stream = _stream_follow( + redis, + cache_key, + source, + cache_ttl, + lock_ttl, + poll_interval, + max_follower_wait, + ) + + response = StreamingHttpResponse(stream, content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_epg_chunk_cache.py new file mode 100644 index 00000000..6ca217af --- /dev/null +++ b/apps/output/test_epg_chunk_cache.py @@ -0,0 +1,187 @@ +import threading +import time +from unittest import TestCase + +from apps.output.epg_chunk_cache import ( + STATUS_BUILDING, + STATUS_READY, + _chunks_key, + _lock_key, + _ready_key, + _status_key, + stream_epg_response, +) + + +class FakeRedis: + """Minimal Redis stand-in for chunk-cache unit tests.""" + + def __init__(self): + self._strings = {} + self._lists = {} + self._expires_at = {} + + def _purge_expired(self): + now = time.monotonic() + expired = [key for key, deadline in self._expires_at.items() if deadline <= now] + for key in expired: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def get(self, key): + self._purge_expired() + return self._strings.get(key) + + def set(self, key, value, nx=False, ex=None): + self._purge_expired() + if nx and key in self._strings: + return None + self._strings[key] = value + if ex is not None: + self._expires_at[key] = time.monotonic() + ex + return True + + def delete(self, *keys): + for key in keys: + self._strings.pop(key, None) + self._lists.pop(key, None) + self._expires_at.pop(key, None) + + def exists(self, key): + self._purge_expired() + return key in self._strings or key in self._lists + + def expire(self, key, ttl): + if key in self._strings or key in self._lists: + self._expires_at[key] = time.monotonic() + ttl + return True + + def rpush(self, key, value): + self._lists.setdefault(key, []).append(value) + + def lindex(self, key, offset): + items = self._lists.get(key, []) + if offset < len(items): + return items[offset] + return None + + def llen(self, key): + return len(self._lists.get(key, [])) + + +def _consume(response): + return b"".join(response.streaming_content).decode("utf-8") + + +class EPGChunkCacheTests(TestCase): + def test_leader_caches_chunks_and_sets_ready(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, [1]) + self.assertEqual(redis.get(_ready_key("epg:test")), "1") + self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) + self.assertFalse(redis.exists(_lock_key("epg:test"))) + + def test_cache_hit_skips_source(self): + redis = FakeRedis() + calls = [] + + def source(): + calls.append(1) + yield "" + yield "" + + _consume(stream_epg_response("epg:test", source, redis=redis)) + calls.clear() + body = _consume(stream_epg_response("epg:test", source, redis=redis)) + + self.assertEqual(body, "") + self.assertEqual(calls, []) + + def test_follower_reads_leader_chunks_without_rebuilding(self): + redis = FakeRedis() + base = "epg:follow" + leader_started = threading.Event() + rebuild_calls = [] + + def slow_source(): + rebuild_calls.append(1) + leader_started.set() + yield "a" + time.sleep(0.05) + yield "b" + + def forbidden_source(): + rebuild_calls.append(2) + yield "SHOULD_NOT_RUN" + + def leader(): + _consume( + stream_epg_response( + base, + slow_source, + redis=redis, + poll_interval=0.01, + ) + ) + + leader_thread = threading.Thread(target=leader) + leader_thread.start() + leader_started.wait(timeout=5) + follower_body = _consume( + stream_epg_response( + base, + forbidden_source, + redis=redis, + poll_interval=0.01, + ) + ) + leader_thread.join(timeout=5) + + self.assertEqual(follower_body, "ab") + self.assertEqual(rebuild_calls, [1]) + + def test_only_one_leader_when_two_clients_start_together(self): + redis = FakeRedis() + build_calls = [] + barrier = threading.Barrier(2) + results = {} + + def source(): + build_calls.append(threading.current_thread().name) + yield "x" + + def worker(): + barrier.wait() + results[threading.current_thread().name] = _consume( + stream_epg_response( + "epg:race", + source, + redis=redis, + poll_interval=0.01, + ) + ) + + threads = [ + threading.Thread(target=worker, name="t1"), + threading.Thread(target=worker, name="t2"), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + self.assertEqual(results["t1"], "x") + self.assertEqual(results["t2"], "x") + self.assertEqual(len(build_calls), 1) diff --git a/apps/output/tests.py b/apps/output/tests.py index b1c7d589..6e2ab395 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,31 +1,114 @@ -from django.test import TestCase, Client +from django.test import TestCase, Client, SimpleTestCase from django.urls import reverse -from apps.channels.models import Channel, ChannelGroup +from unittest.mock import patch +from uuid import uuid4 +from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource import xml.etree.ElementTree as ET +from datetime import timedelta + + +def _response_text(response): + """Read body from HttpResponse or StreamingHttpResponse.""" + if getattr(response, "streaming", False): + return b"".join(response.streaming_content).decode() + return response.content.decode() + + +def _epg_response_without_redis(cache_key, source, **kwargs): + """Test helper: stream EPG directly without Redis chunk caching.""" + from django.http import StreamingHttpResponse + + response = StreamingHttpResponse(source(), content_type="application/xml") + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" + return response + + +class OutputEndpointTestMixin: + """Isolate HTTP endpoint tests from network ACL, logging, DB teardown, and Redis.""" -class OutputM3UTest(TestCase): def setUp(self): + super().setUp() + self._network_patch = patch( + "apps.output.views.network_access_allowed", + return_value=True, + ) + self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._log_event_patch = patch("apps.output.views.log_system_event") + self._close_db_patch = patch("django.db.close_old_connections") + self._epg_cache_patch = patch( + "apps.output.views.stream_epg_response", + side_effect=_epg_response_without_redis, + ) + self._network_patch.start() + self._epg_teardown_patch.start() + self._log_event_patch.start() + self._close_db_patch.start() + self._epg_cache_patch.start() + + def tearDown(self): + from django.core.cache import cache + + cache.clear() + self._epg_cache_patch.stop() + self._close_db_patch.stop() + self._log_event_patch.stop() + self._epg_teardown_patch.stop() + self._network_patch.stop() + super().tearDown() + + def _create_isolated_profile(self, prefix): + """New profiles auto-include every channel via signal; clear that for tests.""" + profile = ChannelProfile.objects.create(name=f"{prefix}-{uuid4().hex[:8]}") + ChannelProfileMembership.objects.filter(channel_profile=profile).delete() + return profile + + def _add_channel_to_profile(self, profile, group, **kwargs): + channel = Channel.objects.create(channel_group=group, **kwargs) + ChannelProfileMembership.objects.create( + channel_profile=profile, + channel=channel, + enabled=True, + ) + return channel + + +class OutputM3UTest(OutputEndpointTestMixin, TestCase): + def setUp(self): + super().setUp() self.client = Client() - + self.group = ChannelGroup.objects.create(name=f"M3U Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("m3u") + self._add_channel_to_profile( + self.profile, + self.group, + channel_number=1.0, + name="Test M3U Channel", + ) + + def _m3u_url(self): + return reverse("output:m3u_endpoint", kwargs={"profile_name": self.profile.name}) + def test_generate_m3u_response(self): """ Test that the M3U endpoint returns a valid M3U file. """ - url = reverse('output:generate_m3u') - response = self.client.get(url) + response = self.client.get(self._m3u_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) self.assertIn("#EXTM3U", content) def test_generate_m3u_response_post_empty_body(self): """ Test that a POST request with an empty body returns 200 OK. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data=None, content_type='application/x-www-form-urlencoded') - content = response.content.decode() + response = self.client.post( + self._m3u_url(), + data=None, + content_type="application/x-www-form-urlencoded", + ) + content = _response_text(response) self.assertEqual(response.status_code, 200, "POST with empty body should return 200 OK") self.assertIn("#EXTM3U", content) @@ -34,35 +117,40 @@ class OutputM3UTest(TestCase): """ Test that a POST request with a non-empty body returns 403 Forbidden. """ - url = reverse('output:generate_m3u') - - response = self.client.post(url, data={'evilstring': 'muhahaha'}) + response = self.client.post(self._m3u_url(), data={"evilstring": "muhahaha"}) self.assertEqual(response.status_code, 403, "POST with body should return 403 Forbidden") - self.assertIn("POST requests with body are not allowed, body is:", response.content.decode()) + self.assertIn("POST requests with body are not allowed", _response_text(response)) -class OutputEPGXMLEscapingTest(TestCase): +class OutputEPGXMLEscapingTest(OutputEndpointTestMixin, TestCase): """Test XML escaping of channel_id attributes in EPG generation""" def setUp(self): + super().setUp() self.client = Client() - self.group = ChannelGroup.objects.create(name="Test Group") + self.group = ChannelGroup.objects.create(name=f"Test Group {uuid4().hex[:8]}") + self.profile = self._create_isolated_profile("epg-xml") + + def _add_channel(self, **kwargs): + return self._add_channel_to_profile(self.profile, self.group, **kwargs) + + def _epg_url(self, query="tvg_id_source=tvg_id&days=0&prev_days=0"): + base = reverse("output:epg_endpoint", kwargs={"profile_name": self.profile.name}) + return f"{base}?{query}" def test_channel_id_with_ampersand(self): """Test channel ID with ampersand is properly escaped""" - channel = Channel.objects.create( + self._add_channel( channel_number=1.0, name="Test Channel", tvg_id="News & Sports", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) self.assertEqual(response.status_code, 200) - content = response.content.decode() + content = _response_text(response) # Should contain escaped ampersand self.assertIn('id="News & Sports"', content) @@ -76,17 +164,15 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_angle_brackets(self): """Test channel ID with < and > characters""" - channel = Channel.objects.create( + self._add_channel( channel_number=2.0, name="HD Channel", tvg_id="Channel ", - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Channel <HD>"', content) try: @@ -96,23 +182,28 @@ class OutputEPGXMLEscapingTest(TestCase): def test_channel_id_with_all_special_chars(self): """Test channel ID with all XML special characters""" - channel = Channel.objects.create( + expected_id = 'Test & "Special" ' + self._add_channel( channel_number=3.0, name="Complex Channel", - tvg_id='Test & "Special" ', - channel_group=self.group + tvg_id=expected_id, ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) self.assertIn('id="Test & "Special" <Chars>"', content) try: tree = ET.fromstring(content) - # Verify we can find the channel with correct ID in parsed tree - channel_elem = tree.find('.//channel[@id="Test & \\"Special\\" "]') + channel_elem = next( + ( + elem + for elem in tree.findall(".//channel") + if elem.get("id") == expected_id + ), + None, + ) self.assertIsNotNone(channel_elem) except ET.ParseError as e: self.fail(f"Generated EPG with all special chars is not valid XML: {e}") @@ -121,25 +212,144 @@ class OutputEPGXMLEscapingTest(TestCase): """Test that programme elements also have escaped channel attributes""" epg_source = EPGSource.objects.create(name="Test EPG", source_type="dummy") epg_data = EPGData.objects.create(name="Test EPG Data", epg_source=epg_source) - channel = Channel.objects.create( + self._add_channel( channel_number=4.0, name="Program Test", tvg_id="News & Sports", epg_data=epg_data, - channel_group=self.group ) - url = reverse('output:generate_epg') + '?tvg_id_source=tvg_id' - response = self.client.get(url) + response = self.client.get(self._epg_url()) - content = response.content.decode() + content = _response_text(response) # Check programme elements have escaped channel attributes self.assertIn('channel="News & Sports"', content) try: tree = ET.fromstring(content) - programmes = tree.findall('.//programme[@channel="News & Sports"]') + programmes = [ + programme + for programme in tree.findall(".//programme") + if programme.get("channel") == "News & Sports" + ] self.assertGreater(len(programmes), 0) except ET.ParseError as e: self.fail(f"Generated EPG with programme elements is not valid XML: {e}") + + def test_programmes_emitted_in_start_time_order(self): + """Programmes for a channel are emitted in start_time order, not insert order.""" + from django.utils import timezone + from apps.epg.models import ProgramData + + epg_source = EPGSource.objects.create(name="Real EPG", source_type="xmltv") + epg_data = EPGData.objects.create(name="Station", epg_source=epg_source, tvg_id="station1") + self._add_channel( + channel_number=149.0, + name="Food Network", + tvg_id="station1", + epg_data=epg_data, + ) + now = timezone.now() + # Insert out of chronological order so id order != start_time order. + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=3), + end_time=now + timedelta(days=3, hours=1), + title="Third", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=1), + end_time=now + timedelta(days=1, hours=1), + title="First", + tvg_id="station1", + ) + ProgramData.objects.create( + epg=epg_data, + start_time=now + timedelta(days=2), + end_time=now + timedelta(days=2, hours=1), + title="Second", + tvg_id="station1", + ) + + content = _response_text(self.client.get(self._epg_url("tvg_id_source=tvg_id&days=7"))) + + self.assertLess(content.find('First'), content.find('Second')) + self.assertLess(content.find('Second'), content.find('Third')) + + +class OutputEPGCustomDummyTest(TestCase): + """Custom dummy EPG must not fall back to default when pattern matched but event is outside window.""" + + def setUp(self): + self.group = ChannelGroup.objects.create(name="Sports Group") + + def test_custom_dummy_outside_window_fills_with_ended_programmes(self): + from django.utils import timezone + from apps.output.views import generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + channel_name = ( + "NHL 01: Washington Capitals vs Philadelphia Flyers @ April 16 07:30 PM ET" + ) + now = timezone.now() + lookback = now - timedelta(days=7) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=7, + epg_source=epg_source, + export_lookback=lookback, + export_cutoff=now + timedelta(days=7), + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all(p['end_time'] >= lookback for p in programs), + "All programmes should fall inside the export window", + ) + self.assertTrue( + any('Ended' in p['description'] for p in programs), + "Past events outside the window should still show ended filler", + ) + for program in programs: + start = program['start_time'] + self.assertEqual(start.second, 0) + self.assertEqual(start.microsecond, 0) + self.assertIn( + start.minute, (0, 30), + "Filler programmes should start on half-hour boundaries", + ) + self.assertGreaterEqual(programs[0]['start_time'], lookback) + + +class OutputEPGHelperTest(SimpleTestCase): + def test_ceil_to_half_hour_on_boundary(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=30, second=0, microsecond=0) + self.assertEqual(_ceil_to_half_hour(dt), dt) + + def test_ceil_to_half_hour_rounds_up(self): + from django.utils import timezone + from apps.output.views import _ceil_to_half_hour + + dt = timezone.now().replace(minute=17, second=42, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) diff --git a/apps/output/views.py b/apps/output/views.py index dba35951..a0875c7f 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,9 +25,55 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib +from apps.output.epg_chunk_cache import stream_epg_response logger = logging.getLogger(__name__) +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -112,6 +158,10 @@ def generate_m3u(request, profile_name=None, user=None): # Check if this is a POST request and the body is not empty (which we don't want to allow) logger.debug("Generating M3U for profile: %s, user: %s, method: %s", profile_name, user.username if user else "Anonymous", request.method) + if request.method == "POST" and request.body: + if request.body.decode() != '{}': + return HttpResponseForbidden("POST requests with body are not allowed.") + # Check cache for recent identical request (helps with double-GET from browsers) from django.core.cache import cache cache_params = f"{profile_name or 'all'}:{user.username if user else 'anonymous'}:{request.GET.urlencode()}" @@ -123,10 +173,6 @@ def generate_m3u(request, profile_name=None, user=None): response = HttpResponse(cached_content, content_type="audio/x-mpegurl") response["Content-Disposition"] = 'attachment; filename="channels.m3u"' return response - # Check if this is a POST request with data (which we don't want to allow) - if request.method == "POST" and request.body: - if request.body.decode() != '{}': - return HttpResponseForbidden("POST requests with body are not allowed.") if user is not None: if user.user_level < 10: @@ -409,7 +455,15 @@ def generate_fallback_programs(channel_id, channel_name, now, num_days, program_ return programs -def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4, epg_source=None): +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): """ Generate dummy EPG programs for channels. @@ -435,29 +489,26 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: custom_programs = generate_custom_dummy_programs( channel_id, channel_name, now, num_days, - epg_source.custom_properties + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, ) - # If custom generation succeeded, return those programs - # If it returned empty (pattern didn't match), check for custom fallback templates - if custom_programs: + if custom_programs is not None: return custom_programs - else: - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # Check if custom fallback templates are provided - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - # If custom fallback templates exist, use them instead of default - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - else: - logger.info(f"No custom fallback templates found, using default dummy EPG") + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") # Default humorous program descriptions based on time of day time_descriptions = { @@ -531,7 +582,15 @@ def generate_dummy_programs(channel_id, channel_name, num_days=1, program_length return programs -def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, custom_properties): +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): """ Generate programs using custom dummy EPG regex patterns. @@ -616,7 +675,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust if not title_pattern: logger.warning(f"No title_pattern in custom_properties, falling back to default") - return [] # Return empty, will use default + return None logger.debug(f"Title pattern from DB: {repr(title_pattern)}") @@ -633,7 +692,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust except Exception as e: logger.error(f"Invalid title regex pattern after conversion: {e}") logger.error(f"Pattern was: {repr(title_pattern)}") - return [] + return None time_regex = None if time_pattern: @@ -665,7 +724,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust title_match = title_regex.search(channel_name) if not title_match: logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return [] # Return empty, will use default + return None groups = title_match.groupdict() logger.debug(f"Title pattern matched. Groups: {groups}") @@ -917,57 +976,93 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust iterations = num_days for day in range(iterations): - # Start from current time (like standard dummy) instead of midnight - # This ensures programs appear in the guide's current viewing window - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - - if time_info: - # We have an extracted event time - this is when the MAIN event starts - # The extracted time is in the SOURCE timezone (e.g., 8PM ET) - # We need to convert it to UTC for storage - - # Determine which date to use - if date_info: - # Use the extracted date from the channel title - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'] - ).date() - logger.debug(f"Using extracted date: {current_date}") - else: - # No date extracted, use day offset from current time in SOURCE timezone - # This ensures we calculate "today" in the event's timezone, not UTC - # For example: 8:30 PM Central (1:30 AM UTC next day) for a 10 PM ET event - # should use today's date in ET, not tomorrow's date in UTC - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - # Create a naive datetime (no timezone info) representing the event in source timezone + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() event_start_naive = datetime.combine( current_date, datetime.min.time().replace( hour=time_info['hour'], - minute=time_info['minute'] - ) + minute=time_info['minute'], + ), ) - - # Use pytz to localize the naive datetime to the source timezone - # This automatically handles DST! try: - event_start_local = source_tz.localize(event_start_naive) - # Convert to UTC - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) except Exception as e: logger.error(f"Error localizing time to {source_tz}: {e}") - # Fallback: treat as UTC event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + # Pre-generate the main event title and description for reuse if title_template: main_event_title = format_template(title_template, all_groups) @@ -994,15 +1089,13 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) - is_event_day = (day == 0) + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window if is_event_day and not event_happened: - # This is THE day the event happens - # Fill programs BEFORE the event current_time = day_start - while current_time < event_start_utc: + while current_time < event_start_utc and current_time < day_end: program_start_utc = current_time program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) @@ -1084,8 +1177,8 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust event_happened = True - # Fill programs AFTER the event until end of day - current_time = event_end_utc + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) while current_time < day_end: program_start_utc = current_time @@ -1325,18 +1418,10 @@ def generate_dummy_epg( def generate_epg(request, profile_name=None, user=None): """ - Dynamically generate an XMLTV (EPG) file using streaming response to handle keep-alives. + Dynamically generate an XMLTV (EPG) file using a streaming response. Since the EPG data is stored independently of Channels, we group programmes by their associated EPGData record. - This version filters data based on the 'days' parameter and sends keep-alives during processing. """ - # Check cache for recent identical request (helps with double-GET from browsers) - from django.core.cache import cache - # Resolve all effective parameter values once here so they are reused for both - # the cache key and inside epg_generator() via closure. - # The cache key is built from resolved values only — not from the raw query string — - # so equivalent requests (e.g. days=7 via URL param vs. user default of 7) share - # the same cache entry regardless of how the value was supplied. user_custom = (user.custom_properties or {}) if user else {} try: num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) @@ -1356,21 +1441,13 @@ def generate_epg(request, profile_name=None, user=None): ) content_cache_key = f"epg_content:{cache_params}" - cached_content = cache.get(content_cache_key) - if cached_content: - logger.debug("Serving EPG from cache") - response = HttpResponse(cached_content, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response - def epg_generator(): """Generator function that yields EPG data with keep-alives during processing.""" - xml_lines = [] - xml_lines.append('') - xml_lines.append( - '' + yield '\n' + yield ( + '\n' ) # Get channels based on user/profile @@ -1416,14 +1493,19 @@ def generate_epg(request, profile_name=None, user=None): # Resolve effective values at SQL level and exclude hidden channels # so output ordering/display honors user overrides. from apps.channels.managers import with_effective_values - channels = ( + channels = list( with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) ) ) + channel_count = len(channels) # For dummy EPG, use either the specified value or default to 3 days dummy_days = num_days if num_days > 0 else 3 @@ -1465,12 +1547,14 @@ def generate_epg(request, profile_name=None, user=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - dummy_epg_ids_for_program_check = set() + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] - # Process channels for the section for channel in channels: effective_name = channel.effective_name effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id effective_logo = channel.effective_logo_obj effective_number = channel.effective_channel_number @@ -1492,8 +1576,6 @@ def generate_epg(request, profile_name=None, user=None): # Check if this is a custom dummy EPG with channel logo URL template if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if channel.effective_epg_data_id: - dummy_epg_ids_for_program_check.add(channel.effective_epg_data_id) epg_source = effective_epg_data.epg_source if epg_source.custom_properties: custom_props = epg_source.custom_properties @@ -1548,51 +1630,16 @@ def generate_epg(request, profile_name=None, user=None): tvg_logo = direct_logo else: tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - display_name = effective_name - xml_lines.append(f' ') - xml_lines.append(f' {html.escape(display_name)}') - xml_lines.append(f' ') - xml_lines.append(" ") + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") - # Send all channel definitions - channel_xml = '\n'.join(xml_lines) + '\n' - yield channel_xml - xml_lines = [] # Clear to save memory + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] - dummy_epg_with_programs = set() - if dummy_epg_ids_for_program_check: - dummy_epg_with_programs = set( - ProgramData.objects.filter(epg_id__in=dummy_epg_ids_for_program_check) - .values_list('epg_id', flat=True) - .distinct() - ) - - # Pre-pass: categorize channels into dummy and real EPG groups - dummy_program_list = [] # (channel_id, pattern_match_name, epg_source_or_None) - real_epg_map = {} # epg_data_id -> [channel_id, ...] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_number = channel.effective_channel_number - - # Determine channel_id (same logic as channel section) - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - display_name = effective_epg_data.name if effective_epg_data else effective_name pattern_match_name = effective_name - - # Check if we should use stream name instead of channel name if effective_epg_data and effective_epg_data.epg_source: epg_source = effective_epg_data.epg_source if epg_source.custom_properties: @@ -1619,48 +1666,19 @@ def generate_epg(request, profile_name=None, user=None): if not effective_epg_data: dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) else: - if effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - if effective_epg_data_id in dummy_epg_with_programs: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - else: - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - continue - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - # Emit dummy programmes - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source - ) - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - yield f' \n' - yield f" {html.escape(program['title'])}\n" - if program.get('sub_title'): - yield f" {html.escape(program['sub_title'])}\n" - yield f" {html.escape(program['description'])}\n" - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - yield f" {html.escape(cat)}\n" - if 'date' in custom_data: - yield f" {html.escape(custom_data['date'])}\n" - if custom_data.get('live', False): - yield f" \n" - if custom_data.get('new', False): - yield f" \n" - if 'icon' in custom_data: - yield f' \n' - yield f" \n" + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - # Emit real programmes: single bulk query, chunked to avoid server-side cursor issues. all_epg_ids = list(real_epg_map.keys()) if all_epg_ids: if num_days > 0: @@ -1682,27 +1700,44 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None - is_multi = False - multi_buffer = [] + pending = [] program_batch = [] - batch_size = 1000 - chunk_size = 5000 - # Keyset pagination: track last (epg_id, id) instead of OFFSET - # to avoid skipping/duplicating rows if the table changes mid-stream. + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE last_epg_id = 0 last_id = 0 _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + while True: program_chunk = list( programs_base_qs.filter(epg_id__gte=last_epg_id) .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] ) - if not program_chunk: break - # Advance keyset cursor to last row in this chunk last_row = program_chunk[-1] last_epg_id = last_row['epg_id'] last_id = last_row['id'] @@ -1710,31 +1745,19 @@ def generate_epg(request, profile_name=None, user=None): for prog in program_chunk: epg_id = prog['epg_id'] - # When epg_id changes, flush multi-channel buffer for previous group if epg_id != current_epg_id: - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - multi_buffer = [] - + yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] - is_multi = len(channel_ids_for_epg) > 1 - # Build programme XML for primary channel_id primary_cid = channel_ids_for_epg[0] - start_str = prog['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = prog['end_time'].strftime("%Y%m%d%H%M%S %z") + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') @@ -1932,67 +1955,101 @@ def generate_epg(request, profile_name=None, user=None): program_xml.append(" ") xml_text = '\n'.join(program_xml) - program_batch.append(xml_text) + pending.append((prog['start_time'], prog['id'], xml_text)) - if is_multi: - multi_buffer.append(xml_text) + del program_chunk - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - - # Final flush of multi-channel buffer - if is_multi and multi_buffer: - escaped_primary = html.escape(channel_ids_for_epg[0]) - for extra_cid in channel_ids_for_epg[1:]: - escaped_extra = html.escape(extra_cid) - for xml_text in multi_buffer: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{escaped_extra}"', - 1, - )) + yield from flush_pending() if program_batch: yield '\n'.join(program_batch) + '\n' - # Send final closing tag and completion message + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + yield "\n" - # Log system event for EPG download after streaming completes (with deduplication based on client) client_id, client_ip, user_agent = get_client_identifier(request) event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - if not cache.get(event_cache_key): - # `len()` reuses the queryset's iteration cache populated above; - # `count()` would issue a separate SELECT COUNT(*). - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=len(channels), - client_ip=client_ip, - user_agent=user_agent, - ) - cache.set(event_cache_key, True, 2) # Prevent duplicate events for 2 seconds - # Wrapper generator that collects content for caching - def caching_generator(): - collected_content = [] - for chunk in epg_generator(): - collected_content.append(chunk) - yield chunk - # After streaming completes, cache the full content - full_content = ''.join(collected_content) - cache.set(content_cache_key, full_content, 300) - logger.debug("Cached EPG content (%d bytes)", len(full_content)) + def _log_epg_download(): + from django.core.cache import cache as event_cache - response = StreamingHttpResponse( - streaming_content=caching_generator(), - content_type="application/xml" - ) - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' - response["Cache-Control"] = "no-cache" - return response + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_epg_response(content_cache_key, build_epg_stream) def xc_get_user(request): diff --git a/core/tests.py b/core/tests.py index 7b2f1986..33475ae7 100644 --- a/core/tests.py +++ b/core/tests.py @@ -1,6 +1,6 @@ from unittest.mock import patch, MagicMock -from django.test import TestCase +from django.test import TestCase, SimpleTestCase from apps.epg.models import EPGSource from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -301,3 +301,14 @@ class DropDBCommandTlsTest(TestCase): host='localhost', port=5432, autocommit=True, ) + + +class MallocTrimTests(SimpleTestCase): + def test_trim_is_noop_when_libc_has_no_malloc_trim(self): + from core.utils import trim_c_allocator_heap + + fake_libc = MagicMock(spec=[]) + with patch('ctypes.util.find_library', return_value='libc.so.6'), patch( + 'ctypes.CDLL', return_value=fake_libc + ): + self.assertFalse(trim_c_allocator_heap()) diff --git a/core/utils.py b/core/utils.py index 29c613b3..89228088 100644 --- a/core/utils.py +++ b/core/utils.py @@ -546,6 +546,25 @@ def monitor_memory_usage(func): return result return wrapper +def trim_c_allocator_heap(): + """Return unused C heap pages to the OS where supported (glibc malloc_trim).""" + try: + import ctypes + import ctypes.util + + libc_name = ctypes.util.find_library("c") + if not libc_name: + return False + libc = ctypes.CDLL(libc_name) + if not hasattr(libc, "malloc_trim"): + return False + libc.malloc_trim(0) + return True + except Exception: + logger.debug("malloc_trim unavailable or failed", exc_info=True) + return False + + def cleanup_memory(log_usage=False, force_collection=True): """ Comprehensive memory cleanup function to reduce memory footprint From bccee9ebc1b36a140103bf3b715e91b8cb1b7def Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 11:50:52 -0500 Subject: [PATCH 10/28] refactor(epg): extract EPG generation logic into dedicated module - Moved all XMLTV output logic from `apps/output/views.py` to `apps/output/epg.py`, streamlining the codebase and maintaining the existing HTTP endpoint functionality. - Updated related tests and references to ensure proper integration with the new module structure. --- CHANGELOG.md | 2 + apps/output/epg.py | 1745 +++++++++++++++++ ...hunk_cache.py => streaming_chunk_cache.py} | 36 +- ...cache.py => test_streaming_chunk_cache.py} | 30 +- apps/output/tests.py | 11 +- apps/output/views.py | 1714 +--------------- 6 files changed, 1790 insertions(+), 1748 deletions(-) create mode 100644 apps/output/epg.py rename apps/output/{epg_chunk_cache.py => streaming_chunk_cache.py} (81%) rename apps/output/{test_epg_chunk_cache.py => test_streaming_chunk_cache.py} (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3315456b..840cb349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. + - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. ### Performance diff --git a/apps/output/epg.py b/apps/output/epg.py new file mode 100644 index 00000000..354d6e5b --- /dev/null +++ b/apps/output/epg.py @@ -0,0 +1,1745 @@ +"""XMLTV (EPG) output generation. + +Consolidates the EPG export logic that backs the `/epg` endpoint and the XC +XMLTV endpoint: real programme streaming, dummy/custom dummy program +generation, and the streaming XMLTV builder. HTTP endpoints live in views.py +and call into this module; Redis chunk caching lives in streaming_chunk_cache.py. +""" + +import html +import logging +from datetime import datetime, timedelta + +import regex + +from django.db.models import Prefetch +from django.http import Http404 +from django.urls import reverse +from django.utils import timezone as django_timezone + +from apps.channels.models import Channel, ChannelProfile, Stream +from apps.channels.utils import format_channel_number +from apps.epg.models import ProgramData +from apps.output.streaming_chunk_cache import stream_cached_response +from core.utils import build_absolute_uri_with_port, log_system_event + +logger = logging.getLogger(__name__) + +_EPG_CHANNEL_XML_BATCH_SIZE = 200 +_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 +_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 + + +def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): + if end_time < lookback_cutoff: + return False + if cutoff_date is not None and start_time >= cutoff_date: + return False + return True + + +def _ceil_to_half_hour(dt): + """Round a datetime up to the next :00 or :30 boundary.""" + dt = dt.replace(second=0, microsecond=0) + remainder = dt.minute % 30 + if remainder == 0: + return dt + return dt + timedelta(minutes=30 - remainder) + + +def _epg_export_teardown(): + from django.db import close_old_connections + + from core.utils import ( + _is_gevent_monkey_patched, + cleanup_memory, + trim_c_allocator_heap, + ) + + close_old_connections() + + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_run) + else: + _run() + + +def _ordered_channel_streams(channel): + """Return a channel's streams ordered by channelstream join order.""" + prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') + if prefetched is not None: + return list(prefetched) + return list(channel.streams.all().order_by('channelstream__order')) + + +def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): + """Name used for custom dummy EPG regex matching (channel or stream title). + + Returns (name, stream_lookup_failed). stream_lookup_failed is True only when + name_source is 'stream' but the configured index is missing or out of range. + """ + if custom_props.get('name_source') != 'stream': + return effective_name, False + stream_index = custom_props.get('stream_index', 1) - 1 + streams = _ordered_channel_streams(channel) + if 0 <= stream_index < len(streams): + return streams[stream_index].name, False + return effective_name, True + + +def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): + """ + Generate dummy programs using custom fallback templates when patterns don't match. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel name to use as fallback in templates + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + fallback_title: Custom fallback title template (empty string if not provided) + fallback_description: Custom fallback description template (empty string if not provided) + + Returns: + List of program dictionaries + """ + programs = [] + + # Use custom fallback title or channel name as default + title = fallback_title if fallback_title else channel_name + + # Use custom fallback description or a simple default message + if fallback_description: + description = fallback_description + else: + description = f"EPG information is currently unavailable for {channel_name}" + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": title, + "description": description, + }) + + return programs + + +def generate_dummy_programs( + channel_id, + channel_name, + num_days=1, + program_length_hours=4, + epg_source=None, + export_lookback=None, + export_cutoff=None, +): + """ + Generate dummy EPG programs for channels. + + If epg_source is provided and it's a custom dummy EPG with patterns, + use those patterns to generate programs from the channel title. + Otherwise, generate default dummy programs. + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title/name + num_days: Number of days to generate programs for + program_length_hours: Length of each program in hours + epg_source: Optional EPGSource for custom dummy EPG with patterns + + Returns: + List of program dictionaries + """ + # Get current time rounded to hour + now = django_timezone.now() + now = now.replace(minute=0, second=0, microsecond=0) + + # Check if this is a custom dummy EPG with regex patterns + if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: + custom_programs = generate_custom_dummy_programs( + channel_id, channel_name, now, num_days, + epg_source.custom_properties, + export_lookback=export_lookback, + export_cutoff=export_cutoff, + ) + if custom_programs is not None: + return custom_programs + + logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") + + custom_props = epg_source.custom_properties + fallback_title = custom_props.get('fallback_title_template', '').strip() + fallback_description = custom_props.get('fallback_description_template', '').strip() + + if fallback_title or fallback_description: + logger.info(f"Using custom fallback templates for '{channel_name}'") + return generate_fallback_programs( + channel_id, channel_name, now, num_days, + program_length_hours, fallback_title, fallback_description + ) + logger.info(f"No custom fallback templates found, using default dummy EPG") + + # Default humorous program descriptions based on time of day + time_descriptions = { + (0, 4): [ + f"Late Night with {channel_name} - Where insomniacs unite!", + f"The 'Why Am I Still Awake?' Show on {channel_name}", + f"Counting Sheep - A {channel_name} production for the sleepless", + ], + (4, 8): [ + f"Dawn Patrol - Rise and shine with {channel_name}!", + f"Early Bird Special - Coffee not included", + f"Morning Zombies - Before coffee viewing on {channel_name}", + ], + (8, 12): [ + f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", + f"The 'I Should Be Working' Hour on {channel_name}", + f"Productivity Killer - {channel_name}'s daytime programming", + ], + (12, 16): [ + f"Lunchtime Laziness with {channel_name}", + f"The Afternoon Slump - Brought to you by {channel_name}", + f"Post-Lunch Food Coma Theater on {channel_name}", + ], + (16, 20): [ + f"Rush Hour - {channel_name}'s alternative to traffic", + f"The 'What's For Dinner?' Debate on {channel_name}", + f"Evening Escapism - {channel_name}'s remedy for reality", + ], + (20, 24): [ + f"Prime Time Placeholder - {channel_name}'s finest not-programming", + f"The 'Netflix Was Too Complicated' Show on {channel_name}", + f"Family Argument Avoider - Courtesy of {channel_name}", + ], + } + + programs = [] + + # Create programs for each day + for day in range(num_days): + day_start = now + timedelta(days=day) + + # Create programs with specified length throughout the day + for hour_offset in range(0, 24, program_length_hours): + # Calculate program start and end times + start_time = day_start + timedelta(hours=hour_offset) + end_time = start_time + timedelta(hours=program_length_hours) + + # Get the hour for selecting a description + hour = start_time.hour + + # Find the appropriate time slot for description + for time_range, descriptions in time_descriptions.items(): + start_range, end_range = time_range + if start_range <= hour < end_range: + # Pick a description using the sum of the hour and day as seed + # This makes it somewhat random but consistent for the same timeslot + description = descriptions[(hour + day) % len(descriptions)] + break + else: + # Fallback description if somehow no range matches + description = f"Placeholder program for {channel_name} - EPG data went on vacation" + + programs.append({ + "channel_id": channel_id, + "start_time": start_time, + "end_time": end_time, + "title": channel_name, + "description": description, + }) + + return programs + + +def generate_custom_dummy_programs( + channel_id, + channel_name, + now, + num_days, + custom_properties, + export_lookback=None, + export_cutoff=None, +): + """ + Generate programs using custom dummy EPG regex patterns. + + Extracts information from channel title using regex patterns and generates + programs based on the extracted data. + + TIMEZONE HANDLING: + ------------------ + The timezone parameter specifies the timezone of the event times in your channel + titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). + DST (Daylight Saving Time) is handled automatically by pytz. + + Examples: + - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" + - Set timezone = "US/Eastern" + - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) + - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) + + Args: + channel_id: Channel ID for the programs + channel_name: Channel title to parse + now: Current datetime (in UTC) + num_days: Number of days to generate programs for + custom_properties: Dict with title_pattern, time_pattern, templates, etc. + - timezone: Timezone name (e.g., 'US/Eastern') + + Returns: + List of program dictionaries with start_time/end_time in UTC + """ + import pytz + + logger.info(f"Generating custom dummy programs for channel: {channel_name}") + + # Extract patterns from custom properties + title_pattern = custom_properties.get('title_pattern', '') + time_pattern = custom_properties.get('time_pattern', '') + date_pattern = custom_properties.get('date_pattern', '') + + # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') + timezone_value = custom_properties.get('timezone', 'UTC') + output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone + program_duration = custom_properties.get('program_duration', 180) # Minutes + title_template = custom_properties.get('title_template', '') + subtitle_template = custom_properties.get('subtitle_template', '') + description_template = custom_properties.get('description_template', '') + + # Templates for upcoming/ended programs + upcoming_title_template = custom_properties.get('upcoming_title_template', '') + upcoming_description_template = custom_properties.get('upcoming_description_template', '') + ended_title_template = custom_properties.get('ended_title_template', '') + ended_description_template = custom_properties.get('ended_description_template', '') + + # Image URL templates + channel_logo_url_template = custom_properties.get('channel_logo_url', '') + program_poster_url_template = custom_properties.get('program_poster_url', '') + + # EPG metadata options + category_string = custom_properties.get('category', '') + # Split comma-separated categories and strip whitespace, filter out empty strings + categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] + include_date = custom_properties.get('include_date', True) + include_live = custom_properties.get('include_live', False) + include_new = custom_properties.get('include_new', False) + + # Parse timezone name + try: + source_tz = pytz.timezone(timezone_value) + logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") + source_tz = pytz.utc + + # Parse output timezone if provided (for display purposes) + output_tz = None + if output_timezone_value: + try: + output_tz = pytz.timezone(output_timezone_value) + logger.debug(f"Using output timezone for display: {output_timezone_value}") + except pytz.exceptions.UnknownTimeZoneError: + logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") + output_tz = None + + if not title_pattern: + logger.warning(f"No title_pattern in custom_properties, falling back to default") + return None + + logger.debug(f"Title pattern from DB: {repr(title_pattern)}") + + # Convert PCRE/JavaScript named groups (?) to Python format (?P) + # This handles patterns created with JavaScript regex syntax + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) + logger.debug(f"Converted title pattern: {repr(title_pattern)}") + + # Compile regex patterns using the enhanced regex module + # (supports variable-width lookbehinds like JavaScript) + try: + title_regex = regex.compile(title_pattern) + except Exception as e: + logger.error(f"Invalid title regex pattern after conversion: {e}") + logger.error(f"Pattern was: {repr(title_pattern)}") + return None + + time_regex = None + if time_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) + logger.debug(f"Converted time pattern: {repr(time_pattern)}") + try: + time_regex = regex.compile(time_pattern) + except Exception as e: + logger.warning(f"Invalid time regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(time_pattern)}") + + # Compile date regex if provided + date_regex = None + if date_pattern: + # Convert PCRE/JavaScript named groups to Python format + # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) + logger.debug(f"Converted date pattern: {repr(date_pattern)}") + try: + date_regex = regex.compile(date_pattern) + except Exception as e: + logger.warning(f"Invalid date regex pattern after conversion: {e}") + logger.warning(f"Pattern was: {repr(date_pattern)}") + + # Try to match the channel name with the title pattern + # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string + title_match = title_regex.search(channel_name) + if not title_match: + logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") + return None + + groups = title_match.groupdict() + logger.debug(f"Title pattern matched. Groups: {groups}") + + # Helper function to format template with matched groups + def format_template(template, groups, url_encode=False): + """Replace {groupname} placeholders with matched group values + + Args: + template: Template string with {groupname} placeholders + groups: Dict of group names to values + url_encode: If True, URL encode the group values for safe use in URLs + """ + if not template: + return '' + result = template + for key, value in groups.items(): + if url_encode and value: + # URL encode the value to handle spaces and special characters + from urllib.parse import quote + encoded_value = quote(str(value), safe='') + result = result.replace(f'{{{key}}}', encoded_value) + else: + result = result.replace(f'{{{key}}}', str(value) if value else '') + return result + + # Extract time from title if time pattern exists + time_info = None + time_groups = {} + if time_regex: + time_match = time_regex.search(channel_name) + if time_match: + time_groups = time_match.groupdict() + try: + hour = int(time_groups.get('hour')) + # Handle optional minute group - could be None if not captured + minute_value = time_groups.get('minute') + minute = int(minute_value) if minute_value is not None else 0 + ampm = time_groups.get('ampm') + ampm = ampm.lower() if ampm else None + + # Determine if this is 12-hour or 24-hour format + if ampm in ('am', 'pm'): + # 12-hour format: convert to 24-hour + if ampm == 'pm' and hour != 12: + hour += 12 + elif ampm == 'am' and hour == 12: + hour = 0 + logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") + else: + # 24-hour format: hour is already in 24-hour format + # Validate that it's actually a 24-hour time (0-23) + if hour > 23: + logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") + hour = hour % 24 # Wrap around just in case + logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") + + time_info = {'hour': hour, 'minute': minute} + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing time: {e}") + + # Extract date from title if date pattern exists + date_info = None + date_groups = {} + if date_regex: + date_match = date_regex.search(channel_name) + if date_match: + date_groups = date_match.groupdict() + try: + # Support various date group names: month, day, year + month_str = date_groups.get('month', '') + day_str = date_groups.get('day', '') + year_str = date_groups.get('year', '') + + # Parse day - default to current day if empty or invalid + day = int(day_str) if day_str else now.day + + # Parse year - default to current year if empty or invalid (matches frontend behavior) + year = int(year_str) if year_str else now.year + + # Parse month - can be numeric (1-12) or text (Jan, January, etc.) + month = None + if month_str: + if month_str.isdigit(): + month = int(month_str) + else: + # Try to parse text month names + import calendar + month_str_lower = month_str.lower() + # Check full month names + for i, month_name in enumerate(calendar.month_name): + if month_name.lower() == month_str_lower: + month = i + break + # Check abbreviated month names if not found + if month is None: + for i, month_abbr in enumerate(calendar.month_abbr): + if month_abbr.lower() == month_str_lower: + month = i + break + + # Default to current month if not extracted or invalid + if month is None: + month = now.month + + if month and 1 <= month <= 12 and 1 <= day <= 31: + date_info = {'year': year, 'month': month, 'day': day} + logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") + else: + logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") + except (ValueError, TypeError) as e: + logger.warning(f"Error parsing date: {e}") + + # Merge title groups, time groups, and date groups for template formatting + all_groups = {**groups, **time_groups, **date_groups} + + # Add normalized versions of all groups for cleaner URLs + # These remove all non-alphanumeric characters and convert to lowercase + for key, value in list(all_groups.items()): + if value: + # Remove all non-alphanumeric characters (except spaces temporarily) + # then replace spaces with nothing, and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + all_groups[f'{key}_normalize'] = normalized + + # Format channel logo URL if template provided (with URL encoding) + channel_logo_url = None + if channel_logo_url_template: + channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted channel logo URL: {channel_logo_url}") + + # Format program poster URL if template provided (with URL encoding) + program_poster_url = None + if program_poster_url_template: + program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) + logger.debug(f"Formatted program poster URL: {program_poster_url}") + + # Add formatted time strings for better display (handles minutes intelligently) + if time_info: + hour_24 = time_info['hour'] + minute = time_info['minute'] + + # Determine the base date to use for placeholders + # If date was extracted, use it; otherwise use current date + if date_info: + base_date = datetime(date_info['year'], date_info['month'], date_info['day']) + else: + base_date = datetime.now() + + # If output_timezone is specified, convert the display time to that timezone + if output_tz: + # Create a datetime in the source timezone using the base date + temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + # Convert to output timezone + temp_date_output = temp_date.astimezone(output_tz) + # Extract converted hour and minute for display + hour_24 = temp_date_output.hour + minute = temp_date_output.minute + logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") + + # Add date placeholders based on the OUTPUT timezone + # This ensures {date}, {month}, {day}, {year} reflect the converted timezone + all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_output.month) + all_groups['day'] = str(temp_date_output.day) + all_groups['year'] = str(temp_date_output.year) + logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") + else: + # No output timezone conversion - use source timezone for date + # Create temp date to get proper date in source timezone using the base date + temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) + all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') + all_groups['month'] = str(temp_date_source.month) + all_groups['day'] = str(temp_date_source.day) + all_groups['year'] = str(temp_date_source.year) + + # Format 24-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime24'] = f"{hour_24}:{minute:02d}" + else: + all_groups['starttime24'] = f"{hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {starttime} placeholder + # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) + ampm = 'AM' if hour_24 < 12 else 'PM' + hour_12 = hour_24 + if hour_24 == 0: + hour_12 = 12 + elif hour_24 > 12: + hour_12 = hour_24 - 12 + + # Format 12-hour start time string - only include minutes if non-zero + if minute > 0: + all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" + else: + all_groups['starttime'] = f"{hour_12} {ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" + + # Calculate end time based on program duration + # Create a datetime for calculations + temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) + temp_end = temp_start + timedelta(minutes=program_duration) + + # Extract end time components (already in correct timezone if output_tz was applied above) + end_hour_24 = temp_end.hour + end_minute = temp_end.minute + + # Format 24-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" + else: + all_groups['endtime24'] = f"{end_hour_24:02d}:00" + + # Convert 24-hour to 12-hour format for {endtime} placeholder + end_ampm = 'AM' if end_hour_24 < 12 else 'PM' + end_hour_12 = end_hour_24 + if end_hour_24 == 0: + end_hour_12 = 12 + elif end_hour_24 > 12: + end_hour_12 = end_hour_24 - 12 + + # Format 12-hour end time string - only include minutes if non-zero + if end_minute > 0: + all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + else: + all_groups['endtime'] = f"{end_hour_12} {end_ampm}" + + # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") + all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" + + # Generate programs + programs = [] + + # If we have extracted time AND date, the event happens on a SPECIFIC date + # If we have time but NO date, generate for multiple days (existing behavior) + # All other days and times show "Upcoming" before or "Ended" after + event_happened = False + + # Determine how many iterations we need + if date_info and time_info: + # Specific date extracted - only generate for that one date + iterations = 1 + logger.debug(f"Date extracted, generating single event for specific date") + else: + # No specific date - use num_days (existing behavior) + iterations = num_days + + for day in range(iterations): + event_overlaps_window = True + if date_info and time_info: + current_date = datetime( + date_info['year'], + date_info['month'], + date_info['day'], + ).date() + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + event_overlaps_window = _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ) + if not event_overlaps_window: + logger.debug( + "Custom dummy event outside export window; filling window only: %s", + channel_name, + ) + event_happened = event_end_utc < lookback + day_start = _ceil_to_half_hour(lookback) + if export_cutoff is not None: + day_end = export_cutoff + else: + day_end = now + timedelta(days=num_days if num_days > 0 else 3) + else: + day_start = source_tz.localize( + datetime.combine(current_date, datetime.min.time()) + ).astimezone(pytz.utc) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + else: + day_start = now + timedelta(days=day) + day_end = day_start + timedelta(days=1) + if export_lookback is not None: + day_start = max(day_start, export_lookback) + if export_cutoff is not None: + day_end = min(day_end, export_cutoff) + + if day_start >= day_end: + continue + + if time_info: + if not date_info: + now_in_source_tz = now.astimezone(source_tz) + current_date = (now_in_source_tz + timedelta(days=day)).date() + logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") + + event_start_naive = datetime.combine( + current_date, + datetime.min.time().replace( + hour=time_info['hour'], + minute=time_info['minute'], + ), + ) + try: + event_start_local = source_tz.localize(event_start_naive) + event_start_utc = event_start_local.astimezone(pytz.utc) + logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") + except Exception as e: + logger.error(f"Error localizing time to {source_tz}: {e}") + event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) + + event_end_utc = event_start_utc + timedelta(minutes=program_duration) + + lookback = export_lookback if export_lookback is not None else now + if not _programme_overlaps_export_window( + event_start_utc, event_end_utc, lookback, export_cutoff + ): + continue + else: + logger.debug(f"Using extracted date: {current_date}") + + # Pre-generate the main event title and description for reuse + if title_template: + main_event_title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + main_event_title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + main_event_subtitle = format_template(subtitle_template, all_groups) + else: + main_event_subtitle = None + + if description_template: + main_event_description = format_template(description_template, all_groups) + else: + main_event_description = main_event_title + + + + # Determine if this day is before, during, or after the event + # Event only happens on day 0 (first day) when it falls inside the window + is_event_day = (day == 0) and event_overlaps_window + + if is_event_day and not event_happened: + current_time = day_start + + while current_time < event_start_utc and current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) + + # Use custom upcoming templates if provided, otherwise use defaults + if upcoming_title_template: + upcoming_title = format_template(upcoming_title_template, all_groups) + else: + upcoming_title = main_event_title + + if upcoming_description_template: + upcoming_description = format_template(upcoming_description_template, all_groups) + else: + upcoming_description = f"Upcoming: {main_event_description}" + + # Build custom_properties for upcoming programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": upcoming_title, + "sub_title": None, # No subtitle for filler programs + "description": upcoming_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + + # Add the MAIN EVENT at the extracted time + # Build custom_properties for main event (includes category and live) + main_event_custom_properties = {} + + # Add categories if provided + if categories: + main_event_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = event_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + main_event_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + main_event_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + main_event_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + main_event_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": event_start_utc, + "end_time": event_end_utc, + "title": main_event_title, + "sub_title": main_event_subtitle, + "description": main_event_description, + "custom_properties": main_event_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + event_happened = True + + # Fill programs AFTER the event until end of export day window + current_time = max(event_end_utc, day_start) + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom ended templates if provided, otherwise use defaults + if ended_title_template: + ended_title = format_template(ended_title_template, all_groups) + else: + ended_title = main_event_title + + if ended_description_template: + ended_description = format_template(ended_description_template, all_groups) + else: + ended_description = f"Ended: {main_event_description}" + + # Build custom_properties for ended programs (only date, no category/live) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": ended_title, + "sub_title": None, # No subtitle for filler programs + "description": ended_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + current_time += timedelta(minutes=program_duration) + else: + # This day is either before the event (future days) or after the event happened + # Fill entire day with appropriate message + current_time = day_start + + # If event already happened, all programs show "Ended" + # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" + is_ended = event_happened + + while current_time < day_end: + program_start_utc = current_time + program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) + + # Use custom templates based on whether event has ended or is upcoming + if is_ended: + if ended_title_template: + program_title = format_template(ended_title_template, all_groups) + else: + program_title = main_event_title + + if ended_description_template: + program_description = format_template(ended_description_template, all_groups) + else: + program_description = f"Ended: {main_event_description}" + else: + if upcoming_title_template: + program_title = format_template(upcoming_title_template, all_groups) + else: + program_title = main_event_title + + if upcoming_description_template: + program_description = format_template(upcoming_description_template, all_groups) + else: + program_description = f"Upcoming: {main_event_description}" + + # Build custom_properties (only date for upcoming/ended filler programs) + program_custom_properties = {} + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": program_title, + "sub_title": None, # No subtitle for filler programs + "description": program_description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, + }) + + current_time += timedelta(minutes=program_duration) + else: + # No extracted time - fill entire day with regular intervals + # day_start and day_end are already in UTC, so no conversion needed + programs_per_day = max(1, int(24 / (program_duration / 60))) + + for program_num in range(programs_per_day): + program_start_utc = day_start + timedelta(minutes=program_num * program_duration) + program_end_utc = program_start_utc + timedelta(minutes=program_duration) + + if title_template: + title = format_template(title_template, all_groups) + else: + title_parts = [] + if 'league' in all_groups and all_groups['league']: + title_parts.append(all_groups['league']) + if 'team1' in all_groups and 'team2' in all_groups: + title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") + elif 'title' in all_groups and all_groups['title']: + title_parts.append(all_groups['title']) + title = ' - '.join(title_parts) if title_parts else channel_name + + if subtitle_template: + subtitle = format_template(subtitle_template, all_groups) + else: + subtitle = None + + if description_template: + description = format_template(description_template, all_groups) + else: + description = title + + # Build custom_properties for this program + program_custom_properties = {} + + # Add categories if provided + if categories: + program_custom_properties['categories'] = categories + + # Add date if requested (YYYY-MM-DD format from start time in event timezone) + if include_date: + # Convert UTC time to event timezone for date calculation + local_time = program_start_utc.astimezone(source_tz) + date_str = local_time.strftime('%Y-%m-%d') + program_custom_properties['date'] = date_str + + # Add live flag if requested + if include_live: + program_custom_properties['live'] = True + + # Add new flag if requested + if include_new: + program_custom_properties['new'] = True + + # Add program poster URL if provided + if program_poster_url: + program_custom_properties['icon'] = program_poster_url + + programs.append({ + "channel_id": channel_id, + "start_time": program_start_utc, + "end_time": program_end_utc, + "title": title, + "sub_title": subtitle, + "description": description, + "custom_properties": program_custom_properties, + "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation + }) + + logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") + return programs + + +def generate_dummy_epg( + channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 +): + """ + Generate dummy EPG programs for channels without EPG data. + Creates program blocks for a specified number of days. + + Args: + channel_id: The channel ID to use in the program entries + channel_name: The name of the channel to use in program titles + xml_lines: Optional list to append lines to, otherwise returns new list + num_days: Number of days to generate EPG data for (default: 1) + program_length_hours: Length of each program block in hours (default: 4) + + Returns: + List of XML lines for the dummy EPG entries + """ + if xml_lines is None: + xml_lines = [] + + for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): + # Format times in XMLTV format + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + + # Create program entry with escaped channel name + xml_lines.append( + f' ' + ) + xml_lines.append(f" {html.escape(program['title'])}") + + # Add subtitle if available + if program.get('sub_title'): + xml_lines.append(f" {html.escape(program['sub_title'])}") + + xml_lines.append(f" {html.escape(program['description'])}") + + # Add custom_properties if present + custom_data = program.get('custom_properties', {}) + + # Categories + if 'categories' in custom_data: + for cat in custom_data['categories']: + xml_lines.append(f" {html.escape(cat)}") + + # Date tag + if 'date' in custom_data: + xml_lines.append(f" {html.escape(custom_data['date'])}") + + # Live tag + if custom_data.get('live', False): + xml_lines.append(f" ") + + # New tag + if custom_data.get('new', False): + xml_lines.append(f" ") + + xml_lines.append(f" ") + + return xml_lines + + +def generate_epg(request, profile_name=None, user=None): + """ + Dynamically generate an XMLTV (EPG) file using a streaming response. + Since the EPG data is stored independently of Channels, we group programmes + by their associated EPGData record. + """ + user_custom = (user.custom_properties or {}) if user else {} + try: + num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) + num_days = max(0, min(num_days, 365)) + except (ValueError, TypeError): + num_days = 0 + try: + prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) + prev_days = max(0, min(prev_days, 30)) + except (ValueError, TypeError): + prev_days = 0 + use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' + tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() + cache_params = ( + f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" + f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" + ) + content_cache_key = f"epg_content:{cache_params}" + + def epg_generator(): + """Generator function that yields EPG data with keep-alives during processing.""" + + yield '\n' + yield ( + '\n' + ) + + # Get channels based on user/profile + if user is not None: + if user.user_level < 10: + user_profile_count = user.channel_profiles.count() + + # If user has ALL profiles or NO profiles, give unrestricted access + if user_profile_count == 0: + # No profile filtering - user sees all channels based on user_level + filters = {"user_level__lte": user.user_level} + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') + else: + # User has specific limited profiles assigned + filters = { + "channelprofilemembership__enabled": True, + "user_level__lte": user.user_level, + "channelprofilemembership__channel_profile__in": user.channel_profiles.all() + } + # Hide adult content if user preference is set + if (user.custom_properties or {}).get('hide_adult_content', False): + filters["is_adult"] = False + base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() + else: + base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') + else: + if profile_name is not None: + try: + channel_profile = ChannelProfile.objects.get(name=profile_name) + except ChannelProfile.DoesNotExist: + logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) + raise Http404(f"Channel profile '{profile_name}' not found") + base_qs = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True, + ).select_related('logo', 'epg_data__epg_source') + else: + base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') + + # Resolve effective values at SQL level and exclude hidden channels + # so output ordering/display honors user overrides. + from apps.channels.managers import with_effective_values + channels = list( + with_effective_values(base_qs, select_related_fks=True) + .exclude(hidden_from_output=True) + .order_by("effective_channel_number") + # programme_index is a multi-MB JSON byte-offset index that EPG + # generation never reads; defer it so it isn't fetched and JSON-parsed + # once per channel (was ~13s of the request on large guides). + .defer("epg_data__epg_source__programme_index") + .prefetch_related( + Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + ) + ) + channel_count = len(channels) + + # For dummy EPG, use either the specified value or default to 3 days + dummy_days = num_days if num_days > 0 else 3 + + # Calculate cutoff dates for EPG data filtering + now = django_timezone.now() + cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None + lookback_cutoff = now - timedelta(days=prev_days) + + # Build collision-free channel number mapping for XC clients (if user is authenticated) + # XC clients require integer channel numbers, so we need to ensure no conflicts + channel_num_map = {} + if user is not None: + # This is an XC client - build collision-free mapping + used_numbers = set() + + # First pass: assign integers for channels that already have integer numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num == int(effective_num): + num = int(effective_num) + channel_num_map[channel.id] = num + used_numbers.add(num) + + # Second pass: assign integers for channels with float numbers + for channel in channels: + effective_num = channel.effective_channel_number + if effective_num is not None and effective_num != int(effective_num): + candidate = int(effective_num) + while candidate in used_numbers: + candidate += 1 + channel_num_map[channel.id] = candidate + used_numbers.add(candidate) + + # Host/port/scheme are constant per request; precompute logo URL prefix once. + _base_url = build_absolute_uri_with_port(request, "") + _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) + _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") + _logo_url_prefix = _base_url + _logo_prefix_raw + "/" + _logo_url_suffix = "/" + _logo_suffix_raw + + dummy_program_list = [] + real_epg_map = {} + channel_xml_batch = [] + + for channel in channels: + effective_name = channel.effective_name + effective_epg_data = channel.effective_epg_data_obj + effective_epg_data_id = channel.effective_epg_data_id + effective_logo = channel.effective_logo_obj + effective_number = channel.effective_channel_number + + # user is set only for XC clients, which require integer channel numbers + if user is not None: + formatted_channel_number = channel_num_map[channel.id] + else: + formatted_channel_number = format_channel_number(effective_number) + + # Determine the channel ID based on the selected source + if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: + channel_id = channel.effective_tvg_id + elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: + channel_id = channel.effective_tvc_guide_stationid + else: + channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) + + tvg_logo = "" + + # Check if this is a custom dummy EPG with channel logo URL template + if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + channel_logo_url_template = custom_props.get('channel_logo_url', '') + + if channel_logo_url_template: + pattern_match_name, _ = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + + # Try to extract groups from the channel/stream name and build the logo URL + title_pattern = custom_props.get('title_pattern', '') + if title_pattern: + try: + # Convert PCRE/JavaScript named groups to Python format + title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) + title_regex = regex.compile(title_pattern) + title_match = title_regex.search(pattern_match_name) + + if title_match: + groups = title_match.groupdict() + + # Add normalized versions of all groups for cleaner URLs + for key, value in list(groups.items()): + if value: + # Remove all non-alphanumeric characters and convert to lowercase + normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) + normalized = regex.sub(r'\s+', '', normalized).lower() + groups[f'{key}_normalize'] = normalized + + # Format the logo URL template with the matched groups (with URL encoding) + from urllib.parse import quote + for key, value in groups.items(): + if value: + encoded_value = quote(str(value), safe='') + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) + else: + channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') + tvg_logo = channel_logo_url_template + logger.debug(f"Built channel logo URL from template: {tvg_logo}") + except Exception as e: + logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") + + # If no custom dummy logo, use regular logo logic + if not tvg_logo and effective_logo: + if use_cached_logos: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + else: + # Use direct URL if available, otherwise fall back to cached version + direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None + if direct_logo: + tvg_logo = direct_logo + else: + tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" + channel_xml_batch.append(f' ') + channel_xml_batch.append(f' {html.escape(effective_name)}') + channel_xml_batch.append(f' ') + channel_xml_batch.append(" ") + + if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: + yield '\n'.join(channel_xml_batch) + '\n' + channel_xml_batch = [] + + pattern_match_name = effective_name + if effective_epg_data and effective_epg_data.epg_source: + epg_source = effective_epg_data.epg_source + if epg_source.custom_properties: + custom_props = epg_source.custom_properties + pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( + channel, effective_name, custom_props + ) + if ( + custom_props.get('name_source') == 'stream' + and not stream_lookup_failed + and pattern_match_name != effective_name + ): + stream_index = custom_props.get('stream_index', 1) - 1 + logger.debug( + f"Using stream name for parsing: {pattern_match_name} " + f"(stream index: {stream_index})" + ) + elif stream_lookup_failed: + stream_index = custom_props.get('stream_index', 1) - 1 + logger.warning( + f"Stream index {stream_index} not found for channel " + f"{effective_name}, falling back to channel name" + ) + + if not effective_epg_data: + dummy_program_list.append((channel_id, pattern_match_name, None)) + elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': + dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) + else: + real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) + + if channel_xml_batch: + yield '\n'.join(channel_xml_batch) + '\n' + + del channels + del channel_num_map + + batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE + + all_epg_ids = list(real_epg_map.keys()) + if all_epg_ids: + if num_days > 0: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + start_time__lt=cutoff_date, + ) + else: + programs_qs = ProgramData.objects.filter( + epg_id__in=all_epg_ids, + end_time__gte=lookback_cutoff, + ) + + programs_base_qs = programs_qs.order_by('epg_id', 'id').values( + 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', + 'description', 'custom_properties', + ) + + current_epg_id = None + channel_ids_for_epg = None + pending = [] + program_batch = [] + chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE + last_epg_id = 0 + last_id = 0 + _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") + + def flush_pending(): + nonlocal program_batch, pending + if not pending: + return + pending.sort(key=lambda row: (row[0], row[1])) + escaped_primary = ( + html.escape(channel_ids_for_epg[0]) + if len(channel_ids_for_epg) > 1 else None + ) + for _, _, xml_text in pending: + program_batch.append(xml_text) + if escaped_primary: + for cid in channel_ids_for_epg[1:]: + program_batch.append(xml_text.replace( + f'channel="{escaped_primary}"', + f'channel="{html.escape(cid)}"', + 1, + )) + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] + pending.clear() + + while True: + program_chunk = list( + programs_base_qs.filter(epg_id__gte=last_epg_id) + .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] + ) + if not program_chunk: + break + + last_row = program_chunk[-1] + last_epg_id = last_row['epg_id'] + last_id = last_row['id'] + + for prog in program_chunk: + epg_id = prog['epg_id'] + + if epg_id != current_epg_id: + yield from flush_pending() + current_epg_id = epg_id + channel_ids_for_epg = real_epg_map[epg_id] + + primary_cid = channel_ids_for_epg[0] + # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format + # directly instead of strftime("%Y%m%d%H%M%S %z"), which is + # ~10x slower and dominates XML build over 750k rows. + st = prog['start_time'] + et = prog['end_time'] + start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" + stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" + + program_xml = [f' '] + program_xml.append(f' {html.escape(prog["title"])}') + + if prog['sub_title']: + program_xml.append(f" {html.escape(prog['sub_title'])}") + + if prog['description']: + program_xml.append(f" {html.escape(prog['description'])}") + + custom_data = prog['custom_properties'] or {} + if custom_data: + + if "categories" in custom_data and custom_data["categories"]: + for category in custom_data["categories"]: + program_xml.append(f" {html.escape(category)}") + + if "keywords" in custom_data and custom_data["keywords"]: + for keyword in custom_data["keywords"]: + program_xml.append(f" {html.escape(keyword)}") + + # onscreen_episode takes priority over episode for the onscreen system + if "onscreen_episode" in custom_data: + program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') + elif "episode" in custom_data: + program_xml.append(f' E{custom_data["episode"]}') + + # Handle dd_progid format + if 'dd_progid' in custom_data: + program_xml.append(f' {html.escape(custom_data["dd_progid"])}') + + # Handle external database IDs + for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: + if f'{system}_id' in custom_data: + program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') + + # Add season and episode numbers in xmltv_ns format if available + if "season" in custom_data and "episode" in custom_data: + season = ( + int(custom_data["season"]) - 1 + if str(custom_data["season"]).isdigit() + else 0 + ) + episode = ( + int(custom_data["episode"]) - 1 + if str(custom_data["episode"]).isdigit() + else 0 + ) + program_xml.append(f' {season}.{episode}.') + + if "language" in custom_data: + program_xml.append(f' {html.escape(custom_data["language"])}') + + if "original_language" in custom_data: + program_xml.append(f' {html.escape(custom_data["original_language"])}') + + if "length" in custom_data and isinstance(custom_data["length"], dict): + length_value = custom_data["length"].get("value", "") + length_units = custom_data["length"].get("units", "minutes") + program_xml.append(f' {html.escape(str(length_value))}') + + if "video" in custom_data and isinstance(custom_data["video"], dict): + program_xml.append(" ") + + if "audio" in custom_data and isinstance(custom_data["audio"], dict): + program_xml.append(" ") + + if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): + for subtitle in custom_data["subtitles"]: + if isinstance(subtitle, dict): + subtitle_type = subtitle.get("type", "") + type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" + program_xml.append(f" ") + if "language" in subtitle: + program_xml.append(f" {html.escape(subtitle['language'])}") + program_xml.append(" ") + + if "rating" in custom_data: + rating_system = custom_data.get("rating_system", "TV Parental Guidelines") + program_xml.append(f' ') + program_xml.append(f' {html.escape(custom_data["rating"])}') + program_xml.append(f" ") + + if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): + for star_rating in custom_data["star_ratings"]: + if isinstance(star_rating, dict) and "value" in star_rating: + system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" + program_xml.append(f" ") + program_xml.append(f" {html.escape(star_rating['value'])}") + program_xml.append(" ") + + if "reviews" in custom_data and isinstance(custom_data["reviews"], list): + for review in custom_data["reviews"]: + if isinstance(review, dict) and "content" in review: + review_type = review.get("type", "text") + attrs = [f'type="{html.escape(review_type)}"'] + if "source" in review: + attrs.append(f'source="{html.escape(review["source"])}"') + if "reviewer" in review: + attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') + attr_str = " ".join(attrs) + program_xml.append(f' {html.escape(review["content"])}') + + if "images" in custom_data and isinstance(custom_data["images"], list): + for image in custom_data["images"]: + if isinstance(image, dict) and "url" in image: + attrs = [] + for attr in ['type', 'size', 'orient', 'system']: + if attr in image: + attrs.append(f'{attr}="{html.escape(image[attr])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f' {html.escape(image["url"])}') + + # Add enhanced credits handling + if "credits" in custom_data: + program_xml.append(" ") + credits = custom_data["credits"] + + for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if role in credits: + people = credits[role] + if isinstance(people, list): + for person in people: + program_xml.append(f" <{role}>{html.escape(person)}") + else: + program_xml.append(f" <{role}>{html.escape(people)}") + + # Handle actors separately to include role and guest attributes + if "actor" in credits: + actors = credits["actor"] + if isinstance(actors, list): + for actor in actors: + if isinstance(actor, dict): + name = actor.get("name", "") + role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" + guest_attr = ' guest="yes"' if actor.get("guest") else "" + program_xml.append(f" {html.escape(name)}") + else: + program_xml.append(f" {html.escape(actor)}") + else: + program_xml.append(f" {html.escape(actors)}") + + program_xml.append(" ") + + if "date" in custom_data: + program_xml.append(f' {html.escape(custom_data["date"])}') + + if "country" in custom_data: + program_xml.append(f' {html.escape(custom_data["country"])}') + + if "icon" in custom_data: + program_xml.append(f' ') + elif "sd_icon" in custom_data: + program_xml.append(f' ') + + # Add special flags as proper tags with enhanced handling + if custom_data.get("previously_shown", False): + prev_shown_details = custom_data.get("previously_shown_details", {}) + attrs = [] + if "start" in prev_shown_details: + attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') + if "channel" in prev_shown_details: + attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f" ") + + if custom_data.get("premiere", False): + premiere_text = custom_data.get("premiere_text", "") + if premiere_text: + program_xml.append(f" {html.escape(premiere_text)}") + else: + program_xml.append(" ") + + if custom_data.get("last_chance", False): + last_chance_text = custom_data.get("last_chance_text", "") + if last_chance_text: + program_xml.append(f" {html.escape(last_chance_text)}") + else: + program_xml.append(" ") + + if custom_data.get("new", False): + program_xml.append(" ") + + if custom_data.get('live', False): + program_xml.append(' ') + + program_xml.append(" ") + + xml_text = '\n'.join(program_xml) + pending.append((prog['start_time'], prog['id'], xml_text)) + + del program_chunk + + yield from flush_pending() + + if program_batch: + yield '\n'.join(program_batch) + '\n' + + del real_epg_map + + for channel_id, pattern_match_name, epg_source in dummy_program_list: + program_length_hours = 4 + dummy_programs = generate_dummy_programs( + channel_id, pattern_match_name, + num_days=dummy_days, + program_length_hours=program_length_hours, + epg_source=epg_source, + export_lookback=lookback_cutoff, + export_cutoff=cutoff_date, + ) + if not dummy_programs: + continue + dummy_batch = [] + for program in dummy_programs: + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + lines = [ + f' ', + f" {html.escape(program['title'])}", + ] + if program.get('sub_title'): + lines.append(f" {html.escape(program['sub_title'])}") + lines.append(f" {html.escape(program['description'])}") + custom_data = program.get('custom_properties', {}) + if 'categories' in custom_data: + for cat in custom_data['categories']: + lines.append(f" {html.escape(cat)}") + if 'date' in custom_data: + lines.append(f" {html.escape(custom_data['date'])}") + if custom_data.get('live', False): + lines.append(" ") + if custom_data.get('new', False): + lines.append(" ") + if 'icon' in custom_data: + lines.append(f' ') + lines.append(" ") + dummy_batch.append('\n'.join(lines)) + if len(dummy_batch) >= batch_size: + yield '\n'.join(dummy_batch) + '\n' + dummy_batch = [] + del dummy_programs + if dummy_batch: + yield '\n'.join(dummy_batch) + '\n' + + del dummy_program_list + + yield "\n" + + from apps.output.views import get_client_identifier + + client_id, client_ip, user_agent = get_client_identifier(request) + event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" + + def _log_epg_download(): + from django.core.cache import cache as event_cache + + if not event_cache.get(event_cache_key): + log_system_event( + event_type='epg_download', + profile=profile_name or 'all', + user=user.username if user else 'anonymous', + channels=channel_count, + client_ip=client_ip, + user_agent=user_agent, + ) + event_cache.set(event_cache_key, True, 2) + + try: + from core.utils import _is_gevent_monkey_patched + + if _is_gevent_monkey_patched(): + import gevent + + gevent.spawn(_log_epg_download) + else: + _log_epg_download() + except Exception: + _log_epg_download() + + def build_epg_stream(): + try: + yield from epg_generator() + finally: + _epg_export_teardown() + + return stream_cached_response( + content_cache_key, + build_epg_stream, + content_type="application/xml", + filename="Dispatcharr.xml", + ) diff --git a/apps/output/epg_chunk_cache.py b/apps/output/streaming_chunk_cache.py similarity index 81% rename from apps/output/epg_chunk_cache.py rename to apps/output/streaming_chunk_cache.py index 92ea6eeb..f86bb060 100644 --- a/apps/output/epg_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -1,4 +1,4 @@ -"""Single-flight Redis chunk cache for XMLTV EPG streaming responses.""" +"""Single-flight Redis chunk cache for large streaming HTTP responses.""" import logging import time @@ -111,7 +111,7 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): try: from django.core.cache import cache as django_cache - django_cache.delete(base_key) # legacy monolithic entry + django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) for chunk in source(): @@ -123,9 +123,9 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): redis.expire(chunks_key, cache_ttl) redis.expire(status_key, cache_ttl) redis.expire(_ready_key(base_key), cache_ttl) - logger.debug("Cached EPG in %s chunks", redis.llen(chunks_key)) + logger.debug("Cached response in %s chunks", redis.llen(chunks_key)) except Exception: - logger.exception("EPG cache build failed for %s", base_key) + logger.exception("Chunk cache build failed for %s", base_key) redis.delete(chunks_key) redis.set(status_key, STATUS_ERROR, ex=60) raise @@ -158,14 +158,14 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - raise RuntimeError("EPG cache build failed") + raise RuntimeError("Chunk cache build failed") if time.monotonic() >= deadline: if offset == 0 and _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache follower timed out; rebuilding %s", base_key) + logger.warning("Chunk cache follower timed out; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return - logger.warning("EPG cache follower timed out after partial read for %s", base_key) + logger.warning("Chunk cache follower timed out after partial read for %s", base_key) break lock_active = bool(redis.exists(lock_key)) @@ -173,7 +173,7 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, idle_polls += 1 if offset == 0 and idle_polls >= max(1, int(1.0 / poll_interval)): if _try_acquire_lock(redis, base_key, lock_ttl): - logger.warning("EPG cache leader lost; rebuilding %s", base_key) + logger.warning("Chunk cache leader lost; rebuilding %s", base_key) yield from _stream_build(redis, base_key, source, cache_ttl, lock_ttl) return else: @@ -182,10 +182,12 @@ def _stream_follow(redis, base_key, source, cache_ttl, lock_ttl, poll_interval, _poll_wait(poll_interval) -def stream_epg_response( +def stream_cached_response( cache_key, source, *, + content_type="application/xml", + filename=None, cache_ttl=DEFAULT_CACHE_TTL, lock_ttl=DEFAULT_LOCK_TTL, poll_interval=DEFAULT_POLL_INTERVAL, @@ -193,16 +195,17 @@ def stream_epg_response( redis=None, ): """ - Stream XMLTV EPG output with single-flight Redis chunk caching. + Stream a large response with single-flight Redis chunk caching. ``source`` must be a callable returning a chunk iterator. Only the leader - invokes it; followers read chunks already written to Redis. + invokes it; concurrent followers replay chunks already written to Redis, so + the expensive ``source`` runs at most once per ``cache_key``. """ if redis is None: redis = _get_redis() if redis.get(_ready_key(cache_key)): - logger.debug("Serving EPG from chunk cache") + logger.debug("Serving response from chunk cache") stream = _stream_ready(redis, cache_key) else: status = _get_status(redis, cache_key) @@ -210,10 +213,10 @@ def stream_epg_response( _clear_build_keys(redis, cache_key) if _try_acquire_lock(redis, cache_key, lock_ttl): - logger.debug("Building EPG (cache leader)") + logger.debug("Building response (cache leader)") stream = _stream_build(redis, cache_key, source, cache_ttl, lock_ttl) else: - logger.debug("Following in-flight EPG build") + logger.debug("Following in-flight cache build") stream = _stream_follow( redis, cache_key, @@ -224,7 +227,8 @@ def stream_epg_response( max_follower_wait, ) - response = StreamingHttpResponse(stream, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response = StreamingHttpResponse(stream, content_type=content_type) + if filename: + response["Content-Disposition"] = f'attachment; filename="{filename}"' response["Cache-Control"] = "no-cache" return response diff --git a/apps/output/test_epg_chunk_cache.py b/apps/output/test_streaming_chunk_cache.py similarity index 84% rename from apps/output/test_epg_chunk_cache.py rename to apps/output/test_streaming_chunk_cache.py index 6ca217af..599501b7 100644 --- a/apps/output/test_epg_chunk_cache.py +++ b/apps/output/test_streaming_chunk_cache.py @@ -2,14 +2,14 @@ import threading import time from unittest import TestCase -from apps.output.epg_chunk_cache import ( +from apps.output.streaming_chunk_cache import ( STATUS_BUILDING, STATUS_READY, _chunks_key, _lock_key, _ready_key, _status_key, - stream_epg_response, + stream_cached_response, ) @@ -74,7 +74,7 @@ def _consume(response): return b"".join(response.streaming_content).decode("utf-8") -class EPGChunkCacheTests(TestCase): +class StreamingChunkCacheTests(TestCase): def test_leader_caches_chunks_and_sets_ready(self): redis = FakeRedis() calls = [] @@ -84,14 +84,14 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, [1]) - self.assertEqual(redis.get(_ready_key("epg:test")), "1") - self.assertEqual(redis.get(_status_key("epg:test")), STATUS_READY) - self.assertEqual(redis.llen(_chunks_key("epg:test")), 2) - self.assertFalse(redis.exists(_lock_key("epg:test"))) + self.assertEqual(redis.get(_ready_key("cache:test")), "1") + self.assertEqual(redis.get(_status_key("cache:test")), STATUS_READY) + self.assertEqual(redis.llen(_chunks_key("cache:test")), 2) + self.assertFalse(redis.exists(_lock_key("cache:test"))) def test_cache_hit_skips_source(self): redis = FakeRedis() @@ -102,16 +102,16 @@ class EPGChunkCacheTests(TestCase): yield "" yield "" - _consume(stream_epg_response("epg:test", source, redis=redis)) + _consume(stream_cached_response("cache:test", source, redis=redis)) calls.clear() - body = _consume(stream_epg_response("epg:test", source, redis=redis)) + body = _consume(stream_cached_response("cache:test", source, redis=redis)) self.assertEqual(body, "") self.assertEqual(calls, []) def test_follower_reads_leader_chunks_without_rebuilding(self): redis = FakeRedis() - base = "epg:follow" + base = "cache:follow" leader_started = threading.Event() rebuild_calls = [] @@ -128,7 +128,7 @@ class EPGChunkCacheTests(TestCase): def leader(): _consume( - stream_epg_response( + stream_cached_response( base, slow_source, redis=redis, @@ -140,7 +140,7 @@ class EPGChunkCacheTests(TestCase): leader_thread.start() leader_started.wait(timeout=5) follower_body = _consume( - stream_epg_response( + stream_cached_response( base, forbidden_source, redis=redis, @@ -165,8 +165,8 @@ class EPGChunkCacheTests(TestCase): def worker(): barrier.wait() results[threading.current_thread().name] = _consume( - stream_epg_response( - "epg:race", + stream_cached_response( + "cache:race", source, redis=redis, poll_interval=0.01, diff --git a/apps/output/tests.py b/apps/output/tests.py index 6e2ab395..5f4fd5f3 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -34,16 +34,18 @@ class OutputEndpointTestMixin: "apps.output.views.network_access_allowed", return_value=True, ) - self._epg_teardown_patch = patch("apps.output.views._epg_export_teardown") + self._epg_teardown_patch = patch("apps.output.epg._epg_export_teardown") self._log_event_patch = patch("apps.output.views.log_system_event") + self._epg_log_event_patch = patch("apps.output.epg.log_system_event") self._close_db_patch = patch("django.db.close_old_connections") self._epg_cache_patch = patch( - "apps.output.views.stream_epg_response", + "apps.output.epg.stream_cached_response", side_effect=_epg_response_without_redis, ) self._network_patch.start() self._epg_teardown_patch.start() self._log_event_patch.start() + self._epg_log_event_patch.start() self._close_db_patch.start() self._epg_cache_patch.start() @@ -53,6 +55,7 @@ class OutputEndpointTestMixin: cache.clear() self._epg_cache_patch.stop() self._close_db_patch.stop() + self._epg_log_event_patch.stop() self._log_event_patch.stop() self._epg_teardown_patch.stop() self._network_patch.stop() @@ -339,14 +342,14 @@ class OutputEPGCustomDummyTest(TestCase): class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=30, second=0, microsecond=0) self.assertEqual(_ceil_to_half_hour(dt), dt) def test_ceil_to_half_hour_rounds_up(self): from django.utils import timezone - from apps.output.views import _ceil_to_half_hour + from apps.output.epg import _ceil_to_half_hour dt = timezone.now().replace(minute=17, second=42, microsecond=123456) aligned = _ceil_to_half_hour(dt) diff --git a/apps/output/views.py b/apps/output/views.py index a0875c7f..4224fb35 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -25,54 +25,10 @@ from apps.proxy.utils import get_user_active_connections import regex from core.utils import log_system_event, build_absolute_uri_with_port import hashlib -from apps.output.epg_chunk_cache import stream_epg_response +from apps.output.epg import generate_epg, generate_dummy_programs logger = logging.getLogger(__name__) -_EPG_CHANNEL_XML_BATCH_SIZE = 200 -_EPG_PROGRAM_YIELD_BATCH_SIZE = 1000 -_EPG_PROGRAM_DB_CHUNK_SIZE = 20000 - - -def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cutoff_date): - if end_time < lookback_cutoff: - return False - if cutoff_date is not None and start_time >= cutoff_date: - return False - return True - - -def _ceil_to_half_hour(dt): - """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) - - -def _epg_export_teardown(): - from django.db import close_old_connections - - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() - def get_client_identifier(request): """Get client information including IP, user agent, and a unique hash identifier @@ -384,1674 +340,6 @@ def generate_m3u(request, profile_name=None, user=None): return response -def _ordered_channel_streams(channel): - """Return a channel's streams ordered by channelstream join order.""" - prefetched = getattr(channel, '_prefetched_objects_cache', {}).get('streams') - if prefetched is not None: - return list(prefetched) - return list(channel.streams.all().order_by('channelstream__order')) - - -def _pattern_match_name_from_custom_props(channel, effective_name, custom_props): - """Name used for custom dummy EPG regex matching (channel or stream title). - - Returns (name, stream_lookup_failed). stream_lookup_failed is True only when - name_source is 'stream' but the configured index is missing or out of range. - """ - if custom_props.get('name_source') != 'stream': - return effective_name, False - stream_index = custom_props.get('stream_index', 1) - 1 - streams = _ordered_channel_streams(channel) - if 0 <= stream_index < len(streams): - return streams[stream_index].name, False - return effective_name, True - - -def generate_fallback_programs(channel_id, channel_name, now, num_days, program_length_hours, fallback_title, fallback_description): - """ - Generate dummy programs using custom fallback templates when patterns don't match. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel name to use as fallback in templates - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - fallback_title: Custom fallback title template (empty string if not provided) - fallback_description: Custom fallback description template (empty string if not provided) - - Returns: - List of program dictionaries - """ - programs = [] - - # Use custom fallback title or channel name as default - title = fallback_title if fallback_title else channel_name - - # Use custom fallback description or a simple default message - if fallback_description: - description = fallback_description - else: - description = f"EPG information is currently unavailable for {channel_name}" - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": title, - "description": description, - }) - - return programs - - -def generate_dummy_programs( - channel_id, - channel_name, - num_days=1, - program_length_hours=4, - epg_source=None, - export_lookback=None, - export_cutoff=None, -): - """ - Generate dummy EPG programs for channels. - - If epg_source is provided and it's a custom dummy EPG with patterns, - use those patterns to generate programs from the channel title. - Otherwise, generate default dummy programs. - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title/name - num_days: Number of days to generate programs for - program_length_hours: Length of each program in hours - epg_source: Optional EPGSource for custom dummy EPG with patterns - - Returns: - List of program dictionaries - """ - # Get current time rounded to hour - now = django_timezone.now() - now = now.replace(minute=0, second=0, microsecond=0) - - # Check if this is a custom dummy EPG with regex patterns - if epg_source and epg_source.source_type == 'dummy' and epg_source.custom_properties: - custom_programs = generate_custom_dummy_programs( - channel_id, channel_name, now, num_days, - epg_source.custom_properties, - export_lookback=export_lookback, - export_cutoff=export_cutoff, - ) - if custom_programs is not None: - return custom_programs - - logger.info(f"Custom pattern didn't match for '{channel_name}', checking for custom fallback templates") - - custom_props = epg_source.custom_properties - fallback_title = custom_props.get('fallback_title_template', '').strip() - fallback_description = custom_props.get('fallback_description_template', '').strip() - - if fallback_title or fallback_description: - logger.info(f"Using custom fallback templates for '{channel_name}'") - return generate_fallback_programs( - channel_id, channel_name, now, num_days, - program_length_hours, fallback_title, fallback_description - ) - logger.info(f"No custom fallback templates found, using default dummy EPG") - - # Default humorous program descriptions based on time of day - time_descriptions = { - (0, 4): [ - f"Late Night with {channel_name} - Where insomniacs unite!", - f"The 'Why Am I Still Awake?' Show on {channel_name}", - f"Counting Sheep - A {channel_name} production for the sleepless", - ], - (4, 8): [ - f"Dawn Patrol - Rise and shine with {channel_name}!", - f"Early Bird Special - Coffee not included", - f"Morning Zombies - Before coffee viewing on {channel_name}", - ], - (8, 12): [ - f"Mid-Morning Meetings - Pretend you're paying attention while watching {channel_name}", - f"The 'I Should Be Working' Hour on {channel_name}", - f"Productivity Killer - {channel_name}'s daytime programming", - ], - (12, 16): [ - f"Lunchtime Laziness with {channel_name}", - f"The Afternoon Slump - Brought to you by {channel_name}", - f"Post-Lunch Food Coma Theater on {channel_name}", - ], - (16, 20): [ - f"Rush Hour - {channel_name}'s alternative to traffic", - f"The 'What's For Dinner?' Debate on {channel_name}", - f"Evening Escapism - {channel_name}'s remedy for reality", - ], - (20, 24): [ - f"Prime Time Placeholder - {channel_name}'s finest not-programming", - f"The 'Netflix Was Too Complicated' Show on {channel_name}", - f"Family Argument Avoider - Courtesy of {channel_name}", - ], - } - - programs = [] - - # Create programs for each day - for day in range(num_days): - day_start = now + timedelta(days=day) - - # Create programs with specified length throughout the day - for hour_offset in range(0, 24, program_length_hours): - # Calculate program start and end times - start_time = day_start + timedelta(hours=hour_offset) - end_time = start_time + timedelta(hours=program_length_hours) - - # Get the hour for selecting a description - hour = start_time.hour - - # Find the appropriate time slot for description - for time_range, descriptions in time_descriptions.items(): - start_range, end_range = time_range - if start_range <= hour < end_range: - # Pick a description using the sum of the hour and day as seed - # This makes it somewhat random but consistent for the same timeslot - description = descriptions[(hour + day) % len(descriptions)] - break - else: - # Fallback description if somehow no range matches - description = f"Placeholder program for {channel_name} - EPG data went on vacation" - - programs.append({ - "channel_id": channel_id, - "start_time": start_time, - "end_time": end_time, - "title": channel_name, - "description": description, - }) - - return programs - - -def generate_custom_dummy_programs( - channel_id, - channel_name, - now, - num_days, - custom_properties, - export_lookback=None, - export_cutoff=None, -): - """ - Generate programs using custom dummy EPG regex patterns. - - Extracts information from channel title using regex patterns and generates - programs based on the extracted data. - - TIMEZONE HANDLING: - ------------------ - The timezone parameter specifies the timezone of the event times in your channel - titles using standard timezone names (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London'). - DST (Daylight Saving Time) is handled automatically by pytz. - - Examples: - - Channel: "NHL 01: Bruins VS Maple Leafs @ 8:00PM ET" - - Set timezone = "US/Eastern" - - In October (DST): 8:00PM EDT → 12:00AM UTC (automatically uses UTC-4) - - In January (no DST): 8:00PM EST → 1:00AM UTC (automatically uses UTC-5) - - Args: - channel_id: Channel ID for the programs - channel_name: Channel title to parse - now: Current datetime (in UTC) - num_days: Number of days to generate programs for - custom_properties: Dict with title_pattern, time_pattern, templates, etc. - - timezone: Timezone name (e.g., 'US/Eastern') - - Returns: - List of program dictionaries with start_time/end_time in UTC - """ - import pytz - - logger.info(f"Generating custom dummy programs for channel: {channel_name}") - - # Extract patterns from custom properties - title_pattern = custom_properties.get('title_pattern', '') - time_pattern = custom_properties.get('time_pattern', '') - date_pattern = custom_properties.get('date_pattern', '') - - # Get timezone name (e.g., 'US/Eastern', 'US/Pacific', 'Europe/London') - timezone_value = custom_properties.get('timezone', 'UTC') - output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone - program_duration = custom_properties.get('program_duration', 180) # Minutes - title_template = custom_properties.get('title_template', '') - subtitle_template = custom_properties.get('subtitle_template', '') - description_template = custom_properties.get('description_template', '') - - # Templates for upcoming/ended programs - upcoming_title_template = custom_properties.get('upcoming_title_template', '') - upcoming_description_template = custom_properties.get('upcoming_description_template', '') - ended_title_template = custom_properties.get('ended_title_template', '') - ended_description_template = custom_properties.get('ended_description_template', '') - - # Image URL templates - channel_logo_url_template = custom_properties.get('channel_logo_url', '') - program_poster_url_template = custom_properties.get('program_poster_url', '') - - # EPG metadata options - category_string = custom_properties.get('category', '') - # Split comma-separated categories and strip whitespace, filter out empty strings - categories = [cat.strip() for cat in category_string.split(',') if cat.strip()] if category_string else [] - include_date = custom_properties.get('include_date', True) - include_live = custom_properties.get('include_live', False) - include_new = custom_properties.get('include_new', False) - - # Parse timezone name - try: - source_tz = pytz.timezone(timezone_value) - logger.debug(f"Using timezone: {timezone_value} (DST will be handled automatically)") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown timezone: {timezone_value}, defaulting to UTC") - source_tz = pytz.utc - - # Parse output timezone if provided (for display purposes) - output_tz = None - if output_timezone_value: - try: - output_tz = pytz.timezone(output_timezone_value) - logger.debug(f"Using output timezone for display: {output_timezone_value}") - except pytz.exceptions.UnknownTimeZoneError: - logger.warning(f"Unknown output timezone: {output_timezone_value}, will use source timezone") - output_tz = None - - if not title_pattern: - logger.warning(f"No title_pattern in custom_properties, falling back to default") - return None - - logger.debug(f"Title pattern from DB: {repr(title_pattern)}") - - # Convert PCRE/JavaScript named groups (?) to Python format (?P) - # This handles patterns created with JavaScript regex syntax - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', title_pattern) - logger.debug(f"Converted title pattern: {repr(title_pattern)}") - - # Compile regex patterns using the enhanced regex module - # (supports variable-width lookbehinds like JavaScript) - try: - title_regex = regex.compile(title_pattern) - except Exception as e: - logger.error(f"Invalid title regex pattern after conversion: {e}") - logger.error(f"Pattern was: {repr(title_pattern)}") - return None - - time_regex = None - if time_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', time_pattern) - logger.debug(f"Converted time pattern: {repr(time_pattern)}") - try: - time_regex = regex.compile(time_pattern) - except Exception as e: - logger.warning(f"Invalid time regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(time_pattern)}") - - # Compile date regex if provided - date_regex = None - if date_pattern: - # Convert PCRE/JavaScript named groups to Python format - # Use negative lookahead to avoid matching lookbehind (?<=) and negative lookbehind (?]+)>', r'(?P<\1>', date_pattern) - logger.debug(f"Converted date pattern: {repr(date_pattern)}") - try: - date_regex = regex.compile(date_pattern) - except Exception as e: - logger.warning(f"Invalid date regex pattern after conversion: {e}") - logger.warning(f"Pattern was: {repr(date_pattern)}") - - # Try to match the channel name with the title pattern - # Use search() instead of match() to match JavaScript behavior where .match() searches anywhere in the string - title_match = title_regex.search(channel_name) - if not title_match: - logger.debug(f"Channel name '{channel_name}' doesn't match title pattern") - return None - - groups = title_match.groupdict() - logger.debug(f"Title pattern matched. Groups: {groups}") - - # Helper function to format template with matched groups - def format_template(template, groups, url_encode=False): - """Replace {groupname} placeholders with matched group values - - Args: - template: Template string with {groupname} placeholders - groups: Dict of group names to values - url_encode: If True, URL encode the group values for safe use in URLs - """ - if not template: - return '' - result = template - for key, value in groups.items(): - if url_encode and value: - # URL encode the value to handle spaces and special characters - from urllib.parse import quote - encoded_value = quote(str(value), safe='') - result = result.replace(f'{{{key}}}', encoded_value) - else: - result = result.replace(f'{{{key}}}', str(value) if value else '') - return result - - # Extract time from title if time pattern exists - time_info = None - time_groups = {} - if time_regex: - time_match = time_regex.search(channel_name) - if time_match: - time_groups = time_match.groupdict() - try: - hour = int(time_groups.get('hour')) - # Handle optional minute group - could be None if not captured - minute_value = time_groups.get('minute') - minute = int(minute_value) if minute_value is not None else 0 - ampm = time_groups.get('ampm') - ampm = ampm.lower() if ampm else None - - # Determine if this is 12-hour or 24-hour format - if ampm in ('am', 'pm'): - # 12-hour format: convert to 24-hour - if ampm == 'pm' and hour != 12: - hour += 12 - elif ampm == 'am' and hour == 12: - hour = 0 - logger.debug(f"Extracted time (12-hour): {hour}:{minute:02d} {ampm}") - else: - # 24-hour format: hour is already in 24-hour format - # Validate that it's actually a 24-hour time (0-23) - if hour > 23: - logger.warning(f"Invalid 24-hour time: {hour}. Must be 0-23.") - hour = hour % 24 # Wrap around just in case - logger.debug(f"Extracted time (24-hour): {hour}:{minute:02d}") - - time_info = {'hour': hour, 'minute': minute} - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing time: {e}") - - # Extract date from title if date pattern exists - date_info = None - date_groups = {} - if date_regex: - date_match = date_regex.search(channel_name) - if date_match: - date_groups = date_match.groupdict() - try: - # Support various date group names: month, day, year - month_str = date_groups.get('month', '') - day_str = date_groups.get('day', '') - year_str = date_groups.get('year', '') - - # Parse day - default to current day if empty or invalid - day = int(day_str) if day_str else now.day - - # Parse year - default to current year if empty or invalid (matches frontend behavior) - year = int(year_str) if year_str else now.year - - # Parse month - can be numeric (1-12) or text (Jan, January, etc.) - month = None - if month_str: - if month_str.isdigit(): - month = int(month_str) - else: - # Try to parse text month names - import calendar - month_str_lower = month_str.lower() - # Check full month names - for i, month_name in enumerate(calendar.month_name): - if month_name.lower() == month_str_lower: - month = i - break - # Check abbreviated month names if not found - if month is None: - for i, month_abbr in enumerate(calendar.month_abbr): - if month_abbr.lower() == month_str_lower: - month = i - break - - # Default to current month if not extracted or invalid - if month is None: - month = now.month - - if month and 1 <= month <= 12 and 1 <= day <= 31: - date_info = {'year': year, 'month': month, 'day': day} - logger.debug(f"Extracted date: {year}-{month:02d}-{day:02d}") - else: - logger.warning(f"Invalid date values: month={month}, day={day}, year={year}") - except (ValueError, TypeError) as e: - logger.warning(f"Error parsing date: {e}") - - # Merge title groups, time groups, and date groups for template formatting - all_groups = {**groups, **time_groups, **date_groups} - - # Add normalized versions of all groups for cleaner URLs - # These remove all non-alphanumeric characters and convert to lowercase - for key, value in list(all_groups.items()): - if value: - # Remove all non-alphanumeric characters (except spaces temporarily) - # then replace spaces with nothing, and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - all_groups[f'{key}_normalize'] = normalized - - # Format channel logo URL if template provided (with URL encoding) - channel_logo_url = None - if channel_logo_url_template: - channel_logo_url = format_template(channel_logo_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted channel logo URL: {channel_logo_url}") - - # Format program poster URL if template provided (with URL encoding) - program_poster_url = None - if program_poster_url_template: - program_poster_url = format_template(program_poster_url_template, all_groups, url_encode=True) - logger.debug(f"Formatted program poster URL: {program_poster_url}") - - # Add formatted time strings for better display (handles minutes intelligently) - if time_info: - hour_24 = time_info['hour'] - minute = time_info['minute'] - - # Determine the base date to use for placeholders - # If date was extracted, use it; otherwise use current date - if date_info: - base_date = datetime(date_info['year'], date_info['month'], date_info['day']) - else: - base_date = datetime.now() - - # If output_timezone is specified, convert the display time to that timezone - if output_tz: - # Create a datetime in the source timezone using the base date - temp_date = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - # Convert to output timezone - temp_date_output = temp_date.astimezone(output_tz) - # Extract converted hour and minute for display - hour_24 = temp_date_output.hour - minute = temp_date_output.minute - logger.debug(f"Converted display time from {source_tz} to {output_tz}: {hour_24}:{minute:02d}") - - # Add date placeholders based on the OUTPUT timezone - # This ensures {date}, {month}, {day}, {year} reflect the converted timezone - all_groups['date'] = temp_date_output.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_output.month) - all_groups['day'] = str(temp_date_output.day) - all_groups['year'] = str(temp_date_output.year) - logger.debug(f"Converted date placeholders to {output_tz}: {all_groups['date']}") - else: - # No output timezone conversion - use source timezone for date - # Create temp date to get proper date in source timezone using the base date - temp_date_source = source_tz.localize(base_date.replace(hour=hour_24, minute=minute, second=0, microsecond=0)) - all_groups['date'] = temp_date_source.strftime('%Y-%m-%d') - all_groups['month'] = str(temp_date_source.month) - all_groups['day'] = str(temp_date_source.day) - all_groups['year'] = str(temp_date_source.year) - - # Format 24-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime24'] = f"{hour_24}:{minute:02d}" - else: - all_groups['starttime24'] = f"{hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {starttime} placeholder - # Note: hour_24 is ALWAYS in 24-hour format at this point (converted earlier if needed) - ampm = 'AM' if hour_24 < 12 else 'PM' - hour_12 = hour_24 - if hour_24 == 0: - hour_12 = 12 - elif hour_24 > 12: - hour_12 = hour_24 - 12 - - # Format 12-hour start time string - only include minutes if non-zero - if minute > 0: - all_groups['starttime'] = f"{hour_12}:{minute:02d} {ampm}" - else: - all_groups['starttime'] = f"{hour_12} {ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['starttime_long'] = f"{hour_12}:{minute:02d} {ampm}" - - # Calculate end time based on program duration - # Create a datetime for calculations - temp_start = datetime.now(source_tz).replace(hour=hour_24, minute=minute, second=0, microsecond=0) - temp_end = temp_start + timedelta(minutes=program_duration) - - # Extract end time components (already in correct timezone if output_tz was applied above) - end_hour_24 = temp_end.hour - end_minute = temp_end.minute - - # Format 24-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime24'] = f"{end_hour_24}:{end_minute:02d}" - else: - all_groups['endtime24'] = f"{end_hour_24:02d}:00" - - # Convert 24-hour to 12-hour format for {endtime} placeholder - end_ampm = 'AM' if end_hour_24 < 12 else 'PM' - end_hour_12 = end_hour_24 - if end_hour_24 == 0: - end_hour_12 = 12 - elif end_hour_24 > 12: - end_hour_12 = end_hour_24 - 12 - - # Format 12-hour end time string - only include minutes if non-zero - if end_minute > 0: - all_groups['endtime'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - else: - all_groups['endtime'] = f"{end_hour_12} {end_ampm}" - - # Format long version that always includes minutes (e.g., "9:00 PM" instead of "9 PM") - all_groups['endtime_long'] = f"{end_hour_12}:{end_minute:02d} {end_ampm}" - - # Generate programs - programs = [] - - # If we have extracted time AND date, the event happens on a SPECIFIC date - # If we have time but NO date, generate for multiple days (existing behavior) - # All other days and times show "Upcoming" before or "Ended" after - event_happened = False - - # Determine how many iterations we need - if date_info and time_info: - # Specific date extracted - only generate for that one date - iterations = 1 - logger.debug(f"Date extracted, generating single event for specific date") - else: - # No specific date - use num_days (existing behavior) - iterations = num_days - - for day in range(iterations): - event_overlaps_window = True - if date_info and time_info: - current_date = datetime( - date_info['year'], - date_info['month'], - date_info['day'], - ).date() - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_utc = source_tz.localize(event_start_naive).astimezone(pytz.utc) - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - event_overlaps_window = _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ) - if not event_overlaps_window: - logger.debug( - "Custom dummy event outside export window; filling window only: %s", - channel_name, - ) - event_happened = event_end_utc < lookback - day_start = _ceil_to_half_hour(lookback) - if export_cutoff is not None: - day_end = export_cutoff - else: - day_end = now + timedelta(days=num_days if num_days > 0 else 3) - else: - day_start = source_tz.localize( - datetime.combine(current_date, datetime.min.time()) - ).astimezone(pytz.utc) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - else: - day_start = now + timedelta(days=day) - day_end = day_start + timedelta(days=1) - if export_lookback is not None: - day_start = max(day_start, export_lookback) - if export_cutoff is not None: - day_end = min(day_end, export_cutoff) - - if day_start >= day_end: - continue - - if time_info: - if not date_info: - now_in_source_tz = now.astimezone(source_tz) - current_date = (now_in_source_tz + timedelta(days=day)).date() - logger.debug(f"No date extracted, using day offset in {source_tz}: {current_date}") - - event_start_naive = datetime.combine( - current_date, - datetime.min.time().replace( - hour=time_info['hour'], - minute=time_info['minute'], - ), - ) - try: - event_start_local = source_tz.localize(event_start_naive) - event_start_utc = event_start_local.astimezone(pytz.utc) - logger.debug(f"Converted {event_start_local} to UTC: {event_start_utc}") - except Exception as e: - logger.error(f"Error localizing time to {source_tz}: {e}") - event_start_utc = django_timezone.make_aware(event_start_naive, pytz.utc) - - event_end_utc = event_start_utc + timedelta(minutes=program_duration) - - lookback = export_lookback if export_lookback is not None else now - if not _programme_overlaps_export_window( - event_start_utc, event_end_utc, lookback, export_cutoff - ): - continue - else: - logger.debug(f"Using extracted date: {current_date}") - - # Pre-generate the main event title and description for reuse - if title_template: - main_event_title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - main_event_title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - main_event_subtitle = format_template(subtitle_template, all_groups) - else: - main_event_subtitle = None - - if description_template: - main_event_description = format_template(description_template, all_groups) - else: - main_event_description = main_event_title - - - - # Determine if this day is before, during, or after the event - # Event only happens on day 0 (first day) when it falls inside the window - is_event_day = (day == 0) and event_overlaps_window - - if is_event_day and not event_happened: - current_time = day_start - - while current_time < event_start_utc and current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), event_start_utc) - - # Use custom upcoming templates if provided, otherwise use defaults - if upcoming_title_template: - upcoming_title = format_template(upcoming_title_template, all_groups) - else: - upcoming_title = main_event_title - - if upcoming_description_template: - upcoming_description = format_template(upcoming_description_template, all_groups) - else: - upcoming_description = f"Upcoming: {main_event_description}" - - # Build custom_properties for upcoming programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": upcoming_title, - "sub_title": None, # No subtitle for filler programs - "description": upcoming_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - - # Add the MAIN EVENT at the extracted time - # Build custom_properties for main event (includes category and live) - main_event_custom_properties = {} - - # Add categories if provided - if categories: - main_event_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = event_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - main_event_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - main_event_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - main_event_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - main_event_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": event_start_utc, - "end_time": event_end_utc, - "title": main_event_title, - "sub_title": main_event_subtitle, - "description": main_event_description, - "custom_properties": main_event_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - event_happened = True - - # Fill programs AFTER the event until end of export day window - current_time = max(event_end_utc, day_start) - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom ended templates if provided, otherwise use defaults - if ended_title_template: - ended_title = format_template(ended_title_template, all_groups) - else: - ended_title = main_event_title - - if ended_description_template: - ended_description = format_template(ended_description_template, all_groups) - else: - ended_description = f"Ended: {main_event_description}" - - # Build custom_properties for ended programs (only date, no category/live) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": ended_title, - "sub_title": None, # No subtitle for filler programs - "description": ended_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - current_time += timedelta(minutes=program_duration) - else: - # This day is either before the event (future days) or after the event happened - # Fill entire day with appropriate message - current_time = day_start - - # If event already happened, all programs show "Ended" - # If event hasn't happened yet (shouldn't occur with day 0 logic), show "Upcoming" - is_ended = event_happened - - while current_time < day_end: - program_start_utc = current_time - program_end_utc = min(current_time + timedelta(minutes=program_duration), day_end) - - # Use custom templates based on whether event has ended or is upcoming - if is_ended: - if ended_title_template: - program_title = format_template(ended_title_template, all_groups) - else: - program_title = main_event_title - - if ended_description_template: - program_description = format_template(ended_description_template, all_groups) - else: - program_description = f"Ended: {main_event_description}" - else: - if upcoming_title_template: - program_title = format_template(upcoming_title_template, all_groups) - else: - program_title = main_event_title - - if upcoming_description_template: - program_description = format_template(upcoming_description_template, all_groups) - else: - program_description = f"Upcoming: {main_event_description}" - - # Build custom_properties (only date for upcoming/ended filler programs) - program_custom_properties = {} - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": program_title, - "sub_title": None, # No subtitle for filler programs - "description": program_description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, - }) - - current_time += timedelta(minutes=program_duration) - else: - # No extracted time - fill entire day with regular intervals - # day_start and day_end are already in UTC, so no conversion needed - programs_per_day = max(1, int(24 / (program_duration / 60))) - - for program_num in range(programs_per_day): - program_start_utc = day_start + timedelta(minutes=program_num * program_duration) - program_end_utc = program_start_utc + timedelta(minutes=program_duration) - - if title_template: - title = format_template(title_template, all_groups) - else: - title_parts = [] - if 'league' in all_groups and all_groups['league']: - title_parts.append(all_groups['league']) - if 'team1' in all_groups and 'team2' in all_groups: - title_parts.append(f"{all_groups['team1']} vs {all_groups['team2']}") - elif 'title' in all_groups and all_groups['title']: - title_parts.append(all_groups['title']) - title = ' - '.join(title_parts) if title_parts else channel_name - - if subtitle_template: - subtitle = format_template(subtitle_template, all_groups) - else: - subtitle = None - - if description_template: - description = format_template(description_template, all_groups) - else: - description = title - - # Build custom_properties for this program - program_custom_properties = {} - - # Add categories if provided - if categories: - program_custom_properties['categories'] = categories - - # Add date if requested (YYYY-MM-DD format from start time in event timezone) - if include_date: - # Convert UTC time to event timezone for date calculation - local_time = program_start_utc.astimezone(source_tz) - date_str = local_time.strftime('%Y-%m-%d') - program_custom_properties['date'] = date_str - - # Add live flag if requested - if include_live: - program_custom_properties['live'] = True - - # Add new flag if requested - if include_new: - program_custom_properties['new'] = True - - # Add program poster URL if provided - if program_poster_url: - program_custom_properties['icon'] = program_poster_url - - programs.append({ - "channel_id": channel_id, - "start_time": program_start_utc, - "end_time": program_end_utc, - "title": title, - "sub_title": subtitle, - "description": description, - "custom_properties": program_custom_properties, - "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation - }) - - logger.info(f"Generated {len(programs)} custom dummy programs for {channel_name}") - return programs - - -def generate_dummy_epg( - channel_id, channel_name, xml_lines=None, num_days=1, program_length_hours=4 -): - """ - Generate dummy EPG programs for channels without EPG data. - Creates program blocks for a specified number of days. - - Args: - channel_id: The channel ID to use in the program entries - channel_name: The name of the channel to use in program titles - xml_lines: Optional list to append lines to, otherwise returns new list - num_days: Number of days to generate EPG data for (default: 1) - program_length_hours: Length of each program block in hours (default: 4) - - Returns: - List of XML lines for the dummy EPG entries - """ - if xml_lines is None: - xml_lines = [] - - for program in generate_dummy_programs(channel_id, channel_name, num_days=1, program_length_hours=4): - # Format times in XMLTV format - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - - # Create program entry with escaped channel name - xml_lines.append( - f' ' - ) - xml_lines.append(f" {html.escape(program['title'])}") - - # Add subtitle if available - if program.get('sub_title'): - xml_lines.append(f" {html.escape(program['sub_title'])}") - - xml_lines.append(f" {html.escape(program['description'])}") - - # Add custom_properties if present - custom_data = program.get('custom_properties', {}) - - # Categories - if 'categories' in custom_data: - for cat in custom_data['categories']: - xml_lines.append(f" {html.escape(cat)}") - - # Date tag - if 'date' in custom_data: - xml_lines.append(f" {html.escape(custom_data['date'])}") - - # Live tag - if custom_data.get('live', False): - xml_lines.append(f" ") - - # New tag - if custom_data.get('new', False): - xml_lines.append(f" ") - - xml_lines.append(f" ") - - return xml_lines - - -def generate_epg(request, profile_name=None, user=None): - """ - Dynamically generate an XMLTV (EPG) file using a streaming response. - Since the EPG data is stored independently of Channels, we group programmes - by their associated EPGData record. - """ - user_custom = (user.custom_properties or {}) if user else {} - try: - num_days = int(request.GET.get('days', user_custom.get('epg_days', 0))) - num_days = max(0, min(num_days, 365)) - except (ValueError, TypeError): - num_days = 0 - try: - prev_days = int(request.GET.get('prev_days', user_custom.get('epg_prev_days', 0))) - prev_days = max(0, min(prev_days, 30)) - except (ValueError, TypeError): - prev_days = 0 - use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' - tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() - cache_params = ( - f"{profile_name or 'all'}:{user.username if user else 'anonymous'}" - f":d={num_days}:p={prev_days}:logos={use_cached_logos}:tvgid={tvg_id_source}" - ) - content_cache_key = f"epg_content:{cache_params}" - - def epg_generator(): - """Generator function that yields EPG data with keep-alives during processing.""" - - yield '\n' - yield ( - '\n' - ) - - # Get channels based on user/profile - if user is not None: - if user.user_level < 10: - user_profile_count = user.channel_profiles.count() - - # If user has ALL profiles or NO profiles, give unrestricted access - if user_profile_count == 0: - # No profile filtering - user sees all channels based on user_level - filters = {"user_level__lte": user.user_level} - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source') - else: - # User has specific limited profiles assigned - filters = { - "channelprofilemembership__enabled": True, - "user_level__lte": user.user_level, - "channelprofilemembership__channel_profile__in": user.channel_profiles.all() - } - # Hide adult content if user preference is set - if (user.custom_properties or {}).get('hide_adult_content', False): - filters["is_adult"] = False - base_qs = Channel.objects.filter(**filters).select_related('logo', 'epg_data__epg_source').distinct() - else: - base_qs = Channel.objects.filter(user_level__lte=user.user_level).select_related('logo', 'epg_data__epg_source') - else: - if profile_name is not None: - try: - channel_profile = ChannelProfile.objects.get(name=profile_name) - except ChannelProfile.DoesNotExist: - logger.warning("Requested channel profile (%s) during epg generation does not exist", profile_name) - raise Http404(f"Channel profile '{profile_name}' not found") - base_qs = Channel.objects.filter( - channelprofilemembership__channel_profile=channel_profile, - channelprofilemembership__enabled=True, - ).select_related('logo', 'epg_data__epg_source') - else: - base_qs = Channel.objects.all().select_related('logo', 'epg_data__epg_source') - - # Resolve effective values at SQL level and exclude hidden channels - # so output ordering/display honors user overrides. - from apps.channels.managers import with_effective_values - channels = list( - with_effective_values(base_qs, select_related_fks=True) - .exclude(hidden_from_output=True) - .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") - .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) - ) - ) - channel_count = len(channels) - - # For dummy EPG, use either the specified value or default to 3 days - dummy_days = num_days if num_days > 0 else 3 - - # Calculate cutoff dates for EPG data filtering - now = django_timezone.now() - cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None - lookback_cutoff = now - timedelta(days=prev_days) - - # Build collision-free channel number mapping for XC clients (if user is authenticated) - # XC clients require integer channel numbers, so we need to ensure no conflicts - channel_num_map = {} - if user is not None: - # This is an XC client - build collision-free mapping - used_numbers = set() - - # First pass: assign integers for channels that already have integer numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num == int(effective_num): - num = int(effective_num) - channel_num_map[channel.id] = num - used_numbers.add(num) - - # Second pass: assign integers for channels with float numbers - for channel in channels: - effective_num = channel.effective_channel_number - if effective_num is not None and effective_num != int(effective_num): - candidate = int(effective_num) - while candidate in used_numbers: - candidate += 1 - channel_num_map[channel.id] = candidate - used_numbers.add(candidate) - - # Host/port/scheme are constant per request; precompute logo URL prefix once. - _base_url = build_absolute_uri_with_port(request, "") - _sample_logo_path = reverse("api:channels:logo-cache", args=[0]) - _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") - _logo_url_prefix = _base_url + _logo_prefix_raw + "/" - _logo_url_suffix = "/" + _logo_suffix_raw - - dummy_program_list = [] - real_epg_map = {} - channel_xml_batch = [] - - for channel in channels: - effective_name = channel.effective_name - effective_epg_data = channel.effective_epg_data_obj - effective_epg_data_id = channel.effective_epg_data_id - effective_logo = channel.effective_logo_obj - effective_number = channel.effective_channel_number - - # user is set only for XC clients, which require integer channel numbers - if user is not None: - formatted_channel_number = channel_num_map[channel.id] - else: - formatted_channel_number = format_channel_number(effective_number) - - # Determine the channel ID based on the selected source - if tvg_id_source == 'tvg_id' and channel.effective_tvg_id: - channel_id = channel.effective_tvg_id - elif tvg_id_source == 'gracenote' and channel.effective_tvc_guide_stationid: - channel_id = channel.effective_tvc_guide_stationid - else: - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - - tvg_logo = "" - - # Check if this is a custom dummy EPG with channel logo URL template - if effective_epg_data and effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - channel_logo_url_template = custom_props.get('channel_logo_url', '') - - if channel_logo_url_template: - pattern_match_name, _ = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - - # Try to extract groups from the channel/stream name and build the logo URL - title_pattern = custom_props.get('title_pattern', '') - if title_pattern: - try: - # Convert PCRE/JavaScript named groups to Python format - title_pattern = regex.sub(r'\(\?<(?![=!])([^>]+)>', r'(?P<\1>', title_pattern) - title_regex = regex.compile(title_pattern) - title_match = title_regex.search(pattern_match_name) - - if title_match: - groups = title_match.groupdict() - - # Add normalized versions of all groups for cleaner URLs - for key, value in list(groups.items()): - if value: - # Remove all non-alphanumeric characters and convert to lowercase - normalized = regex.sub(r'[^a-zA-Z0-9\s]', '', str(value)) - normalized = regex.sub(r'\s+', '', normalized).lower() - groups[f'{key}_normalize'] = normalized - - # Format the logo URL template with the matched groups (with URL encoding) - from urllib.parse import quote - for key, value in groups.items(): - if value: - encoded_value = quote(str(value), safe='') - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', encoded_value) - else: - channel_logo_url_template = channel_logo_url_template.replace(f'{{{key}}}', '') - tvg_logo = channel_logo_url_template - logger.debug(f"Built channel logo URL from template: {tvg_logo}") - except Exception as e: - logger.warning(f"Failed to build channel logo URL for {effective_name}: {e}") - - # If no custom dummy logo, use regular logo logic - if not tvg_logo and effective_logo: - if use_cached_logos: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - else: - # Use direct URL if available, otherwise fall back to cached version - direct_logo = effective_logo.url if effective_logo.url.startswith(('http://', 'https://')) else None - if direct_logo: - tvg_logo = direct_logo - else: - tvg_logo = f"{_logo_url_prefix}{effective_logo.id}{_logo_url_suffix}" - channel_xml_batch.append(f' ') - channel_xml_batch.append(f' {html.escape(effective_name)}') - channel_xml_batch.append(f' ') - channel_xml_batch.append(" ") - - if len(channel_xml_batch) >= _EPG_CHANNEL_XML_BATCH_SIZE * 4: - yield '\n'.join(channel_xml_batch) + '\n' - channel_xml_batch = [] - - pattern_match_name = effective_name - if effective_epg_data and effective_epg_data.epg_source: - epg_source = effective_epg_data.epg_source - if epg_source.custom_properties: - custom_props = epg_source.custom_properties - pattern_match_name, stream_lookup_failed = _pattern_match_name_from_custom_props( - channel, effective_name, custom_props - ) - if ( - custom_props.get('name_source') == 'stream' - and not stream_lookup_failed - and pattern_match_name != effective_name - ): - stream_index = custom_props.get('stream_index', 1) - 1 - logger.debug( - f"Using stream name for parsing: {pattern_match_name} " - f"(stream index: {stream_index})" - ) - elif stream_lookup_failed: - stream_index = custom_props.get('stream_index', 1) - 1 - logger.warning( - f"Stream index {stream_index} not found for channel " - f"{effective_name}, falling back to channel name" - ) - - if not effective_epg_data: - dummy_program_list.append((channel_id, pattern_match_name, None)) - elif effective_epg_data.epg_source and effective_epg_data.epg_source.source_type == 'dummy': - dummy_program_list.append((channel_id, pattern_match_name, effective_epg_data.epg_source)) - else: - real_epg_map.setdefault(effective_epg_data_id, []).append(channel_id) - - if channel_xml_batch: - yield '\n'.join(channel_xml_batch) + '\n' - - del channels - del channel_num_map - - batch_size = _EPG_PROGRAM_YIELD_BATCH_SIZE - - all_epg_ids = list(real_epg_map.keys()) - if all_epg_ids: - if num_days > 0: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - start_time__lt=cutoff_date, - ) - else: - programs_qs = ProgramData.objects.filter( - epg_id__in=all_epg_ids, - end_time__gte=lookback_cutoff, - ) - - programs_base_qs = programs_qs.order_by('epg_id', 'id').values( - 'id', 'epg_id', 'start_time', 'end_time', 'title', 'sub_title', - 'description', 'custom_properties', - ) - - current_epg_id = None - channel_ids_for_epg = None - pending = [] - program_batch = [] - chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE - last_epg_id = 0 - last_id = 0 - _poster_url_base = build_absolute_uri_with_port(request, "/api/epg/programs/") - - def flush_pending(): - nonlocal program_batch, pending - if not pending: - return - pending.sort(key=lambda row: (row[0], row[1])) - escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None - ) - for _, _, xml_text in pending: - program_batch.append(xml_text) - if escaped_primary: - for cid in channel_ids_for_epg[1:]: - program_batch.append(xml_text.replace( - f'channel="{escaped_primary}"', - f'channel="{html.escape(cid)}"', - 1, - )) - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] - pending.clear() - - while True: - program_chunk = list( - programs_base_qs.filter(epg_id__gte=last_epg_id) - .exclude(epg_id=last_epg_id, id__lte=last_id)[:chunk_size] - ) - if not program_chunk: - break - - last_row = program_chunk[-1] - last_epg_id = last_row['epg_id'] - last_id = last_row['id'] - - for prog in program_chunk: - epg_id = prog['epg_id'] - - if epg_id != current_epg_id: - yield from flush_pending() - current_epg_id = epg_id - channel_ids_for_epg = real_epg_map[epg_id] - - primary_cid = channel_ids_for_epg[0] - # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format - # directly instead of strftime("%Y%m%d%H%M%S %z"), which is - # ~10x slower and dominates XML build over 750k rows. - st = prog['start_time'] - et = prog['end_time'] - start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" - stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - - program_xml = [f' '] - program_xml.append(f' {html.escape(prog["title"])}') - - if prog['sub_title']: - program_xml.append(f" {html.escape(prog['sub_title'])}") - - if prog['description']: - program_xml.append(f" {html.escape(prog['description'])}") - - custom_data = prog['custom_properties'] or {} - if custom_data: - - if "categories" in custom_data and custom_data["categories"]: - for category in custom_data["categories"]: - program_xml.append(f" {html.escape(category)}") - - if "keywords" in custom_data and custom_data["keywords"]: - for keyword in custom_data["keywords"]: - program_xml.append(f" {html.escape(keyword)}") - - # onscreen_episode takes priority over episode for the onscreen system - if "onscreen_episode" in custom_data: - program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') - elif "episode" in custom_data: - program_xml.append(f' E{custom_data["episode"]}') - - # Handle dd_progid format - if 'dd_progid' in custom_data: - program_xml.append(f' {html.escape(custom_data["dd_progid"])}') - - # Handle external database IDs - for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: - if f'{system}_id' in custom_data: - program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') - - # Add season and episode numbers in xmltv_ns format if available - if "season" in custom_data and "episode" in custom_data: - season = ( - int(custom_data["season"]) - 1 - if str(custom_data["season"]).isdigit() - else 0 - ) - episode = ( - int(custom_data["episode"]) - 1 - if str(custom_data["episode"]).isdigit() - else 0 - ) - program_xml.append(f' {season}.{episode}.') - - if "language" in custom_data: - program_xml.append(f' {html.escape(custom_data["language"])}') - - if "original_language" in custom_data: - program_xml.append(f' {html.escape(custom_data["original_language"])}') - - if "length" in custom_data and isinstance(custom_data["length"], dict): - length_value = custom_data["length"].get("value", "") - length_units = custom_data["length"].get("units", "minutes") - program_xml.append(f' {html.escape(str(length_value))}') - - if "video" in custom_data and isinstance(custom_data["video"], dict): - program_xml.append(" ") - - if "audio" in custom_data and isinstance(custom_data["audio"], dict): - program_xml.append(" ") - - if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): - for subtitle in custom_data["subtitles"]: - if isinstance(subtitle, dict): - subtitle_type = subtitle.get("type", "") - type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" - program_xml.append(f" ") - if "language" in subtitle: - program_xml.append(f" {html.escape(subtitle['language'])}") - program_xml.append(" ") - - if "rating" in custom_data: - rating_system = custom_data.get("rating_system", "TV Parental Guidelines") - program_xml.append(f' ') - program_xml.append(f' {html.escape(custom_data["rating"])}') - program_xml.append(f" ") - - if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): - for star_rating in custom_data["star_ratings"]: - if isinstance(star_rating, dict) and "value" in star_rating: - system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" - program_xml.append(f" ") - program_xml.append(f" {html.escape(star_rating['value'])}") - program_xml.append(" ") - - if "reviews" in custom_data and isinstance(custom_data["reviews"], list): - for review in custom_data["reviews"]: - if isinstance(review, dict) and "content" in review: - review_type = review.get("type", "text") - attrs = [f'type="{html.escape(review_type)}"'] - if "source" in review: - attrs.append(f'source="{html.escape(review["source"])}"') - if "reviewer" in review: - attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') - attr_str = " ".join(attrs) - program_xml.append(f' {html.escape(review["content"])}') - - if "images" in custom_data and isinstance(custom_data["images"], list): - for image in custom_data["images"]: - if isinstance(image, dict) and "url" in image: - attrs = [] - for attr in ['type', 'size', 'orient', 'system']: - if attr in image: - attrs.append(f'{attr}="{html.escape(image[attr])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f' {html.escape(image["url"])}') - - # Add enhanced credits handling - if "credits" in custom_data: - program_xml.append(" ") - credits = custom_data["credits"] - - for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if role in credits: - people = credits[role] - if isinstance(people, list): - for person in people: - program_xml.append(f" <{role}>{html.escape(person)}") - else: - program_xml.append(f" <{role}>{html.escape(people)}") - - # Handle actors separately to include role and guest attributes - if "actor" in credits: - actors = credits["actor"] - if isinstance(actors, list): - for actor in actors: - if isinstance(actor, dict): - name = actor.get("name", "") - role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" - guest_attr = ' guest="yes"' if actor.get("guest") else "" - program_xml.append(f" {html.escape(name)}") - else: - program_xml.append(f" {html.escape(actor)}") - else: - program_xml.append(f" {html.escape(actors)}") - - program_xml.append(" ") - - if "date" in custom_data: - program_xml.append(f' {html.escape(custom_data["date"])}') - - if "country" in custom_data: - program_xml.append(f' {html.escape(custom_data["country"])}') - - if "icon" in custom_data: - program_xml.append(f' ') - elif "sd_icon" in custom_data: - program_xml.append(f' ') - - # Add special flags as proper tags with enhanced handling - if custom_data.get("previously_shown", False): - prev_shown_details = custom_data.get("previously_shown_details", {}) - attrs = [] - if "start" in prev_shown_details: - attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') - if "channel" in prev_shown_details: - attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f" ") - - if custom_data.get("premiere", False): - premiere_text = custom_data.get("premiere_text", "") - if premiere_text: - program_xml.append(f" {html.escape(premiere_text)}") - else: - program_xml.append(" ") - - if custom_data.get("last_chance", False): - last_chance_text = custom_data.get("last_chance_text", "") - if last_chance_text: - program_xml.append(f" {html.escape(last_chance_text)}") - else: - program_xml.append(" ") - - if custom_data.get("new", False): - program_xml.append(" ") - - if custom_data.get('live', False): - program_xml.append(' ') - - program_xml.append(" ") - - xml_text = '\n'.join(program_xml) - pending.append((prog['start_time'], prog['id'], xml_text)) - - del program_chunk - - yield from flush_pending() - - if program_batch: - yield '\n'.join(program_batch) + '\n' - - del real_epg_map - - for channel_id, pattern_match_name, epg_source in dummy_program_list: - program_length_hours = 4 - dummy_programs = generate_dummy_programs( - channel_id, pattern_match_name, - num_days=dummy_days, - program_length_hours=program_length_hours, - epg_source=epg_source, - export_lookback=lookback_cutoff, - export_cutoff=cutoff_date, - ) - if not dummy_programs: - continue - dummy_batch = [] - for program in dummy_programs: - start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") - stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") - lines = [ - f' ', - f" {html.escape(program['title'])}", - ] - if program.get('sub_title'): - lines.append(f" {html.escape(program['sub_title'])}") - lines.append(f" {html.escape(program['description'])}") - custom_data = program.get('custom_properties', {}) - if 'categories' in custom_data: - for cat in custom_data['categories']: - lines.append(f" {html.escape(cat)}") - if 'date' in custom_data: - lines.append(f" {html.escape(custom_data['date'])}") - if custom_data.get('live', False): - lines.append(" ") - if custom_data.get('new', False): - lines.append(" ") - if 'icon' in custom_data: - lines.append(f' ') - lines.append(" ") - dummy_batch.append('\n'.join(lines)) - if len(dummy_batch) >= batch_size: - yield '\n'.join(dummy_batch) + '\n' - dummy_batch = [] - del dummy_programs - if dummy_batch: - yield '\n'.join(dummy_batch) + '\n' - - del dummy_program_list - - yield "\n" - - client_id, client_ip, user_agent = get_client_identifier(request) - event_cache_key = f"epg_download:{user.username if user else 'anonymous'}:{profile_name or 'all'}:{client_id}" - - def _log_epg_download(): - from django.core.cache import cache as event_cache - - if not event_cache.get(event_cache_key): - log_system_event( - event_type='epg_download', - profile=profile_name or 'all', - user=user.username if user else 'anonymous', - channels=channel_count, - client_ip=client_ip, - user_agent=user_agent, - ) - event_cache.set(event_cache_key, True, 2) - - try: - from core.utils import _is_gevent_monkey_patched - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_log_epg_download) - else: - _log_epg_download() - except Exception: - _log_epg_download() - - def build_epg_stream(): - try: - yield from epg_generator() - finally: - _epg_export_teardown() - - return stream_epg_response(content_cache_key, build_epg_stream) - - def xc_get_user(request): username = request.GET.get("username") password = request.GET.get("password") From 107a891359fc0319bae9389e6fa608b0e3d556b4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 13:20:17 -0500 Subject: [PATCH 11/28] enhancement(epg): optimize XMLTV escaping and channel prefetch for improved performance --- CHANGELOG.md | 1 + apps/output/epg.py | 10 +++++----- apps/output/streaming_chunk_cache.py | 7 ++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840cb349..825a06fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. diff --git a/apps/output/epg.py b/apps/output/epg.py index 354d6e5b..4e11fbe8 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1188,7 +1188,7 @@ def generate_epg(request, profile_name=None, user=None): # once per channel (was ~13s of the request on large guides). .defer("epg_data__epg_source__programme_index") .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) ) channel_count = len(channels) @@ -1386,6 +1386,7 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None + escaped_primary_cid = None pending = [] program_batch = [] chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE @@ -1399,8 +1400,7 @@ def generate_epg(request, profile_name=None, user=None): return pending.sort(key=lambda row: (row[0], row[1])) escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None + escaped_primary_cid if len(channel_ids_for_epg) > 1 else None ) for _, _, xml_text in pending: program_batch.append(xml_text) @@ -1435,8 +1435,8 @@ def generate_epg(request, profile_name=None, user=None): yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] + escaped_primary_cid = html.escape(channel_ids_for_epg[0]) - primary_cid = channel_ids_for_epg[0] # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format # directly instead of strftime("%Y%m%d%H%M%S %z"), which is # ~10x slower and dominates XML build over 750k rows. @@ -1445,7 +1445,7 @@ def generate_epg(request, profile_name=None, user=None): start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - program_xml = [f' '] + program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') if prog['sub_title']: diff --git a/apps/output/streaming_chunk_cache.py b/apps/output/streaming_chunk_cache.py index f86bb060..4373f5ce 100644 --- a/apps/output/streaming_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -114,9 +114,14 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + refresh_interval = max(1, lock_ttl // 4) + last_refresh = 0.0 for chunk in source(): redis.rpush(chunks_key, _encode_chunk(chunk)) - _refresh_build_ttl(redis, base_key, lock_ttl) + now = time.monotonic() + if now - last_refresh >= refresh_interval: + _refresh_build_ttl(redis, base_key, lock_ttl) + last_refresh = now yield chunk redis.set(status_key, STATUS_READY) redis.set(_ready_key(base_key), "1") From 0dc8898e8b413764012ef14e80e0a0329c8dd7d1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:10:27 -0500 Subject: [PATCH 12/28] refactor(epg): separate programme index into dedicated table for optimized data handling - Moved the `programme_index` from the `EPGSource` model to a new `EPGSourceIndex` table, ensuring that the large JSON blob is only loaded when explicitly accessed, thus improving query performance and memory efficiency. - Updated related queries and API views to utilize the new structure, including adjustments to EPG generation and import logic to prevent unnecessary data loading. - Enhanced memory management in the EPG grid endpoint to reduce worker RSS during response handling. --- CHANGELOG.md | 7 ++- apps/epg/api_views.py | 25 ++++++----- apps/epg/migrations/0026_epgsourceindex.py | 52 ++++++++++++++++++++++ apps/epg/models.py | 37 ++++++++++++--- apps/epg/tasks.py | 17 ++++--- apps/epg/tests/test_epg_import_api.py | 7 ++- apps/epg/tests/test_programme_index.py | 1 - apps/output/epg.py | 38 +++++----------- apps/output/tests.py | 12 ++++- core/tasks.py | 8 +++- core/tests.py | 6 +-- core/utils.py | 24 ++++++++++ 12 files changed, 169 insertions(+), 65 deletions(-) create mode 100644 apps/epg/migrations/0026_epgsourceindex.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 825a06fc..0ad1ef3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. +- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed +- **`programme_index` moved off `EPGSource` into a dedicated `EPGSourceIndex` table.** The multi-MB byte-offset index was repeatedly pulled into web and Celery workers by ordinary `EPGSource` queries and `select_related` JOINs (which ignore manager-level `defer()`). It now lives in a one-to-one `EPGSourceIndex` row, read only when explicitly accessed through the `EPGSource.programme_index` property, so no list, detail, or JOIN query can load it by accident. Migration `0026` copies existing indices across. No API or index-build behavior change. + - **EPG generation extracted into `apps/output/epg.py`.** All XMLTV output logic (`generate_epg`, `generate_dummy_programs`, `generate_custom_dummy_programs`, `generate_dummy_epg`, and supporting helpers) moved from `apps/output/views.py` into a dedicated module. `views.py` retains the thin HTTP endpoint wrappers and auth checks; `epg.py` handles all content generation. No behavior change. - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. @@ -29,7 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. -- **`POST /api/epg/import/` no longer loads `programme_index`.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, so import triggers avoid pulling the multi-MB byte-offset index into the uWSGI worker. Request/response shape is unchanged for the UI, plugins, and external importers. +- **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index fbc419c5..1e0bfc41 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -64,8 +64,6 @@ class EPGSourceViewSet(viewsets.ModelViewSet): from apps.channels.models import Channel return EPGSource.objects.select_related( "refresh_task__crontab", "refresh_task__interval" - ).defer( - 'programme_index' ).annotate( has_channels=Exists( Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk')) @@ -1082,13 +1080,19 @@ class EPGGridAPIView(APIView): f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}" ) - # Combine regular and dummy programs - all_programs = list(serialized_programs) + dummy_programs + # Combine regular and dummy programs in place to avoid copying the large list + serialized_programs.extend(dummy_programs) logger.debug( - f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)." + f"EPGGridAPIView: Returning {len(serialized_programs)} total programs (including {len(dummy_programs)} dummy programs)." ) - return Response({"data": all_programs}, status=status.HTTP_200_OK) + # The grid materializes tens of thousands of program dicts plus the + # rendered JSON; trim once the response is sent so worker RSS does not + # ratchet up per request. + from core.utils import spawn_memory_trim + response = Response({"data": serialized_programs}, status=status.HTTP_200_OK) + response._resource_closers.append(spawn_memory_trim) + return response # ───────────────────────────── @@ -1119,7 +1123,7 @@ class EPGImportAPIView(APIView): epg_id = request.data.get("id", None) force = bool(request.data.get("force", False)) - # Reject dummy sources without loading programme_index (multi-MB JSON). + # Reject dummy sources with a narrow existence query, no full row load. if epg_id is not None: from .models import EPGSource @@ -1236,10 +1240,9 @@ class CurrentProgramsAPIView(APIView): # Limit to 50 IDs per request epg_data_ids = epg_data_ids[:50] - # Defer the multi-MB programme_index the JOIN would pull once per row. The lookup reads it via a targeted refresh_from_db - epg_data_entries = EPGData.objects.select_related('epg_source').defer( - 'epg_source__programme_index' - ).filter(id__in=epg_data_ids) + epg_data_entries = EPGData.objects.select_related('epg_source').filter( + id__in=epg_data_ids + ) # Batch-fetch current programs for all requested EPG entries in one query db_programs = ProgramData.objects.filter( diff --git a/apps/epg/migrations/0026_epgsourceindex.py b/apps/epg/migrations/0026_epgsourceindex.py new file mode 100644 index 00000000..75d9d4eb --- /dev/null +++ b/apps/epg/migrations/0026_epgsourceindex.py @@ -0,0 +1,52 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def copy_index_forward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + rows = list( + EPGSource.objects.exclude(programme_index__isnull=True).values_list( + 'id', 'programme_index' + ) + ) + for source_id, data in rows: + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': data} + ) + + +def copy_index_backward(apps, schema_editor): + EPGSource = apps.get_model('epg', 'EPGSource') + EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex') + for source_id, data in EPGSourceIndex.objects.values_list('source_id', 'data'): + EPGSource.objects.filter(id=source_id).update(programme_index=data) + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0025_programdata_epg_id_index'), + ] + + operations = [ + migrations.CreateModel( + name='EPGSourceIndex', + fields=[ + ('source', models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name='index_record', + serialize=False, + to='epg.epgsource', + )), + ('data', models.JSONField(blank=True, default=None, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.RunPython(copy_index_forward, copy_index_backward), + migrations.RemoveField( + model_name='epgsource', + name='programme_index', + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 04c6072c..b85453af 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -62,12 +62,6 @@ class EPGSource(models.Model): blank=True, help_text="Last status message, including success results or error information" ) - programme_index = models.JSONField( - null=True, - blank=True, - default=None, - help_text="Byte-offset index mapping tvg_id to file positions, built after each EPG refresh" - ) created_at = models.DateTimeField( auto_now_add=True, help_text="Time when this source was created" @@ -124,6 +118,37 @@ class EPGSource(models.Model): kwargs['update_fields'].remove('updated_at') super().save(*args, **kwargs) + @property + def programme_index(self): + """Byte-offset index for this source, read on demand from the separate + EPGSourceIndex table so the multi-MB blob is never pulled into EPGSource + queries or select_related JOINs. Returns the stored dict or None.""" + return ( + EPGSourceIndex.objects.filter(source_id=self.pk) + .values_list('data', flat=True) + .first() + ) + + +class EPGSourceIndex(models.Model): + """Byte-offset programme index for an EPGSource, stored in its own table. + + Kept out of EPGSource so the multi-MB JSON blob is only loaded when read + explicitly, never when querying or joining EPGSource rows. + """ + source = models.OneToOneField( + EPGSource, + on_delete=models.CASCADE, + related_name='index_record', + primary_key=True, + ) + data = models.JSONField(null=True, blank=True, default=None) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return f"Programme index for source {self.source_id}" + + class EPGData(models.Model): tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=512) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 9661c5b9..83aaa275 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -26,7 +26,7 @@ from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 +from .models import EPGSource, EPGSourceIndex, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5 from core.utils import ( acquire_task_lock, is_task_lock_held, @@ -653,7 +653,9 @@ def _refresh_epg_data_impl(source_id, force=False): if source.source_type == 'xmltv': # Invalidate the byte-offset index before downloading the new file # so stale offsets are never used during the refresh window. - EPGSource.objects.filter(id=source.id).update(programme_index=None) + EPGSourceIndex.objects.update_or_create( + source_id=source.id, defaults={'data': None} + ) if not fetch_xmltv(source): logger.error(f"Failed to fetch XMLTV for source {source.name}") return @@ -4484,7 +4486,7 @@ def _programme_to_dict(elem, start_time, end_time): def build_programme_index(source_id): """ Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map. - Persists the result to EPGSource.programme_index. Most XMLTV files group programmes + Persists the result to the EPGSourceIndex table. Most XMLTV files group programmes by channel, but some split a channel across multiple non-contiguous blocks, so we record block starts up to _OFFSET_CAP and mark only channels that exceed the cap as interleaved. @@ -4573,7 +4575,9 @@ def build_programme_index(source_id): 'channels': index, 'interleaved_channels': sorted(interleaved_channels), } - EPGSource.objects.filter(id=source_id).update(programme_index=result) + EPGSourceIndex.objects.update_or_create( + source_id=source_id, defaults={'data': result} + ) @shared_task @@ -4620,9 +4624,8 @@ def find_current_program_for_tvg_id(epg_or_id): return None now = timezone.now() - # Force a fresh read of the DB-backed index to avoid using stale related-object - # state when an EPG refresh invalidates/rebuilds the index concurrently. - source.refresh_from_db(fields=['programme_index']) + # The property reads the EPGSourceIndex table fresh on each access, so a + # concurrent refresh invalidating/rebuilding the index can't serve stale state. index = source.programme_index if index is not None: diff --git a/apps/epg/tests/test_epg_import_api.py b/apps/epg/tests/test_epg_import_api.py index c58f2a45..cc96c214 100644 --- a/apps/epg/tests/test_epg_import_api.py +++ b/apps/epg/tests/test_epg_import_api.py @@ -7,7 +7,7 @@ from django.test import TestCase from rest_framework import status from rest_framework.test import APIClient -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex User = get_user_model() @@ -47,7 +47,10 @@ class EPGImportAPITests(TestCase): name="Large Index XMLTV", source_type="xmltv", url="http://example.com/epg.xml", - programme_index={ + ) + EPGSourceIndex.objects.create( + source=source, + data={ "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)}, "interleaved_channels": [], }, diff --git a/apps/epg/tests/test_programme_index.py b/apps/epg/tests/test_programme_index.py index 5ece0ae1..829afc87 100644 --- a/apps/epg/tests/test_programme_index.py +++ b/apps/epg/tests/test_programme_index.py @@ -519,7 +519,6 @@ class FindCurrentProgramTests(TestCase): name="No Index", source_type="xmltv", file_path=FIXTURE_XML, - programme_index=None, ) epg = EPGData.objects.create( tvg_id="channel.current", diff --git a/apps/output/epg.py b/apps/output/epg.py index 4e11fbe8..626b21aa 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -40,34 +40,20 @@ def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cut def _ceil_to_half_hour(dt): """Round a datetime up to the next :00 or :30 boundary.""" - dt = dt.replace(second=0, microsecond=0) - remainder = dt.minute % 30 - if remainder == 0: - return dt - return dt + timedelta(minutes=30 - remainder) + original = dt.replace(microsecond=0) + aligned = dt.replace(second=0, microsecond=0) + remainder = aligned.minute % 30 + if remainder != 0: + aligned += timedelta(minutes=30 - remainder) + if aligned < original: + aligned += timedelta(minutes=30) + return aligned def _epg_export_teardown(): - from django.db import close_old_connections + from core.utils import spawn_memory_trim - from core.utils import ( - _is_gevent_monkey_patched, - cleanup_memory, - trim_c_allocator_heap, - ) - - close_old_connections() - - def _run(): - cleanup_memory(force_collection=True) - trim_c_allocator_heap() - - if _is_gevent_monkey_patched(): - import gevent - - gevent.spawn(_run) - else: - _run() + spawn_memory_trim(close_connections=True) def _ordered_channel_streams(channel): @@ -1183,10 +1169,6 @@ def generate_epg(request, profile_name=None, user=None): with_effective_values(base_qs, select_related_fks=True) .exclude(hidden_from_output=True) .order_by("effective_channel_number") - # programme_index is a multi-MB JSON byte-offset index that EPG - # generation never reads; defer it so it isn't fetched and JSON-parsed - # once per channel (was ~13s of the request on large guides). - .defer("epg_data__epg_source__programme_index") .prefetch_related( Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) diff --git a/apps/output/tests.py b/apps/output/tests.py index 5f4fd5f3..870ff765 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -355,4 +355,14 @@ class OutputEPGHelperTest(SimpleTestCase): aligned = _ceil_to_half_hour(dt) self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) - self.assertGreater(aligned, dt.replace(second=0, microsecond=0)) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + def test_ceil_to_half_hour_past_boundary_second(self): + from django.utils import timezone + from apps.output.epg import _ceil_to_half_hour + + dt = timezone.now().replace(minute=0, second=52, microsecond=123456) + aligned = _ceil_to_half_hour(dt) + self.assertEqual(aligned.minute, 30) + self.assertEqual(aligned.second, 0) + self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) diff --git a/core/tasks.py b/core/tasks.py index b562f335..a64469b4 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -398,12 +398,16 @@ def scan_and_process_files(): def _rebuild_programme_indices(): """Queue index builds for active EPG sources that are missing their DB index.""" try: + from django.db.models import Q from apps.epg.tasks import build_programme_index_task sources = EPGSource.objects.filter( is_active=True, - programme_index__isnull=True, - ).exclude(source_type__in=('dummy', 'schedules_direct')) + ).exclude( + source_type__in=('dummy', 'schedules_direct') + ).filter( + Q(index_record__isnull=True) | Q(index_record__data__isnull=True) + ) count = 0 for source in sources: diff --git a/core/tests.py b/core/tests.py index 33475ae7..2af52f07 100644 --- a/core/tests.py +++ b/core/tests.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock from django.test import TestCase, SimpleTestCase -from apps.epg.models import EPGSource +from apps.epg.models import EPGSource, EPGSourceIndex from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY @@ -37,14 +37,10 @@ class DispatcharrUserAgentTests(TestCase): class ProgrammeIndexRebuildTests(TestCase): def test_startup_rebuild_does_not_lock_out_queued_build_task(self): - EPGSource.objects.update( - programme_index={"channels": {}, "interleaved_channels": []} - ) source = EPGSource.objects.create( name="Missing Index", source_type="xmltv", is_active=True, - programme_index=None, ) class FakeRedis: diff --git a/core/utils.py b/core/utils.py index 89228088..44504b6a 100644 --- a/core/utils.py +++ b/core/utils.py @@ -608,6 +608,30 @@ def cleanup_memory(log_usage=False, force_collection=True): pass logger.trace("Memory cleanup complete for django") + +def spawn_memory_trim(close_connections=False): + """Reclaim a request's heap pages: GC, then return freed C pages to the OS. + + On gevent uWSGI workers the trim runs in a spawned greenlet so it never + blocks the caller; Celery prefork workers (no gevent hub) run it inline. + Set close_connections=True when called from a streaming generator's teardown + so the pooled DB connection is released first. + """ + def _run(): + cleanup_memory(force_collection=True) + trim_c_allocator_heap() + + if close_connections: + from django.db import close_old_connections + close_old_connections() + + if _is_gevent_monkey_patched(): + import gevent + gevent.spawn(_run) + else: + _run() + + def safe_upload_path(filename: str, base_dir) -> str: """Return a safe absolute path for an uploaded file within base_dir. From e984b234e34fd5ad4d5804c2fef8348c8cff3a65 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 15:27:15 -0500 Subject: [PATCH 13/28] enhancement(epg): add export lookback and cutoff parameters to EPG generation - Updated the `generate_dummy_programs` function to include `export_lookback` and `export_cutoff` parameters, allowing for more precise control over the EPG data generation window. - Added a new test case to verify that future events are correctly represented in the grid window, ensuring upcoming fillers are displayed as expected. --- apps/epg/api_views.py | 4 +++- apps/output/tests.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 1e0bfc41..cdb39a47 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -984,7 +984,9 @@ class EPGGridAPIView(APIView): channel_name=name_to_parse, num_days=1, program_length_hours=4, - epg_source=epg_source + epg_source=epg_source, + export_lookback=one_hour_ago, + export_cutoff=twenty_four_hours_later, ) # Custom dummy should always return data (either from patterns or fallback) diff --git a/apps/output/tests.py b/apps/output/tests.py index 870ff765..e713df21 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -338,6 +338,55 @@ class OutputEPGCustomDummyTest(TestCase): ) self.assertGreaterEqual(programs[0]['start_time'], lookback) + def test_custom_dummy_future_event_fills_grid_window_with_upcoming(self): + """Grid-style window: future event should show upcoming filler, not empty.""" + from django.utils import timezone + from apps.output.epg import _programme_overlaps_export_window, generate_dummy_programs + + epg_source = EPGSource.objects.create( + name="NHL Dummy Future", + source_type="dummy", + custom_properties={ + "title_pattern": r"(?.*)\s\d+:\s(?.*?)(?:\s+vs\s+)(?.*?)\s*@.*", + "time_pattern": r"(?\d{1,2}):(?\d{2})\s*(?AM|PM)", + "date_pattern": r"@ (?[A-Za-z]+)\s+(?\d{1,2})", + "timezone": "US/Eastern", + "program_duration": 180, + }, + ) + now = timezone.now() + grid_start = now - timedelta(hours=1) + grid_end = now + timedelta(hours=24) + future = now + timedelta(days=3) + channel_name = ( + f"NHL 01: Washington Capitals vs Philadelphia Flyers @ " + f"{future.strftime('%B')} {future.day} 07:30 PM ET" + ) + + programs = generate_dummy_programs( + channel_id="nhl01", + channel_name=channel_name, + num_days=1, + epg_source=epg_source, + export_lookback=grid_start, + export_cutoff=grid_end, + ) + + self.assertGreater(len(programs), 0) + self.assertTrue( + all( + _programme_overlaps_export_window( + p["start_time"], p["end_time"], grid_start, grid_end + ) + for p in programs + ), + "All programmes should overlap the grid query window", + ) + self.assertTrue( + any("Upcoming" in p.get("description", "") for p in programs), + "Future events outside the window should show upcoming filler", + ) + class OutputEPGHelperTest(SimpleTestCase): def test_ceil_to_half_hour_on_boundary(self): From 7b6adf62d7f84e69f2d121b6d24fa2e17bea499d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:15:29 -0500 Subject: [PATCH 14/28] enhancement(vod): optimize VOD stream retrieval and performance - Improved `xc_get_vod_streams` and `xc_get_series` functions to utilize a new helper method for fetching distinct relations based on account priority, significantly reducing memory usage and response times for large libraries. - Updated the handling of VOD streams to avoid unnecessary ORM instantiation, enhancing performance for endpoints dealing with extensive movie and series data. - Added comprehensive tests to ensure the correctness of the new logic and verify that the highest priority relations are correctly selected. --- CHANGELOG.md | 1 + apps/output/tests.py | 483 ++++++++++++++++++++++++++++++++++++++++++- apps/output/views.py | 189 ++++++++++------- 3 files changed, 595 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad1ef3b..6bc8054f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. diff --git a/apps/output/tests.py b/apps/output/tests.py index e713df21..30b22a18 100644 --- a/apps/output/tests.py +++ b/apps/output/tests.py @@ -1,9 +1,23 @@ -from django.test import TestCase, Client, SimpleTestCase +from django.test import TestCase, Client, SimpleTestCase, RequestFactory from django.urls import reverse +from unittest import skipUnless from unittest.mock import patch from uuid import uuid4 +from django.db import connection +from django.test.utils import CaptureQueriesContext from apps.channels.models import Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership from apps.epg.models import EPGData, EPGSource +from apps.accounts.models import User +from apps.m3u.models import M3UAccount +from apps.output.views import xc_get_series, xc_get_vod_streams +from apps.vod.models import ( + M3UMovieRelation, + M3USeriesRelation, + Movie, + Series, + VODCategory, + VODLogo, +) import xml.etree.ElementTree as ET from datetime import timedelta @@ -415,3 +429,470 @@ class OutputEPGHelperTest(SimpleTestCase): self.assertEqual(aligned.minute, 30) self.assertEqual(aligned.second, 0) self.assertGreaterEqual(aligned, dt.replace(microsecond=0)) + + +class XcVodSeriesDistinctTests(TestCase): + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0, is_active=True): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + is_active=is_active, + ) + + def test_vod_streams_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + movie = Movie.objects.create(name="Shared Movie", year=2020) + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + stream_id="low-stream", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + stream_id="high-stream", + container_extension="mp4", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Shared Movie") + self.assertEqual(streams[0]["container_extension"], "mp4") + + def test_vod_streams_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}", priority=1) + inactive = self._account( + f"inactive-{uuid4().hex[:6]}", priority=99, is_active=False + ) + active_movie = Movie.objects.create(name="Active Movie") + inactive_movie = Movie.objects.create(name="Inactive Only Movie") + M3UMovieRelation.objects.create( + m3u_account=active, + movie=active_movie, + stream_id="active-1", + ) + M3UMovieRelation.objects.create( + m3u_account=inactive, + movie=inactive_movie, + stream_id="inactive-1", + ) + + streams = xc_get_vod_streams(self.request, self.user) + + names = {s["name"] for s in streams} + self.assertEqual(names, {"Active Movie"}) + + def test_vod_streams_category_filter(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + action_movie = Movie.objects.create(name="Action Movie") + comedy_movie = Movie.objects.create(name="Comedy Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=action_movie, + category=action, + stream_id="action-1", + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=comedy_movie, + category=comedy, + stream_id="comedy-1", + ) + + streams = xc_get_vod_streams(self.request, self.user, category_id=action.id) + + self.assertEqual(len(streams), 1) + self.assertEqual(streams[0]["name"], "Action Movie") + self.assertEqual(streams[0]["category_id"], str(action.id)) + + def test_vod_streams_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + zebra = Movie.objects.create(name="Zebra Film") + apple = Movie.objects.create(name="Apple Film") + M3UMovieRelation.objects.create( + m3u_account=account, movie=zebra, stream_id="z-1" + ) + M3UMovieRelation.objects.create( + m3u_account=account, movie=apple, stream_id="a-1" + ) + + streams = xc_get_vod_streams(self.request, self.user) + + self.assertEqual([s["name"] for s in streams], ["Apple Film", "Zebra Film"]) + + def test_vod_streams_includes_metadata_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create( + name="Rich Movie", + description="A plot", + genre="Drama", + year=2021, + rating="8", + custom_properties={ + "director": "Dir", + "actors": "Cast", + "release_date": "2021-01-01", + "youtube_trailer": "yt123", + }, + ) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="rich-1", + container_extension="avi", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["plot"], "A plot") + self.assertEqual(stream["genre"], "Drama") + self.assertEqual(stream["year"], 2021) + self.assertEqual(stream["director"], "Dir") + self.assertEqual(stream["cast"], "Cast") + self.assertEqual(stream["release_date"], "2021-01-01") + self.assertEqual(stream["trailer"], "yt123") + self.assertEqual(stream["container_extension"], "avi") + + def test_vod_streams_stream_icon_uses_logo_id_without_logo_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Poster", url="http://example.com/poster.png") + movie = Movie.objects.create(name="Logo Movie", logo=logo) + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="logo-1", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIn(f"/{logo.id}/", stream["stream_icon"]) + + def test_series_picks_highest_priority_relation(self): + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + series = Series.objects.create(name="Shared Series", year=2019) + M3USeriesRelation.objects.create( + m3u_account=low, + series=series, + external_series_id="low-series", + ) + high_rel = M3USeriesRelation.objects.create( + m3u_account=high, + series=series, + external_series_id="high-series", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["name"], "Shared Series") + self.assertEqual(results[0]["series_id"], high_rel.id) + + def test_series_excludes_inactive_accounts(self): + active = self._account(f"active-{uuid4().hex[:6]}") + inactive = self._account(f"inactive-{uuid4().hex[:6]}", is_active=False) + active_series = Series.objects.create(name="Active Series") + inactive_series = Series.objects.create(name="Inactive Only Series") + M3USeriesRelation.objects.create( + m3u_account=active, + series=active_series, + external_series_id="active-s", + ) + M3USeriesRelation.objects.create( + m3u_account=inactive, + series=inactive_series, + external_series_id="inactive-s", + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual({r["name"] for r in results}, {"Active Series"}) + + def test_series_sorted_alphabetically_by_name(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + z = Series.objects.create(name="Zulu Show") + a = Series.objects.create(name="Alpha Show") + M3USeriesRelation.objects.create( + m3u_account=account, series=z, external_series_id="z" + ) + M3USeriesRelation.objects.create( + m3u_account=account, series=a, external_series_id="a" + ) + + results = xc_get_series(self.request, self.user) + + self.assertEqual([r["name"] for r in results], ["Alpha Show", "Zulu Show"]) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_vod_streams_dedupe_query_avoids_movie_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Query Shape Movie") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="qs-1" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_vod_streams(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_movie"', distinct_queries[0]["sql"]) + self.assertNotIn('"vod_vodlogo"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_movie"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + @skipUnless(connection.vendor == "postgresql", "PostgreSQL-specific query shape") + def test_series_dedupe_query_avoids_series_join(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Query Shape Series") + M3USeriesRelation.objects.create( + m3u_account=account, series=series, external_series_id="qs-s" + ) + + with CaptureQueriesContext(connection) as ctx: + xc_get_series(self.request, self.user) + + distinct_queries = [q for q in ctx.captured_queries if "DISTINCT" in q["sql"]] + self.assertEqual(len(distinct_queries), 1) + self.assertNotIn('"vod_series"', distinct_queries[0]["sql"]) + + fetch_queries = [ + q + for q in ctx.captured_queries + if '"vod_series"' in q["sql"] and "DISTINCT" not in q["sql"] + ] + self.assertGreaterEqual(len(fetch_queries), 1) + fetch_sql = fetch_queries[0]["sql"] + self.assertNotIn('"vod_vodlogo"', fetch_sql) + self.assertNotIn('"vod_vodcategory"', fetch_sql) + + +XC_VOD_STREAM_KEYS = frozenset({ + "num", "name", "stream_type", "stream_id", "stream_icon", "rating", + "rating_5based", "added", "is_adult", "tmdb_id", "imdb_id", "trailer", + "plot", "genre", "year", "director", "cast", "release_date", "category_id", + "category_ids", "container_extension", "custom_sid", "direct_source", +}) + +XC_SERIES_KEYS = frozenset({ + "num", "name", "series_id", "cover", "plot", "cast", "director", "genre", + "release_date", "releaseDate", "last_modified", "rating", "rating_5based", + "backdrop_path", "youtube_trailer", "episode_run_time", "category_id", + "category_ids", "tmdb_id", "imdb_id", +}) + + +class XcVodSeriesRegressionTests(TestCase): + """Full output-shape and edge-case regressions for XC list endpoints.""" + + def setUp(self): + self.factory = RequestFactory() + self.user = User.objects.create_user( + username=f"xc-reg-{uuid4().hex[:8]}", + password="pass", + custom_properties={"xc_password": "xcpass"}, + ) + self.request = self.factory.get("/player_api.php") + + def _account(self, name, *, priority=0): + return M3UAccount.objects.create( + name=name, + server_url="http://example.com", + priority=priority, + ) + + def test_vod_streams_empty_library(self): + self.assertEqual(xc_get_vod_streams(self.request, self.user), []) + + def test_series_empty_library(self): + self.assertEqual(xc_get_series(self.request, self.user), []) + + def test_vod_streams_response_keys(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Schema Movie", rating="10") + M3UMovieRelation.objects.create( + m3u_account=account, movie=movie, stream_id="schema-1" + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(set(stream.keys()), XC_VOD_STREAM_KEYS) + self.assertEqual(stream["stream_type"], "movie") + self.assertEqual(stream["stream_id"], movie.id) + self.assertEqual(stream["rating_5based"], 5.0) + self.assertEqual(stream["custom_sid"], None) + self.assertEqual(stream["direct_source"], "") + + def test_vod_streams_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + movie = Movie.objects.create(name="Sparse Movie") + M3UMovieRelation.objects.create( + m3u_account=account, + movie=movie, + stream_id="sparse-1", + container_extension=None, + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertIsNone(stream["stream_icon"]) + self.assertEqual(stream["category_id"], "0") + self.assertEqual(stream["category_ids"], []) + self.assertEqual(stream["container_extension"], "mp4") + self.assertEqual(stream["plot"], "") + self.assertEqual(stream["trailer"], "") + self.assertEqual(stream["tmdb_id"], "") + self.assertEqual(stream["imdb_id"], "") + + def test_vod_streams_category_from_winning_relation(self): + """Category must come from the highest-priority relation, not any relation.""" + low = self._account(f"low-{uuid4().hex[:6]}", priority=1) + high = self._account(f"high-{uuid4().hex[:6]}", priority=10) + action = VODCategory.objects.create(name="Action", category_type="movie") + comedy = VODCategory.objects.create(name="Comedy", category_type="movie") + movie = Movie.objects.create(name="Dual Category Movie") + M3UMovieRelation.objects.create( + m3u_account=low, + movie=movie, + category=action, + stream_id="low-cat", + ) + M3UMovieRelation.objects.create( + m3u_account=high, + movie=movie, + category=comedy, + stream_id="high-cat", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["category_id"], str(comedy.id)) + self.assertEqual(stream["category_ids"], [comedy.id]) + + def test_series_response_keys_and_metadata(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + logo = VODLogo.objects.create(name="Cover", url="http://example.com/cover.png") + category = VODCategory.objects.create(name="Drama", category_type="series") + series = Series.objects.create( + name="Schema Series", + description="Series plot", + genre="Sci-Fi", + year=2022, + rating="8", + tmdb_id="tm123", + imdb_id="tt123", + logo=logo, + custom_properties={ + "cast": "Actor A", + "director": "Director B", + "release_date": "2022-06-01", + "backdrop_path": ["/img1.jpg"], + "youtube_trailer": "yt-series", + "episode_run_time": "45", + }, + ) + relation = M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + category=category, + external_series_id="schema-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(set(row.keys()), XC_SERIES_KEYS) + self.assertEqual(row["series_id"], relation.id) + self.assertIn(f"/{logo.id}/", row["cover"]) + self.assertEqual(row["plot"], "Series plot") + self.assertEqual(row["cast"], "Actor A") + self.assertEqual(row["director"], "Director B") + self.assertEqual(row["genre"], "Sci-Fi") + self.assertEqual(row["release_date"], "2022-06-01") + self.assertEqual(row["releaseDate"], "2022-06-01") + self.assertEqual(row["backdrop_path"], ["/img1.jpg"]) + self.assertEqual(row["youtube_trailer"], "yt-series") + self.assertEqual(row["episode_run_time"], "45") + self.assertEqual(row["tmdb_id"], "tm123") + self.assertEqual(row["imdb_id"], "tt123") + self.assertEqual(row["category_id"], str(category.id)) + self.assertEqual(row["category_ids"], [category.id]) + self.assertEqual(row["last_modified"], str(int(relation.updated_at.timestamp()))) + + def test_series_null_optional_fields(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Sparse Series") + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="sparse-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertIsNone(row["cover"]) + self.assertEqual(row["category_id"], "0") + self.assertEqual(row["category_ids"], []) + self.assertEqual(row["release_date"], "") + self.assertEqual(row["releaseDate"], "") + self.assertEqual(row["backdrop_path"], []) + self.assertEqual(row["youtube_trailer"], "") + self.assertEqual(row["episode_run_time"], "") + + def test_series_release_date_falls_back_to_year(self): + account = self._account(f"acct-{uuid4().hex[:6]}") + series = Series.objects.create(name="Year Only", year=2018) + M3USeriesRelation.objects.create( + m3u_account=account, + series=series, + external_series_id="year-s", + ) + + row = xc_get_series(self.request, self.user)[0] + + self.assertEqual(row["release_date"], "2018") + self.assertEqual(row["releaseDate"], "2018") + + def test_priority_tiebreaker_uses_lower_relation_id(self): + """Same priority: DISTINCT ON tie-breaks on relation id ascending.""" + a1 = self._account(f"a1-{uuid4().hex[:6]}", priority=5) + a2 = self._account(f"a2-{uuid4().hex[:6]}", priority=5) + movie = Movie.objects.create(name="Tie Movie") + first = M3UMovieRelation.objects.create( + m3u_account=a1, + movie=movie, + stream_id="first", + container_extension="mkv", + ) + M3UMovieRelation.objects.create( + m3u_account=a2, + movie=movie, + stream_id="second", + container_extension="mp4", + ) + + stream = xc_get_vod_streams(self.request, self.user)[0] + + self.assertEqual(stream["container_extension"], first.container_extension) diff --git a/apps/output/views.py b/apps/output/views.py index 4224fb35..699c9f95 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -918,6 +918,73 @@ def xc_get_epg(request, user, short=False): return output +XC_MOVIE_VALUE_FIELDS = ( + 'id', 'movie_id', 'category_id', 'container_extension', + 'movie__id', 'movie__name', 'movie__rating', 'movie__created_at', + 'movie__tmdb_id', 'movie__imdb_id', 'movie__description', 'movie__genre', + 'movie__year', 'movie__custom_properties', 'movie__logo_id', +) + +XC_SERIES_VALUE_FIELDS = ( + 'id', 'series_id', 'category_id', 'updated_at', + 'series__id', 'series__name', 'series__description', 'series__genre', + 'series__year', 'series__rating', 'series__custom_properties', 'series__logo_id', + 'series__tmdb_id', 'series__imdb_id', +) + + +def _xc_fetch_priority_distinct_relations( + *, + manager, + rel_filters, + distinct_field, + value_fields, + order_by_name_field, +): + """ + Return one row dict per distinct content ID (highest account priority wins). + + On PostgreSQL, dedupe on narrow relation rows first, then fetch display + columns via values() (no ORM model instantiation). That avoids sorting + wide joined rows during DISTINCT ON and reduces parallel worker /dev/shm + pressure in Docker. + """ + from django.db import connection, transaction + + narrow_qs = manager.filter(**rel_filters) + + def _fetch_by_ids(ids): + return list( + manager.filter(pk__in=ids) + .values(*value_fields) + .order_by(Lower(order_by_name_field)) + ) + + if connection.vendor == 'postgresql': + winning_ids_qs = ( + narrow_qs + .order_by(distinct_field, '-m3u_account__priority', 'id') + .distinct(distinct_field) + .values('pk') + ) + with transaction.atomic(): + #with connection.cursor() as cursor: + #cursor.execute("SET LOCAL max_parallel_workers_per_gather = 0") + winning_ids = list(winning_ids_qs.values_list('pk', flat=True)) + if not winning_ids: + return [] + return _fetch_by_ids(winning_ids) + + seen = {} + for row in narrow_qs.values(*value_fields).order_by('-m3u_account__priority', 'id'): + key = row[distinct_field] + if key not in seen: + seen[key] = row + rows = list(seen.values()) + rows.sort(key=lambda r: (r[order_by_name_field] or '').lower()) + return rows + + def xc_get_vod_categories(user): """Get VOD categories for XtreamCodes API""" from apps.vod.models import VODCategory, M3UMovieRelation @@ -943,33 +1010,19 @@ def xc_get_vod_categories(user): def xc_get_vod_streams(request, user, category_id=None): """Get VOD streams (movies) for XtreamCodes API""" from apps.vod.models import M3UMovieRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3UMovieRelation.objects - .filter(**rel_filters) - .select_related('movie', 'movie__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3UMovieRelation.objects, + rel_filters=rel_filters, + distinct_field='movie_id', + value_fields=XC_MOVIE_VALUE_FIELDS, + order_by_name_field='movie__name', ) - if connection.vendor == 'postgresql': - # DISTINCT ON returns one row per movie (highest-priority active relation) - # in a single query. ORDER BY must lead with the DISTINCT field. - relations = list( - base_qs.order_by('movie_id', '-m3u_account__priority', 'id') - .distinct('movie_id') - ) - else: - # SQLite fallback: fetch all matching relations, deduplicate in Python. - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.movie_id not in seen: - seen[rel.movie_id] = rel - relations = list(seen.values()) - # Precompute logo URL prefix/suffix once (mirrors _xc_live_streams_setup) # so each row only needs a string concat instead of reverse() + URI build. _base_url = build_absolute_uri_with_port(request, "") @@ -978,44 +1031,40 @@ def xc_get_vod_streams(request, user, category_id=None): _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - # Sort by name (DISTINCT ON forces ORDER BY movie_id; SQLite path is unsorted). - relations.sort(key=lambda r: (r.movie.name or "").lower()) - streams = [] append = streams.append - for num, relation in enumerate(relations, 1): - movie = relation.movie - custom_props = movie.custom_properties or {} - category = relation.category - category_id_str = str(category.id) if category else "0" - category_id_list = [category.id] if category else [] - rating = movie.rating - logo = movie.logo + for num, row in enumerate(relations, 1): + custom_props = row['movie__custom_properties'] or {} + category_id = row['category_id'] + category_id_str = str(category_id) if category_id else "0" + category_id_list = [category_id] if category_id else [] + rating = row['movie__rating'] + logo_id = row['movie__logo_id'] append({ "num": num, - "name": movie.name, + "name": row['movie__name'], "stream_type": "movie", - "stream_id": movie.id, + "stream_id": row['movie__id'], "stream_icon": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), "rating": rating or "0", "rating_5based": round(float(rating or 0) / 2, 2) if rating else 0, - "added": str(int(movie.created_at.timestamp())), + "added": str(int(row['movie__created_at'].timestamp())), "is_adult": 0, - "tmdb_id": movie.tmdb_id or "", - "imdb_id": movie.imdb_id or "", + "tmdb_id": row['movie__tmdb_id'] or "", + "imdb_id": row['movie__imdb_id'] or "", "trailer": custom_props.get('youtube_trailer') or "", - "plot": movie.description or "", - "genre": movie.genre or "", - "year": movie.year or "", + "plot": row['movie__description'] or "", + "genre": row['movie__genre'] or "", + "year": row['movie__year'] or "", "director": custom_props.get('director', ''), "cast": custom_props.get('actors', ''), "release_date": custom_props.get('release_date', ''), "category_id": category_id_str, "category_ids": category_id_list, - "container_extension": relation.container_extension or "mp4", + "container_extension": row['container_extension'] or "mp4", "custom_sid": None, "direct_source": "", }) @@ -1048,72 +1097,58 @@ def xc_get_series_categories(user): def xc_get_series(request, user, category_id=None): """Get series list for XtreamCodes API""" from apps.vod.models import M3USeriesRelation - from django.db import connection rel_filters = {"m3u_account__is_active": True} if category_id: rel_filters["category_id"] = category_id - base_qs = ( - M3USeriesRelation.objects - .filter(**rel_filters) - .select_related('series', 'series__logo', 'category') + relations = _xc_fetch_priority_distinct_relations( + manager=M3USeriesRelation.objects, + rel_filters=rel_filters, + distinct_field='series_id', + value_fields=XC_SERIES_VALUE_FIELDS, + order_by_name_field='series__name', ) - if connection.vendor == 'postgresql': - relations = list( - base_qs.order_by('series_id', '-m3u_account__priority', 'id') - .distinct('series_id') - ) - else: - seen: dict = {} - for rel in base_qs.order_by('-m3u_account__priority', 'id'): - if rel.series_id not in seen: - seen[rel.series_id] = rel - relations = list(seen.values()) - _base_url = build_absolute_uri_with_port(request, "") _sample_logo_path = reverse("api:vod:vodlogo-cache", args=[0]) _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") _logo_url_prefix = _base_url + _logo_prefix_raw + "/" _logo_url_suffix = "/" + _logo_suffix_raw - relations.sort(key=lambda r: (r.series.name or "").lower()) - series_list = [] append = series_list.append - for num, relation in enumerate(relations, 1): - series = relation.series - custom_props = series.custom_properties or {} - category = relation.category - rating = series.rating - logo = series.logo - year_str = str(series.year) if series.year else "" + for num, row in enumerate(relations, 1): + custom_props = row['series__custom_properties'] or {} + category_id = row['category_id'] + rating = row['series__rating'] + logo_id = row['series__logo_id'] + year_str = str(row['series__year']) if row['series__year'] else "" release_date = custom_props.get('release_date', year_str) append({ "num": num, - "name": series.name, - "series_id": relation.id, + "name": row['series__name'], + "series_id": row['id'], "cover": ( - f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None + f"{_logo_url_prefix}{logo_id}{_logo_url_suffix}" if logo_id else None ), - "plot": series.description or "", + "plot": row['series__description'] or "", "cast": custom_props.get('cast', ''), "director": custom_props.get('director', ''), - "genre": series.genre or "", + "genre": row['series__genre'] or "", "release_date": release_date, "releaseDate": release_date, - "last_modified": str(int(relation.updated_at.timestamp())), + "last_modified": str(int(row['updated_at'].timestamp())), "rating": str(rating or "0"), "rating_5based": str(round(float(rating or 0) / 2, 2)) if rating else "0", "backdrop_path": custom_props.get('backdrop_path', []), "youtube_trailer": custom_props.get('youtube_trailer', ''), "episode_run_time": custom_props.get('episode_run_time', ''), - "category_id": str(category.id) if category else "0", - "category_ids": [category.id] if category else [], - "tmdb_id": series.tmdb_id or "", - "imdb_id": series.imdb_id or "", + "category_id": str(category_id) if category_id else "0", + "category_ids": [category_id] if category_id else [], + "tmdb_id": row['series__tmdb_id'] or "", + "imdb_id": row['series__imdb_id'] or "", }) return series_list From 53fa1e42a0a10ae8fcc078d9bb978e464e0190e2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 20:54:12 -0500 Subject: [PATCH 15/28] enhancement(tests): introduce isolated backend test settings and improve test documentation - Added `dispatcharr.settings_test` for isolated testing, automatically creating an empty PostgreSQL test database and ensuring transaction isolation during tests. - Updated `manage.py` to switch to the test settings when running tests. - Enhanced the CONTRIBUTING.md file with detailed instructions on running the backend test suite and handling Celery tasks in tests. - Refactored test cases to use static methods for `worker_id` in `test_process_label.py` and adjusted assertions in `test_user_preferences.py` for better clarity and correctness. --- CHANGELOG.md | 4 +++ CONTRIBUTING.md | 14 +++++++- dispatcharr/settings_test.py | 58 ++++++++++++++++++++++++++++++++++ manage.py | 7 +++- tests/test_process_label.py | 4 +-- tests/test_user_preferences.py | 4 +-- 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 dispatcharr/settings_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc8054f..dfdeeda5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. + ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 725e21e8..6da68a61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,7 +144,19 @@ Untested code is significantly less likely to be merged. - Use Django's `TestCase` for unit/integration tests. - Test files live at `apps//tests/`. -- Run the test suite with: `uv run python manage.py test` +- Run the backend test suite with: + + ```bash + python manage.py test + ``` + + `manage.py` automatically uses `dispatcharr.settings_test`, which creates an empty PostgreSQL database `test_` (same engine as production), runs migrations, and rolls back each test in a transaction. Your live VOD/channels data is not used. + + Optional: `TEST_USE_SQLITE=1` for machines without Postgres (some PostgreSQL-only tests skip automatically). + + Tests that exercise Celery task bodies should use `@override_settings(CELERY_TASK_ALWAYS_EAGER=True)` locally. Global eager mode is off because `post_save` signals on M3U/EPG models call `.delay()` and would break `TestCase` transaction isolation. + +- Do **not** override with `--settings=dispatcharr.settings` on a live instance. ### Frontend diff --git a/dispatcharr/settings_test.py b/dispatcharr/settings_test.py new file mode 100644 index 00000000..fe4225fa --- /dev/null +++ b/dispatcharr/settings_test.py @@ -0,0 +1,58 @@ +""" +Django settings for running the backend test suite in isolation. + +Always use this module instead of dispatcharr.settings when running tests: + + python manage.py test + +`manage.py` selects this module automatically for the ``test`` command. + +Django creates a separate empty database (``test_``) and runs +migrations — your live data under /data/db is not used. + +Why not dispatcharr.settings? +- Production/AIO points at the live ``dispatcharr`` database. +- django-db-geventpool breaks TestCase transaction isolation on pooled connections. + +SQLite (``TEST_USE_SQLITE=1``) is an optional fallback for machines without +Postgres; production and CI should use the default Postgres test database. +""" +import os + +from dispatcharr.settings import * # noqa: F401,F403 + +# Fast password hashing for tests. +PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +# Do NOT run Celery tasks inline during tests. post_save signals on M3UAccount and +# EPGSource call .delay(); eager mode runs them inside TestCase transactions and +# closes/poisons the DB connection for subsequent queries in the same test. +CELERY_TASK_ALWAYS_EAGER = False +CELERY_TASK_EAGER_PROPAGATES = False + +_use_sqlite = os.environ.get("TEST_USE_SQLITE", "").lower() in ("1", "true", "yes") + +if _use_sqlite: + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } + } +else: + # Default: PostgreSQL with Django-managed test_dispatcharr (matches production). + # Uses the standard backend (not geventpool) so TestCase transactions isolate. + _pg_name = os.environ.get("POSTGRES_DB", "dispatcharr") + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": _pg_name, + "USER": os.environ.get("POSTGRES_USER", "dispatch"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), + "HOST": os.environ.get("POSTGRES_HOST", "localhost"), + "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), + "TEST": { + "NAME": "test_" + _pg_name, + }, + } + } diff --git a/manage.py b/manage.py index f3c22dfd..f0a85a39 100644 --- a/manage.py +++ b/manage.py @@ -5,7 +5,12 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') + # Use isolated test DB settings for `manage.py test` (empty test_). + # Override with --settings=... on the command line if needed. + if len(sys.argv) > 1 and sys.argv[1] == "test": + os.environ["DJANGO_SETTINGS_MODULE"] = "dispatcharr.settings_test" + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dispatcharr.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/tests/test_process_label.py b/tests/test_process_label.py index bb4c8429..e4525a85 100644 --- a/tests/test_process_label.py +++ b/tests/test_process_label.py @@ -17,13 +17,13 @@ class ProcessLabelTests(SimpleTestCase): self.assertEqual(role, "uwsgi") def test_uwsgi_labeled_when_worker_module_present(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 2})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 2)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "uwsgi") def test_uwsgi_master_not_labeled_as_uwsgi(self): - fake_uwsgi = type("uwsgi", (), {"worker_id": lambda: 0})() + fake_uwsgi = type("uwsgi", (), {"worker_id": staticmethod(lambda: 0)})() with patch.dict("sys.modules", {"uwsgi": fake_uwsgi}): role = get_process_role(["/dispatcharrpy/bin/python", "-c", "pass"]) self.assertEqual(role, "django") diff --git a/tests/test_user_preferences.py b/tests/test_user_preferences.py index 5749011a..80dcc992 100644 --- a/tests/test_user_preferences.py +++ b/tests/test_user_preferences.py @@ -128,13 +128,13 @@ class UserPreferencesAPITests(TestCase): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) def test_patch_me_cannot_escalate_privileges(self): - """Test PATCH /me/ rejects attempts to change user_level or is_staff""" + """PATCH /me/ ignores privilege fields; they are stripped before save.""" original_level = self.user.user_level data = {"user_level": 99, "is_staff": True, "is_superuser": True} response = self.client.patch(self.me_url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.status_code, status.HTTP_200_OK) self.user.refresh_from_db() self.assertEqual(self.user.user_level, original_level) From 1b7840b71541ca95a6fcb533d48fa7f145c9fdd3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 07:52:13 -0500 Subject: [PATCH 16/28] changelog: Update for pr 1331 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdeeda5..0ea59832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`POST /api/epg/import/` no longer loads the full EPG source row.** The dummy-source guard uses a narrow existence query instead of `objects.get()`, keeping the import trigger lightweight. Request/response shape is unchanged for the UI, plugins, and external importers. - **XC live playback URL building now normalizes account server URLs.** On-demand live URLs (introduced for Server Group profile rotation in 0.27.0) rebuild `/live/{user}/{pass}/{stream_id}.ts` from current credentials instead of reusing the synced `stream.url`. That path now runs `normalize_server_url()` so pasted API URLs (e.g. `/player_api.php` with query params) are stripped while sub-paths like `/server1` are preserved. `get_transformed_credentials()` normalizes the base URL at the source; VOD movie and episode relations use the same helper instead of constructing a throwaway `XCClient` (which also avoided per-request HTTP session setup). (Fixes #1363) - **Live proxy now releases geventpool DB connections on more paths.** `stream_ts()` calls `close_old_connections()` before `StreamingHttpResponse` so clients waiting on channel init do not keep a pool slot while the generator polls Redis. `generate_stream_url()`, `get_stream_info_for_switch()`, `get_alternate_streams()`, and `get_connections_left()` release in `finally` blocks after ORM work on stream-manager failover paths that run outside Django's request cycle. TS client disconnect cleanup matches fMP4, and `ChannelService` metadata helpers release after their ORM lookups. +- **Auto-sync range conflict warning no longer flags a group's own channels when using a channel-group override.** The M3U group settings "Range conflict" check compared occupant channel groups against the source group being configured. Auto-sync with `group_override` creates channels in the override target group, so those channels were misclassified as conflicts. The frontend now resolves the effective target group (override target when set, otherwise the source group) for classification. Genuine conflicts—manual channels, other accounts, different groups, or user-pinned numbers—still surface. (Fixes #1331) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.27.0] - 2026-06-16 From 578c50ea962d12f7d9f956c910accca431fc3e4f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 08:01:13 -0500 Subject: [PATCH 17/28] changelog: corrected --- CHANGELOG.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea59832..ce3d0b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance -- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now dedupe on narrow relation rows first (`SET LOCAL max_parallel_workers_per_gather = 0`), then fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. +- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. - **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. @@ -28,9 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. -### Performance - -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. ### Fixed From 971065b8a874f5837c818fe28267b2da554324bd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:14:00 -0500 Subject: [PATCH 18/28] chore(dependencies): update Django, requests, gevent, torch, sentence-transformers, and lxml to latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a003bc5c..8d4fad78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,18 +6,18 @@ license = "AGPL-3.0-only" requires-python = ">=3.13" dynamic = ["version"] dependencies = [ - "Django==6.0.5", + "Django==6.0.6", "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", - "requests==2.33.1", + "requests==2.34.2", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", "streamlink", "python-vlc", "yt-dlp", - "gevent==26.4.0", + "gevent==26.5.0", "django-db-geventpool", "daphne", "uwsgi", @@ -28,14 +28,14 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.11.0+cpu", - "sentence-transformers==5.4.1", + "torch==2.12.1+cpu", + "sentence-transformers==5.6.0", "channels", "channels-redis==4.3.0", "django-filter", "django-redis", "django-celery-beat>=2.9.0", - "lxml==6.1.0", + "lxml==6.1.1", "packaging", ] From 107246ff0b43d676e4cffc0219b4a047ca1c77d4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:17:38 -0500 Subject: [PATCH 19/28] changelog: Update for dependency updates. --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3d0b06..b2645f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- Updated `Django` 6.0.5 → 6.0.6, resolving the following CVEs: + - **CVE-2026-6873**: Signed cookie salt namespace collision in `HttpRequest.get_signed_cookie()`. + - **CVE-2026-7666**: Potential unencrypted email transmission via STARTTLS in the SMTP backend. + - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. + - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. + - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. + ### Added - **Isolated backend test settings (`dispatcharr.settings_test`).** `python manage.py test` now switches to this module automatically (via `manage.py`). It creates an empty PostgreSQL `test_` database (same engine as production), uses the standard Postgres backend instead of geventpool so `TestCase` transactions isolate correctly, and leaves Celery tasks queued (no eager `post_save` signal runs during tests). Set `TEST_USE_SQLITE=1` for an in-memory SQLite fallback when Postgres is unavailable. @@ -29,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Channel list with nested streams loads faster.** `GET /api/channels/channels/?include_streams=true` (Channels UI and single-channel fetch) now builds nested stream payloads from the prefetched `channelstream_set` instead of issuing one extra streams M2M query per channel. +- Dependency updates: + - `Django` 6.0.5 → 6.0.6 (security patch; see Security section) + - `requests` 2.33.1 → 2.34.2 + - `gevent` 26.4.0 → 26.5.0 + - `torch` 2.11.0+cpu → 2.12.1+cpu + - `sentence-transformers` 5.4.1 → 5.6.0 + - `lxml` 6.1.0 → 6.1.1 + ### Fixed From 6b6eb11cc03ad1bb65b321c2b27c92b869fbe24b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:31:15 -0500 Subject: [PATCH 20/28] chore(dependencies): update Vite, esbuild and js-yaml to latest versions in package.json --- CHANGELOG.md | 4 + frontend/package-lock.json | 240 +++++++++++++++++++------------------ frontend/package.json | 7 +- 3 files changed, 133 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2645f45..f7bedbf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **CVE-2026-8404**: Potential private data exposure via case-sensitive `Cache-Control` directives in `UpdateCacheMiddleware`. - **CVE-2026-35193**: Potential private data exposure via missing `Vary: Authorization` in `UpdateCacheMiddleware`. - **CVE-2026-48587**: Potential private data exposure via whitespace padding in the `Vary` header. +- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (1 low, 2 moderate, 1 high): + - Updated `vite` 7.3.2 → 7.3.5, resolving **moderate** NTLMv2 hash disclosure via UNC path handling on Windows ([GHSA-v6wh-96g9-6wx3](https://github.com/advisories/GHSA-v6wh-96g9-6wx3)) and **high** `server.fs.deny` bypass on Windows alternate paths ([GHSA-fx2h-pf6j-xcff](https://github.com/advisories/GHSA-fx2h-pf6j-xcff)) + - Updated `js-yaml` 4.1.1 → 5.1.0, resolving **moderate** quadratic-complexity DoS in merge key handling via repeated aliases ([GHSA-h67p-54hq-rp68](https://github.com/advisories/GHSA-h67p-54hq-rp68)) + - Updated `esbuild` 0.27.3 → 0.28.1, resolving **low** arbitrary file read when running the development server on Windows ([GHSA-g7r4-m6w7-qqqr](https://github.com/advisories/GHSA-g7r4-m6w7-qqqr)) ### Added diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8ca15bc7..ad8ad268 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -57,7 +57,7 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" } }, @@ -595,9 +595,9 @@ "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", "cpu": [ "ppc64" ], @@ -612,9 +612,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", "cpu": [ "arm" ], @@ -629,9 +629,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", "cpu": [ "arm64" ], @@ -646,9 +646,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", "cpu": [ "x64" ], @@ -663,9 +663,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", "cpu": [ "arm64" ], @@ -680,9 +680,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", "cpu": [ "x64" ], @@ -697,9 +697,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", "cpu": [ "arm64" ], @@ -714,9 +714,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", "cpu": [ "x64" ], @@ -731,9 +731,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", "cpu": [ "arm" ], @@ -748,9 +748,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", "cpu": [ "arm64" ], @@ -765,9 +765,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", "cpu": [ "ia32" ], @@ -782,9 +782,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", "cpu": [ "loong64" ], @@ -799,9 +799,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", "cpu": [ "mips64el" ], @@ -816,9 +816,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", "cpu": [ "ppc64" ], @@ -833,9 +833,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", "cpu": [ "riscv64" ], @@ -850,9 +850,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", "cpu": [ "s390x" ], @@ -867,9 +867,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", "cpu": [ "x64" ], @@ -884,9 +884,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", "cpu": [ "arm64" ], @@ -901,9 +901,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", "cpu": [ "x64" ], @@ -918,9 +918,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", "cpu": [ "arm64" ], @@ -935,9 +935,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", "cpu": [ "x64" ], @@ -952,9 +952,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", "cpu": [ "arm64" ], @@ -969,9 +969,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", "cpu": [ "x64" ], @@ -986,9 +986,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", "cpu": [ "arm64" ], @@ -1003,9 +1003,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", "cpu": [ "ia32" ], @@ -1020,9 +1020,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", "cpu": [ "x64" ], @@ -3163,9 +3163,9 @@ "license": "MIT" }, "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3176,32 +3176,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" } }, "node_modules/escape-string-regexp": { @@ -3814,16 +3814,26 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.1.0.tgz", + "integrity": "sha512-s8VA5jkR8f22S3NAXmhKPFqGUduqZGlsufabVOgN14iTdw/RXcym7bKkbwjxLK9Yw2lEvvmJjFp119+KPeo8Kg==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsdom": { @@ -5425,9 +5435,9 @@ } }, "node_modules/vite": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", - "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.5.tgz", + "integrity": "sha512-KuOaNhcnGFN2zIPGA7wRmzF+lJA1sea7rHq17aiJ++9lzY1WWG6Jpwqwe1KNbRVPIqHmr8GLYx7jbrQcN/7/ww==", "dev": true, "license": "MIT", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 2c1b78f5..8675351d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -61,16 +61,17 @@ "globals": "^15.15.0", "jsdom": "^27.0.0", "prettier": "^3.5.3", - "vite": "^7.1.7", + "vite": "^7.3.5", "vitest": "^4.1.8" }, "resolutions": { - "vite": "7.1.7", + "vite": "7.3.5", "react": "19.1.0", "react-dom": "19.1.0" }, "overrides": { - "js-yaml": "^4.1.1", + "esbuild": "^0.28.1", + "js-yaml": "^5.1.0", "minimatch": "^10.2.1" } } From e2ceef521779467f1f4947f492dc1d040d5404cb Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 16:46:37 -0500 Subject: [PATCH 21/28] changelog: refactor xmltv changes and link issue. --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7bedbf8..059e0d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,11 +27,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **`get_vod_streams` and `get_series` XC API endpoints are faster and no longer exhaust Docker `/dev/shm`.** Large libraries (e.g. 125k movies) previously ran one wide `DISTINCT ON` query with parallel workers, which could fail with `could not resize shared memory segment … No space left on device` on the default 64MB container shm. Both endpoints now fetch display columns via `.values()` (no ORM model instantiation per row). Redundant `category` and `logo` joins were dropped in favor of FK ids; alphabetical sort runs in SQL. Typical full-library response time drops from ~23–28s to ~8–10s with stable shm usage. -- **XMLTV EPG output no longer N+1 queries streams or dummy-program checks.** `generate_epg()` prefetches ordered channel streams once (for custom dummy EPG logo/program parsing when `name_source` is `stream`) and bulk-checks which dummy `EPGData` rows have stored programmes in a single query instead of one `.exists()` per row. Large guides with hundreds of custom-dummy channels issue far fewer SQL round-trips per client refresh. -- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. -- **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. -- **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). The index now lives in its own `EPGSourceIndex` table, so the JOIN never pulls it. -- **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. +- **XMLTV EPG export is faster and no longer balloons worker memory.** `generate_epg()` was reworked end-to-end for large guides. (Fixes #1366) + - Streams incrementally: on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker); repeat requests within 300s stream chunks back from Redis. `malloc_trim` runs after cold builds. + - Channel streams are prefetched once (only `id`/`name`) instead of one query per custom-dummy channel; dummy `EPGData` programme existence is bulk-checked in a single query. + - The primary channel id is escaped once per `epg_id` group instead of once per programme (~750k fewer `html.escape` calls on a large guide). + - The channel query no longer JOINs multi-MB `programme_index` blobs per channel (~13s saved on a ~2000-channel guide; indices live in `EPGSourceIndex`). + - Programme export uses `(epg_id, id)` keyset pagination with a per-source `start_time` sort; a matching composite index on `ProgramData` (created `CONCURRENTLY` on PostgreSQL) lets each chunk use an ordered index range scan instead of re-sorting every chunk. - **EPG grid endpoint releases its payload memory back to the OS.** `/api/epg/grid/` drops the redundant full-list copy when appending dummy programmes and runs `malloc_trim` once the response is sent, so worker RSS no longer ratchets up ~20MB per request. ### Changed From c5e50167285504e03487447d0bb3b2e0f8d12144 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:02:31 -0500 Subject: [PATCH 22/28] fix(dvr): Fix in-progress DVR playback to prevent jumping to live edge and update FloatingVideo component for improved error handling. Added tests to verify HLS configuration behavior. (Fixes #1329) --- CHANGELOG.md | 1 + frontend/src/components/FloatingVideo.jsx | 115 +++++++++--------- .../__tests__/FloatingVideo.test.jsx | 94 ++++++++++++++ 3 files changed, 154 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 059e0d23..5365e79a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **In-progress DVR playback no longer jumps to the live edge.** The floating video player still opens in-progress recordings at the start of the seekable range, but hls.js was configured with `liveMaxLatencyDurationCount: 10`, which forced the playhead forward to "now" shortly after playback began. Live-edge sync is now disabled for recording HLS so users can watch, pause, and scrub from the beginning while the recording continues. (Fixes #1329) - **M3U refresh recovers DB connections and account status after failures.** Celery workers now call `close_old_connections()` before and after every task; `refresh_single_m3u_account` also resets its DB connection at task start and retries account loads once after a connection reset (avoids opaque `list index out of range` errors from poisoned worker connections). Accounts leave `fetching`/`parsing` for a terminal `error` or `success` state when refresh aborts. Error-path status updates use `QuerySet.update()` on a clean connection instead of `save()` on a connection that may already be INTRANS after `refresh_m3u_groups` or batch ORM failures. Failed group downloads now set `error` instead of returning silently. Refresh start replaces stale `last_message` text. `refresh_account_profiles` releases DB connections after each profile failure. (Fixes #1338) - **EPG refresh recovers DB connections and source status after failures.** `refresh_epg_data` now mirrors the M3U hardening: connection reset at task start/end, retried source load after a poisoned-worker connection reset, `QuerySet.update()` for error status on a clean connection, and a `finally` guard that moves sources stuck in `fetching`/`parsing` to `error`. Programme index builds are skipped when programme parsing fails. `parse_channels_only` now persists the error status when a missing file has no URL to refetch. - **M3U `custom_properties` JSON normalization.** Legacy rows and some API writes could store a JSON-encoded string instead of an object in `custom_properties`, causing refresh, profile sync, and auto-sync to fail with `'str' object has no attribute 'get'`. M3U account, profile, and group-relation models normalize on `save()`; group settings bulk writes and read/merge paths use shared helpers in `core.utils` without re-parsing dict values. diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 507e0caf..e49c3bb2 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -333,65 +333,68 @@ export default function FloatingVideo() { let hls = null; if (isHls && Hls.isSupported()) { - hls = new Hls({ - // Open at the very beginning of the recording rather than the live - // edge. Without this, an in-progress recording would start at "now" - // and hide everything already recorded. hls.js applies this AFTER - // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is - // also kept as a safety net for the Safari native-HLS path and for - // edge cases where this initial-position logic loses to the user's - // first interaction. - startPosition: 0, - // Allow seeking back to the start of the recording, regardless of - // current playhead position. Recordings can be hours long and the - // user may want to scrub anywhere; we explicitly disable buffer - // eviction by setting a very large back-buffer length. - backBufferLength: 90 * 60, // 90 minutes - maxBufferLength: 60, - maxMaxBufferLength: 600, - // For an in-progress recording, hls.js refreshes the playlist on - // its target-duration cadence; let it follow the live edge but keep - // the full DVR window seekable. - liveSyncDurationCount: 3, - liveMaxLatencyDurationCount: 10, - enableWorker: true, - lowLatencyMode: false, - // Inject the JWT into every playlist + segment XHR. Read the token - // from the auth store at request time rather than capturing the - // closure value at hls.js init, so a refreshed access token mid- - // playback is picked up on the next segment fetch. - xhrSetup: (xhr) => { - const token = useAuthStore.getState().accessToken; - if (token) { - xhr.setRequestHeader('Authorization', `Bearer ${token}`); - } - }, - }); - hls.on(Hls.Events.ERROR, (_evt, data) => { - if (data.fatal) { - // eslint-disable-next-line no-console - console.error('HLS fatal error:', data.type, data.details); - if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { - try { - hls.startLoad(); - } catch { - // ignore + try { + hls = new Hls({ + // Open at the very beginning of the recording rather than the live + // edge. Without this, an in-progress recording would start at "now" + // and hide everything already recorded. hls.js applies this AFTER + // MEDIA_ATTACHED, so the listener-driven `seekToStart()` above is + // also kept as a safety net for the Safari native-HLS path and for + // edge cases where this initial-position logic loses to the user's + // first interaction. + startPosition: 0, + // Allow seeking back to the start of the recording, regardless of + // current playhead position. Recordings can be hours long and the + // user may want to scrub anywhere; we explicitly disable buffer + // eviction by setting a very large back-buffer length. + backBufferLength: 90 * 60, // 90 minutes + maxBufferLength: 60, + maxMaxBufferLength: 600, + // Leave liveMaxLatencyDurationCount at the hls.js default (Infinity). + // A finite value forces the playhead to the live edge during playback. + enableWorker: true, + lowLatencyMode: false, + // Inject the JWT into every playlist + segment XHR. Read the token + // from the auth store at request time rather than capturing the + // closure value at hls.js init, so a refreshed access token mid- + // playback is picked up on the next segment fetch. + xhrSetup: (xhr) => { + const token = useAuthStore.getState().accessToken; + if (token) { + xhr.setRequestHeader('Authorization', `Bearer ${token}`); } - } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { - try { - hls.recoverMediaError(); - } catch { - // ignore + }, + }); + hls.on(Hls.Events.ERROR, (_evt, data) => { + if (data.fatal) { + // eslint-disable-next-line no-console + console.error('HLS fatal error:', data.type, data.details); + if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { + try { + hls.startLoad(); + } catch { + // ignore + } + } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { + try { + hls.recoverMediaError(); + } catch { + // ignore + } + } else { + setLoadError(`HLS playback error: ${data.details || data.type}`); } - } else { - setLoadError(`HLS playback error: ${data.details || data.type}`); } - } - }); - hls.attachMedia(video); - hls.on(Hls.Events.MEDIA_ATTACHED, () => { - hls.loadSource(streamUrl); - }); + }); + hls.attachMedia(video); + hls.on(Hls.Events.MEDIA_ATTACHED, () => { + hls.loadSource(streamUrl); + }); + } catch (error) { + setIsLoading(false); + setLoadError(`HLS initialization error: ${error.message}`); + return; + } } else if (isHls && video.canPlayType('application/vnd.apple.mpegurl')) { // Safari path: native HLS support, including seekable DVR windows. video.src = streamUrl; diff --git a/frontend/src/components/__tests__/FloatingVideo.test.jsx b/frontend/src/components/__tests__/FloatingVideo.test.jsx index f99a3dcc..9683fc59 100644 --- a/frontend/src/components/__tests__/FloatingVideo.test.jsx +++ b/frontend/src/components/__tests__/FloatingVideo.test.jsx @@ -20,8 +20,49 @@ vi.mock('mpegts.js', () => ({ }, })); +const mockHlsInstance = { + attachMedia: vi.fn(), + loadSource: vi.fn(), + destroy: vi.fn(), + on: vi.fn(), +}; + +let capturedHlsConfig = null; +let forceHlsInitError = false; + +vi.mock('hls.js', () => ({ + default: class MockHls { + static isSupported = vi.fn(() => true); + + static Events = { + ERROR: 'error', + MEDIA_ATTACHED: 'media_attached', + }; + + static ErrorTypes = { + NETWORK_ERROR: 'networkError', + MEDIA_ERROR: 'mediaError', + }; + + constructor(config) { + if (forceHlsInitError) { + throw new Error('Illegal hls.js config'); + } + capturedHlsConfig = config; + Object.assign(this, mockHlsInstance); + } + }, +})); + +vi.mock('../../store/auth', () => ({ + default: { + getState: vi.fn(() => ({ accessToken: 'test-token' })), + }, +})); + // Import the mocked module after mocking const mpegts = (await import('mpegts.js')).default; +const Hls = (await import('hls.js')).default; // Mock react-draggable vi.mock('react-draggable', () => ({ @@ -53,6 +94,8 @@ describe('FloatingVideo', () => { beforeEach(async () => { vi.clearAllMocks(); + capturedHlsConfig = null; + forceHlsInitError = false; // Mock HTMLVideoElement methods HTMLVideoElement.prototype.load = vi.fn(); @@ -239,6 +282,57 @@ describe('FloatingVideo', () => { expect(video.poster).toBe('http://example.com/poster.jpg'); }); + it('should disable live-edge sync for in-progress recording HLS', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + + render(); + + expect(capturedHlsConfig).toEqual( + expect.objectContaining({ + startPosition: 0, + }) + ); + expect(capturedHlsConfig).not.toHaveProperty( + 'liveMaxLatencyDurationCount' + ); + expect(capturedHlsConfig).not.toHaveProperty('liveSyncDurationCount'); + }); + + it('shows an in-player error when hls.js config is invalid', () => { + useVideoStore.mockImplementation((selector) => { + const state = { + isVisible: true, + streamUrl: + 'http://example.com/api/channels/recordings/1/hls/index.m3u8', + contentType: 'vod', + metadata: { name: 'News Recording' }, + hideVideo: mockHideVideo, + }; + return selector ? selector(state) : state; + }); + + Hls.isSupported.mockReturnValue(true); + forceHlsInitError = true; + + render(); + + expect( + screen.getByText(/HLS initialization error: Illegal hls.js config/i) + ).toBeInTheDocument(); + }); + it('should show metadata overlay', () => { const { container } = render(); const video = container.querySelector('video'); From 562393b77e50d56ae51a2772b2d148d5a16aeca1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 24 Jun 2026 17:32:48 -0500 Subject: [PATCH 23/28] fix(recording playback): Enhance completed DVR recording playback by allowing JWT token via query parameter for native