refactor(epg): separate programme index into dedicated table for optimized data handling

- Moved the `programme_index` from the `EPGSource` model to a new `EPGSourceIndex` table, ensuring that the large JSON blob is only loaded when explicitly accessed, thus improving query performance and memory efficiency.
- Updated related queries and API views to utilize the new structure, including adjustments to EPG generation and import logic to prevent unnecessary data loading.
- Enhanced memory management in the EPG grid endpoint to reduce worker RSS during response handling.

apps/epg/api_views.py

@@ -64,8 +64,6 @@ class EPGSourceViewSet(viewsets.ModelViewSet):
         from apps.channels.models import Channel
         return EPGSource.objects.select_related(
             "refresh_task__crontab", "refresh_task__interval"
-        ).defer(
-            'programme_index'
         ).annotate(
             has_channels=Exists(
                 Channel.objects.filter(epg_data__epg_source_id=OuterRef('pk'))
@@ -1082,13 +1080,19 @@ class EPGGridAPIView(APIView):
                     f"Error creating standard dummy programs for channel {channel.name} (ID: {channel.id}): {str(e)}"
                 )
 
-        # Combine regular and dummy programs
-        all_programs = list(serialized_programs) + dummy_programs
+        # Combine regular and dummy programs in place to avoid copying the large list
+        serialized_programs.extend(dummy_programs)
         logger.debug(
-            f"EPGGridAPIView: Returning {len(all_programs)} total programs (including {len(dummy_programs)} dummy programs)."
+            f"EPGGridAPIView: Returning {len(serialized_programs)} total programs (including {len(dummy_programs)} dummy programs)."
         )
 
-        return Response({"data": all_programs}, status=status.HTTP_200_OK)
+        # The grid materializes tens of thousands of program dicts plus the
+        # rendered JSON; trim once the response is sent so worker RSS does not
+        # ratchet up per request.
+        from core.utils import spawn_memory_trim
+        response = Response({"data": serialized_programs}, status=status.HTTP_200_OK)
+        response._resource_closers.append(spawn_memory_trim)
+        return response
 
 
 # ─────────────────────────────
@@ -1119,7 +1123,7 @@ class EPGImportAPIView(APIView):
         epg_id = request.data.get("id", None)
         force = bool(request.data.get("force", False))
 
-        # Reject dummy sources without loading programme_index (multi-MB JSON).
+        # Reject dummy sources with a narrow existence query, no full row load.
         if epg_id is not None:
             from .models import EPGSource
 
@@ -1236,10 +1240,9 @@ class CurrentProgramsAPIView(APIView):
             # Limit to 50 IDs per request
             epg_data_ids = epg_data_ids[:50]
 
-            # Defer the multi-MB programme_index the JOIN would pull once per row. The lookup reads it via a targeted refresh_from_db
-            epg_data_entries = EPGData.objects.select_related('epg_source').defer(
-                'epg_source__programme_index'
-            ).filter(id__in=epg_data_ids)
+            epg_data_entries = EPGData.objects.select_related('epg_source').filter(
+                id__in=epg_data_ids
+            )
 
             # Batch-fetch current programs for all requested EPG entries in one query
             db_programs = ProgramData.objects.filter(

apps/epg/migrations/0026_epgsourceindex.py

@@ -0,0 +1,52 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def copy_index_forward(apps, schema_editor):
+    EPGSource = apps.get_model('epg', 'EPGSource')
+    EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex')
+    rows = list(
+        EPGSource.objects.exclude(programme_index__isnull=True).values_list(
+            'id', 'programme_index'
+        )
+    )
+    for source_id, data in rows:
+        EPGSourceIndex.objects.update_or_create(
+            source_id=source_id, defaults={'data': data}
+        )
+
+
+def copy_index_backward(apps, schema_editor):
+    EPGSource = apps.get_model('epg', 'EPGSource')
+    EPGSourceIndex = apps.get_model('epg', 'EPGSourceIndex')
+    for source_id, data in EPGSourceIndex.objects.values_list('source_id', 'data'):
+        EPGSource.objects.filter(id=source_id).update(programme_index=data)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('epg', '0025_programdata_epg_id_index'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EPGSourceIndex',
+            fields=[
+                ('source', models.OneToOneField(
+                    on_delete=django.db.models.deletion.CASCADE,
+                    primary_key=True,
+                    related_name='index_record',
+                    serialize=False,
+                    to='epg.epgsource',
+                )),
+                ('data', models.JSONField(blank=True, default=None, null=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.RunPython(copy_index_forward, copy_index_backward),
+        migrations.RemoveField(
+            model_name='epgsource',
+            name='programme_index',
+        ),
+    ]

apps/epg/models.py

@@ -62,12 +62,6 @@ class EPGSource(models.Model):
         blank=True,
         help_text="Last status message, including success results or error information"
     )
-    programme_index = models.JSONField(
-        null=True,
-        blank=True,
-        default=None,
-        help_text="Byte-offset index mapping tvg_id to file positions, built after each EPG refresh"
-    )
     created_at = models.DateTimeField(
         auto_now_add=True,
         help_text="Time when this source was created"
@@ -124,6 +118,37 @@ class EPGSource(models.Model):
                 kwargs['update_fields'].remove('updated_at')
         super().save(*args, **kwargs)
 
+    @property
+    def programme_index(self):
+        """Byte-offset index for this source, read on demand from the separate
+        EPGSourceIndex table so the multi-MB blob is never pulled into EPGSource
+        queries or select_related JOINs. Returns the stored dict or None."""
+        return (
+            EPGSourceIndex.objects.filter(source_id=self.pk)
+            .values_list('data', flat=True)
+            .first()
+        )
+
+
+class EPGSourceIndex(models.Model):
+    """Byte-offset programme index for an EPGSource, stored in its own table.
+
+    Kept out of EPGSource so the multi-MB JSON blob is only loaded when read
+    explicitly, never when querying or joining EPGSource rows.
+    """
+    source = models.OneToOneField(
+        EPGSource,
+        on_delete=models.CASCADE,
+        related_name='index_record',
+        primary_key=True,
+    )
+    data = models.JSONField(null=True, blank=True, default=None)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    def __str__(self):
+        return f"Programme index for source {self.source_id}"
+
+
 class EPGData(models.Model):
     tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True)
     name = models.CharField(max_length=512)

apps/epg/tasks.py

@@ -26,7 +26,7 @@ from core.models import UserAgent, CoreSettings
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
 
-from .models import EPGSource, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5
+from .models import EPGSource, EPGSourceIndex, EPGData, ProgramData, SDScheduleMD5, SDProgramMD5
 from core.utils import (
     acquire_task_lock,
     is_task_lock_held,
@@ -653,7 +653,9 @@ def _refresh_epg_data_impl(source_id, force=False):
     if source.source_type == 'xmltv':
         # Invalidate the byte-offset index before downloading the new file
         # so stale offsets are never used during the refresh window.
-        EPGSource.objects.filter(id=source.id).update(programme_index=None)
+        EPGSourceIndex.objects.update_or_create(
+            source_id=source.id, defaults={'data': None}
+        )
         if not fetch_xmltv(source):
             logger.error(f"Failed to fetch XMLTV for source {source.name}")
             return
@@ -4484,7 +4486,7 @@ def _programme_to_dict(elem, start_time, end_time):
 def build_programme_index(source_id):
     """
     Scan the XML file with raw binary I/O to build a {tvg_id: [byte_offset, ...]} map.
-    Persists the result to EPGSource.programme_index. Most XMLTV files group programmes
+    Persists the result to the EPGSourceIndex table. Most XMLTV files group programmes
     by channel, but some split a channel across multiple non-contiguous blocks, so we
     record block starts up to _OFFSET_CAP and mark only channels that exceed the cap
     as interleaved.
@@ -4573,7 +4575,9 @@ def build_programme_index(source_id):
         'channels': index,
         'interleaved_channels': sorted(interleaved_channels),
     }
-    EPGSource.objects.filter(id=source_id).update(programme_index=result)
+    EPGSourceIndex.objects.update_or_create(
+        source_id=source_id, defaults={'data': result}
+    )
 
 
 @shared_task
@@ -4620,9 +4624,8 @@ def find_current_program_for_tvg_id(epg_or_id):
         return None
 
     now = timezone.now()
-    # Force a fresh read of the DB-backed index to avoid using stale related-object
-    # state when an EPG refresh invalidates/rebuilds the index concurrently.
-    source.refresh_from_db(fields=['programme_index'])
+    # The property reads the EPGSourceIndex table fresh on each access, so a
+    # concurrent refresh invalidating/rebuilding the index can't serve stale state.
     index = source.programme_index
 
     if index is not None:

apps/epg/tests/test_epg_import_api.py

@@ -7,7 +7,7 @@ from django.test import TestCase
 from rest_framework import status
 from rest_framework.test import APIClient
 
-from apps.epg.models import EPGSource
+from apps.epg.models import EPGSource, EPGSourceIndex
 
 User = get_user_model()
 
@@ -47,7 +47,10 @@ class EPGImportAPITests(TestCase):
             name="Large Index XMLTV",
             source_type="xmltv",
             url="http://example.com/epg.xml",
-            programme_index={
+        )
+        EPGSourceIndex.objects.create(
+            source=source,
+            data={
                 "channels": {f"ch.{i}": {"offsets": [0, 100]} for i in range(200)},
                 "interleaved_channels": [],
             },

apps/epg/tests/test_programme_index.py

@@ -519,7 +519,6 @@ class FindCurrentProgramTests(TestCase):
             name="No Index",
             source_type="xmltv",
             file_path=FIXTURE_XML,
-            programme_index=None,
         )
         epg = EPGData.objects.create(
             tvg_id="channel.current",

apps/output/epg.py

@@ -40,34 +40,20 @@ def _programme_overlaps_export_window(start_time, end_time, lookback_cutoff, cut
 
 def _ceil_to_half_hour(dt):
     """Round a datetime up to the next :00 or :30 boundary."""
-    dt = dt.replace(second=0, microsecond=0)
-    remainder = dt.minute % 30
-    if remainder == 0:
-        return dt
-    return dt + timedelta(minutes=30 - remainder)
+    original = dt.replace(microsecond=0)
+    aligned = dt.replace(second=0, microsecond=0)
+    remainder = aligned.minute % 30
+    if remainder != 0:
+        aligned += timedelta(minutes=30 - remainder)
+    if aligned < original:
+        aligned += timedelta(minutes=30)
+    return aligned
 
 
 def _epg_export_teardown():
-    from django.db import close_old_connections
+    from core.utils import spawn_memory_trim
 
-    from core.utils import (
-        _is_gevent_monkey_patched,
-        cleanup_memory,
-        trim_c_allocator_heap,
-    )
-
-    close_old_connections()
-
-    def _run():
-        cleanup_memory(force_collection=True)
-        trim_c_allocator_heap()
-
-    if _is_gevent_monkey_patched():
-        import gevent
-
-        gevent.spawn(_run)
-    else:
-        _run()
+    spawn_memory_trim(close_connections=True)
 
 
 def _ordered_channel_streams(channel):
@@ -1183,10 +1169,6 @@ def generate_epg(request, profile_name=None, user=None):
             with_effective_values(base_qs, select_related_fks=True)
             .exclude(hidden_from_output=True)
             .order_by("effective_channel_number")
-            # programme_index is a multi-MB JSON byte-offset index that EPG
-            # generation never reads; defer it so it isn't fetched and JSON-parsed
-            # once per channel (was ~13s of the request on large guides).
-            .defer("epg_data__epg_source__programme_index")
             .prefetch_related(
                 Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order'))
             )

apps/output/tests.py

@@ -355,4 +355,14 @@ class OutputEPGHelperTest(SimpleTestCase):
         aligned = _ceil_to_half_hour(dt)
         self.assertEqual(aligned.minute, 30)
         self.assertEqual(aligned.second, 0)
-        self.assertGreater(aligned, dt.replace(second=0, microsecond=0))
+        self.assertGreaterEqual(aligned, dt.replace(microsecond=0))
+
+    def test_ceil_to_half_hour_past_boundary_second(self):
+        from django.utils import timezone
+        from apps.output.epg import _ceil_to_half_hour
+
+        dt = timezone.now().replace(minute=0, second=52, microsecond=123456)
+        aligned = _ceil_to_half_hour(dt)
+        self.assertEqual(aligned.minute, 30)
+        self.assertEqual(aligned.second, 0)
+        self.assertGreaterEqual(aligned, dt.replace(microsecond=0))