From 107a891359fc0319bae9389e6fa608b0e3d556b4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 23 Jun 2026 13:20:17 -0500 Subject: [PATCH] enhancement(epg): optimize XMLTV escaping and channel prefetch for improved performance --- CHANGELOG.md | 1 + apps/output/epg.py | 10 +++++----- apps/output/streaming_chunk_cache.py | 7 ++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840cb349..825a06fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Lighter EPG export: fewer escape calls and a slimmer channel prefetch.** The primary channel id is escaped once per `epg_id` group instead of once per programme (saves ~750k `html.escape` calls on a large guide). The per-channel `streams` prefetch now loads only `id`/`name` (the only fields the export reads) instead of full `Stream` rows, reducing worker RSS during the channel phase. - **XMLTV EPG export streams without holding the full guide in the worker.** `generate_epg()` streams incrementally; on a cache miss each chunk is pushed to a Redis list as it is yielded (no `''.join()` in the worker). Repeat requests within 300s stream chunks back one at a time from Redis. Post-export `malloc_trim` runs after cold builds. Real programmes use fast `(epg_id, id)` keyset pagination with a per-source `start_time` sort before emit. - **EPG export no longer fetches the multi-MB `programme_index` per channel.** The channel query `select_related`s `epg_data__epg_source`, which pulled and JSON-parsed each source's byte-offset `programme_index` blob once per channel (~13s of the request on a ~2000-channel guide). It is now `defer()`red since EPG generation never reads it. - **`ProgramData` composite index `(epg_id, id)`.** The EPG export scans hundreds of thousands of programmes with keyset pagination on `(epg_id, id)`; without a matching index PostgreSQL re-sorted every chunk. A composite index (created `CONCURRENTLY` on PostgreSQL so it does not lock the table) lets each chunk use an ordered index range scan. diff --git a/apps/output/epg.py b/apps/output/epg.py index 354d6e5b..4e11fbe8 100644 --- a/apps/output/epg.py +++ b/apps/output/epg.py @@ -1188,7 +1188,7 @@ def generate_epg(request, profile_name=None, user=None): # once per channel (was ~13s of the request on large guides). .defer("epg_data__epg_source__programme_index") .prefetch_related( - Prefetch('streams', queryset=Stream.objects.order_by('channelstream__order')) + Prefetch('streams', queryset=Stream.objects.only('id', 'name').order_by('channelstream__order')) ) ) channel_count = len(channels) @@ -1386,6 +1386,7 @@ def generate_epg(request, profile_name=None, user=None): current_epg_id = None channel_ids_for_epg = None + escaped_primary_cid = None pending = [] program_batch = [] chunk_size = _EPG_PROGRAM_DB_CHUNK_SIZE @@ -1399,8 +1400,7 @@ def generate_epg(request, profile_name=None, user=None): return pending.sort(key=lambda row: (row[0], row[1])) escaped_primary = ( - html.escape(channel_ids_for_epg[0]) - if len(channel_ids_for_epg) > 1 else None + escaped_primary_cid if len(channel_ids_for_epg) > 1 else None ) for _, _, xml_text in pending: program_batch.append(xml_text) @@ -1435,8 +1435,8 @@ def generate_epg(request, profile_name=None, user=None): yield from flush_pending() current_epg_id = epg_id channel_ids_for_epg = real_epg_map[epg_id] + escaped_primary_cid = html.escape(channel_ids_for_epg[0]) - primary_cid = channel_ids_for_epg[0] # DB datetimes are UTC (USE_TZ=True, TIME_ZONE=UTC); format # directly instead of strftime("%Y%m%d%H%M%S %z"), which is # ~10x slower and dominates XML build over 750k rows. @@ -1445,7 +1445,7 @@ def generate_epg(request, profile_name=None, user=None): start_str = f"{st.year:04d}{st.month:02d}{st.day:02d}{st.hour:02d}{st.minute:02d}{st.second:02d} +0000" stop_str = f"{et.year:04d}{et.month:02d}{et.day:02d}{et.hour:02d}{et.minute:02d}{et.second:02d} +0000" - program_xml = [f' '] + program_xml = [f' '] program_xml.append(f' {html.escape(prog["title"])}') if prog['sub_title']: diff --git a/apps/output/streaming_chunk_cache.py b/apps/output/streaming_chunk_cache.py index f86bb060..4373f5ce 100644 --- a/apps/output/streaming_chunk_cache.py +++ b/apps/output/streaming_chunk_cache.py @@ -114,9 +114,14 @@ def _stream_build(redis, base_key, source, cache_ttl, lock_ttl): django_cache.delete(base_key) # clear any non-chunked entry under this key redis.delete(chunks_key, _ready_key(base_key)) redis.set(status_key, STATUS_BUILDING, ex=lock_ttl) + refresh_interval = max(1, lock_ttl // 4) + last_refresh = 0.0 for chunk in source(): redis.rpush(chunks_key, _encode_chunk(chunk)) - _refresh_build_ttl(redis, base_key, lock_ttl) + now = time.monotonic() + if now - last_refresh >= refresh_interval: + _refresh_build_ttl(redis, base_key, lock_ttl) + last_refresh = now yield chunk redis.set(status_key, STATUS_READY) redis.set(_ready_key(base_key), "1")