From a83b9fdf624a718896d32e5af026e81f53d94826 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 12 Apr 2026 15:37:28 -0500 Subject: [PATCH] Enhancement: EPG channel scanning now automatically removes stale `EPGData` entries. --- CHANGELOG.md | 1 + apps/epg/tasks.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c948a2a7..aad0afec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- EPG channel scanning now automatically removes stale `EPGData` entries. tvg-ids that were present in a previous scan but are no longer found in the upstream source, provided they are not mapped to any channel. This prevents unbounded database bloat over time. Entries mapped to at least one channel are always preserved. - Rewrote the M3U line parser as an `iter_m3u_entries` generator that owns the full per-entry state machine. Intermediate directive lines between `#EXTINF` and the stream URL are now handled correctly rather than corrupting the pending entry or being silently misassigned. A `#EXTINF` with no following URL is discarded with a warning instead of carrying over a `url`-less entry into batch processing. Attribute keys are normalised to lowercase during parsing (provider attribute names remain case-insensitive end-to-end). The `#EXTINF` attribute regex is pre-compiled at module load, and attribute lookups use O(1) `dict.get()` instead of linear scans — approximately 10% faster parsing on large M3U files. - Added support for the `#EXTGRP` directive in M3U files. When a `group-title` attribute is absent from the `#EXTINF` line, the value from a following `#EXTGRP:` line is used as the group. An explicit `group-title` attribute always takes priority. (Closes #1088) - Added accumulation of `#EXTVLCOPT` directives per entry. Options are stored as a list under `vlc_opts` inside the stream's `custom_properties`, available for downstream use (e.g. passing VLC-specific options to the player). This is for a planned future enhancement and can also be utlized with the API. diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 9c877f4b..7700ad1b 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -938,7 +938,8 @@ def parse_channels_only(source): # Replace full dictionary load with more efficient lookup set existing_tvg_ids = set() - existing_epgs = {} # Initialize the dictionary that will lazily load objects + existing_epgs = {} + scanned_tvg_ids = set() # Track tvg_ids seen in the current scan for stale cleanup last_id = 0 chunk_size = 5000 @@ -1006,6 +1007,7 @@ def parse_channels_only(source): channel_count += 1 tvg_id = elem.get('id', '').strip() if tvg_id: + scanned_tvg_ids.add(tvg_id) display_name = None icon_url = None for child in elem: @@ -1153,6 +1155,16 @@ def parse_channels_only(source): if epgs_to_update: EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") + + # Clean up stale EPGData: entries that existed before the scan but weren't seen, and aren't mapped to any channel. + # Use existing_tvg_ids - scanned_tvg_ids to avoid a full-table scan with a large EXCLUDE list. + potentially_stale = existing_tvg_ids - scanned_tvg_ids + if potentially_stale: + stale_qs = EPGData.objects.filter(epg_source=source, tvg_id__in=potentially_stale, channels__isnull=True) + deleted_count, _ = stale_qs.delete() + if deleted_count: + logger.info(f"[parse_channels_only] Cleaned up {deleted_count} stale EPG entries not in current scan and unmapped to any channel") + if process: logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") @@ -1219,6 +1231,9 @@ def parse_channels_only(source): existing_epgs = None epgs_to_create = None epgs_to_update = None + if 'scanned_tvg_ids' in locals() and scanned_tvg_ids is not None: + scanned_tvg_ids.clear() + scanned_tvg_ids = None cleanup_memory(log_usage=should_log_memory, force_collection=True) except Exception as e: logger.warning(f"Cleanup error: {e}")