From bfec229ca9103e2abe847e1d8b5b1d1037a127be Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 11 Apr 2026 14:57:52 -0500 Subject: [PATCH] =?UTF-8?q?Enhancements:=20-=20Rewrote=20the=20M3U=20line?= =?UTF-8?q?=20parser=20as=20an=20`iter=5Fm3u=5Fentries`=20generator=20that?= =?UTF-8?q?=20owns=20the=20full=20per-entry=20state=20machine.=20Intermedi?= =?UTF-8?q?ate=20directive=20lines=20between=20`#EXTINF`=20and=20the=20str?= =?UTF-8?q?eam=20URL=20are=20now=20handled=20correctly=20rather=20than=20c?= =?UTF-8?q?orrupting=20the=20pending=20entry=20or=20being=20silently=20mis?= =?UTF-8?q?assigned.=20A=20`#EXTINF`=20with=20no=20following=20URL=20is=20?= =?UTF-8?q?discarded=20with=20a=20warning=20instead=20of=20carrying=20over?= =?UTF-8?q?=20a=20`url`-less=20entry=20into=20batch=20processing.=20Attrib?= =?UTF-8?q?ute=20keys=20are=20normalised=20to=20lowercase=20during=20parsi?= =?UTF-8?q?ng=20(provider=20attribute=20names=20remain=20case-insensitive?= =?UTF-8?q?=20end-to-end).=20The=20`#EXTINF`=20attribute=20regex=20is=20pr?= =?UTF-8?q?e-compiled=20at=20module=20load,=20and=20attribute=20lookups=20?= =?UTF-8?q?use=20O(1)=20`dict.get()`=20instead=20of=20linear=20scans=20?= =?UTF-8?q?=E2=80=94=20approximately=2010%=20faster=20parsing=20on=20large?= =?UTF-8?q?=20M3U=20files.=20-=20Added=20support=20for=20the=20`#EXTGRP`?= =?UTF-8?q?=20directive=20in=20M3U=20files.=20When=20a=20`group-title`=20a?= =?UTF-8?q?ttribute=20is=20absent=20from=20the=20`#EXTINF`=20line,=20the?= =?UTF-8?q?=20value=20from=20a=20following=20`#EXTGRP:`=20line=20is=20used?= =?UTF-8?q?=20as=20the=20group.=20An=20explicit=20`group-title`=20attribut?= =?UTF-8?q?e=20always=20takes=20priority.=20(Closes=20#1088)=20-=20Added?= =?UTF-8?q?=20accumulation=20of=20`#EXTVLCOPT`=20directives=20per=20entry.?= =?UTF-8?q?=20Options=20are=20stored=20as=20a=20list=20under=20`vlc=5Fopts?= =?UTF-8?q?`=20inside=20the=20stream's=20`custom=5Fproperties`,=20availabl?= =?UTF-8?q?e=20for=20downstream=20use=20(e.g.=20passing=20VLC-specific=20o?= =?UTF-8?q?ptions=20to=20the=20player).=20This=20is=20for=20a=20planned=20?= =?UTF-8?q?future=20enhancement=20and=20can=20also=20be=20utlized=20with?= =?UTF-8?q?=20the=20API.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 + apps/m3u/tasks.py | 158 +++++++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f0d28f..09c7d675 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Rewrote the M3U line parser as an `iter_m3u_entries` generator that owns the full per-entry state machine. Intermediate directive lines between `#EXTINF` and the stream URL are now handled correctly rather than corrupting the pending entry or being silently misassigned. A `#EXTINF` with no following URL is discarded with a warning instead of carrying over a `url`-less entry into batch processing. Attribute keys are normalised to lowercase during parsing (provider attribute names remain case-insensitive end-to-end). The `#EXTINF` attribute regex is pre-compiled at module load, and attribute lookups use O(1) `dict.get()` instead of linear scans — approximately 10% faster parsing on large M3U files. +- Added support for the `#EXTGRP` directive in M3U files. When a `group-title` attribute is absent from the `#EXTINF` line, the value from a following `#EXTGRP:` line is used as the group. An explicit `group-title` attribute always takes priority. (Closes #1088) +- Added accumulation of `#EXTVLCOPT` directives per entry. Options are stored as a list under `vlc_opts` inside the stream's `custom_properties`, available for downstream use (e.g. passing VLC-specific options to the player). This is for a planned future enhancement and can also be utlized with the API. - M3U stream name parsing now uses the comma text (the canonical display title per the base `#EXTINF` spec) as the primary stream name, falling back to `tvc-guide-title`, then `tvg-name`, rather than preferring `tvg-name` first. Providers that use `tvg-name` as an EPG key and put the human-readable title after the comma will now display the correct name. Providers that duplicate the same value in both fields are unaffected. (Fixes #1081) - FloatingVideo player: the native video controls (timeline, play/pause, volume) are now hidden by default when a live stream starts and only appear when the user hovers over the player. - Enhanced Swagger UI authorization dialog: registered a custom `OpenApiAuthenticationExtension` for `ApiKeyAuthentication` so drf-spectacular now generates an `ApiKeyAuth (apiKey)` entry alongside `jwtAuth`. Both entries include descriptive text linking to the relevant endpoints (`/api/accounts/token/`, `/api/accounts/api-keys/generate/`, `/api/accounts/api-keys/revoke/`). diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index a2b21ee0..8f446b6e 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -39,6 +39,8 @@ logger = logging.getLogger(__name__) BATCH_SIZE = 1500 # Optimized batch size for threading m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") +_EXTINF_ATTR_RE = re.compile(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2') + def fetch_m3u_lines(account, use_cache=False): os.makedirs(m3u_dir, exist_ok=True) @@ -485,18 +487,13 @@ def parse_extinf_line(line: str) -> dict: return None content = line[len("#EXTINF:") :].strip() - # Single pass: extract all attributes AND track the last attribute position - # This regex matches both key="value" and key='value' patterns + # Single pass: extract all attributes AND track the last attribute position. + # Keys are normalised to lowercase so downstream code can use plain dict.get() attrs = {} last_attr_end = 0 - # Use a single regex that handles both quote types. - # Keys must stop at '=' so values like base64-padded URLs ending with '==' - # don't get folded into the preceding attribute name. - for match in re.finditer(r'([^\s=]+)\s*=\s*(["\'])(.*?)\2', content): - key = match.group(1) - value = match.group(3) - attrs[key] = value + for match in _EXTINF_ATTR_RE.finditer(content): + attrs[match.group(1).lower()] = match.group(3) last_attr_end = match.end() # Everything after the last attribute (skipping leading comma and whitespace) is the display name @@ -517,10 +514,77 @@ def parse_extinf_line(line: str) -> dict: # Per the base #EXTINF spec, the comma text is the canonical human-readable title. # Fall back to tvc-guide-title, then tvg-name (which some providers use as an EPG key, # not a display label), and finally the raw content if everything else is empty. - name = display_name or get_case_insensitive_attr(attrs, "tvc-guide-title", None) or get_case_insensitive_attr(attrs, "tvg-name", None) or content.strip() + name = display_name or attrs.get("tvc-guide-title") or attrs.get("tvg-name") or content.strip() return {"attributes": attrs, "display_name": display_name, "name": name} +def iter_m3u_entries(lines): + """ + Generator that yields fully-assembled M3U stream entries from raw lines. + + Each yielded dict is guaranteed to contain a ``url`` key in addition to the + fields produced by :func:`parse_extinf_line` (``attributes``, ``display_name``, + ``name``). Recognised extended-tag lines that appear *between* an ``#EXTINF`` + and its URL are accumulated into the pending entry so they are available for + downstream processing: + + - ``#EXTGRP`` — sets ``attributes["group-title"]`` when no ``group-title`` + attribute was present on the ``#EXTINF`` line (explicit attribute wins). + - ``#EXTVLCOPT`` — stored as a list under the ``vlc_opts`` key. + + Unknown directives (``#KODIPROP``, etc.) and blank lines are + silently skipped while keeping the pending entry intact. A second ``#EXTINF`` + before a URL discards the first entry with a warning. A trailing ``#EXTINF`` + at end-of-file with no URL is also discarded. + + Adding support for a new directive requires only a new ``elif`` branch here; + no other code needs to change. + """ + pending = None + pending_line_num = None + for line_num, raw_line in enumerate(lines, 1): + line = raw_line.strip() + if not line: + continue + + if line.startswith("#EXTINF"): + if pending is not None: + logger.warning( + f"Line {pending_line_num}: #EXTINF had no URL (next #EXTINF at line {line_num}); " + f"discarding entry: {list(pending['attributes'].items())[:3]}" + ) + parsed = parse_extinf_line(line) + if parsed is None: + logger.warning(f"Line {line_num}: Failed to parse #EXTINF: {line[:200]}") + pending = parsed # None if malformed; URL branch guards on `pending is not None` + pending_line_num = line_num + + elif line.startswith("#EXTGRP:"): + # Only apply when group-title is absent — explicit attribute wins. + if pending is not None and "group-title" not in pending["attributes"]: + pending["attributes"]["group-title"] = line[len("#EXTGRP:"):].strip() + # else: #EXTGRP outside an entry, or group-title already set — silently skip + + elif line.startswith("#EXTVLCOPT:"): + if pending is not None: + pending.setdefault("vlc_opts", []).append(line[len("#EXTVLCOPT:"):]) + # else: #EXTVLCOPT outside an entry — silently skip + + elif pending is not None and line.startswith(("http", "rtsp", "rtp", "udp")): + pending["url"] = normalize_stream_url(line) if line.startswith("udp") else line + yield pending + pending = None + pending_line_num = None + + # else: unknown directive or bare content — skip, keeping pending intact + + if pending is not None: + logger.warning( + f"Line {pending_line_num}: #EXTINF at end of file had no URL; " + f"discarding entry: {list(pending['attributes'].items())[:3]}" + ) + + @shared_task def refresh_m3u_accounts(): """Queue background parse for all active M3UAccounts.""" @@ -1113,7 +1177,7 @@ def process_m3u_batch_direct(account_id, batch, groups, hash_keys): "m3u_account": account, "channel_group_id": int(groups.get(group_title)), "stream_hash": stream_hash, - "custom_properties": stream_info["attributes"], + "custom_properties": {**stream_info["attributes"], "vlc_opts": stream_info["vlc_opts"]} if "vlc_opts" in stream_info else stream_info["attributes"], "is_adult": parse_is_adult(stream_info["attributes"].get("is_adult", 0)), "is_stale": False, "stream_id": provider_stream_id, @@ -1481,7 +1545,6 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_sta release_task_lock("refresh_m3u_account_groups", account_id) return error_msg, None else: - # Here's the key change - use the success flag from fetch_m3u_lines lines, success = fetch_m3u_lines(account, use_cache) if not success: # If fetch failed, don't continue processing @@ -1492,71 +1555,20 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False, scan_sta # Log basic file structure for debugging logger.debug(f"Processing {len(lines)} lines from M3U file") - line_count = 0 - extinf_count = 0 - url_count = 0 valid_stream_count = 0 - problematic_lines = [] - for line_index, line in enumerate(lines): - line_count += 1 - line = line.strip() + for entry in iter_m3u_entries(lines): + valid_stream_count += 1 + group_title_attr = get_case_insensitive_attr(entry["attributes"], "group-title", "") + if group_title_attr and group_title_attr not in groups: + logger.debug(f"Found new group for M3U account {account_id}: '{group_title_attr}'") + groups[group_title_attr] = {} + extinf_data.append(entry) - if line.startswith("#EXTINF"): - extinf_count += 1 - parsed = parse_extinf_line(line) - if parsed: - group_title_attr = get_case_insensitive_attr( - parsed["attributes"], "group-title", "" - ) - if group_title_attr: - group_name = group_title_attr - # Log new groups as they're discovered - if group_name not in groups: - logger.debug( - f"Found new group for M3U account {account_id}: '{group_name}'" - ) - groups[group_name] = {} + if valid_stream_count % 1000 == 0: + logger.debug(f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}") - extinf_data.append(parsed) - else: - # Log problematic EXTINF lines - logger.warning( - f"Failed to parse EXTINF at line {line_index+1}: {line[:200]}" - ) - problematic_lines.append((line_index + 1, line[:200])) - - elif extinf_data and (line.startswith("http") or line.startswith("rtsp") or line.startswith("rtp") or line.startswith("udp")): - url_count += 1 - # Normalize UDP URLs only (e.g., remove VLC-specific @ prefix) - normalized_url = normalize_stream_url(line) if line.startswith("udp") else line - # Associate URL with the last EXTINF line - extinf_data[-1]["url"] = normalized_url - valid_stream_count += 1 - - # Periodically log progress for large files - if valid_stream_count % 1000 == 0: - logger.debug( - f"Processed {valid_stream_count} valid streams so far for M3U account: {account_id}" - ) - - # Log summary statistics - logger.info( - f"M3U parsing complete - Lines: {line_count}, EXTINF: {extinf_count}, URLs: {url_count}, Valid streams: {valid_stream_count}" - ) - - if problematic_lines: - logger.warning( - f"Found {len(problematic_lines)} problematic lines during parsing" - ) - for i, (line_num, content) in enumerate( - problematic_lines[:10] - ): # Log max 10 examples - logger.warning(f"Problematic line #{i+1} at line {line_num}: {content}") - if len(problematic_lines) > 10: - logger.warning( - f"... and {len(problematic_lines) - 10} more problematic lines" - ) + logger.info(f"M3U parsing complete - Valid streams: {valid_stream_count}") # Log group statistics logger.info(