diff --git a/CHANGELOG.md b/CHANGELOG.md index 1790f1e9..55962c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U refresh completion counts now reflect actual stream changes.** The "updated" count previously included every existing stream because the summary treated `last_seen` touch-only rows as updates; it now counts only streams whose provider metadata changed. "Total processed" includes unchanged streams separately. The completion message, WebSocket payload, and parsing notification now also report how many streams were **marked stale** (missing from this refresh, pending retention-gated deletion) versus **removed** (deleted this run). - **M3U group processing retries on poisoned DB connections.** `_db_query_with_retry` now treats psycopg desync errors (`DatabaseError`, e.g. `lost synchronization with server`) as transient; `process_groups` relationship loading uses it so a stale Celery worker connection resets once instead of failing the whole refresh. +- **Auto Channel Sync's Find and Replace preview now matches the rename it performs.** The preview rendered the literal `$1` instead of substituting numbered capture groups, and compiled patterns with a stricter engine than the rename, so patterns like `^*` previewed a change the sync silently skipped. The live rename now uses the same `regex` engine as the preview (with a substitution timeout to bound catastrophic backtracking), both apply the `$1`→`\1` conversion through one shared helper, and a rename whose result exceeds the channel-name column length is truncated instead of aborting the whole sync. (Fixes #1332) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.27.2] - 2026-06-30 diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index fd8a4758..3f28903c 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -27,6 +27,7 @@ from apps.accounts.permissions import ( from core.models import UserAgent, CoreSettings from core.utils import RedisClient, safe_upload_path +from apps.m3u.utils import convert_js_numbered_backreferences from .models import ( Stream, @@ -368,6 +369,17 @@ class StreamViewSet(viewsets.ModelViewSet): except re.error as e: exclude_error = str(e) + # The replace field accepts JS-style $1 backreferences, but the regex + # engine honors \1. Convert once so the preview's "after" matches the + # name the live rename produces (apps/m3u/tasks.py sync_auto_channels + # applies the same conversion on the same engine). + replace_repl = convert_js_numbered_backreferences(replace_pat) + + # The live rename caps the result at Channel.name's column length + # before bulk_create; mirror that cap so the preview never shows a + # name the sync would truncate. + name_max_len = Channel._meta.get_field("name").max_length + # Capped at SCAN_CAP to bound memory on huge groups; the # separate COUNT lets the client surface scan_limit_hit when # the preview covers only a sample. @@ -390,11 +402,12 @@ class StreamViewSet(viewsets.ModelViewSet): total_scanned += 1 if find_re is not None: try: - new_name = find_re.sub(replace_pat, name, timeout=REGEX_TIMEOUT) + new_name = find_re.sub(replace_repl, name, timeout=REGEX_TIMEOUT) except (TimeoutError, re.error) as e: find_error = find_error or f"Pattern timed out: {e}" find_re = None continue + new_name = new_name[:name_max_len] if new_name != name: find_match_count += 1 if len(find_matches) < limit: diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 5d9172c7..18365113 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -26,7 +26,7 @@ from core.utils import ( from core.models import CoreSettings, UserAgent from core.xtream_codes import Client as XCClient from core.utils import send_websocket_update -from .utils import normalize_stream_url +from .utils import convert_js_numbered_backreferences, normalize_stream_url logger = logging.getLogger(__name__) @@ -2047,6 +2047,11 @@ def sync_auto_channels(account_id, scan_start_time=None): from apps.epg.models import EPGData from django.utils import timezone + channel_name_max_len = Channel._meta.get_field("name").max_length + # Per-call cap on the rename substitution; bounds catastrophic + # backtracking on a user-supplied pattern so it cannot hang the worker. + rename_regex_timeout = 0.1 + try: account = M3UAccount.objects.get(id=account_id) logger.info(f"Starting auto channel sync for M3U account {account.name}") @@ -2600,17 +2605,30 @@ def sync_auto_channels(account_id, scan_start_time=None): else "" ) try: - # Convert $1, $2, etc. to \1, \2, etc. for consistency with M3U profiles - safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace) - new_name = re.sub( - name_regex_pattern, safe_replace_pattern, original_name + # Use the regex module (not stdlib re) so rename + # patterns match the JS-style semantics the UI + # authors and the preview uses; the timeout bounds + # catastrophic backtracking. + safe_replace_pattern = convert_js_numbered_backreferences(replace) + new_name = regex.sub( + name_regex_pattern, + safe_replace_pattern, + original_name, + timeout=rename_regex_timeout, ) - except re.error as e: + except (regex.error, TimeoutError) as e: logger.warning( f"Regex error for group '{channel_group.name}': {e}. Using original name." ) new_name = original_name + # Channel.name is bounded by the column length; a rename + # that expands past it would otherwise fail the whole + # bulk_create and abort the sync. Cap it so one overlong + # result cannot break the batch, and so the preview (which + # applies the same cap) stays faithful. + new_name = new_name[:channel_name_max_len] + # Check if we already have a channel for this stream existing_channel = existing_channel_map.get(stream.id) diff --git a/apps/m3u/tests/test_rename_preview_parity.py b/apps/m3u/tests/test_rename_preview_parity.py new file mode 100644 index 00000000..4650c50b --- /dev/null +++ b/apps/m3u/tests/test_rename_preview_parity.py @@ -0,0 +1,212 @@ +""" +Differential parity tests: the auto-sync rename preview must predict the exact +name the live rename produces, across a broad matrix of regex strategies. + +Both the preview and the live rename compile with the third-party `regex` +module (for its JS-aligned syntax and per-call timeout). They can only be +trusted together if, for every find/replace a user might author, the preview's +predicted `after` equals the channel name the sync actually writes. + +Each case is run end-to-end: real streams, the real sync_auto_channels rename, +and the real regex-preview endpoint, compared per original stream name. +""" +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import Channel, ChannelGroup, ChannelStream, Stream +from apps.m3u.models import M3UAccount +from apps.m3u.tasks import sync_auto_channels + + +# Diverse names: distinct word-prefixes (collision-resistant), with quality +# tags, brackets, pipes, ampersands, extra whitespace, underscores, dots, CJK, +# and emoji, to exercise anchors, classes, boundaries, and Unicode handling. +NAMES = [ + "Alpha Channel 11", + "Bravo Sports HD", + "Charlie News FHD", + "Delta Movie (2024)", + "Echo [UK] 4K", + "Foxtrot spaced", + "Golf|Pipe|Name", + "Hotel & Inn ", + "India 日本語 77", + "Juliet 📺 88", + "Kilo_under_9", + "Lima.dot.name", +] + +# (find, replace) pairs spanning common user strategies and edge cases. +STRATEGIES = [ + # --- capture groups --- + (r"(.+) Channel (\d+)", r"$1 #$2"), + (r"(\w+) (\w+)", r"$2 $1"), + (r"(.+)", r"$1 - $1"), + (r"(.+)", r"[$1]"), + (r"(.+) (\d+)$", r"$2 $1"), + # --- strip / delete --- + (r" (HD|FHD|4K|SD)\b", r""), + (r"\s+", r" "), + (r"[\[\(].*?[\]\)]", r""), + (r"\d+", r""), + (r"[_.]", r" "), + # --- anchors / inserts --- + (r"^", r"NEW "), + (r"$", r" LIVE"), + (r"^(\w+)", r"<$1>"), + # --- char classes / Unicode (divergence hunters) --- + (r"\w+", r"W"), + (r"\b\w", r"_"), + (r"[A-Z]", r"*"), + (r"\s", r"_"), + (r"[^\x00-\x7F]+", r"?"), + # --- non-capturing / lookaround / in-pattern backref --- + (r"(?:Channel|Movie|News) ", r""), + (r"(\w)\1", r"$1"), + (r"\w+(?= )", r"X"), + (r"(?<=\d)\d", r"#"), + # --- literal $ and odd replacements --- + (r" ", r" $ "), + (r"o", r"0"), + # --- invalid group references (rejected by both engines) --- + (r"(.+)", r"$2"), + (r"(.+)", r"$10"), + # --- rename that expands past Channel.name's column length --- + (r"(.+)", r"$1" * 40), + # --- regex-module syntax: quantified anchor, JS-style and duplicate + # named groups (these transform; both paths use the regex module) --- + (r"^*", r"$"), + (r"(?\d+)", r"S$1"), + (r"(?Px)(?Py)", r"z"), +] + + +class RenamePreviewParityTests(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_rename_parity", password="pw", user_level=10 + ) + cls.client_api = APIClient() + cls.client_api.force_authenticate(user=admin) + cls.account = M3UAccount.objects.create( + name="Rename Parity Provider", + server_url="http://example.com/test.m3u", + ) + + def _sync(self): + return sync_auto_channels( + self.account.id, + scan_start_time=( + timezone.now() - timezone.timedelta(minutes=1) + ).isoformat(), + ) + + def _run_case(self, group_name, find, replace): + """Returns a list of human-readable mismatch strings (empty == parity).""" + group = ChannelGroup.objects.create(name=group_name) + from apps.channels.models import ChannelGroupM3UAccount + + ChannelGroupM3UAccount.objects.create( + m3u_account=self.account, + channel_group=group, + enabled=True, + auto_channel_sync=True, + auto_sync_channel_start=1000, + custom_properties={ + "name_regex_pattern": find, + "name_replace_pattern": replace, + }, + ) + for i, name in enumerate(NAMES): + Stream.objects.create( + name=name, + url=f"http://example.com/{group_name}_{i}.m3u8", + m3u_account=self.account, + channel_group=group, + tvg_id=f"{group_name}-{i}", + last_seen=timezone.now(), + ) + + # --- live rename via sync --- + result = self._sync() + if result.get("status") != "ok": + return [f"[{find!r} -> {replace!r}] sync status={result.get('status')}"] + + channels = Channel.objects.filter( + auto_created_by=self.account, channel_group=group + ) + cs_rows = ChannelStream.objects.filter( + channel__in=channels + ).select_related("channel", "stream") + sync_map = {row.stream.name: row.channel.name for row in cs_rows} + + # --- preview endpoint --- + response = self.client_api.get( + "/api/channels/streams/regex-preview/", + { + "channel_group": group_name, + "find": find, + "replace": replace, + "limit": 50, + }, + ) + if response.status_code != 200: + return [f"[{find!r} -> {replace!r}] preview HTTP {response.status_code}"] + data = response.data + # When the preview reports find_error it predicts no rename at all. + preview_map = {name: name for name in NAMES} + if "find_error" not in data: + for m in data.get("find_matches", []): + preview_map[m["before"]] = m["after"] + + mismatches = [] + for name in NAMES: + sync_after = sync_map.get(name) + if sync_after is None: + mismatches.append( + f"[{find!r} -> {replace!r}] {name!r}: no channel created" + ) + continue + preview_after = preview_map[name] + if preview_after != sync_after: + mismatches.append( + f"[{find!r} -> {replace!r}] {name!r}: " + f"preview={preview_after!r} sync={sync_after!r} " + f"(find_error={data.get('find_error')!r})" + ) + return mismatches + + def test_preview_predicts_rename_across_strategies(self): + all_mismatches = [] + for idx, (find, replace) in enumerate(STRATEGIES): + all_mismatches.extend( + self._run_case(f"ParityG{idx}", find, replace) + ) + self.assertEqual( + all_mismatches, + [], + "Preview diverged from the live rename:\n" + + "\n".join(all_mismatches), + ) + + def test_overlong_rename_is_bounded_not_aborting_sync(self): + # A rename that expands past Channel.name's column length must not + # abort the bulk_create sync. Both sync and preview cap at the column + # length so the channel is created (truncated) and the preview shows + # the same bounded name. + max_len = Channel._meta.get_field("name").max_length + mismatches = self._run_case("OverlongG", r"(.+)", "$1" * 40) + self.assertEqual(mismatches, [], "\n".join(mismatches)) + + group = ChannelGroup.objects.get(name="OverlongG") + channels = Channel.objects.filter( + auto_created_by=self.account, channel_group=group + ) + self.assertEqual(channels.count(), len(NAMES)) + self.assertTrue(all(len(c.name) <= max_len for c in channels)) + self.assertTrue(any(len(c.name) == max_len for c in channels)) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 8ae09fa8..75bc5d03 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -763,6 +763,128 @@ class RegexPreviewTests(TestCase): self.assertEqual(response.data["total_scanned"], 3) self.assertFalse(response.data["scan_limit_hit"]) + def test_find_replace_applies_numbered_capture_group(self): + # The replace field accepts JS-style $1 backreferences, but the regex + # engine expects \1. Without the conversion the preview echoes the + # literal "$1", so the previewed "after" disagrees with the name the + # live rename produces. + account = self._make_account() + group = _make_group(name="Sports") + Stream.objects.create( + name="High Limit Racing at Eagle @ Jun 9 7:00 PM", + url="http://example.com/hlr.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Sports", "find": r"(.+) @.*", "replace": "$1"}, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["find_match_count"], 1) + after = response.data["find_matches"][0]["after"] + self.assertEqual(after, "High Limit Racing at Eagle") + self.assertNotIn("$1", after) + + def test_preview_after_matches_live_sync_rename(self): + # Guards the defect class: the preview and the live rename are + # separate code paths that must convert the replacement identically, + # so the preview can never promise an output the sync would not yield. + name = "High Limit Racing at Eagle @ Jun 9 7:00 PM" + account = self._make_account() + group = _make_group(name="Racing") + _attach_group_to_account( + account, + group, + custom_properties={ + "name_regex_pattern": r"(.+) @.*", + "name_replace_pattern": "$1", + }, + ) + _make_stream(account, group, name=name, tvg_id="hlr") + + result = _sync(account) + self.assertEqual(result.get("status"), "ok") + channel = Channel.objects.get(auto_created=True, auto_created_by=account) + live_name = channel.name + + client = self._client() + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Racing", "find": r"(.+) @.*", "replace": "$1"}, + ) + + self.assertEqual(response.status_code, 200) + preview_after = response.data["find_matches"][0]["after"] + self.assertEqual(preview_after, live_name) + self.assertEqual(preview_after, "High Limit Racing at Eagle") + + def test_regex_engine_pattern_transforms_in_preview(self): + # Both the preview and the live rename use the regex module, which is + # more permissive than stdlib re and matches the JS-style syntax the UI + # authors. A quantified anchor like "^*" (which stdlib re rejects) + # compiles and transforms rather than reporting an error. + account = self._make_account() + group = _make_group(name="Sports") + Stream.objects.create( + name="Doc95", + url="http://example.com/doc95.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Sports", "find": "^*", "replace": "$"}, + ) + + self.assertEqual(response.status_code, 200) + self.assertNotIn("find_error", response.data) + self.assertEqual(response.data["find_match_count"], 1) + # ^* matches the empty string at every position, so the literal $ + # replacement is inserted between characters. + self.assertEqual( + response.data["find_matches"][0]["after"], "$D$o$c$9$5$" + ) + + def test_preview_and_sync_agree_on_regex_only_pattern(self): + # Parity guard for the engine alignment: a pattern valid in regex but + # not stdlib re must transform identically in the sync and the preview, + # rather than diverging (the sync no longer silently keeps the + # original name for these patterns). + name = "Doc95" + account = self._make_account() + group = _make_group(name="Docs") + _attach_group_to_account( + account, + group, + custom_properties={ + "name_regex_pattern": "^*", + "name_replace_pattern": "$", + }, + ) + _make_stream(account, group, name=name, tvg_id="doc95") + + result = _sync(account) + self.assertEqual(result.get("status"), "ok") + channel = Channel.objects.get(auto_created=True, auto_created_by=account) + live_name = channel.name + self.assertNotEqual(live_name, name) + + client = self._client() + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Docs", "find": "^*", "replace": "$"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["find_matches"][0]["after"], live_name) + def test_filter_returns_matched_names_with_count(self): account = self._make_account() group = _make_group(name="Sports") diff --git a/apps/m3u/utils.py b/apps/m3u/utils.py index 598ef713..dbf8d91a 100644 --- a/apps/m3u/utils.py +++ b/apps/m3u/utils.py @@ -1,4 +1,5 @@ # apps/m3u/utils.py +import regex import threading import logging from django.db import models @@ -9,6 +10,18 @@ active_streams_map = {} logger = logging.getLogger(__name__) +def convert_js_numbered_backreferences(replacement): + """Translate JS-style ``$1``/``$2`` backreferences to Python ``\\1``/``\\2``. + + Auto-sync replace patterns are authored in JS regex syntax, but Python's + regex engines honor backslash backreferences, not ``$1``. The live rename + and the UI preview must convert identically, so both call this single + helper and cannot drift apart (otherwise the preview promises an output + the sync would never produce). + """ + return regex.sub(r"\$(\d+)", r"\\\1", replacement) + + def normalize_stream_url(url): """ Normalize stream URLs for compatibility with FFmpeg.