From 40ec6b6339915a5ad4487beea5d4ff1497170984 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:45:42 -0500 Subject: [PATCH 1/2] fix(channels): align the auto-sync rename preview and live rename on the regex module The Find and Replace preview did not correctly reflect the rename the sync performs, and the rename engine differed from the preview engine. - The preview rendered the literal $1 instead of the substituted capture group, because the replacement was passed straight into the regex engine, which honors \1, not the JS-style $1 the field accepts. - The preview compiled patterns with the regex module while the live rename used stdlib re, so patterns valid in regex but not re (for example ^*) previewed a transform the sync silently skipped. - A rename that expanded a name past the Channel.name column length aborted the whole bulk_create sync, while the preview showed the full name. - Convert JS-style $1 backreferences to \1 via a shared helper used by both the preview and the live rename. - Switch the live rename from re.sub to regex.sub, matching the preview engine and the sync's own include/exclude filters, with a timeout to bound catastrophic backtracking on user patterns. - Cap the rename result at the Channel.name column length in both paths, so an over-length result cannot abort the sync. - Add unit, integration, and differential parity tests covering the above. --- CHANGELOG.md | 1 + apps/channels/api_views.py | 15 +- apps/m3u/tasks.py | 30 ++- apps/m3u/tests/test_rename_preview_parity.py | 212 +++++++++++++++++++ apps/m3u/tests/test_sync_correctness.py | 122 +++++++++++ apps/m3u/utils.py | 13 ++ 6 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 apps/m3u/tests/test_rename_preview_parity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a20b9032..81a4248d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **M3U group processing retries on poisoned DB connections.** `_db_query_with_retry` now treats psycopg desync errors (`DatabaseError`, e.g. `lost synchronization with server`) as transient; `process_groups` relationship loading uses it so a stale Celery worker connection resets once instead of failing the whole refresh. +- **Auto Channel Sync's Find and Replace preview now matches the rename it performs.** The preview rendered the literal `$1` instead of substituting numbered capture groups, and compiled patterns with a stricter engine than the rename, so patterns like `^*` previewed a change the sync silently skipped. The live rename now uses the same `regex` engine as the preview (with a substitution timeout to bound catastrophic backtracking), both apply the `$1`→`\1` conversion through one shared helper, and a rename whose result exceeds the channel-name column length is truncated instead of aborting the whole sync. (Fixes #1332) ## [0.27.1] - 2026-06-25 diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index fd8a4758..3f28903c 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -27,6 +27,7 @@ from apps.accounts.permissions import ( from core.models import UserAgent, CoreSettings from core.utils import RedisClient, safe_upload_path +from apps.m3u.utils import convert_js_numbered_backreferences from .models import ( Stream, @@ -368,6 +369,17 @@ class StreamViewSet(viewsets.ModelViewSet): except re.error as e: exclude_error = str(e) + # The replace field accepts JS-style $1 backreferences, but the regex + # engine honors \1. Convert once so the preview's "after" matches the + # name the live rename produces (apps/m3u/tasks.py sync_auto_channels + # applies the same conversion on the same engine). + replace_repl = convert_js_numbered_backreferences(replace_pat) + + # The live rename caps the result at Channel.name's column length + # before bulk_create; mirror that cap so the preview never shows a + # name the sync would truncate. + name_max_len = Channel._meta.get_field("name").max_length + # Capped at SCAN_CAP to bound memory on huge groups; the # separate COUNT lets the client surface scan_limit_hit when # the preview covers only a sample. @@ -390,11 +402,12 @@ class StreamViewSet(viewsets.ModelViewSet): total_scanned += 1 if find_re is not None: try: - new_name = find_re.sub(replace_pat, name, timeout=REGEX_TIMEOUT) + new_name = find_re.sub(replace_repl, name, timeout=REGEX_TIMEOUT) except (TimeoutError, re.error) as e: find_error = find_error or f"Pattern timed out: {e}" find_re = None continue + new_name = new_name[:name_max_len] if new_name != name: find_match_count += 1 if len(find_matches) < limit: diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index a5894892..4b570d93 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -26,7 +26,7 @@ from core.utils import ( from core.models import CoreSettings, UserAgent from core.xtream_codes import Client as XCClient from core.utils import send_websocket_update -from .utils import normalize_stream_url +from .utils import convert_js_numbered_backreferences, normalize_stream_url logger = logging.getLogger(__name__) @@ -2019,6 +2019,11 @@ def sync_auto_channels(account_id, scan_start_time=None): from apps.epg.models import EPGData from django.utils import timezone + channel_name_max_len = Channel._meta.get_field("name").max_length + # Per-call cap on the rename substitution; bounds catastrophic + # backtracking on a user-supplied pattern so it cannot hang the worker. + rename_regex_timeout = 0.1 + try: account = M3UAccount.objects.get(id=account_id) logger.info(f"Starting auto channel sync for M3U account {account.name}") @@ -2572,17 +2577,30 @@ def sync_auto_channels(account_id, scan_start_time=None): else "" ) try: - # Convert $1, $2, etc. to \1, \2, etc. for consistency with M3U profiles - safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace) - new_name = re.sub( - name_regex_pattern, safe_replace_pattern, original_name + # Use the regex module (not stdlib re) so rename + # patterns match the JS-style semantics the UI + # authors and the preview uses; the timeout bounds + # catastrophic backtracking. + safe_replace_pattern = convert_js_numbered_backreferences(replace) + new_name = regex.sub( + name_regex_pattern, + safe_replace_pattern, + original_name, + timeout=rename_regex_timeout, ) - except re.error as e: + except (regex.error, TimeoutError) as e: logger.warning( f"Regex error for group '{channel_group.name}': {e}. Using original name." ) new_name = original_name + # Channel.name is bounded by the column length; a rename + # that expands past it would otherwise fail the whole + # bulk_create and abort the sync. Cap it so one overlong + # result cannot break the batch, and so the preview (which + # applies the same cap) stays faithful. + new_name = new_name[:channel_name_max_len] + # Check if we already have a channel for this stream existing_channel = existing_channel_map.get(stream.id) diff --git a/apps/m3u/tests/test_rename_preview_parity.py b/apps/m3u/tests/test_rename_preview_parity.py new file mode 100644 index 00000000..4650c50b --- /dev/null +++ b/apps/m3u/tests/test_rename_preview_parity.py @@ -0,0 +1,212 @@ +""" +Differential parity tests: the auto-sync rename preview must predict the exact +name the live rename produces, across a broad matrix of regex strategies. + +Both the preview and the live rename compile with the third-party `regex` +module (for its JS-aligned syntax and per-call timeout). They can only be +trusted together if, for every find/replace a user might author, the preview's +predicted `after` equals the channel name the sync actually writes. + +Each case is run end-to-end: real streams, the real sync_auto_channels rename, +and the real regex-preview endpoint, compared per original stream name. +""" +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import Channel, ChannelGroup, ChannelStream, Stream +from apps.m3u.models import M3UAccount +from apps.m3u.tasks import sync_auto_channels + + +# Diverse names: distinct word-prefixes (collision-resistant), with quality +# tags, brackets, pipes, ampersands, extra whitespace, underscores, dots, CJK, +# and emoji, to exercise anchors, classes, boundaries, and Unicode handling. +NAMES = [ + "Alpha Channel 11", + "Bravo Sports HD", + "Charlie News FHD", + "Delta Movie (2024)", + "Echo [UK] 4K", + "Foxtrot spaced", + "Golf|Pipe|Name", + "Hotel & Inn ", + "India 日本語 77", + "Juliet 📺 88", + "Kilo_under_9", + "Lima.dot.name", +] + +# (find, replace) pairs spanning common user strategies and edge cases. +STRATEGIES = [ + # --- capture groups --- + (r"(.+) Channel (\d+)", r"$1 #$2"), + (r"(\w+) (\w+)", r"$2 $1"), + (r"(.+)", r"$1 - $1"), + (r"(.+)", r"[$1]"), + (r"(.+) (\d+)$", r"$2 $1"), + # --- strip / delete --- + (r" (HD|FHD|4K|SD)\b", r""), + (r"\s+", r" "), + (r"[\[\(].*?[\]\)]", r""), + (r"\d+", r""), + (r"[_.]", r" "), + # --- anchors / inserts --- + (r"^", r"NEW "), + (r"$", r" LIVE"), + (r"^(\w+)", r"<$1>"), + # --- char classes / Unicode (divergence hunters) --- + (r"\w+", r"W"), + (r"\b\w", r"_"), + (r"[A-Z]", r"*"), + (r"\s", r"_"), + (r"[^\x00-\x7F]+", r"?"), + # --- non-capturing / lookaround / in-pattern backref --- + (r"(?:Channel|Movie|News) ", r""), + (r"(\w)\1", r"$1"), + (r"\w+(?= )", r"X"), + (r"(?<=\d)\d", r"#"), + # --- literal $ and odd replacements --- + (r" ", r" $ "), + (r"o", r"0"), + # --- invalid group references (rejected by both engines) --- + (r"(.+)", r"$2"), + (r"(.+)", r"$10"), + # --- rename that expands past Channel.name's column length --- + (r"(.+)", r"$1" * 40), + # --- regex-module syntax: quantified anchor, JS-style and duplicate + # named groups (these transform; both paths use the regex module) --- + (r"^*", r"$"), + (r"(?\d+)", r"S$1"), + (r"(?Px)(?Py)", r"z"), +] + + +class RenamePreviewParityTests(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + from rest_framework.test import APIClient + from apps.accounts.models import User + + admin = User.objects.create_superuser( + username="admin_rename_parity", password="pw", user_level=10 + ) + cls.client_api = APIClient() + cls.client_api.force_authenticate(user=admin) + cls.account = M3UAccount.objects.create( + name="Rename Parity Provider", + server_url="http://example.com/test.m3u", + ) + + def _sync(self): + return sync_auto_channels( + self.account.id, + scan_start_time=( + timezone.now() - timezone.timedelta(minutes=1) + ).isoformat(), + ) + + def _run_case(self, group_name, find, replace): + """Returns a list of human-readable mismatch strings (empty == parity).""" + group = ChannelGroup.objects.create(name=group_name) + from apps.channels.models import ChannelGroupM3UAccount + + ChannelGroupM3UAccount.objects.create( + m3u_account=self.account, + channel_group=group, + enabled=True, + auto_channel_sync=True, + auto_sync_channel_start=1000, + custom_properties={ + "name_regex_pattern": find, + "name_replace_pattern": replace, + }, + ) + for i, name in enumerate(NAMES): + Stream.objects.create( + name=name, + url=f"http://example.com/{group_name}_{i}.m3u8", + m3u_account=self.account, + channel_group=group, + tvg_id=f"{group_name}-{i}", + last_seen=timezone.now(), + ) + + # --- live rename via sync --- + result = self._sync() + if result.get("status") != "ok": + return [f"[{find!r} -> {replace!r}] sync status={result.get('status')}"] + + channels = Channel.objects.filter( + auto_created_by=self.account, channel_group=group + ) + cs_rows = ChannelStream.objects.filter( + channel__in=channels + ).select_related("channel", "stream") + sync_map = {row.stream.name: row.channel.name for row in cs_rows} + + # --- preview endpoint --- + response = self.client_api.get( + "/api/channels/streams/regex-preview/", + { + "channel_group": group_name, + "find": find, + "replace": replace, + "limit": 50, + }, + ) + if response.status_code != 200: + return [f"[{find!r} -> {replace!r}] preview HTTP {response.status_code}"] + data = response.data + # When the preview reports find_error it predicts no rename at all. + preview_map = {name: name for name in NAMES} + if "find_error" not in data: + for m in data.get("find_matches", []): + preview_map[m["before"]] = m["after"] + + mismatches = [] + for name in NAMES: + sync_after = sync_map.get(name) + if sync_after is None: + mismatches.append( + f"[{find!r} -> {replace!r}] {name!r}: no channel created" + ) + continue + preview_after = preview_map[name] + if preview_after != sync_after: + mismatches.append( + f"[{find!r} -> {replace!r}] {name!r}: " + f"preview={preview_after!r} sync={sync_after!r} " + f"(find_error={data.get('find_error')!r})" + ) + return mismatches + + def test_preview_predicts_rename_across_strategies(self): + all_mismatches = [] + for idx, (find, replace) in enumerate(STRATEGIES): + all_mismatches.extend( + self._run_case(f"ParityG{idx}", find, replace) + ) + self.assertEqual( + all_mismatches, + [], + "Preview diverged from the live rename:\n" + + "\n".join(all_mismatches), + ) + + def test_overlong_rename_is_bounded_not_aborting_sync(self): + # A rename that expands past Channel.name's column length must not + # abort the bulk_create sync. Both sync and preview cap at the column + # length so the channel is created (truncated) and the preview shows + # the same bounded name. + max_len = Channel._meta.get_field("name").max_length + mismatches = self._run_case("OverlongG", r"(.+)", "$1" * 40) + self.assertEqual(mismatches, [], "\n".join(mismatches)) + + group = ChannelGroup.objects.get(name="OverlongG") + channels = Channel.objects.filter( + auto_created_by=self.account, channel_group=group + ) + self.assertEqual(channels.count(), len(NAMES)) + self.assertTrue(all(len(c.name) <= max_len for c in channels)) + self.assertTrue(any(len(c.name) == max_len for c in channels)) diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 8ae09fa8..75bc5d03 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -763,6 +763,128 @@ class RegexPreviewTests(TestCase): self.assertEqual(response.data["total_scanned"], 3) self.assertFalse(response.data["scan_limit_hit"]) + def test_find_replace_applies_numbered_capture_group(self): + # The replace field accepts JS-style $1 backreferences, but the regex + # engine expects \1. Without the conversion the preview echoes the + # literal "$1", so the previewed "after" disagrees with the name the + # live rename produces. + account = self._make_account() + group = _make_group(name="Sports") + Stream.objects.create( + name="High Limit Racing at Eagle @ Jun 9 7:00 PM", + url="http://example.com/hlr.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Sports", "find": r"(.+) @.*", "replace": "$1"}, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["find_match_count"], 1) + after = response.data["find_matches"][0]["after"] + self.assertEqual(after, "High Limit Racing at Eagle") + self.assertNotIn("$1", after) + + def test_preview_after_matches_live_sync_rename(self): + # Guards the defect class: the preview and the live rename are + # separate code paths that must convert the replacement identically, + # so the preview can never promise an output the sync would not yield. + name = "High Limit Racing at Eagle @ Jun 9 7:00 PM" + account = self._make_account() + group = _make_group(name="Racing") + _attach_group_to_account( + account, + group, + custom_properties={ + "name_regex_pattern": r"(.+) @.*", + "name_replace_pattern": "$1", + }, + ) + _make_stream(account, group, name=name, tvg_id="hlr") + + result = _sync(account) + self.assertEqual(result.get("status"), "ok") + channel = Channel.objects.get(auto_created=True, auto_created_by=account) + live_name = channel.name + + client = self._client() + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Racing", "find": r"(.+) @.*", "replace": "$1"}, + ) + + self.assertEqual(response.status_code, 200) + preview_after = response.data["find_matches"][0]["after"] + self.assertEqual(preview_after, live_name) + self.assertEqual(preview_after, "High Limit Racing at Eagle") + + def test_regex_engine_pattern_transforms_in_preview(self): + # Both the preview and the live rename use the regex module, which is + # more permissive than stdlib re and matches the JS-style syntax the UI + # authors. A quantified anchor like "^*" (which stdlib re rejects) + # compiles and transforms rather than reporting an error. + account = self._make_account() + group = _make_group(name="Sports") + Stream.objects.create( + name="Doc95", + url="http://example.com/doc95.m3u8", + m3u_account=account, + channel_group=group, + last_seen=timezone.now(), + ) + client = self._client() + + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Sports", "find": "^*", "replace": "$"}, + ) + + self.assertEqual(response.status_code, 200) + self.assertNotIn("find_error", response.data) + self.assertEqual(response.data["find_match_count"], 1) + # ^* matches the empty string at every position, so the literal $ + # replacement is inserted between characters. + self.assertEqual( + response.data["find_matches"][0]["after"], "$D$o$c$9$5$" + ) + + def test_preview_and_sync_agree_on_regex_only_pattern(self): + # Parity guard for the engine alignment: a pattern valid in regex but + # not stdlib re must transform identically in the sync and the preview, + # rather than diverging (the sync no longer silently keeps the + # original name for these patterns). + name = "Doc95" + account = self._make_account() + group = _make_group(name="Docs") + _attach_group_to_account( + account, + group, + custom_properties={ + "name_regex_pattern": "^*", + "name_replace_pattern": "$", + }, + ) + _make_stream(account, group, name=name, tvg_id="doc95") + + result = _sync(account) + self.assertEqual(result.get("status"), "ok") + channel = Channel.objects.get(auto_created=True, auto_created_by=account) + live_name = channel.name + self.assertNotEqual(live_name, name) + + client = self._client() + response = client.get( + "/api/channels/streams/regex-preview/", + {"channel_group": "Docs", "find": "^*", "replace": "$"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["find_matches"][0]["after"], live_name) + def test_filter_returns_matched_names_with_count(self): account = self._make_account() group = _make_group(name="Sports") diff --git a/apps/m3u/utils.py b/apps/m3u/utils.py index 598ef713..dbf8d91a 100644 --- a/apps/m3u/utils.py +++ b/apps/m3u/utils.py @@ -1,4 +1,5 @@ # apps/m3u/utils.py +import regex import threading import logging from django.db import models @@ -9,6 +10,18 @@ active_streams_map = {} logger = logging.getLogger(__name__) +def convert_js_numbered_backreferences(replacement): + """Translate JS-style ``$1``/``$2`` backreferences to Python ``\\1``/``\\2``. + + Auto-sync replace patterns are authored in JS regex syntax, but Python's + regex engines honor backslash backreferences, not ``$1``. The live rename + and the UI preview must convert identically, so both call this single + helper and cannot drift apart (otherwise the preview promises an output + the sync would never produce). + """ + return regex.sub(r"\$(\d+)", r"\\\1", replacement) + + def normalize_stream_url(url): """ Normalize stream URLs for compatibility with FFmpeg. From eac3849486e5ce39a1b9ae367dc493cde46afcb4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 30 Jun 2026 11:57:38 -0500 Subject: [PATCH 2/2] changelog: add thanks to submitter. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6a2687..55962c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **M3U refresh completion counts now reflect actual stream changes.** The "updated" count previously included every existing stream because the summary treated `last_seen` touch-only rows as updates; it now counts only streams whose provider metadata changed. "Total processed" includes unchanged streams separately. The completion message, WebSocket payload, and parsing notification now also report how many streams were **marked stale** (missing from this refresh, pending retention-gated deletion) versus **removed** (deleted this run). - **M3U group processing retries on poisoned DB connections.** `_db_query_with_retry` now treats psycopg desync errors (`DatabaseError`, e.g. `lost synchronization with server`) as transient; `process_groups` relationship loading uses it so a stale Celery worker connection resets once instead of failing the whole refresh. -- **Auto Channel Sync's Find and Replace preview now matches the rename it performs.** The preview rendered the literal `$1` instead of substituting numbered capture groups, and compiled patterns with a stricter engine than the rename, so patterns like `^*` previewed a change the sync silently skipped. The live rename now uses the same `regex` engine as the preview (with a substitution timeout to bound catastrophic backtracking), both apply the `$1`→`\1` conversion through one shared helper, and a rename whose result exceeds the channel-name column length is truncated instead of aborting the whole sync. (Fixes #1332) +- **Auto Channel Sync's Find and Replace preview now matches the rename it performs.** The preview rendered the literal `$1` instead of substituting numbered capture groups, and compiled patterns with a stricter engine than the rename, so patterns like `^*` previewed a change the sync silently skipped. The live rename now uses the same `regex` engine as the preview (with a substitution timeout to bound catastrophic backtracking), both apply the `$1`→`\1` conversion through one shared helper, and a rename whose result exceeds the channel-name column length is truncated instead of aborting the whole sync. (Fixes #1332) — Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.27.2] - 2026-06-30