Merge pull request #1367 from CodeBormen/fix/1332-rename-preview

fix(m3u): align auto-sync rename preview with the live rename

CHANGELOG.md

@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - **M3U refresh completion counts now reflect actual stream changes.** The "updated" count previously included every existing stream because the summary treated `last_seen` touch-only rows as updates; it now counts only streams whose provider metadata changed. "Total processed" includes unchanged streams separately. The completion message, WebSocket payload, and parsing notification now also report how many streams were **marked stale** (missing from this refresh, pending retention-gated deletion) versus **removed** (deleted this run).
 - **M3U group processing retries on poisoned DB connections.** `_db_query_with_retry` now treats psycopg desync errors (`DatabaseError`, e.g. `lost synchronization with server`) as transient; `process_groups` relationship loading uses it so a stale Celery worker connection resets once instead of failing the whole refresh.
+- **Auto Channel Sync's Find and Replace preview now matches the rename it performs.** The preview rendered the literal `$1` instead of substituting numbered capture groups, and compiled patterns with a stricter engine than the rename, so patterns like `^*` previewed a change the sync silently skipped. The live rename now uses the same `regex` engine as the preview (with a substitution timeout to bound catastrophic backtracking), both apply the `$1`→`\1` conversion through one shared helper, and a rename whose result exceeds the channel-name column length is truncated instead of aborting the whole sync. (Fixes #1332) — Thanks [@CodeBormen](https://github.com/CodeBormen)
 
 ## [0.27.2] - 2026-06-30
 

apps/channels/api_views.py

@@ -27,6 +27,7 @@ from apps.accounts.permissions import (
 
 from core.models import UserAgent, CoreSettings
 from core.utils import RedisClient, safe_upload_path
+from apps.m3u.utils import convert_js_numbered_backreferences
 
 from .models import (
     Stream,
@@ -368,6 +369,17 @@ class StreamViewSet(viewsets.ModelViewSet):
             except re.error as e:
                 exclude_error = str(e)
 
+        # The replace field accepts JS-style $1 backreferences, but the regex
+        # engine honors \1. Convert once so the preview's "after" matches the
+        # name the live rename produces (apps/m3u/tasks.py sync_auto_channels
+        # applies the same conversion on the same engine).
+        replace_repl = convert_js_numbered_backreferences(replace_pat)
+
+        # The live rename caps the result at Channel.name's column length
+        # before bulk_create; mirror that cap so the preview never shows a
+        # name the sync would truncate.
+        name_max_len = Channel._meta.get_field("name").max_length
+
         # Capped at SCAN_CAP to bound memory on huge groups; the
         # separate COUNT lets the client surface scan_limit_hit when
         # the preview covers only a sample.
@@ -390,11 +402,12 @@ class StreamViewSet(viewsets.ModelViewSet):
             total_scanned += 1
             if find_re is not None:
                 try:
-                    new_name = find_re.sub(replace_pat, name, timeout=REGEX_TIMEOUT)
+                    new_name = find_re.sub(replace_repl, name, timeout=REGEX_TIMEOUT)
                 except (TimeoutError, re.error) as e:
                     find_error = find_error or f"Pattern timed out: {e}"
                     find_re = None
                     continue
+                new_name = new_name[:name_max_len]
                 if new_name != name:
                     find_match_count += 1
                     if len(find_matches) < limit:

apps/m3u/tasks.py

@@ -26,7 +26,7 @@ from core.utils import (
 from core.models import CoreSettings, UserAgent
 from core.xtream_codes import Client as XCClient
 from core.utils import send_websocket_update
-from .utils import normalize_stream_url
+from .utils import convert_js_numbered_backreferences, normalize_stream_url
 
 logger = logging.getLogger(__name__)
 
@@ -2047,6 +2047,11 @@ def sync_auto_channels(account_id, scan_start_time=None):
     from apps.epg.models import EPGData
     from django.utils import timezone
 
+    channel_name_max_len = Channel._meta.get_field("name").max_length
+    # Per-call cap on the rename substitution; bounds catastrophic
+    # backtracking on a user-supplied pattern so it cannot hang the worker.
+    rename_regex_timeout = 0.1
+
     try:
         account = M3UAccount.objects.get(id=account_id)
         logger.info(f"Starting auto channel sync for M3U account {account.name}")
@@ -2600,17 +2605,30 @@ def sync_auto_channels(account_id, scan_start_time=None):
                             else ""
                         )
                         try:
-                            # Convert $1, $2, etc. to \1, \2, etc. for consistency with M3U profiles
-                            safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace)
-                            new_name = re.sub(
-                                name_regex_pattern, safe_replace_pattern, original_name
+                            # Use the regex module (not stdlib re) so rename
+                            # patterns match the JS-style semantics the UI
+                            # authors and the preview uses; the timeout bounds
+                            # catastrophic backtracking.
+                            safe_replace_pattern = convert_js_numbered_backreferences(replace)
+                            new_name = regex.sub(
+                                name_regex_pattern,
+                                safe_replace_pattern,
+                                original_name,
+                                timeout=rename_regex_timeout,
                             )
-                        except re.error as e:
+                        except (regex.error, TimeoutError) as e:
                             logger.warning(
                                 f"Regex error for group '{channel_group.name}': {e}. Using original name."
                             )
                             new_name = original_name
 
+                    # Channel.name is bounded by the column length; a rename
+                    # that expands past it would otherwise fail the whole
+                    # bulk_create and abort the sync. Cap it so one overlong
+                    # result cannot break the batch, and so the preview (which
+                    # applies the same cap) stays faithful.
+                    new_name = new_name[:channel_name_max_len]
+
                     # Check if we already have a channel for this stream
                     existing_channel = existing_channel_map.get(stream.id)
 

apps/m3u/tests/test_rename_preview_parity.py

@@ -0,0 +1,212 @@
+"""
+Differential parity tests: the auto-sync rename preview must predict the exact
+name the live rename produces, across a broad matrix of regex strategies.
+
+Both the preview and the live rename compile with the third-party `regex`
+module (for its JS-aligned syntax and per-call timeout). They can only be
+trusted together if, for every find/replace a user might author, the preview's
+predicted `after` equals the channel name the sync actually writes.
+
+Each case is run end-to-end: real streams, the real sync_auto_channels rename,
+and the real regex-preview endpoint, compared per original stream name.
+"""
+from django.test import TestCase
+from django.utils import timezone
+
+from apps.channels.models import Channel, ChannelGroup, ChannelStream, Stream
+from apps.m3u.models import M3UAccount
+from apps.m3u.tasks import sync_auto_channels
+
+
+# Diverse names: distinct word-prefixes (collision-resistant), with quality
+# tags, brackets, pipes, ampersands, extra whitespace, underscores, dots, CJK,
+# and emoji, to exercise anchors, classes, boundaries, and Unicode handling.
+NAMES = [
+    "Alpha Channel 11",
+    "Bravo Sports HD",
+    "Charlie News FHD",
+    "Delta Movie (2024)",
+    "Echo [UK] 4K",
+    "Foxtrot   spaced",
+    "Golf|Pipe|Name",
+    "Hotel & Inn <x>",
+    "India 日本語 77",
+    "Juliet 📺 88",
+    "Kilo_under_9",
+    "Lima.dot.name",
+]
+
+# (find, replace) pairs spanning common user strategies and edge cases.
+STRATEGIES = [
+    # --- capture groups ---
+    (r"(.+) Channel (\d+)", r"$1 #$2"),
+    (r"(\w+) (\w+)", r"$2 $1"),
+    (r"(.+)", r"$1 - $1"),
+    (r"(.+)", r"[$1]"),
+    (r"(.+) (\d+)$", r"$2 $1"),
+    # --- strip / delete ---
+    (r" (HD|FHD|4K|SD)\b", r""),
+    (r"\s+", r" "),
+    (r"[\[\(].*?[\]\)]", r""),
+    (r"\d+", r""),
+    (r"[_.]", r" "),
+    # --- anchors / inserts ---
+    (r"^", r"NEW "),
+    (r"$", r" LIVE"),
+    (r"^(\w+)", r"<$1>"),
+    # --- char classes / Unicode (divergence hunters) ---
+    (r"\w+", r"W"),
+    (r"\b\w", r"_"),
+    (r"[A-Z]", r"*"),
+    (r"\s", r"_"),
+    (r"[^\x00-\x7F]+", r"?"),
+    # --- non-capturing / lookaround / in-pattern backref ---
+    (r"(?:Channel|Movie|News) ", r""),
+    (r"(\w)\1", r"$1"),
+    (r"\w+(?= )", r"X"),
+    (r"(?<=\d)\d", r"#"),
+    # --- literal $ and odd replacements ---
+    (r" ", r" $ "),
+    (r"o", r"0"),
+    # --- invalid group references (rejected by both engines) ---
+    (r"(.+)", r"$2"),
+    (r"(.+)", r"$10"),
+    # --- rename that expands past Channel.name's column length ---
+    (r"(.+)", r"$1" * 40),
+    # --- regex-module syntax: quantified anchor, JS-style and duplicate
+    #     named groups (these transform; both paths use the regex module) ---
+    (r"^*", r"$"),
+    (r"(?<season>\d+)", r"S$1"),
+    (r"(?P<n>x)(?P<n>y)", r"z"),
+]
+
+
+class RenamePreviewParityTests(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        from rest_framework.test import APIClient
+        from apps.accounts.models import User
+
+        admin = User.objects.create_superuser(
+            username="admin_rename_parity", password="pw", user_level=10
+        )
+        cls.client_api = APIClient()
+        cls.client_api.force_authenticate(user=admin)
+        cls.account = M3UAccount.objects.create(
+            name="Rename Parity Provider",
+            server_url="http://example.com/test.m3u",
+        )
+
+    def _sync(self):
+        return sync_auto_channels(
+            self.account.id,
+            scan_start_time=(
+                timezone.now() - timezone.timedelta(minutes=1)
+            ).isoformat(),
+        )
+
+    def _run_case(self, group_name, find, replace):
+        """Returns a list of human-readable mismatch strings (empty == parity)."""
+        group = ChannelGroup.objects.create(name=group_name)
+        from apps.channels.models import ChannelGroupM3UAccount
+
+        ChannelGroupM3UAccount.objects.create(
+            m3u_account=self.account,
+            channel_group=group,
+            enabled=True,
+            auto_channel_sync=True,
+            auto_sync_channel_start=1000,
+            custom_properties={
+                "name_regex_pattern": find,
+                "name_replace_pattern": replace,
+            },
+        )
+        for i, name in enumerate(NAMES):
+            Stream.objects.create(
+                name=name,
+                url=f"http://example.com/{group_name}_{i}.m3u8",
+                m3u_account=self.account,
+                channel_group=group,
+                tvg_id=f"{group_name}-{i}",
+                last_seen=timezone.now(),
+            )
+
+        # --- live rename via sync ---
+        result = self._sync()
+        if result.get("status") != "ok":
+            return [f"[{find!r} -> {replace!r}] sync status={result.get('status')}"]
+
+        channels = Channel.objects.filter(
+            auto_created_by=self.account, channel_group=group
+        )
+        cs_rows = ChannelStream.objects.filter(
+            channel__in=channels
+        ).select_related("channel", "stream")
+        sync_map = {row.stream.name: row.channel.name for row in cs_rows}
+
+        # --- preview endpoint ---
+        response = self.client_api.get(
+            "/api/channels/streams/regex-preview/",
+            {
+                "channel_group": group_name,
+                "find": find,
+                "replace": replace,
+                "limit": 50,
+            },
+        )
+        if response.status_code != 200:
+            return [f"[{find!r} -> {replace!r}] preview HTTP {response.status_code}"]
+        data = response.data
+        # When the preview reports find_error it predicts no rename at all.
+        preview_map = {name: name for name in NAMES}
+        if "find_error" not in data:
+            for m in data.get("find_matches", []):
+                preview_map[m["before"]] = m["after"]
+
+        mismatches = []
+        for name in NAMES:
+            sync_after = sync_map.get(name)
+            if sync_after is None:
+                mismatches.append(
+                    f"[{find!r} -> {replace!r}] {name!r}: no channel created"
+                )
+                continue
+            preview_after = preview_map[name]
+            if preview_after != sync_after:
+                mismatches.append(
+                    f"[{find!r} -> {replace!r}] {name!r}: "
+                    f"preview={preview_after!r} sync={sync_after!r} "
+                    f"(find_error={data.get('find_error')!r})"
+                )
+        return mismatches
+
+    def test_preview_predicts_rename_across_strategies(self):
+        all_mismatches = []
+        for idx, (find, replace) in enumerate(STRATEGIES):
+            all_mismatches.extend(
+                self._run_case(f"ParityG{idx}", find, replace)
+            )
+        self.assertEqual(
+            all_mismatches,
+            [],
+            "Preview diverged from the live rename:\n"
+            + "\n".join(all_mismatches),
+        )
+
+    def test_overlong_rename_is_bounded_not_aborting_sync(self):
+        # A rename that expands past Channel.name's column length must not
+        # abort the bulk_create sync. Both sync and preview cap at the column
+        # length so the channel is created (truncated) and the preview shows
+        # the same bounded name.
+        max_len = Channel._meta.get_field("name").max_length
+        mismatches = self._run_case("OverlongG", r"(.+)", "$1" * 40)
+        self.assertEqual(mismatches, [], "\n".join(mismatches))
+
+        group = ChannelGroup.objects.get(name="OverlongG")
+        channels = Channel.objects.filter(
+            auto_created_by=self.account, channel_group=group
+        )
+        self.assertEqual(channels.count(), len(NAMES))
+        self.assertTrue(all(len(c.name) <= max_len for c in channels))
+        self.assertTrue(any(len(c.name) == max_len for c in channels))

apps/m3u/tests/test_sync_correctness.py

@@ -763,6 +763,128 @@ class RegexPreviewTests(TestCase):
         self.assertEqual(response.data["total_scanned"], 3)
         self.assertFalse(response.data["scan_limit_hit"])
 
+    def test_find_replace_applies_numbered_capture_group(self):
+        # The replace field accepts JS-style $1 backreferences, but the regex
+        # engine expects \1. Without the conversion the preview echoes the
+        # literal "$1", so the previewed "after" disagrees with the name the
+        # live rename produces.
+        account = self._make_account()
+        group = _make_group(name="Sports")
+        Stream.objects.create(
+            name="High Limit Racing at Eagle @ Jun 9 7:00 PM",
+            url="http://example.com/hlr.m3u8",
+            m3u_account=account,
+            channel_group=group,
+            last_seen=timezone.now(),
+        )
+        client = self._client()
+
+        response = client.get(
+            "/api/channels/streams/regex-preview/",
+            {"channel_group": "Sports", "find": r"(.+) @.*", "replace": "$1"},
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["find_match_count"], 1)
+        after = response.data["find_matches"][0]["after"]
+        self.assertEqual(after, "High Limit Racing at Eagle")
+        self.assertNotIn("$1", after)
+
+    def test_preview_after_matches_live_sync_rename(self):
+        # Guards the defect class: the preview and the live rename are
+        # separate code paths that must convert the replacement identically,
+        # so the preview can never promise an output the sync would not yield.
+        name = "High Limit Racing at Eagle @ Jun 9 7:00 PM"
+        account = self._make_account()
+        group = _make_group(name="Racing")
+        _attach_group_to_account(
+            account,
+            group,
+            custom_properties={
+                "name_regex_pattern": r"(.+) @.*",
+                "name_replace_pattern": "$1",
+            },
+        )
+        _make_stream(account, group, name=name, tvg_id="hlr")
+
+        result = _sync(account)
+        self.assertEqual(result.get("status"), "ok")
+        channel = Channel.objects.get(auto_created=True, auto_created_by=account)
+        live_name = channel.name
+
+        client = self._client()
+        response = client.get(
+            "/api/channels/streams/regex-preview/",
+            {"channel_group": "Racing", "find": r"(.+) @.*", "replace": "$1"},
+        )
+
+        self.assertEqual(response.status_code, 200)
+        preview_after = response.data["find_matches"][0]["after"]
+        self.assertEqual(preview_after, live_name)
+        self.assertEqual(preview_after, "High Limit Racing at Eagle")
+
+    def test_regex_engine_pattern_transforms_in_preview(self):
+        # Both the preview and the live rename use the regex module, which is
+        # more permissive than stdlib re and matches the JS-style syntax the UI
+        # authors. A quantified anchor like "^*" (which stdlib re rejects)
+        # compiles and transforms rather than reporting an error.
+        account = self._make_account()
+        group = _make_group(name="Sports")
+        Stream.objects.create(
+            name="Doc95",
+            url="http://example.com/doc95.m3u8",
+            m3u_account=account,
+            channel_group=group,
+            last_seen=timezone.now(),
+        )
+        client = self._client()
+
+        response = client.get(
+            "/api/channels/streams/regex-preview/",
+            {"channel_group": "Sports", "find": "^*", "replace": "$"},
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn("find_error", response.data)
+        self.assertEqual(response.data["find_match_count"], 1)
+        # ^* matches the empty string at every position, so the literal $
+        # replacement is inserted between characters.
+        self.assertEqual(
+            response.data["find_matches"][0]["after"], "$D$o$c$9$5$"
+        )
+
+    def test_preview_and_sync_agree_on_regex_only_pattern(self):
+        # Parity guard for the engine alignment: a pattern valid in regex but
+        # not stdlib re must transform identically in the sync and the preview,
+        # rather than diverging (the sync no longer silently keeps the
+        # original name for these patterns).
+        name = "Doc95"
+        account = self._make_account()
+        group = _make_group(name="Docs")
+        _attach_group_to_account(
+            account,
+            group,
+            custom_properties={
+                "name_regex_pattern": "^*",
+                "name_replace_pattern": "$",
+            },
+        )
+        _make_stream(account, group, name=name, tvg_id="doc95")
+
+        result = _sync(account)
+        self.assertEqual(result.get("status"), "ok")
+        channel = Channel.objects.get(auto_created=True, auto_created_by=account)
+        live_name = channel.name
+        self.assertNotEqual(live_name, name)
+
+        client = self._client()
+        response = client.get(
+            "/api/channels/streams/regex-preview/",
+            {"channel_group": "Docs", "find": "^*", "replace": "$"},
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["find_matches"][0]["after"], live_name)
+
     def test_filter_returns_matched_names_with_count(self):
         account = self._make_account()
         group = _make_group(name="Sports")

apps/m3u/utils.py

@@ -1,4 +1,5 @@
 # apps/m3u/utils.py
+import regex
 import threading
 import logging
 from django.db import models
@@ -9,6 +10,18 @@ active_streams_map = {}
 logger = logging.getLogger(__name__)
 
 
+def convert_js_numbered_backreferences(replacement):
+    """Translate JS-style ``$1``/``$2`` backreferences to Python ``\\1``/``\\2``.
+
+    Auto-sync replace patterns are authored in JS regex syntax, but Python's
+    regex engines honor backslash backreferences, not ``$1``. The live rename
+    and the UI preview must convert identically, so both call this single
+    helper and cannot drift apart (otherwise the preview promises an output
+    the sync would never produce).
+    """
+    return regex.sub(r"\$(\d+)", r"\\\1", replacement)
+
+
 def normalize_stream_url(url):
     """
     Normalize stream URLs for compatibility with FFmpeg.