From b9b3ded71c2a77fb242c2606c3e47f81d0b104fa Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 1 May 2026 18:32:23 -0500 Subject: [PATCH] refactor(migrations): merge channels FR #1196 migrations into single 0037 Consolidates 0037_channeloverride_and_user_hidden, 0038_backfill_auto_created_by_null, 0039_channelgroupm3uaccount_auto_sync_channel_end, and 0040_channel_channel_number_nullable into a single 0037_auto_sync_overhaul migration. The merged migration runs operations in dependency-respecting order: 1. AddField Channel.user_hidden 2. CreateModel ChannelOverride 3. AddField ChannelGroupM3UAccount.auto_sync_channel_end 4. RunPython backfill_auto_created_by_null 5. AlterField Channel.channel_number (nullable) 6. RunPython noop / reverse_backfill_channel_number_nulls Also, update CHANGELOG.md and update migration references in test --- CHANGELOG.md | 4 +- .../migrations/0037_auto_sync_overhaul.py | 156 ++++++++++++++++++ .../0037_channeloverride_and_user_hidden.py | 40 ----- .../0038_backfill_auto_created_by_null.py | 71 -------- ...elgroupm3uaccount_auto_sync_channel_end.py | 18 -- .../0040_channel_channel_number_nullable.py | 64 ------- apps/m3u/tests/test_sync_correctness.py | 14 +- 7 files changed, 165 insertions(+), 202 deletions(-) create mode 100644 apps/channels/migrations/0037_auto_sync_overhaul.py delete mode 100644 apps/channels/migrations/0037_channeloverride_and_user_hidden.py delete mode 100644 apps/channels/migrations/0038_backfill_auto_created_by_null.py delete mode 100644 apps/channels/migrations/0039_channelgroupm3uaccount_auto_sync_channel_end.py delete mode 100644 apps/channels/migrations/0040_channel_channel_number_nullable.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d0b965..fd83d37a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Auto-sync overhaul (FR #1196)**: comprehensive rebuild of the M3U auto-channel-sync flow. Introduces a per-field override system, hide-from-output flag, range-bounded auto-numbering with a re-pack helper, multi-stream channel safety, multi-provider shared-range merging, and an across-the-board move from per-row writes to bulk operations. Migrations included: `apps/channels/0037_channeloverride_and_user_hidden`, `0038_backfill_auto_created_by_null`, `0039_channelgroupm3uaccount_auto_sync_channel_end`, `0040_channel_channel_number_nullable`, and `apps/m3u/0020_m3uaccount_auto_cleanup_unused_channels`. (Closes #1196) +- **Auto-sync overhaul (FR #1196)**: comprehensive rebuild of the M3U auto-channel-sync flow. Introduces a per-field override system, hide-from-output flag, range-bounded auto-numbering with a re-pack helper, multi-stream channel safety, multi-provider shared-range merging, and an across-the-board move from per-row writes to bulk operations. Migrations included: `apps/channels/0037_auto_sync_overhaul` (bundled CreateModel + AddField + AlterField + RunPython operations) and `apps/m3u/0020_m3uaccount_auto_cleanup_unused_channels`. (Closes #1196) - **Per-field channel overrides for auto-synced channels.** A new `ChannelOverride` table (one-to-one with `Channel`) holds user-specified values for `name`, `channel_number`, `channel_group`, `logo`, `tvg_id`, `tvc_guide_stationid`, `epg_data`, and `stream_profile`. Sync writes only to `Channel.*` columns; the override row takes precedence at read time via a new `with_effective_values()` queryset helper that coalesces both sources at the SQL layer. Every output surface that consumes channel data reads through this helper so overrides surface consistently: HDHR (`/hdhr/lineup.json` and per-profile variants, `/hdhr/discover.json`), M3U (`/output/m3u`, per-profile variants), EPG (`/output/epg`, per-profile variants), XC API (`get_live_streams`, `xmltv.php`), the channels list/detail endpoints, and the TV Guide summary endpoint (`GET /api/channels/channels/summary/`). Channel API responses now include both the raw `name`/`channel_number`/etc. fields AND new `effective_*` annotations plus the `override` object, so frontend consumers can show both "what the user picked" and "what the provider sent" simultaneously. - **Per-field reset-to-provider icon.** FK pickers (channel group, logo, EPG, stream profile) and scalar inputs (name, channel_number, tvg_id, tvc_guide_stationid) display a small reset icon ("Provider: " subtext + Undo2 button) when the field has an active override on an auto-synced channel. Clicking it sets the form value back to the provider value; the override for that field is cleared on save while other overrides remain intact. - **Auto-created channel source attribution.** The channel edit form shows an "Auto-created from: / " label at the top of auto-synced channels so the user can see which M3U account and which stream produced the row. @@ -101,7 +101,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **`Channel.channel_number` is now nullable.** Auto-sync may produce channels without an assigned number (for example, a sync run where the configured range was exhausted before this stream could be slotted, or a hidden channel in compact mode whose slot was released for visible channels). Existing channels are unchanged. HDHR / M3U / EPG / XC output endpoints filter out channels with `null` channel_number, so a number-less row is invisible to clients but visible in the channels page where the user can assign one. Migration `0040_channel_channel_number_nullable` includes a reverse-direction backfill so a rollback that re-imposes NOT NULL still succeeds even if NULLs are present. +- **`Channel.channel_number` is now nullable.** Auto-sync may produce channels without an assigned number (for example, a sync run where the configured range was exhausted before this stream could be slotted, or a hidden channel in compact mode whose slot was released for visible channels). Existing channels are unchanged. HDHR / M3U / EPG / XC output endpoints filter out channels with `null` channel_number, so a number-less row is invisible to clients but visible in the channels page where the user can assign one. The 0037 auto-sync overhaul migration includes a reverse-direction backfill on the `channel_number` AlterField step so a rollback that re-imposes NOT NULL still succeeds even if NULLs are present. - **M3U account delete always cascades auto-created channels.** The delete dialog is a single confirmation showing the count of affected channels (fetched from the new auto-created-channels-count endpoint). Auto-created channels are removed regardless of `user_hidden` or override state: override rows cascade via the one-to-one FK; hidden status does not preserve the channel because there is no provider left to populate it on the next sync. Manual channels are unaffected; their non-provider streams remain intact, and only streams owned by the deleted account are removed. The destroy response now returns `{"deleted_channels": N}` (HTTP 200) instead of an empty 204 so the confirmation toast can show the actual count. - **Bulk channel edit auto-routes auto-created channels through the override path.** When a bulk edit includes auto-synced channels, the changed fields are written to each channel's `ChannelOverride` row instead of the raw `Channel.*` columns, so the edit survives the next sync. Manual channels in the same selection write directly to `Channel.*`. A "Clear all overrides" affordance pre-clears overrides on the selection before applying new edits in the same submit (clear runs before the routing PATCH so a same-submit clear cannot wipe just-written overrides). - **Inline edits on the channels table route through the override row for auto-synced channels.** Editing a cell directly in the table (number, name, group, EPG, logo) now writes to `ChannelOverride.` for auto-synced rows so the edit survives the next refresh; manual rows continue to write directly. The save mechanic and validation are unchanged from the user's perspective. diff --git a/apps/channels/migrations/0037_auto_sync_overhaul.py b/apps/channels/migrations/0037_auto_sync_overhaul.py new file mode 100644 index 00000000..17dd601f --- /dev/null +++ b/apps/channels/migrations/0037_auto_sync_overhaul.py @@ -0,0 +1,156 @@ +""" +Auto-sync overhaul (FR #1196): per-field channel overrides, hide-from-output +flag, configurable auto-sync number range, and nullable channel_number for +compact-numbering slot release. + +Bundled operations (in forward order; reversed on rollback): + 1. AddField Channel.user_hidden + 2. CreateModel ChannelOverride (one-to-one Channel) + 3. AddField ChannelGroupM3UAccount.auto_sync_channel_end + 4. RunPython backfill_auto_created_by_null (orphan re-attribution / demotion) + 5. AlterField Channel.channel_number (nullable) + 6. RunPython noop / reverse_backfill_channel_number_nulls + (rollback-safety hook; runs FIRST on un-apply, fills NULLs + so step 5 reverse can re-impose NOT NULL) +""" + +import django.db.models.deletion +from django.db import migrations, models +from django.db.models import Max + + +def backfill_auto_created_by_null(apps, schema_editor): + """ + Re-attribute or demote `auto_created=True, auto_created_by=NULL` rows. + + Sync only touches rows where `auto_created_by=account`, so orphans + accumulate indefinitely. Best-effort re-attribute via the channel's + streams' single owning account; otherwise demote to manual by clearing + `auto_created`. The channel and any user customization survive; sync + will not touch a demoted row again. + """ + Channel = apps.get_model("dispatcharr_channels", "Channel") + ChannelStream = apps.get_model("dispatcharr_channels", "ChannelStream") + + orphans = Channel.objects.filter(auto_created=True, auto_created_by__isnull=True) + total = orphans.count() + if total == 0: + return + + print(f"\n Found {total} auto_created channels with NULL auto_created_by") + reattributed = 0 + demoted = 0 + + for channel in orphans.iterator(chunk_size=200): + account_ids = set( + ChannelStream.objects.filter(channel=channel) + .values_list("stream__m3u_account_id", flat=True) + ) + account_ids.discard(None) + + if len(account_ids) == 1: + channel.auto_created_by_id = next(iter(account_ids)) + channel.save(update_fields=["auto_created_by"]) + reattributed += 1 + else: + channel.auto_created = False + channel.save(update_fields=["auto_created"]) + demoted += 1 + + print( + f" Re-attributed: {reattributed}, demoted to manual " + f"(ambiguous/no streams): {demoted}" + ) + + +def reverse_auto_created_by_null(apps, schema_editor): + # Forward decisions cannot be cleanly reverted (no record of the + # original NULL state). Leaving the re-attributions and demotions in + # place is safer than restoring NULLs the schema may not accept. + pass + + +def noop(apps, schema_editor): + pass + + +def reverse_backfill_channel_number_nulls(apps, schema_editor): + """ + Rollback-safety hook for the channel_number nullable AlterField. + + Runs FIRST on un-apply (operations reverse in list order) and assigns + sequential channel numbers above the current max to any NULL rows so + the AlterField reverse (nullable to NOT NULL) succeeds without a + constraint violation. The user can re-hide or re-number these + channels after they have rolled back. + """ + Channel = apps.get_model("dispatcharr_channels", "Channel") + null_qs = Channel.objects.filter(channel_number__isnull=True) + null_count = null_qs.count() + if null_count == 0: + return + + max_num = Channel.objects.aggregate(m=Max("channel_number"))["m"] or 0.0 + next_num = float(max_num) + 1.0 + print( + f"\n Backfilling channel_number on {null_count} NULL row(s) " + f"starting at {int(next_num)} so rollback can re-impose NOT NULL" + ) + for ch in null_qs.order_by("id"): + ch.channel_number = next_num + ch.save(update_fields=["channel_number"]) + next_num += 1.0 + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '022_default_user_limit_settings'), + ('dispatcharr_channels', '0036_alter_stream_name'), + ('epg', '0022_alter_epgdata_name'), + ] + + operations = [ + migrations.AddField( + model_name='channel', + name='user_hidden', + field=models.BooleanField( + db_index=True, + default=False, + help_text='Exclude this channel from downstream client output (HDHR, M3U, EPG, XC). Auto-sync still updates provider metadata.', + ), + ), + migrations.CreateModel( + name='ChannelOverride', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=512, null=True)), + ('channel_number', models.FloatField(blank=True, null=True)), + ('tvg_id', models.CharField(blank=True, max_length=255, null=True)), + ('tvc_guide_stationid', models.CharField(blank=True, max_length=255, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('channel', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='override', to='dispatcharr_channels.channel')), + ('channel_group', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.channelgroup')), + ('epg_data', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='epg.epgdata')), + ('logo', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.logo')), + ('stream_profile', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.streamprofile')), + ], + ), + migrations.AddField( + model_name='channelgroupm3uaccount', + name='auto_sync_channel_end', + field=models.FloatField( + blank=True, + help_text='Optional upper bound for auto-created channel numbers in this group. Leave blank for unlimited fill. Overflow streams are skipped and reported in the completion notification.', + null=True, + ), + ), + migrations.RunPython(backfill_auto_created_by_null, reverse_auto_created_by_null), + migrations.AlterField( + model_name='channel', + name='channel_number', + field=models.FloatField(blank=True, db_index=True, null=True), + ), + migrations.RunPython(noop, reverse_backfill_channel_number_nulls), + ] diff --git a/apps/channels/migrations/0037_channeloverride_and_user_hidden.py b/apps/channels/migrations/0037_channeloverride_and_user_hidden.py deleted file mode 100644 index 9a1913e3..00000000 --- a/apps/channels/migrations/0037_channeloverride_and_user_hidden.py +++ /dev/null @@ -1,40 +0,0 @@ -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '022_default_user_limit_settings'), - ('dispatcharr_channels', '0036_alter_stream_name'), - ('epg', '0022_alter_epgdata_name'), - ] - - operations = [ - migrations.AddField( - model_name='channel', - name='user_hidden', - field=models.BooleanField( - db_index=True, - default=False, - help_text='Exclude this channel from downstream client output (HDHR, M3U, EPG, XC). Auto-sync still updates provider metadata.', - ), - ), - migrations.CreateModel( - name='ChannelOverride', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(blank=True, max_length=512, null=True)), - ('channel_number', models.FloatField(blank=True, null=True)), - ('tvg_id', models.CharField(blank=True, max_length=255, null=True)), - ('tvc_guide_stationid', models.CharField(blank=True, max_length=255, null=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('channel', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='override', to='dispatcharr_channels.channel')), - ('channel_group', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.channelgroup')), - ('epg_data', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='epg.epgdata')), - ('logo', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dispatcharr_channels.logo')), - ('stream_profile', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.streamprofile')), - ], - ), - ] diff --git a/apps/channels/migrations/0038_backfill_auto_created_by_null.py b/apps/channels/migrations/0038_backfill_auto_created_by_null.py deleted file mode 100644 index fb28cb42..00000000 --- a/apps/channels/migrations/0038_backfill_auto_created_by_null.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Data migration: re-attribute `auto_created=True, auto_created_by=NULL` channels -or demote them to manual. - -Sync logic only touches rows where `auto_created_by=account`, so orphaned -rows accumulate indefinitely and clutter the admin table. - -Strategy: -1. Best-effort re-attribute: if the channel's streams all live under a - single M3U account, set `auto_created_by` to that account. -2. Otherwise (zero streams or multiple accounts), demote to manual by - clearing `auto_created`. The channel and any user customization survive; - sync will not touch it again. - -Each decision is logged so operators can see the outcome at migrate time. -""" - -from django.db import migrations - - -def backfill(apps, schema_editor): - Channel = apps.get_model("dispatcharr_channels", "Channel") - ChannelStream = apps.get_model("dispatcharr_channels", "ChannelStream") - - orphans = Channel.objects.filter(auto_created=True, auto_created_by__isnull=True) - total = orphans.count() - if total == 0: - return - - print(f"\n Found {total} auto_created channels with NULL auto_created_by") - reattributed = 0 - demoted = 0 - - for channel in orphans.iterator(chunk_size=200): - account_ids = set( - ChannelStream.objects.filter(channel=channel) - .values_list("stream__m3u_account_id", flat=True) - ) - account_ids.discard(None) - - if len(account_ids) == 1: - channel.auto_created_by_id = next(iter(account_ids)) - channel.save(update_fields=["auto_created_by"]) - reattributed += 1 - else: - channel.auto_created = False - channel.save(update_fields=["auto_created"]) - demoted += 1 - - print( - f" Re-attributed: {reattributed}, demoted to manual " - f"(ambiguous/no streams): {demoted}" - ) - - -def reverse(apps, schema_editor): - # Irreversible data fix - leaving the forward decisions in place is - # safer than trying to re-null the auto_created_by field (we cannot - # recreate the deleted rows). - pass - - -class Migration(migrations.Migration): - - dependencies = [ - ("dispatcharr_channels", "0037_channeloverride_and_user_hidden"), - ] - - operations = [ - migrations.RunPython(backfill, reverse), - ] diff --git a/apps/channels/migrations/0039_channelgroupm3uaccount_auto_sync_channel_end.py b/apps/channels/migrations/0039_channelgroupm3uaccount_auto_sync_channel_end.py deleted file mode 100644 index 50291f5f..00000000 --- a/apps/channels/migrations/0039_channelgroupm3uaccount_auto_sync_channel_end.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 6.0.4 on 2026-04-24 02:38 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('dispatcharr_channels', '0038_backfill_auto_created_by_null'), - ] - - operations = [ - migrations.AddField( - model_name='channelgroupm3uaccount', - name='auto_sync_channel_end', - field=models.FloatField(blank=True, help_text='Optional upper bound for auto-created channel numbers in this group. Leave blank for unlimited fill. Overflow streams are skipped and reported in the completion notification.', null=True), - ), - ] diff --git a/apps/channels/migrations/0040_channel_channel_number_nullable.py b/apps/channels/migrations/0040_channel_channel_number_nullable.py deleted file mode 100644 index e431e35d..00000000 --- a/apps/channels/migrations/0040_channel_channel_number_nullable.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated for compact-numbering support. -# Allowing NULL on Channel.channel_number is required so the compact -# numbering pass can release a hidden channel's slot back to the pool. -# Existing rows are unaffected (NOT NULL -> NULL is a column-only ALTER). -# -# Rollback safety: reverting nullable -> NOT NULL would fail with a -# constraint violation on any row with NULL channel_number. A pre-revert -# RunPython step backfills those NULLs with sequential numbers above the -# current max so the schema can shrink back to NOT NULL cleanly. The -# operation list is ordered so that on REVERSE, the RunPython runs -# BEFORE the AlterField (operations reverse in list order on un-apply). - -from django.db import migrations, models -from django.db.models import Max - - -def forward_noop(apps, schema_editor): - pass - - -def reverse_backfill_nulls(apps, schema_editor): - """ - Assign sequential channel numbers to rows whose channel_number is NULL, - so the subsequent reverse AlterField can re-impose NOT NULL. - - Numbers are placed above the current max to avoid collision with any - other channel's existing assignment. The user can re-hide or re-number - these channels after they have rolled back. - """ - Channel = apps.get_model("dispatcharr_channels", "Channel") - null_qs = Channel.objects.filter(channel_number__isnull=True) - null_count = null_qs.count() - if null_count == 0: - return - - max_num = Channel.objects.aggregate(m=Max("channel_number"))["m"] or 0.0 - next_num = float(max_num) + 1.0 - print( - f"\n Backfilling channel_number on {null_count} NULL row(s) " - f"starting at {int(next_num)} so rollback can re-impose NOT NULL" - ) - for ch in null_qs.order_by("id"): - ch.channel_number = next_num - ch.save(update_fields=["channel_number"]) - next_num += 1.0 - - -class Migration(migrations.Migration): - - dependencies = [ - ("dispatcharr_channels", "0039_channelgroupm3uaccount_auto_sync_channel_end"), - ] - - operations = [ - migrations.AlterField( - model_name="channel", - name="channel_number", - field=models.FloatField(blank=True, db_index=True, null=True), - ), - # Forward: no-op (no NULL rows exist before AlterField runs). - # Reverse: runs FIRST on un-apply (list-reversed) and clears NULLs - # so the AlterField reverse (nullable -> NOT NULL) succeeds. - migrations.RunPython(forward_noop, reverse_backfill_nulls), - ] diff --git a/apps/m3u/tests/test_sync_correctness.py b/apps/m3u/tests/test_sync_correctness.py index 1b7011f8..b1c84563 100644 --- a/apps/m3u/tests/test_sync_correctness.py +++ b/apps/m3u/tests/test_sync_correctness.py @@ -1931,15 +1931,15 @@ class EPGDispatchExistingChannelTests(TestCase): class Migration0037DemoteOrphansTests(TestCase): """ - Migration 0037 originally deleted orphaned `auto_created=True, - auto_created_by=NULL` channels. The fix demotes them to - `auto_created=False` instead, preserving the channel and any - overrides that may exist on it. + The 0037 auto-sync overhaul migration's `backfill_auto_created_by_null` + step demotes orphaned `auto_created=True, auto_created_by=NULL` channels + to `auto_created=False` instead of deleting them, preserving the + channel and any overrides that may exist on it. """ def test_orphan_with_no_streams_is_demoted_not_deleted(self): # Create an orphaned auto-created channel with no streams. This - # is the case that the original migration would silently delete. + # is the case that a delete-on-orphan strategy would silently lose. ch = Channel.objects.create( name="OrphanGhost", channel_number=999, @@ -1955,13 +1955,13 @@ class Migration0037DemoteOrphansTests(TestCase): from django.apps import apps as django_apps module = import_module( - "apps.channels.migrations.0038_backfill_auto_created_by_null" + "apps.channels.migrations.0037_auto_sync_overhaul" ) # The migration function takes (apps, schema_editor); apps is # the historical app registry. For this unit test we call with # the live registry. - module.backfill(django_apps, None) + module.backfill_auto_created_by_null(django_apps, None) ch.refresh_from_db() self.assertFalse(