m3u modifications so streams are identified by hash, configurable, also streams now have channel_groups instead of a string for groups

apps/channels/admin.py

@@ -6,13 +6,16 @@ class StreamAdmin(admin.ModelAdmin):
     list_display = (
         'id',  # Primary Key
         'name',
-        'group_name',
+        'channel_group',
         'url',
         'current_viewers',
         'updated_at',
     )
-    list_filter = ('group_name',)
-    search_fields = ('id', 'name', 'url', 'group_name')  # Added 'id' for searching by ID
+
+    list_filter = ('channel_group',)  # Filter by 'channel_group' (foreign key)
+
+    search_fields = ('id', 'name', 'url', 'channel_group__name')  # Search by 'ChannelGroup' name
+
     ordering = ('-updated_at',)
 
 @admin.register(Channel)

apps/channels/api_views.py

@@ -23,14 +23,14 @@ class StreamPagination(PageNumberPagination):
 
 class StreamFilter(django_filters.FilterSet):
     name = django_filters.CharFilter(lookup_expr='icontains')
-    group_name = django_filters.CharFilter(lookup_expr='icontains')
+    channel_group_name = django_filters.CharFilter(field_name="channel_group__name", lookup_expr="icontains")
     m3u_account = django_filters.NumberFilter(field_name="m3u_account__id")
     m3u_account_name = django_filters.CharFilter(field_name="m3u_account__name", lookup_expr="icontains")
     m3u_account_is_active = django_filters.BooleanFilter(field_name="m3u_account__is_active")
 
     class Meta:
         model = Stream
-        fields = ['name', 'group_name', 'm3u_account', 'm3u_account_name', 'm3u_account_is_active']
+        fields = ['name', 'channel_group_name', 'm3u_account', 'm3u_account_name', 'm3u_account_is_active']
 
 # ─────────────────────────────────────────────────────────
 # 1) Stream API (CRUD)
@@ -43,20 +43,27 @@ class StreamViewSet(viewsets.ModelViewSet):
 
     filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
     filterset_class = StreamFilter
-    search_fields = ['name', 'group_name']
-    ordering_fields = ['name', 'group_name']
+    search_fields = ['name', 'channel_group__name']
+    ordering_fields = ['name', 'channel_group__name']
     ordering = ['-name']
 
     def get_queryset(self):
         qs = super().get_queryset()
         # Exclude streams from inactive M3U accounts
         qs = qs.exclude(m3u_account__is_active=False)
+
         assigned = self.request.query_params.get('assigned')
         if assigned is not None:
             qs = qs.filter(channels__id=assigned)
+
         unassigned = self.request.query_params.get('unassigned')
         if unassigned == '1':
             qs = qs.filter(channels__isnull=True)
+
+        channel_group = self.request.query_params.get('channel_group')
+        if channel_group:
+            qs = qs.filter(channel_group__name=channel_group)
+
         return qs
 
     @action(detail=False, methods=['get'], url_path='ids')
@@ -75,7 +82,8 @@ class StreamViewSet(viewsets.ModelViewSet):
 
     @action(detail=False, methods=['get'], url_path='groups')
     def get_groups(self, request, *args, **kwargs):
-        group_names = Stream.objects.exclude(group_name__isnull=True).exclude(group_name="").order_by().values_list('group_name', flat=True).distinct()
+        # Get unique ChannelGroup names that are linked to streams
+        group_names = ChannelGroup.objects.filter(streams__isnull=False).order_by('name').values_list('name', flat=True).distinct()
 
         # Return the response with the list of unique group names
         return Response(list(group_names))
@@ -158,7 +166,7 @@ class ChannelViewSet(viewsets.ModelViewSet):
         if not stream_id:
             return Response({"error": "Missing stream_id"}, status=status.HTTP_400_BAD_REQUEST)
         stream = get_object_or_404(Stream, pk=stream_id)
-        channel_group, _ = ChannelGroup.objects.get_or_create(name=stream.group_name)
+        channel_group = stream.channel_group
 
         # Check if client provided a channel_number; if not, auto-assign one.
         provided_number = request.data.get('channel_number')

apps/channels/forms.py

@@ -42,5 +42,5 @@ class StreamForm(forms.ModelForm):
             'logo_url',
             'tvg_id',
             'local_file',
-            'group_name',
+            'channel_group',
         ]

apps/channels/migrations/0005_stream_channel_group_stream_last_seen_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 5.1.6 on 2025-03-19 16:33
+
+import datetime
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dispatcharr_channels', '0004_stream_is_custom'),
+        ('m3u', '0003_create_custom_account'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='stream',
+            name='channel_group',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='streams', to='dispatcharr_channels.channelgroup'),
+        ),
+        migrations.AddField(
+            model_name='stream',
+            name='last_seen',
+            field=models.DateTimeField(db_index=True, default=datetime.datetime.now),
+        ),
+        migrations.AlterField(
+            model_name='channel',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True),
+        ),
+        migrations.CreateModel(
+            name='ChannelGroupM3UAccount',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('enabled', models.BooleanField(default=True)),
+                ('channel_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='m3u_account', to='dispatcharr_channels.channelgroup')),
+                ('m3u_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='channel_group', to='m3u.m3uaccount')),
+            ],
+            options={
+                'unique_together': {('channel_group', 'm3u_account')},
+            },
+        ),
+    ]

apps/channels/migrations/0006_migrate_stream_groups.py

@@ -0,0 +1,51 @@
+# In your app's migrations folder, create a new migration file
+# e.g., migrations/000X_migrate_channel_group_to_foreign_key.py
+
+from django.db import migrations
+
+def migrate_channel_group(apps, schema_editor):
+    Stream = apps.get_model('dispatcharr_channels', 'Stream')
+    ChannelGroup = apps.get_model('dispatcharr_channels', 'ChannelGroup')
+    ChannelGroupM3UAccount = apps.get_model('dispatcharr_channels', 'ChannelGroup')
+    M3UAccount = apps.get_model('m3u', 'M3UAccount')
+
+    streams_to_update = []
+    for stream in Stream.objects.all():
+        # If the stream has a 'channel_group' string, try to find or create the ChannelGroup
+        if stream.group_name:  # group_name holds the channel group string
+            channel_group_name = stream.group_name.strip()
+
+            # Try to find the ChannelGroup by name
+            channel_group, created = ChannelGroup.objects.get_or_create(name=channel_group_name)
+
+            # Set the foreign key to the found or newly created ChannelGroup
+            stream.channel_group = channel_group
+
+            streams_to_update.append(stream)
+
+            # If the stream has an M3U account, ensure the M3U account is linked
+            if stream.m3u_account:
+                ChannelGroupM3UAccount.objects.get_or_create(
+                    channel_group=channel_group,
+                    m3u_account=stream.m3u_account,
+                    enabled=True  # Or set it to whatever the default logic is
+                )
+
+    Stream.objects.bulk_update(streams_to_update, ['channel_group'])
+
+def reverse_migration(apps, schema_editor):
+    # This reverse migration would undo the changes, setting `channel_group` to `None` and clearing any relationships.
+    Stream = apps.get_model('yourapp', 'Stream')
+    for stream in Stream.objects.all():
+        stream.channel_group = None
+        stream.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dispatcharr_channels', '0005_stream_channel_group_stream_last_seen_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_channel_group, reverse_code=reverse_migration),
+    ]

apps/channels/migrations/0007_remove_stream_group_name.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.1.6 on 2025-03-19 16:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dispatcharr_channels', '0006_migrate_stream_groups'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='stream',
+            name='group_name',
+        ),
+    ]

apps/channels/migrations/0008_stream_stream_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.6 on 2025-03-19 18:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dispatcharr_channels', '0007_remove_stream_group_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='stream',
+            name='stream_hash',
+            field=models.CharField(help_text='Unique hash for this stream from the M3U account', max_length=255, null=True, unique=True),
+        ),
+    ]

apps/channels/models.py

@@ -6,12 +6,25 @@ from core.models import StreamProfile, CoreSettings
 from core.utils import redis_client
 import logging
 import uuid
+from datetime import datetime
+import hashlib
+import json
 
 logger = logging.getLogger(__name__)
 
 # If you have an M3UAccount model in apps.m3u, you can still import it:
 from apps.m3u.models import M3UAccount
 
+class ChannelGroup(models.Model):
+    name = models.CharField(max_length=100, unique=True)
+
+    def related_channels(self):
+        # local import if needed to avoid cyc. Usually fine in a single file though
+        return Channel.objects.filter(channel_group=self)
+
+    def __str__(self):
+        return self.name
+
 class Stream(models.Model):
     """
     Represents a single stream (e.g. from an M3U source or custom URL).
@@ -30,7 +43,13 @@ class Stream(models.Model):
     local_file = models.FileField(upload_to='uploads/', blank=True, null=True)
     current_viewers = models.PositiveIntegerField(default=0)
     updated_at = models.DateTimeField(auto_now=True)
-    group_name = models.CharField(max_length=255, blank=True, null=True)
+    channel_group = models.ForeignKey(
+        ChannelGroup,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name='streams'
+    )
     stream_profile = models.ForeignKey(
         StreamProfile,
         null=True,
@@ -42,6 +61,13 @@ class Stream(models.Model):
         default=False,
         help_text="Whether this is a user-created stream or from an M3U account"
     )
+    stream_hash = models.CharField(
+        max_length=255,
+        null=True,
+        unique=True,
+        help_text="Unique hash for this stream from the M3U account"
+    )
+    last_seen = models.DateTimeField(db_index=True, default=datetime.now)
 
     class Meta:
         # If you use m3u_account, you might do unique_together = ('name','url','m3u_account')
@@ -52,6 +78,43 @@ class Stream(models.Model):
     def __str__(self):
         return self.name or self.url or f"Stream ID {self.id}"
 
+    @classmethod
+    def generate_hash_key(cls, stream):
+        # Check if the passed object is an instance or a dictionary
+        if isinstance(stream, dict):
+            # Handle dictionary case (e.g., when the input is a dict of stream data)
+            hash_parts = {key: stream[key] for key in CoreSettings.get_m3u_hash_key().split(",") if key in stream}
+            if 'm3u_account_id' in stream:
+                hash_parts['m3u_account_id'] = stream['m3u_account_id']
+        elif isinstance(stream, Stream):
+            # Handle the case where the input is a Stream instance
+            key_parts = CoreSettings.get_m3u_hash_key().split(",")
+            hash_parts = {key: getattr(stream, key) for key in key_parts if hasattr(stream, key)}
+            if stream.m3u_account:
+                hash_parts['m3u_account_id'] = stream.m3u_account.id
+        else:
+            raise ValueError("stream must be either a dictionary or a Stream instance")
+
+        # Serialize and hash the dictionary
+        serialized_obj = json.dumps(hash_parts, sort_keys=True)  # sort_keys ensures consistent ordering
+        hash_object = hashlib.sha256(serialized_obj.encode())
+        return hash_object.hexdigest()
+
+    @classmethod
+    def update_or_create_by_hash(cls, hash_value, **fields_to_update):
+        try:
+            # Try to find the Stream object with the given hash
+            stream = cls.objects.get(stream_hash=hash_value)
+            # If it exists, update the fields
+            for field, value in fields_to_update.items():
+                setattr(stream, field, value)
+            stream.save()  # Save the updated object
+            return stream, False  # False means it was updated, not created
+        except cls.DoesNotExist:
+            # If it doesn't exist, create a new object with the given hash
+            fields_to_update['stream_hash'] = hash_value  # Make sure the hash field is set
+            stream = cls.objects.create(**fields_to_update)
+            return stream, True  # True means it was created
 
 class ChannelManager(models.Manager):
     def active(self):
@@ -95,7 +158,7 @@ class Channel(models.Model):
         related_name='channels'
     )
 
-    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
+    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True, db_index=True)
 
     def clean(self):
         # Enforce unique channel_number within a given group
@@ -198,16 +261,6 @@ class Channel(models.Model):
         if current_count > 0:
             redis_client.decr(profile_connections_key)
 
-class ChannelGroup(models.Model):
-    name = models.CharField(max_length=100, unique=True)
-
-    def related_channels(self):
-        # local import if needed to avoid cyc. Usually fine in a single file though
-        return Channel.objects.filter(channel_group=self)
-
-    def __str__(self):
-        return self.name
-
 class ChannelStream(models.Model):
     channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
     stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
@@ -215,3 +268,22 @@ class ChannelStream(models.Model):
 
     class Meta:
         ordering = ['order']  # Ensure streams are retrieved in order
+
+class ChannelGroupM3UAccount(models.Model):
+    channel_group = models.ForeignKey(
+        ChannelGroup,
+        on_delete=models.CASCADE,
+        related_name='m3u_account'
+    )
+    m3u_account = models.ForeignKey(
+        M3UAccount,
+        on_delete=models.CASCADE,
+        related_name='channel_group'
+    )
+    enabled = models.BooleanField(default=True)
+
+    class Meta:
+        unique_together = ('channel_group', 'm3u_account')
+
+    def __str__(self):
+        return f"{self.channel_group.name} - {self.m3u_account.name} (Enabled: {self.enabled})"

apps/channels/serializers.py

@@ -1,5 +1,5 @@
 from rest_framework import serializers
-from .models import Stream, Channel, ChannelGroup, ChannelStream
+from .models import Stream, Channel, ChannelGroup, ChannelStream, ChannelGroupM3UAccount
 from core.models import StreamProfile
 
 #
@@ -26,9 +26,9 @@ class StreamSerializer(serializers.ModelSerializer):
             'local_file',
             'current_viewers',
             'updated_at',
-            'group_name',
             'stream_profile_id',
             'is_custom',
+            'channel_group',
         ]
 
     def get_fields(self):
@@ -41,7 +41,7 @@ class StreamSerializer(serializers.ModelSerializer):
             fields['url'].read_only = True
             fields['m3u_account'].read_only = True
             fields['tvg_id'].read_only = True
-            fields['group_name'].read_only = True
+            fields['channel_group'].read_only = True
 
 
         return fields
@@ -146,3 +146,11 @@ class ChannelSerializer(serializers.ModelSerializer):
                 ChannelStream.objects.create(channel=instance, stream_id=stream.id, order=index)
 
         return instance
+
+class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ChannelGroupM3UAccount
+        fields = ['channel_group', 'enabled']
+
+    # Optionally, if you only need the id of the ChannelGroup, you can customize it like this:
+    channel_group = serializers.PrimaryKeyRelatedField(queryset=ChannelGroup.objects.all())

apps/channels/views.py

@@ -16,7 +16,7 @@ class StreamDashboardView(View):
     def get(self, request, *args, **kwargs):
         streams = Stream.objects.values(
             'id', 'name', 'url',
-            'group_name', 'current_viewers'
+            'channel_group', 'current_viewers'
         )
         return JsonResponse({'data': list(streams)}, safe=False)
 

apps/m3u/migrations/0004_m3uaccount_stream_profile.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.1.6 on 2025-03-19 16:33
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_m3u_hash_settings'),
+        ('m3u', '0003_create_custom_account'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='m3uaccount',
+            name='stream_profile',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='m3u_accounts', to='core.streamprofile'),
+        ),
+    ]

apps/m3u/models.py

@@ -3,6 +3,7 @@ from django.core.exceptions import ValidationError
 from core.models import UserAgent
 import re
 from django.dispatch import receiver
+from apps.channels.models import StreamProfile
 
 CUSTOM_M3U_ACCOUNT_NAME="custom"
 
@@ -59,6 +60,13 @@ class M3UAccount(models.Model):
         default=False,
         help_text="Protected - can't be deleted or modified"
     )
+    stream_profile = models.ForeignKey(
+        StreamProfile,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name='m3u_accounts'
+    )
 
     def __str__(self):
         return self.name
@@ -86,6 +94,16 @@ class M3UAccount(models.Model):
     def get_custom_account(cls):
         return cls.objects.get(name=CUSTOM_M3U_ACCOUNT_NAME, locked=True)
 
+    # def get_channel_groups(self):
+    #     return ChannelGroup.objects.filter(m3u_account__m3u_account=self)
+
+    # def is_channel_group_enabled(self, channel_group):
+    #     """Check if the specified ChannelGroup is enabled for this M3UAccount."""
+    #     return self.channel_group.filter(channel_group=channel_group, enabled=True).exists()
+
+    # def get_enabled_streams(self):
+    #     """Return all streams linked to this account with enabled ChannelGroups."""
+    #     return self.streams.filter(channel_group__in=ChannelGroup.objects.filter(m3u_account__enabled=True))
 
 class M3UFilter(models.Model):
     """Defines filters for M3U accounts based on stream name or group title."""

apps/m3u/serializers.py

@@ -1,13 +1,16 @@
 from rest_framework import serializers
 from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile
 from core.models import UserAgent
+from apps.channels.models import ChannelGroup
+from apps.channels.serializers import ChannelGroupM3UAccountSerializer
 
 class M3UFilterSerializer(serializers.ModelSerializer):
     """Serializer for M3U Filters"""
+    channel_groups = ChannelGroupM3UAccountSerializer(source='m3u_account', many=True)
 
     class Meta:
         model = M3UFilter
-        fields = ['id', 'filter_type', 'regex_pattern', 'exclude']
+        fields = ['id', 'filter_type', 'regex_pattern', 'exclude', 'channel_groups']
 
 from rest_framework import serializers
 from .models import M3UAccountProfile

apps/m3u/tasks.py

@@ -8,9 +8,10 @@ from celery import shared_task, current_app
 from django.conf import settings
 from django.core.cache import cache
 from .models import M3UAccount
-from apps.channels.models import Stream
+from apps.channels.models import Stream, ChannelGroup, ChannelGroupM3UAccount
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.utils import timezone
 
 logger = logging.getLogger(__name__)
 
@@ -184,13 +185,23 @@ def refresh_single_m3u_account(account_id):
                 "tvg_id": current_info["tvg_id"]
             }
             try:
-                obj, created = Stream.objects.update_or_create(
-                    name=current_info["name"],
-                    url=line,
+                channel_group, created = ChannelGroup.objects.get_or_create(name=current_info["group_title"])
+                ChannelGroupM3UAccount.objects.get_or_create(
+                    channel_group=channel_group,
                     m3u_account=account,
-                    group_name=current_info["group_title"],
-                    defaults=defaults
                 )
+
+                stream_props = defaults | {
+                    "name": current_info["name"],
+                    "url": line,
+                    "m3u_account": account,
+                    "channel_group": channel_group,
+                    "last_seen": timezone.now(),
+                }
+
+                stream_hash = Stream.generate_hash_key(stream_props)
+                obj, created = Stream.update_or_create_by_hash(stream_hash, **stream_props)
+
                 if created:
                     created_count += 1
                 else:

apps/proxy/ts_proxy/views.py

@@ -101,7 +101,7 @@ def stream_ts(request, channel_id):
             success = proxy_server.initialize_channel(stream_url, channel_id, stream_user_agent, transcode)
             if proxy_server.redis_client:
                 metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
-                profile_value = str(stream_profile)
+                profile_value = stream_profile.id
                 proxy_server.redis_client.hset(metadata_key, "profile", profile_value)
             if not success:
                 return JsonResponse({'error': 'Failed to initialize channel'}, status=500)

core/migrations/0009_m3u_hash_settings.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.6 on 2025-03-01 14:01
+
+from django.db import migrations
+from django.utils.text import slugify
+
+def preload_core_settings(apps, schema_editor):
+    CoreSettings = apps.get_model("core", "CoreSettings")
+    CoreSettings.objects.create(
+        key=slugify("M3U Hash Key"),
+        name="M3U Hash Key",
+        value="name,url,tvg_id",
+    )
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_rename_profile_name_streamprofile_name_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(preload_core_settings),
+    ]

core/models.py

@@ -142,6 +142,7 @@ class StreamProfile(models.Model):
 
 DEFAULT_USER_AGENT_KEY= slugify("Default User-Agent")
 DEFAULT_STREAM_PROFILE_KEY = slugify("Default Stream Profile")
+STREAM_HASH_KEY = slugify("M3U Hash Key")
 
 class CoreSettings(models.Model):
     key = models.CharField(
@@ -166,3 +167,7 @@ class CoreSettings(models.Model):
     @classmethod
     def get_default_stream_profile_id(cls):
         return cls.objects.get(key=DEFAULT_STREAM_PROFILE_KEY).value
+
+    @classmethod
+    def get_m3u_hash_key(cls):
+        return cls.objects.get(key=STREAM_HASH_KEY).value

dispatcharr/settings.py

@@ -151,6 +151,19 @@ AUTH_USER_MODEL = 'accounts.User'
 CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0')
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 
+# Configure Redis key prefix
+CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = {
+    'prefix': 'celery-task:',  # Set the Redis key prefix for Celery
+}
+
+# Set TTL (Time-to-Live) for task results (in seconds)
+CELERY_RESULT_EXPIRES = 3600  # 1 hour TTL for task results
+
+# Optionally, set visibility timeout for task retries (if using Redis)
+CELERY_BROKER_TRANSPORT_OPTIONS = {
+    'visibility_timeout': 3600,  # Time in seconds that a task remains invisible during retries
+}
+
 CELERY_BEAT_SCHEDULE = {
     'fetch-channel-statuses': {
         'task': 'apps.proxy.tasks.fetch_channel_stats',