diff --git a/apps/accounts/api_views.py b/apps/accounts/api_views.py index c4b544b0..bf87c2ab 100644 --- a/apps/accounts/api_views.py +++ b/apps/accounts/api_views.py @@ -134,7 +134,7 @@ class AuthViewSet(viewsets.ViewSet): class UserViewSet(viewsets.ModelViewSet): """Handles CRUD operations for Users""" - queryset = User.objects.all() + queryset = User.objects.all().prefetch_related('channel_profiles') serializer_class = UserSerializer def get_permissions(self): diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 0cdd716a..7a3d5135 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -39,7 +39,7 @@ from .serializers import ( ChannelProfileSerializer, RecordingSerializer, ) -from .tasks import match_epg_channels, evaluate_series_rules, evaluate_series_rules_impl +from .tasks import match_epg_channels, evaluate_series_rules, evaluate_series_rules_impl, match_single_channel_epg, match_selected_channels_epg import django_filters from django_filters.rest_framework import DjangoFilterBackend from rest_framework.filters import SearchFilter, OrderingFilter @@ -178,6 +178,35 @@ class StreamViewSet(viewsets.ModelViewSet): # Return the response with the list of unique group names return Response(list(group_names)) + @swagger_auto_schema( + method="post", + operation_description="Retrieve streams by a list of IDs using POST to avoid URL length limitations", + request_body=openapi.Schema( + type=openapi.TYPE_OBJECT, + required=["ids"], + properties={ + "ids": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Items(type=openapi.TYPE_INTEGER), + description="List of stream IDs to retrieve" + ), + }, + ), + responses={200: StreamSerializer(many=True)}, + ) + @action(detail=False, methods=["post"], url_path="by-ids") + def get_by_ids(self, request, *args, **kwargs): + ids = request.data.get("ids", []) + if not isinstance(ids, list): + return Response( + {"error": "ids must be a list of integers"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + streams = Stream.objects.filter(id__in=ids) + serializer = self.get_serializer(streams, many=True) + return Response(serializer.data) + # ───────────────────────────────────────────────────────── # 2) Channel Group Management (CRUD) @@ -464,6 +493,68 @@ class ChannelViewSet(viewsets.ModelViewSet): "channels": serialized_channels }) + @action(detail=False, methods=["post"], url_path="set-names-from-epg") + def set_names_from_epg(self, request): + """ + Trigger a Celery task to set channel names from EPG data + """ + from .tasks import set_channels_names_from_epg + + data = request.data + channel_ids = data.get("channel_ids", []) + + if not channel_ids: + return Response( + {"error": "channel_ids is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if not isinstance(channel_ids, list): + return Response( + {"error": "channel_ids must be a list"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Start the Celery task + task = set_channels_names_from_epg.delay(channel_ids) + + return Response({ + "message": f"Started EPG name setting task for {len(channel_ids)} channels", + "task_id": task.id, + "channel_count": len(channel_ids) + }) + + @action(detail=False, methods=["post"], url_path="set-logos-from-epg") + def set_logos_from_epg(self, request): + """ + Trigger a Celery task to set channel logos from EPG data + """ + from .tasks import set_channels_logos_from_epg + + data = request.data + channel_ids = data.get("channel_ids", []) + + if not channel_ids: + return Response( + {"error": "channel_ids is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if not isinstance(channel_ids, list): + return Response( + {"error": "channel_ids must be a list"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Start the Celery task + task = set_channels_logos_from_epg.delay(channel_ids) + + return Response({ + "message": f"Started EPG logo setting task for {len(channel_ids)} channels", + "task_id": task.id, + "channel_count": len(channel_ids) + }) + @action(detail=False, methods=["get"], url_path="ids") def get_ids(self, request, *args, **kwargs): # Get the filtered queryset @@ -559,40 +650,42 @@ class ChannelViewSet(viewsets.ModelViewSet): channel_group = stream.channel_group name = request.data.get("name") + + if name is None: name = stream.name # Check if client provided a channel_number; if not, auto-assign one. stream_custom_props = stream.custom_properties or {} - - channel_number = None - if "tvg-chno" in stream_custom_props: - channel_number = float(stream_custom_props["tvg-chno"]) - elif "channel-number" in stream_custom_props: - channel_number = float(stream_custom_props["channel-number"]) - elif "num" in stream_custom_props: - channel_number = float(stream_custom_props["num"]) + channel_number = request.data.get("channel_number") if channel_number is None: - provided_number = request.data.get("channel_number") - if provided_number is None: - channel_number = Channel.get_next_available_channel_number() - else: - try: - channel_number = float(provided_number) - except ValueError: - return Response( - {"error": "channel_number must be an integer."}, - status=status.HTTP_400_BAD_REQUEST, - ) - # If the provided number is already used, return an error. - if Channel.objects.filter(channel_number=channel_number).exists(): - return Response( - { - "error": f"Channel number {channel_number} is already in use. Please choose a different number." - }, - status=status.HTTP_400_BAD_REQUEST, - ) + # Channel number not provided by client, check stream properties or auto-assign + if "tvg-chno" in stream_custom_props: + channel_number = float(stream_custom_props["tvg-chno"]) + elif "channel-number" in stream_custom_props: + channel_number = float(stream_custom_props["channel-number"]) + elif "num" in stream_custom_props: + channel_number = float(stream_custom_props["num"]) + elif channel_number == 0: + # Special case: 0 means ignore provider numbers and auto-assign + channel_number = None + + if channel_number is None: + # Still None, auto-assign the next available channel number + channel_number = Channel.get_next_available_channel_number() + + + try: + channel_number = float(channel_number) + except ValueError: + return Response( + {"error": "channel_number must be an integer."}, + status=status.HTTP_400_BAD_REQUEST, + ) + # If the provided number is already used, return an error. + if Channel.objects.filter(channel_number=channel_number).exists(): + channel_number = Channel.get_next_available_channel_number(channel_number) # Get the tvc_guide_stationid from custom properties if it exists tvc_guide_stationid = None if "tvc-guide-stationid" in stream_custom_props: @@ -667,283 +760,146 @@ class ChannelViewSet(viewsets.ModelViewSet): for profile in profiles ]) + # Send WebSocket notification for single channel creation + from core.utils import send_websocket_update + send_websocket_update('updates', 'update', { + 'type': 'channels_created', + 'count': 1, + 'channel_id': channel.id, + 'channel_name': channel.name, + 'channel_number': channel.channel_number + }) + return Response(serializer.data, status=status.HTTP_201_CREATED) @swagger_auto_schema( method="post", operation_description=( - "Bulk create channels from existing streams. For each object, if 'channel_number' is provided, " - "it is used (if available); otherwise, the next available number is auto-assigned. " - "Each object must include 'stream_id' and 'name'. " - "Supports single profile ID or array of profile IDs in 'channel_profile_ids'." + "Asynchronously bulk create channels from stream IDs. " + "Returns a task ID to track progress via WebSocket. " + "This is the recommended approach for large bulk operations." ), request_body=openapi.Schema( - type=openapi.TYPE_ARRAY, - items=openapi.Schema( - type=openapi.TYPE_OBJECT, - required=["stream_id"], - properties={ - "stream_id": openapi.Schema( - type=openapi.TYPE_INTEGER, - description="ID of the stream to link", - ), - "channel_number": openapi.Schema( - type=openapi.TYPE_NUMBER, - description="(Optional) Desired channel number. Must not be in use.", - ), - "name": openapi.Schema( - type=openapi.TYPE_STRING, description="Desired channel name" - ), - "channel_profile_ids": openapi.Schema( - type=openapi.TYPE_ARRAY, - items=openapi.Items(type=openapi.TYPE_INTEGER), - description="(Optional) Channel profile ID(s) to add the channel to. Can be a single ID or array of IDs." - ), - }, - ), + type=openapi.TYPE_OBJECT, + required=["stream_ids"], + properties={ + "stream_ids": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Items(type=openapi.TYPE_INTEGER), + description="List of stream IDs to create channels from" + ), + "channel_profile_ids": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Items(type=openapi.TYPE_INTEGER), + description="(Optional) Channel profile ID(s) to add the channels to. If not provided, channels are added to all profiles." + ), + "starting_channel_number": openapi.Schema( + type=openapi.TYPE_INTEGER, + description="(Optional) Starting channel number mode: null=use provider numbers, 0=lowest available, other=start from specified number" + ), + }, ), - responses={201: "Bulk channels created"}, + responses={202: "Task started successfully"}, ) @action(detail=False, methods=["post"], url_path="from-stream/bulk") def from_stream_bulk(self, request): - data_list = request.data - if not isinstance(data_list, list): + from .tasks import bulk_create_channels_from_streams + + stream_ids = request.data.get("stream_ids", []) + channel_profile_ids = request.data.get("channel_profile_ids") + starting_channel_number = request.data.get("starting_channel_number") + + if not stream_ids: return Response( - {"error": "Expected a list of channel objects"}, + {"error": "stream_ids is required and cannot be empty"}, status=status.HTTP_400_BAD_REQUEST, ) - created_channels = [] - errors = [] - - # Gather current used numbers once. - used_numbers = set( - Channel.objects.all().values_list("channel_number", flat=True) - ) - next_number = 1 - - def get_auto_number(): - nonlocal next_number - while next_number in used_numbers: - next_number += 1 - used_numbers.add(next_number) - return next_number - - logos_to_create = [] - channels_to_create = [] - streams_map = [] - logo_map = [] - profile_map = [] # Track which profiles each channel should be added to - - for item in data_list: - stream_id = item.get("stream_id") - if not stream_id: - errors.append( - { - "item": item, - "error": "Missing required field: stream_id is required.", - } - ) - continue - - try: - stream = get_object_or_404(Stream, pk=stream_id) - except Exception as e: - errors.append({"item": item, "error": str(e)}) - continue - - name = item.get("name") - if name is None: - name = stream.name - - channel_group = stream.channel_group - - stream_custom_props = stream.custom_properties or {} - - channel_number = None - if "tvg-chno" in stream_custom_props: - channel_number = float(stream_custom_props["tvg-chno"]) - elif "channel-number" in stream_custom_props: - channel_number = float(stream_custom_props["channel-number"]) - elif "num" in stream_custom_props: - channel_number = float(stream_custom_props["num"]) - # Get the tvc_guide_stationid from custom properties if it exists - tvc_guide_stationid = None - if "tvc-guide-stationid" in stream_custom_props: - tvc_guide_stationid = stream_custom_props["tvc-guide-stationid"] - - # Determine channel number: if provided, use it (if free); else auto assign. - if channel_number is None: - provided_number = item.get("channel_number") - if provided_number is None: - channel_number = get_auto_number() - else: - try: - channel_number = float(provided_number) - except ValueError: - errors.append( - { - "item": item, - "error": "channel_number must be a number.", - } - ) - continue - if ( - channel_number in used_numbers - or Channel.objects.filter( - channel_number=channel_number - ).exists() - ): - errors.append( - { - "item": item, - "error": f"Channel number {channel_number} is already in use.", - } - ) - continue - used_numbers.add(channel_number) - - channel_data = { - "channel_number": channel_number, - "name": name, - "tvc_guide_stationid": tvc_guide_stationid, - "tvg_id": stream.tvg_id, - } - - # Only add channel_group_id if the stream has a channel group - if channel_group: - channel_data["channel_group_id"] = channel_group.id - - # Attempt to find existing EPGs with the same tvg-id - epgs = EPGData.objects.filter(tvg_id=stream.tvg_id) - if epgs: - channel_data["epg_data_id"] = epgs.first().id - - serializer = self.get_serializer(data=channel_data) - if serializer.is_valid(): - validated_data = serializer.validated_data - channel = Channel(**validated_data) - channels_to_create.append(channel) - - streams_map.append([stream_id]) - # Store which profiles this channel should be added to - normalize to array - channel_profile_ids = item.get("channel_profile_ids") - if channel_profile_ids is not None: - # Normalize single ID to array - if not isinstance(channel_profile_ids, list): - channel_profile_ids = [channel_profile_ids] - - profile_map.append(channel_profile_ids) - - if stream.logo_url: - logos_to_create.append( - Logo( - url=stream.logo_url, - name=stream.name or stream.tvg_id, - ) - ) - logo_map.append(stream.logo_url) - else: - logo_map.append(None) - - else: - errors.append({"item": item, "error": serializer.errors}) - - if logos_to_create: - Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True) - - channel_logos = { - logo.url: logo - for logo in Logo.objects.filter( - url__in=[url for url in logo_map if url is not None] + if not isinstance(stream_ids, list): + return Response( + {"error": "stream_ids must be a list of integers"}, + status=status.HTTP_400_BAD_REQUEST, ) - } - # Get all profiles for default assignment - all_profiles = ChannelProfile.objects.all() - channel_profile_memberships = [] + # Normalize channel_profile_ids to array if single ID provided + if channel_profile_ids is not None: + if not isinstance(channel_profile_ids, list): + channel_profile_ids = [channel_profile_ids] - if channels_to_create: - with transaction.atomic(): - created_channels = Channel.objects.bulk_create(channels_to_create) + # Start the async task + task = bulk_create_channels_from_streams.delay(stream_ids, channel_profile_ids, starting_channel_number) - update = [] - for channel, stream_ids, logo_url, channel_profile_ids in zip( - created_channels, streams_map, logo_map, profile_map - ): - if logo_url: - channel.logo = channel_logos[logo_url] - update.append(channel) - - # Handle channel profile membership based on channel_profile_ids - if channel_profile_ids: - # Add channel only to the specified profiles - try: - specific_profiles = ChannelProfile.objects.filter(id__in=channel_profile_ids) - channel_profile_memberships.extend([ - ChannelProfileMembership( - channel_profile=profile, - channel=channel, - enabled=True - ) - for profile in specific_profiles - ]) - except Exception: - # If profiles don't exist, add to all profiles as fallback - channel_profile_memberships.extend([ - ChannelProfileMembership( - channel_profile=profile, - channel=channel, - enabled=True - ) - for profile in all_profiles - ]) - else: - # Default behavior: add to all profiles - channel_profile_memberships.extend([ - ChannelProfileMembership( - channel_profile=profile, - channel=channel, - enabled=True - ) - for profile in all_profiles - ]) - - # Bulk create profile memberships - if channel_profile_memberships: - ChannelProfileMembership.objects.bulk_create( - channel_profile_memberships - ) - - # Update logos - if update: - Channel.objects.bulk_update(update, ["logo"]) - - # Set stream relationships - for channel, stream_ids in zip(created_channels, streams_map): - channel.streams.set(stream_ids) - - response_data = {"created": ChannelSerializer(created_channels, many=True).data} - if errors: - response_data["errors"] = errors - - return Response(response_data, status=status.HTTP_201_CREATED) + return Response({ + "task_id": task.id, + "message": f"Bulk channel creation task started for {len(stream_ids)} streams", + "stream_count": len(stream_ids), + "status": "started" + }, status=status.HTTP_202_ACCEPTED) # ───────────────────────────────────────────────────────── # 6) EPG Fuzzy Matching # ───────────────────────────────────────────────────────── @swagger_auto_schema( method="post", - operation_description="Kick off a Celery task that tries to fuzzy-match channels with EPG data.", + operation_description="Kick off a Celery task that tries to fuzzy-match channels with EPG data. If channel_ids are provided, only those channels will be processed.", + request_body=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + 'channel_ids': openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema(type=openapi.TYPE_INTEGER), + description='List of channel IDs to process. If empty or not provided, all channels without EPG will be processed.' + ) + } + ), responses={202: "EPG matching task initiated"}, ) @action(detail=False, methods=["post"], url_path="match-epg") def match_epg(self, request): - match_epg_channels.delay() + # Get channel IDs from request body if provided + channel_ids = request.data.get('channel_ids', []) + + if channel_ids: + # Process only selected channels + from .tasks import match_selected_channels_epg + match_selected_channels_epg.delay(channel_ids) + message = f"EPG matching task initiated for {len(channel_ids)} selected channel(s)." + else: + # Process all channels without EPG (original behavior) + match_epg_channels.delay() + message = "EPG matching task initiated for all channels without EPG." + return Response( - {"message": "EPG matching task initiated."}, status=status.HTTP_202_ACCEPTED + {"message": message}, status=status.HTTP_202_ACCEPTED ) + @swagger_auto_schema( + method="post", + operation_description="Try to auto-match this specific channel with EPG data.", + responses={200: "EPG matching completed", 202: "EPG matching task initiated"}, + ) + @action(detail=True, methods=["post"], url_path="match-epg") + def match_channel_epg(self, request, pk=None): + channel = self.get_object() + + # Import the matching logic + from apps.channels.tasks import match_single_channel_epg + + try: + # Try to match this specific channel - call synchronously for immediate response + result = match_single_channel_epg.apply_async(args=[channel.id]).get(timeout=30) + + # Refresh the channel from DB to get any updates + channel.refresh_from_db() + + return Response({ + "message": result.get("message", "Channel matching completed"), + "matched": result.get("matched", False), + "channel": self.get_serializer(channel).data + }) + except Exception as e: + return Response({"error": str(e)}, status=400) + # ───────────────────────────────────────────────────────── # 7) Set EPG and Refresh # ───────────────────────────────────────────────────────── diff --git a/apps/channels/models.py b/apps/channels/models.py index 0a4f72e7..af66178d 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -379,14 +379,17 @@ class Channel(models.Model): if not m3u_account: logger.debug(f"Stream {stream.id} has no M3U account") continue + if m3u_account.is_active == False: + logger.debug(f"M3U account {m3u_account.id} is inactive, skipping.") + continue - m3u_profiles = m3u_account.profiles.all() + m3u_profiles = m3u_account.profiles.filter(is_active=True) default_profile = next( (obj for obj in m3u_profiles if obj.is_default), None ) if not default_profile: - logger.debug(f"M3U account {m3u_account.id} has no default profile") + logger.debug(f"M3U account {m3u_account.id} has no active default profile") continue profiles = [default_profile] + [ @@ -394,11 +397,6 @@ class Channel(models.Model): ] for profile in profiles: - # Skip inactive profiles - if not profile.is_active: - logger.debug(f"Skipping inactive profile {profile.id}") - continue - has_active_profiles = True profile_connections_key = f"profile_connections:{profile.id}" @@ -433,9 +431,9 @@ class Channel(models.Model): # No available streams - determine specific reason if has_streams_but_maxed_out: - error_reason = "All M3U profiles have reached maximum connection limits" + error_reason = "All active M3U profiles have reached maximum connection limits" elif has_active_profiles: - error_reason = "No compatible profile found for any assigned stream" + error_reason = "No compatible active profile found for any assigned stream" else: error_reason = "No active profiles found for any assigned stream" diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 5135ddaf..2760d1a7 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -12,6 +12,7 @@ import gc from celery import shared_task from django.utils.text import slugify +from rapidfuzz import fuzz from apps.channels.models import Channel from apps.epg.models import EPGData @@ -27,6 +28,87 @@ from urllib.parse import quote logger = logging.getLogger(__name__) +def send_epg_matching_progress(total_channels, matched_channels, current_channel_name="", stage="matching"): + """ + Send EPG matching progress via WebSocket + """ + try: + channel_layer = get_channel_layer() + if channel_layer: + progress_data = { + 'type': 'epg_matching_progress', + 'total': total_channels, + 'matched': len(matched_channels) if isinstance(matched_channels, list) else matched_channels, + 'remaining': total_channels - (len(matched_channels) if isinstance(matched_channels, list) else matched_channels), + 'current_channel': current_channel_name, + 'stage': stage, + 'progress_percent': round((len(matched_channels) if isinstance(matched_channels, list) else matched_channels) / total_channels * 100, 1) if total_channels > 0 else 0 + } + + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": { + "type": "epg_matching_progress", + **progress_data + } + } + ) + except Exception as e: + logger.warning(f"Failed to send EPG matching progress: {e}") + +# Lazy loading for ML models - only imported/loaded when needed +_ml_model_cache = { + 'sentence_transformer': None +} + +def get_sentence_transformer(): + """Lazy load the sentence transformer model only when needed""" + if _ml_model_cache['sentence_transformer'] is None: + try: + from sentence_transformers import SentenceTransformer + from sentence_transformers import util + + model_name = "sentence-transformers/all-MiniLM-L6-v2" + cache_dir = "/data/models" + + # Check environment variable to disable downloads + disable_downloads = os.environ.get('DISABLE_ML_DOWNLOADS', 'false').lower() == 'true' + + if disable_downloads: + # Check if model exists before attempting to load + hf_model_path = os.path.join(cache_dir, f"models--{model_name.replace('/', '--')}") + if not os.path.exists(hf_model_path): + logger.warning("ML model not found and downloads disabled (DISABLE_ML_DOWNLOADS=true). Skipping ML matching.") + return None, None + + # Ensure cache directory exists + os.makedirs(cache_dir, exist_ok=True) + + # Let sentence-transformers handle all cache detection and management + logger.info(f"Loading sentence transformer model (cache: {cache_dir})") + _ml_model_cache['sentence_transformer'] = SentenceTransformer( + model_name, + cache_folder=cache_dir + ) + + return _ml_model_cache['sentence_transformer'], util + except ImportError: + logger.warning("sentence-transformers not available - ML-enhanced matching disabled") + return None, None + except Exception as e: + logger.error(f"Failed to load sentence transformer: {e}") + return None, None + else: + from sentence_transformers import util + return _ml_model_cache['sentence_transformer'], util + +# ML matching thresholds (same as original script) +BEST_FUZZY_THRESHOLD = 85 +LOWER_FUZZY_THRESHOLD = 40 +EMBED_SIM_THRESHOLD = 0.65 + # Words we remove to help with fuzzy + embedding matching COMMON_EXTRANEOUS_WORDS = [ "tv", "channel", "network", "television", @@ -49,155 +131,353 @@ def normalize_name(name: str) -> str: norm = name.lower() norm = re.sub(r"\[.*?\]", "", norm) + + # Extract and preserve important call signs from parentheses before removing them + # This captures call signs like (KVLY), (KING), (KARE), etc. + call_sign_match = re.search(r"\(([A-Z]{3,5})\)", name) + preserved_call_sign = "" + if call_sign_match: + preserved_call_sign = " " + call_sign_match.group(1).lower() + + # Now remove all parentheses content norm = re.sub(r"\(.*?\)", "", norm) + + # Add back the preserved call sign + norm = norm + preserved_call_sign + norm = re.sub(r"[^\w\s]", "", norm) tokens = norm.split() tokens = [t for t in tokens if t not in COMMON_EXTRANEOUS_WORDS] norm = " ".join(tokens).strip() return norm +def match_channels_to_epg(channels_data, epg_data, region_code=None, use_ml=True, send_progress=True): + """ + EPG matching logic that finds the best EPG matches for channels using + multiple matching strategies including fuzzy matching and ML models. + + Automatically uses conservative thresholds for bulk matching (multiple channels) + to avoid bad matches that create user cleanup work, and aggressive thresholds + for single channel matching where users specifically requested a match attempt. + """ + channels_to_update = [] + matched_channels = [] + total_channels = len(channels_data) + + # Send initial progress + if send_progress: + send_epg_matching_progress(total_channels, 0, stage="starting") + + # Try to get ML models if requested (but don't load yet - lazy loading) + st_model, util = None, None + epg_embeddings = None + ml_available = use_ml + + # Automatically determine matching strategy based on number of channels + is_bulk_matching = len(channels_data) > 1 + + # Adjust matching thresholds based on operation type + if is_bulk_matching: + # Conservative thresholds for bulk matching to avoid creating cleanup work + FUZZY_HIGH_CONFIDENCE = 90 # Only very high fuzzy scores + FUZZY_MEDIUM_CONFIDENCE = 70 # Higher threshold for ML enhancement + ML_HIGH_CONFIDENCE = 0.75 # Higher ML confidence required + ML_LAST_RESORT = 0.65 # More conservative last resort + FUZZY_LAST_RESORT_MIN = 50 # Higher fuzzy minimum for last resort + logger.info(f"Using conservative thresholds for bulk matching ({total_channels} channels)") + else: + # More aggressive thresholds for single channel matching (user requested specific match) + FUZZY_HIGH_CONFIDENCE = 85 # Original threshold + FUZZY_MEDIUM_CONFIDENCE = 40 # Original threshold + ML_HIGH_CONFIDENCE = 0.65 # Original threshold + ML_LAST_RESORT = 0.50 # Original desperate threshold + FUZZY_LAST_RESORT_MIN = 20 # Original minimum + logger.info("Using aggressive thresholds for single channel matching") # Process each channel + for index, chan in enumerate(channels_data): + normalized_tvg_id = chan.get("tvg_id", "") + fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] + + # Send progress update every 5 channels or for the first few + if send_progress and (index < 5 or index % 5 == 0 or index == total_channels - 1): + send_epg_matching_progress( + total_channels, + len(matched_channels), + current_channel_name=chan["name"][:50], # Truncate long names + stage="matching" + ) + normalized_tvg_id = chan.get("tvg_id", "") + fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] + + # Step 1: Exact TVG ID match + epg_by_tvg_id = next((epg for epg in epg_data if epg["tvg_id"] == normalized_tvg_id), None) + if normalized_tvg_id and epg_by_tvg_id: + chan["epg_data_id"] = epg_by_tvg_id["id"] + channels_to_update.append(chan) + matched_channels.append((chan['id'], fallback_name, epg_by_tvg_id["tvg_id"])) + logger.info(f"Channel {chan['id']} '{fallback_name}' => EPG found by exact tvg_id={epg_by_tvg_id['tvg_id']}") + continue + + # Step 2: Secondary TVG ID check (legacy compatibility) + if chan["tvg_id"]: + epg_match = [epg["id"] for epg in epg_data if epg["tvg_id"] == chan["tvg_id"]] + if epg_match: + chan["epg_data_id"] = epg_match[0] + channels_to_update.append(chan) + matched_channels.append((chan['id'], fallback_name, chan["tvg_id"])) + logger.info(f"Channel {chan['id']} '{chan['name']}' => EPG found by secondary tvg_id={chan['tvg_id']}") + continue + + # Step 3: Name-based fuzzy matching + if not chan["norm_chan"]: + logger.debug(f"Channel {chan['id']} '{chan['name']}' => empty after normalization, skipping") + continue + + best_score = 0 + best_epg = None + + # Debug: show what we're matching against + logger.debug(f"Fuzzy matching '{chan['norm_chan']}' against EPG entries...") + + # Find best fuzzy match + for row in epg_data: + if not row.get("norm_name"): + continue + + base_score = fuzz.ratio(chan["norm_chan"], row["norm_name"]) + bonus = 0 + + # Apply region-based bonus/penalty + if region_code and row.get("tvg_id"): + combined_text = row["tvg_id"].lower() + " " + row["name"].lower() + dot_regions = re.findall(r'\.([a-z]{2})', combined_text) + + if dot_regions: + if region_code in dot_regions: + bonus = 15 # Bigger bonus for matching region + else: + bonus = -15 # Penalty for different region + elif region_code in combined_text: + bonus = 10 + + score = base_score + bonus + + # Debug the best few matches + if score > 50: # Only show decent matches + logger.debug(f" EPG '{row['name']}' (norm: '{row['norm_name']}') => score: {score} (base: {base_score}, bonus: {bonus})") + + if score > best_score: + best_score = score + best_epg = row + + # Log the best score we found + if best_epg: + logger.info(f"Channel {chan['id']} '{chan['name']}' => best match: '{best_epg['name']}' (score: {best_score})") + else: + logger.debug(f"Channel {chan['id']} '{chan['name']}' => no EPG entries with valid norm_name found") + continue + + # High confidence match - accept immediately + if best_score >= FUZZY_HIGH_CONFIDENCE: + chan["epg_data_id"] = best_epg["id"] + channels_to_update.append(chan) + matched_channels.append((chan['id'], chan['name'], best_epg["tvg_id"])) + logger.info(f"Channel {chan['id']} '{chan['name']}' => matched tvg_id={best_epg['tvg_id']} (score={best_score})") + + # Medium confidence - use ML if available (lazy load models here) + elif best_score >= FUZZY_MEDIUM_CONFIDENCE and ml_available: + # Lazy load ML models only when we actually need them + if st_model is None: + st_model, util = get_sentence_transformer() + + # Lazy generate embeddings only when we actually need them + if epg_embeddings is None and st_model and any(row.get("norm_name") for row in epg_data): + try: + logger.info("Generating embeddings for EPG data using ML model (lazy loading)") + epg_embeddings = st_model.encode( + [row["norm_name"] for row in epg_data if row.get("norm_name")], + convert_to_tensor=True + ) + except Exception as e: + logger.warning(f"Failed to generate embeddings: {e}") + epg_embeddings = None + + if epg_embeddings is not None and st_model: + try: + # Generate embedding for this channel + chan_embedding = st_model.encode(chan["norm_chan"], convert_to_tensor=True) + + # Calculate similarity with all EPG embeddings + sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] + top_index = int(sim_scores.argmax()) + top_value = float(sim_scores[top_index]) + + if top_value >= ML_HIGH_CONFIDENCE: + # Find the EPG entry that corresponds to this embedding index + epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] + matched_epg = epg_with_names[top_index] + + chan["epg_data_id"] = matched_epg["id"] + channels_to_update.append(chan) + matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) + logger.info(f"Channel {chan['id']} '{chan['name']}' => matched EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") + else: + logger.info(f"Channel {chan['id']} '{chan['name']}' => fuzzy={best_score}, ML-sim={top_value:.2f} < {ML_HIGH_CONFIDENCE}, trying last resort...") + + # Last resort: try ML with very low fuzzy threshold + if top_value >= ML_LAST_RESORT: # Dynamic last resort threshold + epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] + matched_epg = epg_with_names[top_index] + + chan["epg_data_id"] = matched_epg["id"] + channels_to_update.append(chan) + matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) + logger.info(f"Channel {chan['id']} '{chan['name']}' => LAST RESORT match EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") + else: + logger.info(f"Channel {chan['id']} '{chan['name']}' => even last resort ML-sim {top_value:.2f} < {ML_LAST_RESORT}, skipping") + + except Exception as e: + logger.warning(f"ML matching failed for channel {chan['id']}: {e}") + # Fall back to non-ML decision + logger.info(f"Channel {chan['id']} '{chan['name']}' => fuzzy score {best_score} below threshold, skipping") + + # Last resort: Try ML matching even with very low fuzzy scores + elif best_score >= FUZZY_LAST_RESORT_MIN and ml_available: + # Lazy load ML models for last resort attempts + if st_model is None: + st_model, util = get_sentence_transformer() + + # Lazy generate embeddings for last resort attempts + if epg_embeddings is None and st_model and any(row.get("norm_name") for row in epg_data): + try: + logger.info("Generating embeddings for EPG data using ML model (last resort lazy loading)") + epg_embeddings = st_model.encode( + [row["norm_name"] for row in epg_data if row.get("norm_name")], + convert_to_tensor=True + ) + except Exception as e: + logger.warning(f"Failed to generate embeddings for last resort: {e}") + epg_embeddings = None + + if epg_embeddings is not None and st_model: + try: + logger.info(f"Channel {chan['id']} '{chan['name']}' => trying ML as last resort (fuzzy={best_score})") + # Generate embedding for this channel + chan_embedding = st_model.encode(chan["norm_chan"], convert_to_tensor=True) + + # Calculate similarity with all EPG embeddings + sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] + top_index = int(sim_scores.argmax()) + top_value = float(sim_scores[top_index]) + + if top_value >= ML_LAST_RESORT: # Dynamic threshold for desperate attempts + # Find the EPG entry that corresponds to this embedding index + epg_with_names = [epg for epg in epg_data if epg.get("norm_name")] + matched_epg = epg_with_names[top_index] + + chan["epg_data_id"] = matched_epg["id"] + channels_to_update.append(chan) + matched_channels.append((chan['id'], chan['name'], matched_epg["tvg_id"])) + logger.info(f"Channel {chan['id']} '{chan['name']}' => DESPERATE LAST RESORT match EPG tvg_id={matched_epg['tvg_id']} (fuzzy={best_score}, ML-sim={top_value:.2f})") + else: + logger.info(f"Channel {chan['id']} '{chan['name']}' => desperate last resort ML-sim {top_value:.2f} < {ML_LAST_RESORT}, giving up") + except Exception as e: + logger.warning(f"Last resort ML matching failed for channel {chan['id']}: {e}") + logger.info(f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} < {FUZZY_MEDIUM_CONFIDENCE}, giving up") + else: + # No ML available or very low fuzzy score + logger.info(f"Channel {chan['id']} '{chan['name']}' => best fuzzy score={best_score} < {FUZZY_MEDIUM_CONFIDENCE}, no ML fallback available") + + # Clean up ML models from memory after matching (infrequent operation) + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory") + _ml_model_cache['sentence_transformer'] = None + gc.collect() + + # Send final progress update + if send_progress: + send_epg_matching_progress( + total_channels, + len(matched_channels), + stage="completed" + ) + + return { + "channels_to_update": channels_to_update, + "matched_channels": matched_channels + } + @shared_task def match_epg_channels(): """ - Goes through all Channels and tries to find a matching EPGData row by: - 1) If channel.tvg_id is valid in EPGData, skip. - 2) If channel has a tvg_id but not found in EPGData, attempt direct EPGData lookup. - 3) Otherwise, perform name-based fuzzy matching with optional region-based bonus. - 4) If a match is found, we set channel.tvg_id - 5) Summarize and log results. + Uses integrated EPG matching instead of external script. + Provides the same functionality with better performance and maintainability. """ try: - logger.info("Starting EPG matching logic...") + logger.info("Starting integrated EPG matching...") - # Attempt to retrieve a "preferred-region" if configured + # Get region preference try: region_obj = CoreSettings.objects.get(key="preferred-region") region_code = region_obj.value.strip().lower() except CoreSettings.DoesNotExist: region_code = None - matched_channels = [] - channels_to_update = [] - # Get channels that don't have EPG data assigned channels_without_epg = Channel.objects.filter(epg_data__isnull=True) logger.info(f"Found {channels_without_epg.count()} channels without EPG data") - channels_json = [] + channels_data = [] for channel in channels_without_epg: - # Normalize TVG ID - strip whitespace and convert to lowercase normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" - if normalized_tvg_id: - logger.info(f"Processing channel {channel.id} '{channel.name}' with TVG ID='{normalized_tvg_id}'") - - channels_json.append({ + channels_data.append({ "id": channel.id, "name": channel.name, - "tvg_id": normalized_tvg_id, # Use normalized TVG ID - "original_tvg_id": channel.tvg_id, # Keep original for reference + "tvg_id": normalized_tvg_id, + "original_tvg_id": channel.tvg_id, "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, - "norm_chan": normalize_name(normalized_tvg_id if normalized_tvg_id else channel.name) + "norm_chan": normalize_name(channel.name) # Always use channel name for fuzzy matching! }) - # Similarly normalize EPG data TVG IDs - epg_json = [] + # Get all EPG data + epg_data = [] for epg in EPGData.objects.all(): normalized_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" - epg_json.append({ + epg_data.append({ 'id': epg.id, - 'tvg_id': normalized_tvg_id, # Use normalized TVG ID - 'original_tvg_id': epg.tvg_id, # Keep original for reference + 'tvg_id': normalized_tvg_id, + 'original_tvg_id': epg.tvg_id, 'name': epg.name, 'norm_name': normalize_name(epg.name), 'epg_source_id': epg.epg_source.id if epg.epg_source else None, }) - # Log available EPG data TVG IDs for debugging - unique_epg_tvg_ids = set(e['tvg_id'] for e in epg_json if e['tvg_id']) - logger.info(f"Available EPG TVG IDs: {', '.join(sorted(unique_epg_tvg_ids))}") + logger.info(f"Processing {len(channels_data)} channels against {len(epg_data)} EPG entries") - payload = { - "channels": channels_json, - "epg_data": epg_json, - "region_code": region_code, - } - - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_file.write(json.dumps(payload).encode('utf-8')) - temp_file_path = temp_file.name - - # After writing to the file but before subprocess - # Explicitly delete the large data structures - del payload - gc.collect() - - process = subprocess.Popen( - ['python', '/app/scripts/epg_match.py', temp_file_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - stdout = '' - block_size = 1024 - - while True: - # Monitor stdout and stderr for readability - readable, _, _ = select.select([process.stdout, process.stderr], [], [], 1) # timeout of 1 second - - if not readable: # timeout expired - if process.poll() is not None: # check if process finished - break - else: # process still running, continue - continue - - for stream in readable: - if stream == process.stdout: - stdout += stream.read(block_size) - elif stream == process.stderr: - error = stream.readline() - if error: - logger.info(error.strip()) - - if process.poll() is not None: - break - - process.wait() - os.remove(temp_file_path) - - if process.returncode != 0: - return f"Failed to process EPG matching" - - result = json.loads(stdout) - # This returns lists of dicts, not model objects + # Run EPG matching with progress updates - automatically uses conservative thresholds for bulk operations + result = match_channels_to_epg(channels_data, epg_data, region_code, use_ml=True, send_progress=True) channels_to_update_dicts = result["channels_to_update"] matched_channels = result["matched_channels"] - # Explicitly clean up large objects - del stdout, result - gc.collect() - - # Convert your dict-based 'channels_to_update' into real Channel objects + # Update channels in database if channels_to_update_dicts: - # Extract IDs of the channels that need updates channel_ids = [d["id"] for d in channels_to_update_dicts] - - # Fetch them from DB channels_qs = Channel.objects.filter(id__in=channel_ids) channels_list = list(channels_qs) - # Build a map from channel_id -> epg_data_id (or whatever fields you need) - epg_mapping = { - d["id"]: d["epg_data_id"] for d in channels_to_update_dicts - } + # Create mapping from channel_id to epg_data_id + epg_mapping = {d["id"]: d["epg_data_id"] for d in channels_to_update_dicts} - # Populate each Channel object with the updated epg_data_id + # Update each channel with matched EPG data for channel_obj in channels_list: - # The script sets 'epg_data_id' in the returned dict - # We either assign directly, or fetch the EPGData instance if needed. - channel_obj.epg_data_id = epg_mapping.get(channel_obj.id) + epg_data_id = epg_mapping.get(channel_obj.id) + if epg_data_id: + try: + epg_data_obj = EPGData.objects.get(id=epg_data_id) + channel_obj.epg_data = epg_data_obj + except EPGData.DoesNotExist: + logger.error(f"EPG data {epg_data_id} not found for channel {channel_obj.id}") - # Now we have real model objects, so bulk_update will work + # Bulk update all channels Channel.objects.bulk_update(channels_list, ["epg_data"]) total_matched = len(matched_channels) @@ -208,9 +488,9 @@ def match_epg_channels(): else: logger.info("No new channels were matched.") - logger.info("Finished EPG matching logic.") + logger.info("Finished integrated EPG matching.") - # Send update with additional information for refreshing UI + # Send WebSocket update channel_layer = get_channel_layer() associations = [ {"channel_id": chan["id"], "epg_data_id": chan["epg_data_id"]} @@ -224,23 +504,311 @@ def match_epg_channels(): "data": { "success": True, "type": "epg_match", - "refresh_channels": True, # Flag to tell frontend to refresh channels + "refresh_channels": True, "matches_count": total_matched, "message": f"EPG matching complete: {total_matched} channel(s) matched", - "associations": associations # Add the associations data + "associations": associations } } ) return f"Done. Matched {total_matched} channel(s)." + finally: - # Final cleanup + # Clean up ML models from memory after bulk matching + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory") + _ml_model_cache['sentence_transformer'] = None + + # Memory cleanup gc.collect() - # Use our standardized cleanup function for more thorough memory management from core.utils import cleanup_memory cleanup_memory(log_usage=True, force_collection=True) +@shared_task +def match_selected_channels_epg(channel_ids): + """ + Match EPG data for only the specified selected channels. + Uses the same integrated EPG matching logic but processes only selected channels. + """ + try: + logger.info(f"Starting integrated EPG matching for {len(channel_ids)} selected channels...") + + # Get region preference + try: + region_obj = CoreSettings.objects.get(key="preferred-region") + region_code = region_obj.value.strip().lower() + except CoreSettings.DoesNotExist: + region_code = None + + # Get only the specified channels that don't have EPG data assigned + channels_without_epg = Channel.objects.filter( + id__in=channel_ids, + epg_data__isnull=True + ) + logger.info(f"Found {channels_without_epg.count()} selected channels without EPG data") + + if not channels_without_epg.exists(): + logger.info("No selected channels need EPG matching.") + + # Send WebSocket update + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": { + "success": True, + "type": "epg_match", + "refresh_channels": True, + "matches_count": 0, + "message": "No selected channels need EPG matching", + "associations": [] + } + } + ) + return "No selected channels needed EPG matching." + + channels_data = [] + for channel in channels_without_epg: + normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" + channels_data.append({ + "id": channel.id, + "name": channel.name, + "tvg_id": normalized_tvg_id, + "original_tvg_id": channel.tvg_id, + "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, + "norm_chan": normalize_name(channel.name) + }) + + # Get all EPG data + epg_data = [] + for epg in EPGData.objects.all(): + normalized_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" + epg_data.append({ + 'id': epg.id, + 'tvg_id': normalized_tvg_id, + 'original_tvg_id': epg.tvg_id, + 'name': epg.name, + 'norm_name': normalize_name(epg.name), + 'epg_source_id': epg.epg_source.id if epg.epg_source else None, + }) + + logger.info(f"Processing {len(channels_data)} selected channels against {len(epg_data)} EPG entries") + + # Run EPG matching with progress updates - automatically uses appropriate thresholds + result = match_channels_to_epg(channels_data, epg_data, region_code, use_ml=True, send_progress=True) + channels_to_update_dicts = result["channels_to_update"] + matched_channels = result["matched_channels"] + + # Update channels in database + if channels_to_update_dicts: + channel_ids_to_update = [d["id"] for d in channels_to_update_dicts] + channels_qs = Channel.objects.filter(id__in=channel_ids_to_update) + channels_list = list(channels_qs) + + # Create mapping from channel_id to epg_data_id + epg_mapping = {d["id"]: d["epg_data_id"] for d in channels_to_update_dicts} + + # Update each channel with matched EPG data + for channel_obj in channels_list: + epg_data_id = epg_mapping.get(channel_obj.id) + if epg_data_id: + try: + epg_data_obj = EPGData.objects.get(id=epg_data_id) + channel_obj.epg_data = epg_data_obj + except EPGData.DoesNotExist: + logger.error(f"EPG data {epg_data_id} not found for channel {channel_obj.id}") + + # Bulk update all channels + Channel.objects.bulk_update(channels_list, ["epg_data"]) + + total_matched = len(matched_channels) + if total_matched: + logger.info(f"Selected Channel Match Summary: {total_matched} channel(s) matched.") + for (cid, cname, tvg) in matched_channels: + logger.info(f" - Channel ID={cid}, Name='{cname}' => tvg_id='{tvg}'") + else: + logger.info("No selected channels were matched.") + + logger.info("Finished integrated EPG matching for selected channels.") + + # Send WebSocket update + channel_layer = get_channel_layer() + associations = [ + {"channel_id": chan["id"], "epg_data_id": chan["epg_data_id"]} + for chan in channels_to_update_dicts + ] + + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": { + "success": True, + "type": "epg_match", + "refresh_channels": True, + "matches_count": total_matched, + "message": f"EPG matching complete: {total_matched} selected channel(s) matched", + "associations": associations + } + } + ) + + return f"Done. Matched {total_matched} selected channel(s)." + + finally: + # Clean up ML models from memory after bulk matching + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory") + _ml_model_cache['sentence_transformer'] = None + + # Memory cleanup + gc.collect() + from core.utils import cleanup_memory + cleanup_memory(log_usage=True, force_collection=True) + + +@shared_task +def match_single_channel_epg(channel_id): + """ + Try to match a single channel with EPG data using the integrated matching logic + that includes both fuzzy and ML-enhanced matching. Returns a dict with match status and message. + """ + try: + from apps.channels.models import Channel + from apps.epg.models import EPGData + + logger.info(f"Starting integrated single channel EPG matching for channel ID {channel_id}") + + # Get the channel + try: + channel = Channel.objects.get(id=channel_id) + except Channel.DoesNotExist: + return {"matched": False, "message": "Channel not found"} + + # If channel already has EPG data, skip + if channel.epg_data: + return {"matched": False, "message": f"Channel '{channel.name}' already has EPG data assigned"} + + # Prepare single channel data for matching (same format as bulk matching) + normalized_tvg_id = channel.tvg_id.strip().lower() if channel.tvg_id else "" + channel_data = { + "id": channel.id, + "name": channel.name, + "tvg_id": normalized_tvg_id, + "original_tvg_id": channel.tvg_id, + "fallback_name": normalized_tvg_id if normalized_tvg_id else channel.name, + "norm_chan": normalize_name(channel.name) # Always use channel name for fuzzy matching! + } + + logger.info(f"Channel data prepared: name='{channel.name}', tvg_id='{normalized_tvg_id}', norm_chan='{channel_data['norm_chan']}'") + + # Debug: Test what the normalization does to preserve call signs + test_name = "NBC 11 (KVLY) - Fargo" # Example for testing + test_normalized = normalize_name(test_name) + logger.debug(f"DEBUG normalization example: '{test_name}' → '{test_normalized}' (call sign preserved)") + + # Get all EPG data for matching - must include norm_name field + epg_data_list = [] + for epg in EPGData.objects.filter(name__isnull=False).exclude(name=''): + normalized_epg_tvg_id = epg.tvg_id.strip().lower() if epg.tvg_id else "" + epg_data_list.append({ + 'id': epg.id, + 'tvg_id': normalized_epg_tvg_id, + 'original_tvg_id': epg.tvg_id, + 'name': epg.name, + 'norm_name': normalize_name(epg.name), + 'epg_source_id': epg.epg_source.id if epg.epg_source else None, + }) + + if not epg_data_list: + return {"matched": False, "message": "No EPG data available for matching"} + + logger.info(f"Matching single channel '{channel.name}' against {len(epg_data_list)} EPG entries") + + # Send progress for single channel matching + send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="matching") + + # Use the EPG matching function - automatically uses aggressive thresholds for single channel + result = match_channels_to_epg([channel_data], epg_data_list, send_progress=False) + channels_to_update = result.get("channels_to_update", []) + matched_channels = result.get("matched_channels", []) + + if channels_to_update: + # Find our channel in the results + channel_match = None + for update in channels_to_update: + if update["id"] == channel.id: + channel_match = update + break + + if channel_match: + # Apply the match to the channel + try: + epg_data = EPGData.objects.get(id=channel_match['epg_data_id']) + channel.epg_data = epg_data + channel.save(update_fields=["epg_data"]) + + # Find match details from matched_channels for better reporting + match_details = None + for match_info in matched_channels: + if match_info[0] == channel.id: # matched_channels format: (channel_id, channel_name, epg_info) + match_details = match_info + break + + success_msg = f"Channel '{channel.name}' matched with EPG '{epg_data.name}'" + if match_details: + success_msg += f" (matched via: {match_details[2]})" + + logger.info(success_msg) + + # Send completion progress for single channel + send_epg_matching_progress(1, 1, current_channel_name=channel.name, stage="completed") + + # Clean up ML models from memory after single channel matching + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory") + _ml_model_cache['sentence_transformer'] = None + gc.collect() + + return { + "matched": True, + "message": success_msg, + "epg_name": epg_data.name, + "epg_id": epg_data.id + } + except EPGData.DoesNotExist: + return {"matched": False, "message": "Matched EPG data not found"} + + # No match found + # Send completion progress for single channel (failed) + send_epg_matching_progress(1, 0, current_channel_name=channel.name, stage="completed") + + # Clean up ML models from memory after single channel matching + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory") + _ml_model_cache['sentence_transformer'] = None + gc.collect() + + return { + "matched": False, + "message": f"No suitable EPG match found for channel '{channel.name}'" + } + + except Exception as e: + logger.error(f"Error in integrated single channel EPG matching: {e}", exc_info=True) + + # Clean up ML models from memory even on error + if _ml_model_cache['sentence_transformer'] is not None: + logger.info("Cleaning up ML models from memory after error") + _ml_model_cache['sentence_transformer'] = None + gc.collect() + + return {"matched": False, "message": f"Error during matching: {str(e)}"} + + def evaluate_series_rules_impl(tvg_id: str | None = None): """Synchronous implementation of series rule evaluation; returns details for debugging.""" from django.utils import timezone @@ -1597,3 +2165,527 @@ def prefetch_recording_artwork(recording_id): except Exception as e: logger.debug(f"prefetch_recording_artwork failed: {e}") return f"error: {e}" + + +@shared_task(bind=True) +def bulk_create_channels_from_streams(self, stream_ids, channel_profile_ids=None, starting_channel_number=None): + """ + Asynchronously create channels from a list of stream IDs. + Provides progress updates via WebSocket. + + Args: + stream_ids: List of stream IDs to create channels from + channel_profile_ids: Optional list of channel profile IDs to assign channels to + starting_channel_number: Optional starting channel number behavior: + - None: Use provider channel numbers, then auto-assign from 1 + - 0: Start with lowest available number and increment by 1 + - Other number: Use as starting number for auto-assignment + """ + from apps.channels.models import Stream, Channel, ChannelGroup, ChannelProfile, ChannelProfileMembership, Logo + from apps.epg.models import EPGData + from django.db import transaction + from django.shortcuts import get_object_or_404 + from core.utils import send_websocket_update + + task_id = self.request.id + total_streams = len(stream_ids) + created_channels = [] + errors = [] + + try: + # Send initial progress update + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_streams, + 'status': 'starting', + 'message': f'Starting bulk creation of {total_streams} channels...' + }) + + # Gather current used numbers once + used_numbers = set(Channel.objects.all().values_list("channel_number", flat=True)) + + # Initialize next_number based on starting_channel_number mode + if starting_channel_number is None: + # Mode 1: Use provider numbers when available, auto-assign when not + next_number = 1 + elif starting_channel_number == 0: + # Mode 2: Start from lowest available number + next_number = 1 + else: + # Mode 3: Start from specified number + next_number = starting_channel_number + + def get_auto_number(): + nonlocal next_number + while next_number in used_numbers: + next_number += 1 + used_numbers.add(next_number) + return next_number + + logos_to_create = [] + channels_to_create = [] + streams_map = [] + logo_map = [] + profile_map = [] + + # Process streams in batches to avoid memory issues + batch_size = 100 + processed = 0 + + for i in range(0, total_streams, batch_size): + batch_stream_ids = stream_ids[i:i + batch_size] + batch_streams = Stream.objects.filter(id__in=batch_stream_ids) + + # Send progress update + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': processed, + 'total': total_streams, + 'status': 'processing', + 'message': f'Processing streams {processed + 1}-{min(processed + batch_size, total_streams)} of {total_streams}...' + }) + + for stream in batch_streams: + try: + name = stream.name + channel_group = stream.channel_group + stream_custom_props = stream.custom_properties or {} + + # Determine channel number based on starting_channel_number mode + channel_number = None + + if starting_channel_number is None: + # Mode 1: Use provider numbers when available + if "tvg-chno" in stream_custom_props: + channel_number = float(stream_custom_props["tvg-chno"]) + elif "channel-number" in stream_custom_props: + channel_number = float(stream_custom_props["channel-number"]) + elif "num" in stream_custom_props: + channel_number = float(stream_custom_props["num"]) + + # For modes 2 and 3 (starting_channel_number == 0 or specific number), + # ignore provider numbers and use sequential assignment + + # Get TVC guide station ID + tvc_guide_stationid = None + if "tvc-guide-stationid" in stream_custom_props: + tvc_guide_stationid = stream_custom_props["tvc-guide-stationid"] + + # Check if the determined/provider number is available + if channel_number is not None and ( + channel_number in used_numbers + or Channel.objects.filter(channel_number=channel_number).exists() + ): + # Provider number is taken, use auto-assignment + channel_number = get_auto_number() + elif channel_number is not None: + # Provider number is available, use it + used_numbers.add(channel_number) + else: + # No provider number or ignoring provider numbers, use auto-assignment + channel_number = get_auto_number() + + channel_data = { + "channel_number": channel_number, + "name": name, + "tvc_guide_stationid": tvc_guide_stationid, + "tvg_id": stream.tvg_id, + } + + # Only add channel_group_id if the stream has a channel group + if channel_group: + channel_data["channel_group_id"] = channel_group.id + + # Attempt to find existing EPGs with the same tvg-id + epgs = EPGData.objects.filter(tvg_id=stream.tvg_id) + if epgs: + channel_data["epg_data_id"] = epgs.first().id + + channel = Channel(**channel_data) + channels_to_create.append(channel) + streams_map.append([stream.id]) + + # Store profile IDs for this channel + profile_map.append(channel_profile_ids) + + # Handle logo + if stream.logo_url: + logos_to_create.append( + Logo( + url=stream.logo_url, + name=stream.name or stream.tvg_id, + ) + ) + logo_map.append(stream.logo_url) + else: + logo_map.append(None) + + processed += 1 + + except Exception as e: + errors.append({ + 'stream_id': stream.id if 'stream' in locals() else 'unknown', + 'error': str(e) + }) + processed += 1 + + # Create logos first + if logos_to_create: + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': processed, + 'total': total_streams, + 'status': 'creating_logos', + 'message': f'Creating {len(logos_to_create)} logos...' + }) + Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True) + + # Get logo objects for association + channel_logos = { + logo.url: logo + for logo in Logo.objects.filter( + url__in=[url for url in logo_map if url is not None] + ) + } + + # Create channels in database + if channels_to_create: + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': processed, + 'total': total_streams, + 'status': 'creating_channels', + 'message': f'Creating {len(channels_to_create)} channels in database...' + }) + + with transaction.atomic(): + created_channels = Channel.objects.bulk_create(channels_to_create) + + # Update channels with logos and create stream associations + update = [] + channel_stream_associations = [] + channel_profile_memberships = [] + + for channel, stream_ids, logo_url, profile_ids in zip( + created_channels, streams_map, logo_map, profile_map + ): + if logo_url: + channel.logo = channel_logos[logo_url] + update.append(channel) + + # Create stream associations + for stream_id in stream_ids: + from apps.channels.models import ChannelStream + channel_stream_associations.append( + ChannelStream(channel=channel, stream_id=stream_id, order=0) + ) + + # Handle channel profile membership + if profile_ids: + try: + specific_profiles = ChannelProfile.objects.filter(id__in=profile_ids) + channel_profile_memberships.extend([ + ChannelProfileMembership( + channel_profile=profile, + channel=channel, + enabled=True + ) + for profile in specific_profiles + ]) + except Exception as e: + errors.append({ + 'channel_id': channel.id, + 'error': f'Failed to add to profiles: {str(e)}' + }) + else: + # Add to all profiles by default + all_profiles = ChannelProfile.objects.all() + channel_profile_memberships.extend([ + ChannelProfileMembership( + channel_profile=profile, + channel=channel, + enabled=True + ) + for profile in all_profiles + ]) + + # Bulk update channels with logos + if update: + Channel.objects.bulk_update(update, ["logo"]) + + # Bulk create channel-stream associations + if channel_stream_associations: + from apps.channels.models import ChannelStream + ChannelStream.objects.bulk_create(channel_stream_associations, ignore_conflicts=True) + + # Bulk create profile memberships + if channel_profile_memberships: + ChannelProfileMembership.objects.bulk_create(channel_profile_memberships, ignore_conflicts=True) + + # Send completion update + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': total_streams, + 'total': total_streams, + 'status': 'completed', + 'message': f'Successfully created {len(created_channels)} channels', + 'created_count': len(created_channels), + 'error_count': len(errors), + 'errors': errors[:10] # Send first 10 errors only + }) + + # Send general channel update notification + send_websocket_update('updates', 'update', { + 'type': 'channels_created', + 'count': len(created_channels) + }) + + return { + 'status': 'completed', + 'created_count': len(created_channels), + 'error_count': len(errors), + 'errors': errors + } + + except Exception as e: + logger.error(f"Bulk channel creation failed: {e}") + send_websocket_update('updates', 'update', { + 'type': 'bulk_channel_creation_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_streams, + 'status': 'failed', + 'message': f'Task failed: {str(e)}', + 'error': str(e) + }) + raise + + +@shared_task(bind=True) +def set_channels_names_from_epg(self, channel_ids): + """ + Celery task to set channel names from EPG data for multiple channels + """ + from core.utils import send_websocket_update + + task_id = self.request.id + total_channels = len(channel_ids) + updated_count = 0 + errors = [] + + try: + logger.info(f"Starting EPG name setting task for {total_channels} channels") + + # Send initial progress + send_websocket_update('updates', 'update', { + 'type': 'epg_name_setting_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_channels, + 'status': 'running', + 'message': 'Starting EPG name setting...' + }) + + batch_size = 100 + for i in range(0, total_channels, batch_size): + batch_ids = channel_ids[i:i + batch_size] + batch_updates = [] + + # Get channels and their EPG data + channels = Channel.objects.filter(id__in=batch_ids).select_related('epg_data') + + for channel in channels: + try: + if channel.epg_data and channel.epg_data.name: + if channel.name != channel.epg_data.name: + channel.name = channel.epg_data.name + batch_updates.append(channel) + updated_count += 1 + except Exception as e: + errors.append(f"Channel {channel.id}: {str(e)}") + logger.error(f"Error processing channel {channel.id}: {e}") + + # Bulk update the batch + if batch_updates: + Channel.objects.bulk_update(batch_updates, ['name']) + + # Send progress update + progress = min(i + batch_size, total_channels) + send_websocket_update('updates', 'update', { + 'type': 'epg_name_setting_progress', + 'task_id': task_id, + 'progress': progress, + 'total': total_channels, + 'status': 'running', + 'message': f'Updated {updated_count} channel names...', + 'updated_count': updated_count + }) + + # Send completion notification + send_websocket_update('updates', 'update', { + 'type': 'epg_name_setting_progress', + 'task_id': task_id, + 'progress': total_channels, + 'total': total_channels, + 'status': 'completed', + 'message': f'Successfully updated {updated_count} channel names from EPG data', + 'updated_count': updated_count, + 'error_count': len(errors), + 'errors': errors + }) + + logger.info(f"EPG name setting task completed. Updated {updated_count} channels") + return { + 'status': 'completed', + 'updated_count': updated_count, + 'error_count': len(errors), + 'errors': errors + } + + except Exception as e: + logger.error(f"EPG name setting task failed: {e}") + send_websocket_update('updates', 'update', { + 'type': 'epg_name_setting_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_channels, + 'status': 'failed', + 'message': f'Task failed: {str(e)}', + 'error': str(e) + }) + raise + + +@shared_task(bind=True) +def set_channels_logos_from_epg(self, channel_ids): + """ + Celery task to set channel logos from EPG data for multiple channels + Creates logos from EPG icon URLs if they don't exist + """ + from .models import Logo + from core.utils import send_websocket_update + import requests + from urllib.parse import urlparse + + task_id = self.request.id + total_channels = len(channel_ids) + updated_count = 0 + created_logos_count = 0 + errors = [] + + try: + logger.info(f"Starting EPG logo setting task for {total_channels} channels") + + # Send initial progress + send_websocket_update('updates', 'update', { + 'type': 'epg_logo_setting_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_channels, + 'status': 'running', + 'message': 'Starting EPG logo setting...' + }) + + batch_size = 50 # Smaller batch for logo processing + for i in range(0, total_channels, batch_size): + batch_ids = channel_ids[i:i + batch_size] + batch_updates = [] + + # Get channels and their EPG data + channels = Channel.objects.filter(id__in=batch_ids).select_related('epg_data', 'logo') + + for channel in channels: + try: + if channel.epg_data and channel.epg_data.icon_url: + icon_url = channel.epg_data.icon_url.strip() + + # Try to find existing logo with this URL + try: + logo = Logo.objects.get(url=icon_url) + except Logo.DoesNotExist: + # Create new logo from EPG icon URL + try: + # Generate a name for the logo + logo_name = channel.epg_data.name or f"Logo for {channel.epg_data.tvg_id}" + + # Create the logo record + logo = Logo.objects.create( + name=logo_name, + url=icon_url + ) + created_logos_count += 1 + logger.info(f"Created new logo from EPG: {logo_name} - {icon_url}") + + except Exception as create_error: + errors.append(f"Channel {channel.id}: Failed to create logo from {icon_url}: {str(create_error)}") + logger.error(f"Failed to create logo for channel {channel.id}: {create_error}") + continue + + # Update channel logo if different + if channel.logo != logo: + channel.logo = logo + batch_updates.append(channel) + updated_count += 1 + + except Exception as e: + errors.append(f"Channel {channel.id}: {str(e)}") + logger.error(f"Error processing channel {channel.id}: {e}") + + # Bulk update the batch + if batch_updates: + Channel.objects.bulk_update(batch_updates, ['logo']) + + # Send progress update + progress = min(i + batch_size, total_channels) + send_websocket_update('updates', 'update', { + 'type': 'epg_logo_setting_progress', + 'task_id': task_id, + 'progress': progress, + 'total': total_channels, + 'status': 'running', + 'message': f'Updated {updated_count} channel logos, created {created_logos_count} new logos...', + 'updated_count': updated_count, + 'created_logos_count': created_logos_count + }) + + # Send completion notification + send_websocket_update('updates', 'update', { + 'type': 'epg_logo_setting_progress', + 'task_id': task_id, + 'progress': total_channels, + 'total': total_channels, + 'status': 'completed', + 'message': f'Successfully updated {updated_count} channel logos and created {created_logos_count} new logos from EPG data', + 'updated_count': updated_count, + 'created_logos_count': created_logos_count, + 'error_count': len(errors), + 'errors': errors + }) + + logger.info(f"EPG logo setting task completed. Updated {updated_count} channels, created {created_logos_count} logos") + return { + 'status': 'completed', + 'updated_count': updated_count, + 'created_logos_count': created_logos_count, + 'error_count': len(errors), + 'errors': errors + } + + except Exception as e: + logger.error(f"EPG logo setting task failed: {e}") + send_websocket_update('updates', 'update', { + 'type': 'epg_logo_setting_progress', + 'task_id': task_id, + 'progress': 0, + 'total': total_channels, + 'status': 'failed', + 'message': f'Task failed: {str(e)}', + 'error': str(e) + }) + raise diff --git a/apps/epg/migrations/0016_epgdata_icon_url.py b/apps/epg/migrations/0016_epgdata_icon_url.py new file mode 100644 index 00000000..b934b024 --- /dev/null +++ b/apps/epg/migrations/0016_epgdata_icon_url.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.4 on 2025-09-16 22:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0015_alter_programdata_custom_properties'), + ] + + operations = [ + migrations.AddField( + model_name='epgdata', + name='icon_url', + field=models.URLField(blank=True, max_length=500, null=True), + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 22f2bd28..3ed33ab2 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -127,6 +127,7 @@ class EPGData(models.Model): # and a name (which might simply be the tvg_id if no real channel exists). tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=255) + icon_url = models.URLField(max_length=500, null=True, blank=True) epg_source = models.ForeignKey( EPGSource, on_delete=models.CASCADE, diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index 2f97cebf..85186cae 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -52,5 +52,6 @@ class EPGDataSerializer(serializers.ModelSerializer): 'id', 'tvg_id', 'name', + 'icon_url', 'epg_source', ] diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 0d0ebbb3..45b17bb7 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -873,10 +873,14 @@ def parse_channels_only(source): tvg_id = elem.get('id', '').strip() if tvg_id: display_name = None + icon_url = None for child in elem: if child.tag == 'display-name' and child.text: display_name = child.text.strip() - break + elif child.tag == 'icon': + icon_url = child.get('src', '').strip() + if display_name and icon_url: + break # No need to continue if we have both if not display_name: display_name = tvg_id @@ -894,17 +898,24 @@ def parse_channels_only(source): epgs_to_create.append(EPGData( tvg_id=tvg_id, name=display_name, + icon_url=icon_url, epg_source=source, )) logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 1: {tvg_id} - {display_name}") processed_channels += 1 continue - # We use the cached object to check if the name has changed + # We use the cached object to check if the name or icon_url has changed epg_obj = existing_epgs[tvg_id] + needs_update = False if epg_obj.name != display_name: - # Only update if the name actually changed epg_obj.name = display_name + needs_update = True + if epg_obj.icon_url != icon_url: + epg_obj.icon_url = icon_url + needs_update = True + + if needs_update: epgs_to_update.append(epg_obj) logger.debug(f"[parse_channels_only] Added channel to update to epgs_to_update: {tvg_id} - {display_name}") else: @@ -915,6 +926,7 @@ def parse_channels_only(source): epgs_to_create.append(EPGData( tvg_id=tvg_id, name=display_name, + icon_url=icon_url, epg_source=source, )) logger.debug(f"[parse_channels_only] Added new channel to epgs_to_create 2: {tvg_id} - {display_name}") @@ -937,7 +949,7 @@ def parse_channels_only(source): logger.info(f"[parse_channels_only] Bulk updating {len(epgs_to_update)} EPG entries") if process: logger.info(f"[parse_channels_only] Memory before bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") - EPGData.objects.bulk_update(epgs_to_update, ["name"]) + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) if process: logger.info(f"[parse_channels_only] Memory after bulk_update: {process.memory_info().rss / 1024 / 1024:.2f} MB") epgs_to_update = [] @@ -1004,7 +1016,7 @@ def parse_channels_only(source): logger.debug(f"[parse_channels_only] Created final batch of {len(epgs_to_create)} EPG entries") if epgs_to_update: - EPGData.objects.bulk_update(epgs_to_update, ["name"]) + EPGData.objects.bulk_update(epgs_to_update, ["name", "icon_url"]) logger.debug(f"[parse_channels_only] Updated final batch of {len(epgs_to_update)} EPG entries") if process: logger.debug(f"[parse_channels_only] Memory after final batch creation: {process.memory_info().rss / 1024 / 1024:.2f} MB") diff --git a/apps/output/views.py b/apps/output/views.py index 902c8f71..bbf79acf 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -19,6 +19,7 @@ from tzlocal import get_localzone from urllib.parse import urlparse import base64 import logging +from django.db.models.functions import Lower import os from apps.m3u.utils import calculate_tuner_count @@ -859,11 +860,11 @@ def xc_get_live_categories(user): channel_profiles ) - channel_groups = ChannelGroup.objects.filter(**filters).distinct() + channel_groups = ChannelGroup.objects.filter(**filters).distinct().order_by(Lower("name")) else: channel_groups = ChannelGroup.objects.filter( channels__isnull=False, channels__user_level__lte=user.user_level - ).distinct() + ).distinct().order_by(Lower("name")) for group in channel_groups: response.append( @@ -1007,31 +1008,11 @@ def xc_get_vod_categories(user): response = [] - # Filter categories based on user's M3U accounts - if user.user_level == 0: - # For regular users, get categories from their accessible M3U accounts - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - # Get M3U accounts accessible through user's profiles - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - else: - m3u_accounts = [] - - # Get categories that have movie relations with these accounts - categories = VODCategory.objects.filter( - category_type='movie', - m3umovierelation__m3u_account__in=m3u_accounts - ).distinct() - else: - # Admins can see all categories that have active movie relations - categories = VODCategory.objects.filter( - category_type='movie', - m3umovierelation__m3u_account__is_active=True - ).distinct() + # All authenticated users get access to VOD from all active M3U accounts + categories = VODCategory.objects.filter( + category_type='movie', + m3umovierelation__m3u_account__is_active=True + ).distinct().order_by(Lower("name")) for category in categories: response.append({ @@ -1050,22 +1031,9 @@ def xc_get_vod_streams(request, user, category_id=None): streams = [] - # Build filters for movies based on user access + # All authenticated users get access to VOD from all active M3U accounts filters = {"m3u_relations__m3u_account__is_active": True} - if user.user_level == 0: - # For regular users, filter by accessible M3U accounts - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_relations__m3u_account__in"] = m3u_accounts - else: - return [] # No accessible accounts - if category_id: filters["m3u_relations__category_id"] = category_id @@ -1126,28 +1094,11 @@ def xc_get_series_categories(user): response = [] - # Similar filtering as VOD categories but for series - if user.user_level == 0: - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - else: - m3u_accounts = [] - - # Get categories that have series relations with these accounts - categories = VODCategory.objects.filter( - category_type='series', - m3useriesrelation__m3u_account__in=m3u_accounts - ).distinct() - else: - categories = VODCategory.objects.filter( - category_type='series', - m3useriesrelation__m3u_account__is_active=True - ).distinct() + # All authenticated users get access to series from all active M3U accounts + categories = VODCategory.objects.filter( + category_type='series', + m3useriesrelation__m3u_account__is_active=True + ).distinct().order_by(Lower("name")) for category in categories: response.append({ @@ -1165,21 +1116,9 @@ def xc_get_series(request, user, category_id=None): series_list = [] - # Build filters based on user access + # All authenticated users get access to series from all active M3U accounts filters = {"m3u_account__is_active": True} - if user.user_level == 0: - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_account__in"] = m3u_accounts - else: - return [] - if category_id: filters["category_id"] = category_id @@ -1227,21 +1166,9 @@ def xc_get_series_info(request, user, series_id): if not series_id: raise Http404() - # Get series relation with user access filtering + # All authenticated users get access to series from all active M3U accounts filters = {"id": series_id, "m3u_account__is_active": True} - if user.user_level == 0: - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_account__in"] = m3u_accounts - else: - raise Http404() - try: series_relation = M3USeriesRelation.objects.select_related('series', 'series__logo').get(**filters) series = series_relation.series @@ -1438,21 +1365,9 @@ def xc_get_vod_info(request, user, vod_id): if not vod_id: raise Http404() - # Get movie relation with user access filtering - use movie ID instead of relation ID + # All authenticated users get access to VOD from all active M3U accounts filters = {"movie_id": vod_id, "m3u_account__is_active": True} - if user.user_level == 0: - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_account__in"] = m3u_accounts - else: - raise Http404() - try: # Order by account priority to get the best relation when multiple exist movie_relation = M3UMovieRelation.objects.select_related('movie', 'movie__logo').filter(**filters).order_by('-m3u_account__priority', 'id').first() @@ -1601,22 +1516,9 @@ def xc_movie_stream(request, username, password, stream_id, extension): if custom_properties["xc_password"] != password: return JsonResponse({"error": "Invalid credentials"}, status=401) - # Get movie relation based on user access level - use movie ID instead of relation ID + # All authenticated users get access to VOD from all active M3U accounts filters = {"movie_id": stream_id, "m3u_account__is_active": True} - if user.user_level < 10: - # For regular users, filter by accessible M3U accounts - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_account__in"] = m3u_accounts - else: - return JsonResponse({"error": "No accessible content"}, status=403) - try: # Order by account priority to get the best relation when multiple exist movie_relation = M3UMovieRelation.objects.select_related('movie').filter(**filters).order_by('-m3u_account__priority', 'id').first() @@ -1651,22 +1553,9 @@ def xc_series_stream(request, username, password, stream_id, extension): if custom_properties["xc_password"] != password: return JsonResponse({"error": "Invalid credentials"}, status=401) - # Get episode relation based on user access level - use episode ID instead of stream_id + # All authenticated users get access to series/episodes from all active M3U accounts filters = {"episode_id": stream_id, "m3u_account__is_active": True} - if user.user_level < 10: - # For regular users, filter by accessible M3U accounts - if user.channel_profiles.count() > 0: - channel_profiles = user.channel_profiles.all() - from apps.m3u.models import M3UAccount - m3u_accounts = M3UAccount.objects.filter( - is_active=True, - profiles__in=channel_profiles - ).distinct() - filters["m3u_account__in"] = m3u_accounts - else: - return JsonResponse({"error": "No accessible content"}, status=403) - try: episode_relation = M3UEpisodeRelation.objects.select_related('episode').get(**filters) except M3UEpisodeRelation.DoesNotExist: diff --git a/apps/proxy/ts_proxy/url_utils.py b/apps/proxy/ts_proxy/url_utils.py index 0c8dfee2..75e7653e 100644 --- a/apps/proxy/ts_proxy/url_utils.py +++ b/apps/proxy/ts_proxy/url_utils.py @@ -142,7 +142,7 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] if not m3u_account: return {'error': 'Stream has no M3U account'} - m3u_profiles = m3u_account.profiles.all() + m3u_profiles = m3u_account.profiles.filter(is_active=True) default_profile = next((obj for obj in m3u_profiles if obj.is_default), None) if not default_profile: @@ -153,10 +153,6 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] selected_profile = None for profile in profiles: - # Skip inactive profiles - if not profile.is_active: - logger.debug(f"Skipping inactive profile {profile.id}") - continue # Check connection availability if redis_client: @@ -281,8 +277,10 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No if not m3u_account: logger.debug(f"Stream {stream.id} has no M3U account") continue - - m3u_profiles = m3u_account.profiles.all() + if m3u_account.is_active == False: + logger.debug(f"M3U account {m3u_account.id} is inactive, skipping.") + continue + m3u_profiles = m3u_account.profiles.filter(is_active=True) default_profile = next((obj for obj in m3u_profiles if obj.is_default), None) if not default_profile: @@ -294,11 +292,6 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No selected_profile = None for profile in profiles: - # Skip inactive profiles - if not profile.is_active: - logger.debug(f"Skipping inactive profile {profile.id}") - continue - # Check connection availability if redis_client: profile_connections_key = f"profile_connections:{profile.id}" diff --git a/apps/vod/api_urls.py b/apps/vod/api_urls.py index b49e79e3..ffccc3f5 100644 --- a/apps/vod/api_urls.py +++ b/apps/vod/api_urls.py @@ -5,6 +5,7 @@ from .api_views import ( EpisodeViewSet, SeriesViewSet, VODCategoryViewSet, + UnifiedContentViewSet, ) app_name = 'vod' @@ -14,5 +15,6 @@ router.register(r'movies', MovieViewSet, basename='movie') router.register(r'episodes', EpisodeViewSet, basename='episode') router.register(r'series', SeriesViewSet, basename='series') router.register(r'categories', VODCategoryViewSet, basename='vodcategory') +router.register(r'all', UnifiedContentViewSet, basename='unified-content') urlpatterns = router.urls diff --git a/apps/vod/api_views.py b/apps/vod/api_views.py index b72ae035..517038a6 100644 --- a/apps/vod/api_views.py +++ b/apps/vod/api_views.py @@ -469,3 +469,203 @@ class VODCategoryViewSet(viewsets.ReadOnlyModelViewSet): return [perm() for perm in permission_classes_by_action[self.action]] except KeyError: return [Authenticated()] + + +class UnifiedContentViewSet(viewsets.ReadOnlyModelViewSet): + """ViewSet that combines Movies and Series for unified 'All' view""" + queryset = Movie.objects.none() # Empty queryset, we override list method + serializer_class = MovieSerializer # Default serializer, overridden in list + pagination_class = VODPagination + + filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter] + search_fields = ['name', 'description', 'genre'] + ordering_fields = ['name', 'year', 'created_at'] + ordering = ['name'] + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + def list(self, request, *args, **kwargs): + """Override list to handle unified content properly - database-level approach""" + import logging + from django.db import connection + + logger = logging.getLogger(__name__) + logger.error("=== UnifiedContentViewSet.list() called ===") + + try: + # Get pagination parameters + page_size = int(request.query_params.get('page_size', 24)) + page_number = int(request.query_params.get('page', 1)) + + logger.error(f"Page {page_number}, page_size {page_size}") + + # Calculate offset for unified pagination + offset = (page_number - 1) * page_size + + # For high page numbers, use raw SQL for efficiency + # This avoids loading and sorting massive amounts of data in Python + + search = request.query_params.get('search', '') + category = request.query_params.get('category', '') + + # Build WHERE clauses + where_conditions = [ + # Only active content + "movies.id IN (SELECT DISTINCT movie_id FROM vod_m3umovierelation mmr JOIN m3u_m3uaccount ma ON mmr.m3u_account_id = ma.id WHERE ma.is_active = true)", + "series.id IN (SELECT DISTINCT series_id FROM vod_m3useriesrelation msr JOIN m3u_m3uaccount ma ON msr.m3u_account_id = ma.id WHERE ma.is_active = true)" + ] + + params = [] + + if search: + where_conditions[0] += " AND LOWER(movies.name) LIKE %s" + where_conditions[1] += " AND LOWER(series.name) LIKE %s" + search_param = f"%{search.lower()}%" + params.extend([search_param, search_param]) + + if category: + if '|' in category: + cat_name, cat_type = category.split('|', 1) + if cat_type == 'movie': + where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)" + where_conditions[1] = "1=0" # Exclude series + params.append(cat_name) + elif cat_type == 'series': + where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)" + where_conditions[0] = "1=0" # Exclude movies + params.append(cat_name) + else: + where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)" + where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)" + params.extend([category, category]) + + # Use UNION ALL with ORDER BY and LIMIT/OFFSET for true unified pagination + # This is much more efficient than Python sorting + sql = f""" + WITH unified_content AS ( + SELECT + movies.id, + movies.uuid, + movies.name, + movies.description, + movies.year, + movies.rating, + movies.genre, + movies.duration_secs as duration, + movies.created_at, + movies.updated_at, + movies.custom_properties, + movies.logo_id, + logo.name as logo_name, + logo.url as logo_url, + 'movie' as content_type + FROM vod_movie movies + LEFT JOIN dispatcharr_channels_logo logo ON movies.logo_id = logo.id + WHERE {where_conditions[0]} + + UNION ALL + + SELECT + series.id, + series.uuid, + series.name, + series.description, + series.year, + series.rating, + series.genre, + NULL as duration, + series.created_at, + series.updated_at, + series.custom_properties, + series.logo_id, + logo.name as logo_name, + logo.url as logo_url, + 'series' as content_type + FROM vod_series series + LEFT JOIN dispatcharr_channels_logo logo ON series.logo_id = logo.id + WHERE {where_conditions[1]} + ) + SELECT * FROM unified_content + ORDER BY LOWER(name), id + LIMIT %s OFFSET %s + """ + + params.extend([page_size, offset]) + + logger.error(f"Executing SQL with LIMIT {page_size} OFFSET {offset}") + + with connection.cursor() as cursor: + cursor.execute(sql, params) + columns = [col[0] for col in cursor.description] + results = [] + + for row in cursor.fetchall(): + item_dict = dict(zip(columns, row)) + + # Build logo object in the format expected by frontend + logo_data = None + if item_dict['logo_id']: + logo_data = { + 'id': item_dict['logo_id'], + 'name': item_dict['logo_name'], + 'url': item_dict['logo_url'], + 'cache_url': f"/media/logo_cache/{item_dict['logo_id']}.png" if item_dict['logo_id'] else None, + 'channel_count': 0, # We don't need this for VOD + 'is_used': True, + 'channel_names': [] # We don't need this for VOD + } + + # Convert to the format expected by frontend + formatted_item = { + 'id': item_dict['id'], + 'uuid': str(item_dict['uuid']), + 'name': item_dict['name'], + 'description': item_dict['description'] or '', + 'year': item_dict['year'], + 'rating': float(item_dict['rating']) if item_dict['rating'] else 0.0, + 'genre': item_dict['genre'] or '', + 'duration': item_dict['duration'], + 'created_at': item_dict['created_at'].isoformat() if item_dict['created_at'] else None, + 'updated_at': item_dict['updated_at'].isoformat() if item_dict['updated_at'] else None, + 'custom_properties': item_dict['custom_properties'] or {}, + 'logo': logo_data, + 'content_type': item_dict['content_type'] + } + results.append(formatted_item) + + logger.error(f"Retrieved {len(results)} results via SQL") + + # Get total count estimate (for pagination info) + # Use a separate efficient count query + count_sql = f""" + SELECT COUNT(*) FROM ( + SELECT 1 FROM vod_movie movies WHERE {where_conditions[0]} + UNION ALL + SELECT 1 FROM vod_series series WHERE {where_conditions[1]} + ) as total_count + """ + + count_params = params[:-2] # Remove LIMIT and OFFSET params + + with connection.cursor() as cursor: + cursor.execute(count_sql, count_params) + total_count = cursor.fetchone()[0] + + response_data = { + 'count': total_count, + 'next': offset + page_size < total_count, + 'previous': page_number > 1, + 'results': results + } + + return Response(response_data) + + except Exception as e: + logger.error(f"Error in UnifiedContentViewSet.list(): {e}") + import traceback + logger.error(traceback.format_exc()) + return Response({'error': str(e)}, status=500) \ No newline at end of file diff --git a/apps/vod/tasks.py b/apps/vod/tasks.py index 93cef3e1..504b09fc 100644 --- a/apps/vod/tasks.py +++ b/apps/vod/tasks.py @@ -62,9 +62,9 @@ def refresh_vod_content(account_id): logger.info(f"Batch VOD refresh completed for account {account.name} in {duration:.2f} seconds") - # Cleanup orphaned VOD content after refresh - logger.info("Starting cleanup of orphaned VOD content") - cleanup_result = cleanup_orphaned_vod_content(scan_start_time=start_time) + # Cleanup orphaned VOD content after refresh (scoped to this account only) + logger.info(f"Starting cleanup of orphaned VOD content for account {account.name}") + cleanup_result = cleanup_orphaned_vod_content(account_id=account_id, scan_start_time=start_time) logger.info(f"VOD cleanup completed: {cleanup_result}") # Send completion notification @@ -135,11 +135,11 @@ def refresh_movies(client, account, categories_by_provider, relations, scan_star # Process movies in chunks using the simple approach chunk_size = 1000 total_movies = len(all_movies_data) + total_chunks = (total_movies + chunk_size - 1) // chunk_size if total_movies > 0 else 0 for i in range(0, total_movies, chunk_size): chunk = all_movies_data[i:i + chunk_size] chunk_num = (i // chunk_size) + 1 - total_chunks = (total_movies + chunk_size - 1) // chunk_size logger.info(f"Processing movie chunk {chunk_num}/{total_chunks} ({len(chunk)} movies)") process_movie_batch(account, chunk, categories_by_provider, relations, scan_start_time) @@ -158,11 +158,11 @@ def refresh_series(client, account, categories_by_provider, relations, scan_star # Process series in chunks using the simple approach chunk_size = 1000 total_series = len(all_series_data) + total_chunks = (total_series + chunk_size - 1) // chunk_size if total_series > 0 else 0 for i in range(0, total_series, chunk_size): chunk = all_series_data[i:i + chunk_size] chunk_num = (i // chunk_size) + 1 - total_chunks = (total_series + chunk_size - 1) // chunk_size logger.info(f"Processing series chunk {chunk_num}/{total_chunks} ({len(chunk)} series)") process_series_batch(account, chunk, categories_by_provider, relations, scan_start_time) @@ -427,17 +427,18 @@ def process_movie_batch(account, batch, categories, relations, scan_start_time=N updated = True # Handle logo assignment for existing movies + logo_updated = False if logo_url and len(logo_url) <= 500 and logo_url in existing_logos: new_logo = existing_logos[logo_url] if movie.logo != new_logo: - movie.logo = new_logo - updated = True + movie._logo_to_update = new_logo + logo_updated = True elif (not logo_url or len(logo_url) > 500) and movie.logo: # Clear logo if no logo URL provided or URL is too long - movie.logo = None - updated = True + movie._logo_to_update = None + logo_updated = True - if updated: + if updated or logo_updated: movies_to_update.append(movie) else: # Create new movie @@ -509,11 +510,18 @@ def process_movie_batch(account, batch, categories, relations, scan_start_time=N # Update existing movies if movies_to_update: + # First, update all fields except logo to avoid unsaved related object issues Movie.objects.bulk_update(movies_to_update, [ 'description', 'rating', 'genre', 'year', 'tmdb_id', 'imdb_id', - 'duration_secs', 'custom_properties', 'logo' + 'duration_secs', 'custom_properties' ]) + # Handle logo updates separately to avoid bulk_update issues + for movie in movies_to_update: + if hasattr(movie, '_logo_to_update'): + movie.logo = movie._logo_to_update + movie.save(update_fields=['logo']) + # Update relations to reference the correct movie objects for relation in relations_to_create: if id(relation.movie) in created_movies: @@ -741,17 +749,18 @@ def process_series_batch(account, batch, categories, relations, scan_start_time= updated = True # Handle logo assignment for existing series + logo_updated = False if logo_url and len(logo_url) <= 500 and logo_url in existing_logos: new_logo = existing_logos[logo_url] if series.logo != new_logo: - series.logo = new_logo - updated = True + series._logo_to_update = new_logo + logo_updated = True elif (not logo_url or len(logo_url) > 500) and series.logo: # Clear logo if no logo URL provided or URL is too long - series.logo = None - updated = True + series._logo_to_update = None + logo_updated = True - if updated: + if updated or logo_updated: series_to_update.append(series) else: # Create new series @@ -823,11 +832,18 @@ def process_series_batch(account, batch, categories, relations, scan_start_time= # Update existing series if series_to_update: + # First, update all fields except logo to avoid unsaved related object issues Series.objects.bulk_update(series_to_update, [ 'description', 'rating', 'genre', 'year', 'tmdb_id', 'imdb_id', - 'custom_properties', 'logo' + 'custom_properties' ]) + # Handle logo updates separately to avoid bulk_update issues + for series in series_to_update: + if hasattr(series, '_logo_to_update'): + series.logo = series._logo_to_update + series.save(update_fields=['logo']) + # Update relations to reference the correct series objects for relation in relations_to_create: if id(relation.series) in created_series: @@ -1292,7 +1308,7 @@ def batch_refresh_series_episodes(account_id, series_ids=None): @shared_task -def cleanup_orphaned_vod_content(stale_days=0, scan_start_time=None): +def cleanup_orphaned_vod_content(stale_days=0, scan_start_time=None, account_id=None): """Clean up VOD content that has no M3U relations or has stale relations""" from datetime import timedelta @@ -1302,30 +1318,44 @@ def cleanup_orphaned_vod_content(stale_days=0, scan_start_time=None): # Calculate cutoff date for stale relations cutoff_date = reference_time - timedelta(days=stale_days) + # Build base query filters + base_filters = {'last_seen__lt': cutoff_date} + if account_id: + base_filters['m3u_account_id'] = account_id + logger.info(f"Cleaning up stale VOD content for account {account_id}") + else: + logger.info("Cleaning up stale VOD content across all accounts") + # Clean up stale movie relations (haven't been seen in the specified days) - stale_movie_relations = M3UMovieRelation.objects.filter(last_seen__lt=cutoff_date) + stale_movie_relations = M3UMovieRelation.objects.filter(**base_filters) stale_movie_count = stale_movie_relations.count() stale_movie_relations.delete() # Clean up stale series relations - stale_series_relations = M3USeriesRelation.objects.filter(last_seen__lt=cutoff_date) + stale_series_relations = M3USeriesRelation.objects.filter(**base_filters) stale_series_count = stale_series_relations.count() stale_series_relations.delete() # Clean up stale episode relations - stale_episode_relations = M3UEpisodeRelation.objects.filter(last_seen__lt=cutoff_date) + stale_episode_relations = M3UEpisodeRelation.objects.filter(**base_filters) stale_episode_count = stale_episode_relations.count() stale_episode_relations.delete() - # Clean up movies with no relations (orphaned) - orphaned_movies = Movie.objects.filter(m3u_relations__isnull=True) - orphaned_movie_count = orphaned_movies.count() - orphaned_movies.delete() + # Clean up movies with no relations (orphaned) - only if no account_id specified (global cleanup) + if not account_id: + orphaned_movies = Movie.objects.filter(m3u_relations__isnull=True) + orphaned_movie_count = orphaned_movies.count() + orphaned_movies.delete() - # Clean up series with no relations (orphaned) - orphaned_series = Series.objects.filter(m3u_relations__isnull=True) - orphaned_series_count = orphaned_series.count() - orphaned_series.delete() + # Clean up series with no relations (orphaned) - only if no account_id specified (global cleanup) + orphaned_series = Series.objects.filter(m3u_relations__isnull=True) + orphaned_series_count = orphaned_series.count() + orphaned_series.delete() + else: + # When cleaning up for specific account, we don't remove orphaned content + # as other accounts might still reference it + orphaned_movie_count = 0 + orphaned_series_count = 0 # Episodes will be cleaned up via CASCADE when series are deleted diff --git a/apps/vod/urls.py b/apps/vod/urls.py index f90e3fb6..3cea96a5 100644 --- a/apps/vod/urls.py +++ b/apps/vod/urls.py @@ -1,6 +1,6 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .api_views import MovieViewSet, EpisodeViewSet, SeriesViewSet, VODCategoryViewSet, VODConnectionViewSet +from .api_views import MovieViewSet, EpisodeViewSet, SeriesViewSet, VODCategoryViewSet, UnifiedContentViewSet app_name = 'vod' @@ -9,7 +9,7 @@ router.register(r'movies', MovieViewSet) router.register(r'episodes', EpisodeViewSet) router.register(r'series', SeriesViewSet) router.register(r'categories', VODCategoryViewSet) -router.register(r'connections', VODConnectionViewSet) +router.register(r'all', UnifiedContentViewSet, basename='unified-content') urlpatterns = [ path('api/', include(router.urls)), diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index 819a923f..1101c9f8 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -330,6 +330,173 @@ export const WebsocketProvider = ({ children }) => { } break; + case 'epg_matching_progress': { + const progress = parsedEvent.data; + const id = 'epg-matching-progress'; + + if (progress.stage === 'starting') { + notifications.show({ + id, + title: 'EPG Matching in Progress', + message: `Starting to match ${progress.total} channels...`, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.stage === 'matching') { + let message = `Matched ${progress.matched} of ${progress.total} channels`; + if (progress.remaining > 0) { + message += ` (${progress.remaining} remaining)`; + } + if (progress.current_channel) { + message += `\nCurrently processing: ${progress.current_channel}`; + } + + notifications.update({ + id, + title: 'EPG Matching in Progress', + message, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.stage === 'completed') { + notifications.update({ + id, + title: 'EPG Matching Complete', + message: `Successfully matched ${progress.matched} of ${progress.total} channels (${progress.progress_percent}%)`, + color: progress.matched > 0 ? 'green.5' : 'orange', + loading: false, + autoClose: 6000, + }); + } + break; + } + + case 'epg_logo_setting_progress': { + const progress = parsedEvent.data; + const id = 'epg-logo-setting-progress'; + + if (progress.status === 'running' && progress.progress === 0) { + // Initial message + notifications.show({ + id, + title: 'Setting Logos from EPG', + message: `Processing ${progress.total} channels...`, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.status === 'running') { + // Progress update + let message = `Processed ${progress.progress} of ${progress.total} channels`; + if (progress.updated_count !== undefined) { + message += ` (${progress.updated_count} updated)`; + } + if (progress.created_logos_count !== undefined) { + message += `, created ${progress.created_logos_count} logos`; + } + + notifications.update({ + id, + title: 'Setting Logos from EPG', + message, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.status === 'completed') { + notifications.update({ + id, + title: 'Logo Setting Complete', + message: `Successfully updated ${progress.updated_count || 0} channel logos${progress.created_logos_count ? `, created ${progress.created_logos_count} new logos` : ''}`, + color: progress.updated_count > 0 ? 'green.5' : 'orange', + loading: false, + autoClose: 6000, + }); + // Refresh channels data and logos + try { + await API.requeryChannels(); + await useChannelsStore.getState().fetchChannels(); + + // Get updated channel data and extract logo IDs to load + const channels = useChannelsStore.getState().channels; + const logoIds = Object.values(channels) + .filter((channel) => channel.logo_id) + .map((channel) => channel.logo_id); + + // Fetch the specific logos that were just assigned + if (logoIds.length > 0) { + await useLogosStore.getState().fetchLogosByIds(logoIds); + } + } catch (e) { + console.warn( + 'Failed to refresh channels after logo setting:', + e + ); + } + } + break; + } + + case 'epg_name_setting_progress': { + const progress = parsedEvent.data; + const id = 'epg-name-setting-progress'; + + if (progress.status === 'running' && progress.progress === 0) { + // Initial message + notifications.show({ + id, + title: 'Setting Names from EPG', + message: `Processing ${progress.total} channels...`, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.status === 'running') { + // Progress update + let message = `Processed ${progress.progress} of ${progress.total} channels`; + if (progress.updated_count !== undefined) { + message += ` (${progress.updated_count} updated)`; + } + + notifications.update({ + id, + title: 'Setting Names from EPG', + message, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (progress.status === 'completed') { + notifications.update({ + id, + title: 'Name Setting Complete', + message: `Successfully updated ${progress.updated_count || 0} channel names from EPG data`, + color: progress.updated_count > 0 ? 'green.5' : 'orange', + loading: false, + autoClose: 6000, + }); + // Refresh channels data + try { + await API.requeryChannels(); + await useChannelsStore.getState().fetchChannels(); + } catch (e) { + console.warn( + 'Failed to refresh channels after name setting:', + e + ); + } + } + break; + } + case 'm3u_profile_test': setProfilePreview( parsedEvent.data.search_preview, @@ -549,6 +716,87 @@ export const WebsocketProvider = ({ children }) => { }); break; + case 'channels_created': + // General notification for channel creation + notifications.show({ + title: 'Channels Created', + message: `Successfully created ${parsedEvent.data.count || 'multiple'} channel(s)`, + color: 'green', + autoClose: 4000, + }); + + // Refresh the channels table to show new channels + try { + await API.requeryChannels(); + await useChannelsStore.getState().fetchChannels(); + console.log('Channels refreshed after bulk creation'); + } catch (error) { + console.error( + 'Error refreshing channels after creation:', + error + ); + } + + break; + + case 'bulk_channel_creation_progress': { + // Handle progress updates with persistent notifications like stream rehash + const data = parsedEvent.data; + + if (data.status === 'starting') { + notifications.show({ + id: 'bulk-channel-creation-progress', // Persistent ID + title: 'Bulk Channel Creation Started', + message: data.message || 'Starting bulk channel creation...', + color: 'blue.5', + autoClose: false, // Don't auto-close + withCloseButton: false, // No close button during processing + loading: true, // Show loading indicator + }); + } else if ( + data.status === 'processing' || + data.status === 'creating_logos' || + data.status === 'creating_channels' + ) { + // Calculate progress percentage + const progressPercent = + data.total > 0 + ? Math.round((data.progress / data.total) * 100) + : 0; + + // Update the existing notification with progress + notifications.update({ + id: 'bulk-channel-creation-progress', + title: 'Bulk Channel Creation in Progress', + message: `${progressPercent}% complete - ${data.message}`, + color: 'blue.5', + autoClose: false, + withCloseButton: false, + loading: true, + }); + } else if (data.status === 'completed') { + // Hide the progress notification since channels_created will show success + notifications.hide('bulk-channel-creation-progress'); + } else if (data.status === 'failed') { + // Update to error state + notifications.update({ + id: 'bulk-channel-creation-progress', + title: 'Bulk Channel Creation Failed', + message: + data.error || + 'An error occurred during bulk channel creation', + color: 'red.5', + autoClose: 12000, // Auto-close after longer delay for errors + withCloseButton: true, // Allow manual close + loading: false, // Remove loading indicator + }); + } + + // Pass through to individual components for any additional handling + setVal(parsedEvent); + break; + } + default: console.error( `Unknown websocket event type: ${parsedEvent.data?.type}` diff --git a/frontend/src/api.js b/frontend/src/api.js index 110d6b75..01186bf6 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -323,14 +323,27 @@ export default class API { static async addChannel(channel) { try { let body = null; + // Prepare a copy to safely mutate + const channelData = { ...channel }; + + // Remove channel_number if empty, null, or not a valid number + if ( + channelData.channel_number === '' || + channelData.channel_number === null || + channelData.channel_number === undefined || + (typeof channelData.channel_number === 'string' && channelData.channel_number.trim() === '') + ) { + delete channelData.channel_number; + } + if (channel.logo_file) { // Must send FormData for file upload body = new FormData(); - for (const prop in channel) { - body.append(prop, channel[prop]); + for (const prop in channelData) { + body.append(prop, channelData[prop]); } } else { - body = { ...channel }; + body = { ...channelData }; delete body.logo_file; } @@ -503,6 +516,52 @@ export default class API { } } + static async setChannelNamesFromEpg(channelIds) { + try { + const response = await request( + `${host}/api/channels/channels/set-names-from-epg/`, + { + method: 'POST', + body: { channel_ids: channelIds }, + } + ); + + notifications.show({ + title: 'Task Started', + message: response.message, + color: 'blue', + }); + + return response; + } catch (e) { + errorNotification('Failed to start EPG name setting task', e); + throw e; + } + } + + static async setChannelLogosFromEpg(channelIds) { + try { + const response = await request( + `${host}/api/channels/channels/set-logos-from-epg/`, + { + method: 'POST', + body: { channel_ids: channelIds }, + } + ); + + notifications.show({ + title: 'Task Started', + message: response.message, + color: 'blue', + }); + + return response; + } catch (e) { + errorNotification('Failed to start EPG logo setting task', e); + throw e; + } + } + static async assignChannelNumbers(channelIds, startingNum = 1) { try { const response = await request(`${host}/api/channels/channels/assign/`, { @@ -536,24 +595,32 @@ export default class API { } } - static async createChannelsFromStreams(values) { + static async createChannelsFromStreamsAsync(streamIds, channelProfileIds = null, startingChannelNumber = null) { try { + const requestBody = { + stream_ids: streamIds, + }; + + if (channelProfileIds !== null) { + requestBody.channel_profile_ids = channelProfileIds; + } + + if (startingChannelNumber !== null) { + requestBody.starting_channel_number = startingChannelNumber; + } + const response = await request( `${host}/api/channels/channels/from-stream/bulk/`, { method: 'POST', - body: values, + body: requestBody, } ); - if (response.created && response.created.length > 0) { - useChannelsStore.getState().addChannels(response.created); - } - return response; } catch (e) { - errorNotification('Failed to create channels', e); - throw e; // Re-throw to allow proper error handling in calling code + errorNotification('Failed to start bulk channel creation task', e); + throw e; } } @@ -1416,12 +1483,18 @@ export default class API { } } - static async matchEpg() { + static async matchEpg(channelIds = null) { try { + const requestBody = channelIds ? { channel_ids: channelIds } : {}; + const response = await request( `${host}/api/channels/channels/match-epg/`, { method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), } ); @@ -1431,6 +1504,26 @@ export default class API { } } + static async matchChannelEpg(channelId) { + try { + const response = await request( + `${host}/api/channels/channels/${channelId}/match-epg/`, + { + method: 'POST', + } + ); + + // Update the channel in the store with the refreshed data if provided + if (response.channel) { + useChannelsStore.getState().updateChannel(response.channel); + } + + return response; + } catch (e) { + errorNotification('Failed to run EPG auto-match for channel', e); + } + } + static async fetchActiveChannelStats() { try { const response = await request(`${host}/proxy/ts/status`); @@ -2023,15 +2116,28 @@ export default class API { static async getStreamsByIds(ids) { try { - const params = new URLSearchParams(); - params.append('ids', ids.join(',')); - const response = await request( - `${host}/api/channels/streams/?${params.toString()}` - ); - - return response.results || response; + // Use POST for large ID lists to avoid URL length limitations + if (ids.length > 50) { + const response = await request( + `${host}/api/channels/streams/by-ids/`, + { + method: 'POST', + body: { ids }, + } + ); + return response; + } else { + // Use GET for small ID lists for backward compatibility + const params = new URLSearchParams(); + params.append('ids', ids.join(',')); + const response = await request( + `${host}/api/channels/streams/?${params.toString()}` + ); + return response.results || response; + } } catch (e) { errorNotification('Failed to retrieve streams by IDs', e); + throw e; // Re-throw to allow proper error handling in calling code } } @@ -2043,7 +2149,14 @@ export default class API { ); return response; } catch (e) { - errorNotification('Failed to retrieve movies', e); + // Don't show error notification for "Invalid page" errors as they're handled gracefully + const isInvalidPage = e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); + + if (!isInvalidPage) { + errorNotification('Failed to retrieve movies', e); + } + throw e; } } @@ -2054,7 +2167,39 @@ export default class API { ); return response; } catch (e) { - errorNotification('Failed to retrieve series', e); + // Don't show error notification for "Invalid page" errors as they're handled gracefully + const isInvalidPage = e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); + + if (!isInvalidPage) { + errorNotification('Failed to retrieve series', e); + } + throw e; + } + } + + static async getAllContent(params = new URLSearchParams()) { + try { + console.log('Calling getAllContent with URL:', `${host}/api/vod/all/?${params.toString()}`); + const response = await request( + `${host}/api/vod/all/?${params.toString()}` + ); + console.log('getAllContent raw response:', response); + return response; + } catch (e) { + console.error('getAllContent error:', e); + console.error('Error status:', e.status); + console.error('Error body:', e.body); + console.error('Error message:', e.message); + + // Don't show error notification for "Invalid page" errors as they're handled gracefully + const isInvalidPage = e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); + + if (!isInvalidPage) { + errorNotification('Failed to retrieve content', e); + } + throw e; } } diff --git a/frontend/src/components/forms/Channel.jsx b/frontend/src/components/forms/Channel.jsx index 574c5d3a..53ecddd2 100644 --- a/frontend/src/components/forms/Channel.jsx +++ b/frontend/src/components/forms/Channel.jsx @@ -9,6 +9,7 @@ import ChannelGroupForm from './ChannelGroup'; import usePlaylistsStore from '../../store/playlists'; import logo from '../../images/logo.png'; import { useChannelLogoSelection } from '../../hooks/useSmartLogos'; +import useLogosStore from '../../store/logos'; import LazyLogo from '../LazyLogo'; import { Box, @@ -34,7 +35,7 @@ import { UnstyledButton, } from '@mantine/core'; import { notifications } from '@mantine/notifications'; -import { ListOrdered, SquarePlus, SquareX, X } from 'lucide-react'; +import { ListOrdered, SquarePlus, SquareX, X, Zap } from 'lucide-react'; import useEPGsStore from '../../store/epgs'; import { Dropzone } from '@mantine/dropzone'; import { FixedSizeList as List } from 'react-window'; @@ -51,11 +52,14 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { const canEditChannelGroup = useChannelsStore((s) => s.canEditChannelGroup); const { - logos, + logos: channelLogos, ensureLogosLoaded, isLoading: logosLoading, } = useChannelLogoSelection(); + // Import the full logos store for duplicate checking + const allLogos = useLogosStore((s) => s.logos); + // Ensure logos are loaded when component mounts useEffect(() => { ensureLogosLoaded(); @@ -78,6 +82,7 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { const [groupPopoverOpened, setGroupPopoverOpened] = useState(false); const [groupFilter, setGroupFilter] = useState(''); + const [autoMatchLoading, setAutoMatchLoading] = useState(false); const groupOptions = Object.values(channelGroups); const addStream = (stream) => { @@ -121,6 +126,163 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { } }; + const handleAutoMatchEpg = async () => { + // Only attempt auto-match for existing channels (editing mode) + if (!channel || !channel.id) { + notifications.show({ + title: 'Info', + message: 'Auto-match is only available when editing existing channels.', + color: 'blue', + }); + return; + } + + setAutoMatchLoading(true); + try { + const response = await API.matchChannelEpg(channel.id); + + if (response.matched) { + // Update the form with the new EPG data + if (response.channel && response.channel.epg_data_id) { + formik.setFieldValue('epg_data_id', response.channel.epg_data_id); + } + + notifications.show({ + title: 'Success', + message: response.message, + color: 'green', + }); + } else { + notifications.show({ + title: 'No Match Found', + message: response.message, + color: 'orange', + }); + } + } catch (error) { + notifications.show({ + title: 'Error', + message: 'Failed to auto-match EPG data', + color: 'red', + }); + console.error('Auto-match error:', error); + } finally { + setAutoMatchLoading(false); + } + }; + + const handleSetNameFromEpg = () => { + const epgDataId = formik.values.epg_data_id; + if (!epgDataId) { + notifications.show({ + title: 'No EPG Selected', + message: 'Please select an EPG source first.', + color: 'orange', + }); + return; + } + + const tvg = tvgsById[epgDataId]; + if (tvg && tvg.name) { + formik.setFieldValue('name', tvg.name); + notifications.show({ + title: 'Success', + message: `Channel name set to "${tvg.name}"`, + color: 'green', + }); + } else { + notifications.show({ + title: 'No Name Available', + message: 'No name found in the selected EPG data.', + color: 'orange', + }); + } + }; + + const handleSetLogoFromEpg = async () => { + const epgDataId = formik.values.epg_data_id; + if (!epgDataId) { + notifications.show({ + title: 'No EPG Selected', + message: 'Please select an EPG source first.', + color: 'orange', + }); + return; + } + + const tvg = tvgsById[epgDataId]; + if (!tvg || !tvg.icon_url) { + notifications.show({ + title: 'No EPG Icon', + message: 'EPG data does not have an icon URL.', + color: 'orange', + }); + return; + } + + try { + // Try to find a logo that matches the EPG icon URL - check ALL logos to avoid duplicates + let matchingLogo = Object.values(allLogos).find( + (logo) => logo.url === tvg.icon_url + ); + + if (matchingLogo) { + formik.setFieldValue('logo_id', matchingLogo.id); + notifications.show({ + title: 'Success', + message: `Logo set to "${matchingLogo.name}"`, + color: 'green', + }); + } else { + // Logo doesn't exist - create it + notifications.show({ + id: 'creating-logo', + title: 'Creating Logo', + message: `Creating new logo from EPG icon URL...`, + loading: true, + }); + + try { + const newLogoData = { + name: tvg.name || `Logo for ${tvg.icon_url}`, + url: tvg.icon_url, + }; + + // Create logo by calling the Logo API directly + const newLogo = await API.createLogo(newLogoData); + + formik.setFieldValue('logo_id', newLogo.id); + + notifications.update({ + id: 'creating-logo', + title: 'Success', + message: `Created and assigned new logo "${newLogo.name}"`, + loading: false, + color: 'green', + autoClose: 5000, + }); + } catch (createError) { + notifications.update({ + id: 'creating-logo', + title: 'Error', + message: 'Failed to create logo from EPG icon URL', + loading: false, + color: 'red', + autoClose: 5000, + }); + throw createError; + } + } + } catch (error) { + notifications.show({ + title: 'Error', + message: 'Failed to set logo from EPG data', + color: 'red', + }); + console.error('Set logo from EPG error:', error); + } + }; + const formik = useFormik({ initialValues: { name: '', @@ -242,14 +404,17 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { formik.resetForm(); setTvgFilter(''); setLogoFilter(''); + setChannelStreams([]); // Ensure streams are cleared when adding a new channel } }, [channel, tvgsById, channelGroups]); // Memoize logo options to prevent infinite re-renders during background loading const logoOptions = useMemo(() => { - const options = [{ id: '0', name: 'Default' }].concat(Object.values(logos)); + const options = [{ id: '0', name: 'Default' }].concat( + Object.values(channelLogos) + ); return options; - }, [logos]); // Only depend on logos object + }, [channelLogos]); // Only depend on channelLogos object // Update the handler for when channel group modal is closed const handleChannelGroupModalClose = (newGroup) => { @@ -305,11 +470,28 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { + Channel Name + {formik.values.epg_data_id && ( + + )} + + } value={formik.values.name} onChange={formik.handleChange} error={formik.errors.name ? formik.touched.name : ''} size="xs" + style={{ flex: 1 }} /> @@ -491,9 +673,27 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { + Logo + {formik.values.epg_data_id && ( + + )} + + } readOnly - value={logos[formik.values.logo_id]?.name || 'Default'} + value={ + channelLogos[formik.values.logo_id]?.name || 'Default' + } onClick={() => { console.log( 'Logo input clicked, setting popover opened to true' @@ -600,11 +800,13 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { - + + + @@ -706,6 +908,25 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { > Use Dummy + } readOnly @@ -717,8 +938,6 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { return `${epgSource.name} - ${tvgLabel}`; } else if (tvgLabel) { return tvgLabel; - } else if (formik.values.name) { - return formik.values.name; } else { return 'Dummy'; } @@ -768,6 +987,7 @@ const ChannelForm = ({ channel = null, isOpen, onClose }) => { } mb="xs" size="xs" + autoFocus /> diff --git a/frontend/src/components/forms/ChannelBatch.jsx b/frontend/src/components/forms/ChannelBatch.jsx index 9973ea57..ad61fb26 100644 --- a/frontend/src/components/forms/ChannelBatch.jsx +++ b/frontend/src/components/forms/ChannelBatch.jsx @@ -27,6 +27,7 @@ import { import { ListOrdered, SquarePlus, SquareX, X } from 'lucide-react'; import { FixedSizeList as List } from 'react-window'; import { useForm } from '@mantine/form'; +import { notifications } from '@mantine/notifications'; import { USER_LEVELS, USER_LEVEL_LABELS } from '../../constants'; const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { @@ -35,7 +36,6 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { const groupListRef = useRef(null); const channelGroups = useChannelsStore((s) => s.channelGroups); - const canEditChannelGroup = useChannelsStore((s) => s.canEditChannelGroup); const streamProfiles = useStreamProfilesStore((s) => s.profiles); @@ -134,6 +134,74 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { } }; + const handleSetNamesFromEpg = async () => { + if (!channelIds || channelIds.length === 0) { + notifications.show({ + title: 'No Channels Selected', + message: 'No channels to update.', + color: 'orange', + }); + return; + } + + try { + // Start the backend task + await API.setChannelNamesFromEpg(channelIds); + + // The task will send WebSocket updates for progress + // Just show that it started successfully + notifications.show({ + title: 'Task Started', + message: `Started setting names from EPG for ${channelIds.length} channels. Progress will be shown in notifications.`, + color: 'blue', + }); + + // Close the modal since the task is now running in background + onClose(); + } catch (error) { + console.error('Failed to start EPG name setting task:', error); + notifications.show({ + title: 'Error', + message: 'Failed to start EPG name setting task.', + color: 'red', + }); + } + }; + + const handleSetLogosFromEpg = async () => { + if (!channelIds || channelIds.length === 0) { + notifications.show({ + title: 'No Channels Selected', + message: 'No channels to update.', + color: 'orange', + }); + return; + } + + try { + // Start the backend task + await API.setChannelLogosFromEpg(channelIds); + + // The task will send WebSocket updates for progress + // Just show that it started successfully + notifications.show({ + title: 'Task Started', + message: `Started setting logos from EPG for ${channelIds.length} channels. Progress will be shown in notifications.`, + color: 'blue', + }); + + // Close the modal since the task is now running in background + onClose(); + } catch (error) { + console.error('Failed to start EPG logo setting task:', error); + notifications.show({ + title: 'Error', + message: 'Failed to start EPG logo setting task.', + color: 'red', + }); + } + }; + // useEffect(() => { // // const sameStreamProfile = channels.every( // // (channel) => channel.stream_profile_id == channels[0].stream_profile_id @@ -183,7 +251,7 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { @@ -197,7 +265,9 @@ const ChannelBatchForm = ({ channelIds, isOpen, onClose }) => { - Channel Name + + Channel Name + { /> + + + + EPG Operations + + + + + + + + Updates channel names and logos based on their assigned EPG + data + + + { export default ChannelBatchForm; // Lightweight inline preview component to visualize rename results for a subset -const RegexPreview = ({ channelIds, find, replace}) => { +const RegexPreview = ({ channelIds, find, replace }) => { const channelsMap = useChannelsStore((s) => s.channels); const previewItems = useMemo(() => { const items = []; @@ -412,7 +512,8 @@ const RegexPreview = ({ channelIds, find, replace}) => { let re; try { re = new RegExp(find, flags); - } catch (e) { + } catch (error) { + console.error('Invalid regex:', error); return [{ before: 'Invalid regex', after: '' }]; } for (let i = 0; i < Math.min(channelIds.length, 25); i++) { @@ -431,20 +532,41 @@ const RegexPreview = ({ channelIds, find, replace}) => { return ( - Preview (first {Math.min(channelIds.length, 25)} of {channelIds.length} selected) + Preview (first {Math.min(channelIds.length, 25)} of {channelIds.length}{' '} + selected) {previewItems.length === 0 ? ( - No changes with current pattern. + + No changes with current pattern. + ) : ( previewItems.map((row, idx) => ( - + {row.before} - → - + + → + + {row.after} diff --git a/frontend/src/components/tables/ChannelsTable.jsx b/frontend/src/components/tables/ChannelsTable.jsx index 313d2b4b..b9da2318 100644 --- a/frontend/src/components/tables/ChannelsTable.jsx +++ b/frontend/src/components/tables/ChannelsTable.jsx @@ -306,6 +306,12 @@ const ChannelsTable = ({}) => { const [isBulkDelete, setIsBulkDelete] = useState(false); const [channelToDelete, setChannelToDelete] = useState(null); + // Column sizing state for resizable columns + const [columnSizing, setColumnSizing] = useLocalStorage( + 'channels-table-column-sizing', + {} + ); + // M3U and EPG URL configuration state const [m3uParams, setM3uParams] = useState({ cachedlogos: true, @@ -427,7 +433,13 @@ const ChannelsTable = ({}) => { })); }; - const editChannel = async (ch = null) => { + const editChannel = async (ch = null, opts = {}) => { + // If forceAdd is set, always open a blank form + if (opts.forceAdd) { + setChannel(null); + setChannelModalOpen(true); + return; + } // Use table's selected state instead of store state to avoid stale selections const currentSelection = table ? table.selectedTableIds : []; console.log('editChannel called with:', { @@ -697,14 +709,17 @@ const ChannelsTable = ({}) => { { id: 'expand', size: 20, + enableResizing: false, }, { id: 'select', size: 30, + enableResizing: false, }, { id: 'enabled', size: 45, + enableResizing: false, cell: ({ row, table }) => { return ( { { id: 'channel_number', accessorKey: 'channel_number', - size: 40, + size: columnSizing.channel_number || 40, + minSize: 30, + maxSize: 100, cell: ({ getValue }) => { const value = getValue(); // Format as integer if no decimal component @@ -739,6 +756,9 @@ const ChannelsTable = ({}) => { { id: 'name', accessorKey: 'name', + size: columnSizing.name || 200, + minSize: 100, + grow: true, cell: ({ getValue }) => ( { cell: ({ getValue }) => { const epgDataId = getValue(); const epgObj = epgDataId ? tvgsById[epgDataId] : null; + const tvgName = epgObj?.name; + const tvgId = epgObj?.tvg_id; const epgName = epgObj && epgObj.epg_source ? epgs[epgObj.epg_source]?.name || epgObj.epg_source : null; - const epgDataName = epgObj?.name; - const tvgId = epgObj?.tvg_id; + const tooltip = epgObj - ? `${epgName ? `EPG Name: ${epgName}\n` : ''}${epgDataName ? `TVG Name: ${epgDataName}\n` : ''}${tvgId ? `TVG-ID: ${tvgId}` : ''}`.trim() + ? `${epgName ? `EPG Name: ${epgName}\n` : ''}${tvgName ? `TVG Name: ${tvgName}\n` : ''}${tvgId ? `TVG-ID: ${tvgId}` : ''}`.trim() : ''; return ( { withArrow position="top" > - {epgName} + + {epgObj.epg_source} - {tvgId} + ) : epgObj ? ( {epgObj.name} ) : ( - Not linked + Not Assigned )} ); }, - size: 120, + size: columnSizing.epg || 200, + minSize: 80, }, { id: 'channel_group', @@ -812,7 +836,8 @@ const ChannelsTable = ({}) => { {getValue()} ), - size: 175, + size: columnSizing.channel_group || 175, + minSize: 100, }, { id: 'logo', @@ -821,6 +846,9 @@ const ChannelsTable = ({}) => { return row.logo_id; }, size: 75, + minSize: 50, + maxSize: 120, + enableResizing: false, header: '', cell: ({ getValue }) => { const logoId = getValue(); @@ -839,6 +867,7 @@ const ChannelsTable = ({}) => { { id: 'actions', size: tableSize == 'compact' ? 75 : 100, + enableResizing: false, header: '', cell: ({ row }) => ( { ), }, ], - [selectedProfileId, channelGroups, logos, theme] + [selectedProfileId, channelGroups, logos, theme, columnSizing] ); const renderHeaderCell = (header) => { @@ -873,6 +902,7 @@ const ChannelsTable = ({}) => { placeholder="EPG" variant="unstyled" data={epgOptions} + className="table-input-header" size="xs" searchable clearable @@ -927,6 +957,7 @@ const ChannelsTable = ({}) => { return ( { manualFiltering: true, enableRowSelection: true, onRowSelectionChange: onRowSelectionChange, + state: { + columnSizing, + pagination, + sorting, + }, + onColumnSizingChange: setColumnSizing, getExpandedRowHeight: (row) => { return 20 + 28 * row.original.streams.length; }, @@ -1301,7 +1338,7 @@ const ChannelsTable = ({}) => { style={{ flex: 1, overflowY: 'auto', - overflowX: 'hidden', + overflowX: 'auto', border: 'solid 1px rgb(68,68,68)', borderRadius: 'var(--mantine-radius-default)', }} diff --git a/frontend/src/components/tables/ChannelsTable/ChannelTableHeader.jsx b/frontend/src/components/tables/ChannelsTable/ChannelTableHeader.jsx index c2d2cb51..b7e04d7d 100644 --- a/frontend/src/components/tables/ChannelsTable/ChannelTableHeader.jsx +++ b/frontend/src/components/tables/ChannelsTable/ChannelTableHeader.jsx @@ -143,11 +143,18 @@ const ChannelTableHeader = ({ const matchEpg = async () => { try { // Hit our new endpoint that triggers the fuzzy matching Celery task - await API.matchEpg(); - - notifications.show({ - title: 'EPG matching task started!', - }); + // If channels are selected, only match those; otherwise match all + if (selectedTableIds.length > 0) { + await API.matchEpg(selectedTableIds); + notifications.show({ + title: `EPG matching task started for ${selectedTableIds.length} selected channel(s)!`, + }); + } else { + await API.matchEpg(); + notifications.show({ + title: 'EPG matching task started for all channels without EPG!', + }); + } } catch (err) { notifications.show(`Error: ${err.message}`); } @@ -259,7 +266,7 @@ const ChannelTableHeader = ({ leftSection={} variant="light" size="xs" - onClick={() => editChannel()} + onClick={() => editChannel(null, { forceAdd: true })} disabled={authUser.user_level != USER_LEVELS.ADMIN} p={5} color={theme.tailwind.green[5]} @@ -298,7 +305,11 @@ const ChannelTableHeader = ({ disabled={authUser.user_level != USER_LEVELS.ADMIN} onClick={matchEpg} > - Auto-Match + + {selectedTableIds.length > 0 + ? `Auto-Match (${selectedTableIds.length} selected)` + : 'Auto-Match EPG'} + { const [tableSize, _] = useLocalStorage('table-size', 'default'); + // Get column sizing state for dependency tracking + const columnSizing = table.getState().columnSizing; + + // Calculate minimum table width reactively based on column sizes + const minTableWidth = useMemo(() => { + const headerGroups = table.getHeaderGroups(); + if (!headerGroups || headerGroups.length === 0) return 0; + + const width = + headerGroups[0]?.headers.reduce((total, header) => { + return total + header.getSize(); + }, 0) || 0; + + return width; + }, [table, columnSizing]); + return ( { + if (rows.length === 0) return 0; + + return rows[0].getVisibleCells().reduce((total, cell) => { + // Only count columns with fixed sizes, flexible columns will expand + const columnSize = cell.column.columnDef.size + ? cell.column.getSize() + : cell.column.columnDef.minSize || 150; // Default min for flexible columns + return total + columnSize; + }, 0); + }, [rows]); + const renderTableBodyContents = () => { const virtualized = false; @@ -94,6 +108,7 @@ const CustomTableBody = ({ style={{ display: 'flex', width: '100%', + minWidth: '100%', // Force full width ...(row.getIsSelected() && { backgroundColor: '#163632', }), @@ -101,16 +116,24 @@ const CustomTableBody = ({ }} > {row.getVisibleCells().map((cell) => { + const hasFixedSize = cell.column.columnDef.size; + const isFlexible = !hasFixedSize; + return ( diff --git a/frontend/src/components/tables/CustomTable/CustomTableHeader.jsx b/frontend/src/components/tables/CustomTable/CustomTableHeader.jsx index 1e32e258..92643fc9 100644 --- a/frontend/src/components/tables/CustomTable/CustomTableHeader.jsx +++ b/frontend/src/components/tables/CustomTable/CustomTableHeader.jsx @@ -1,6 +1,6 @@ import { Box, Center, Checkbox, Flex } from '@mantine/core'; import { flexRender } from '@tanstack/react-table'; -import { useCallback } from 'react'; +import { useCallback, useMemo } from 'react'; const CustomTableHeader = ({ getHeaderGroups, @@ -40,6 +40,25 @@ const CustomTableHeader = ({ } }; + // Get header groups for dependency tracking + const headerGroups = getHeaderGroups(); + + // Calculate minimum width based only on fixed-size columns + const minTableWidth = useMemo(() => { + if (!headerGroups || headerGroups.length === 0) return 0; + + const width = + headerGroups[0]?.headers.reduce((total, header) => { + // Only count columns with fixed sizes, flexible columns will expand + const columnSize = header.column.columnDef.size + ? header.getSize() + : header.column.columnDef.minSize || 150; // Default min for flexible columns + return total + columnSize; + }, 0) || 0; + + return width; + }, [headerGroups]); + return ( {headerGroup.headers.map((header) => { return ( @@ -62,11 +85,17 @@ const CustomTableHeader = ({ className="th" key={header.id} style={{ - flex: header.column.columnDef.size ? '0 0 auto' : '1 1 0', - width: header.column.columnDef.size - ? header.getSize() - : undefined, - minWidth: 0, + ...(header.column.columnDef.grow + ? { + flex: '1 1 0%', + minWidth: 0, + } + : { + flex: `0 0 ${header.getSize ? header.getSize() : 150}px`, + width: `${header.getSize ? header.getSize() : 150}px`, + maxWidth: `${header.getSize ? header.getSize() : 150}px`, + }), + position: 'relative', // ...(tableCellProps && tableCellProps({ cell: header })), }} > @@ -76,10 +105,46 @@ const CustomTableHeader = ({ ...(header.column.columnDef.style && header.column.columnDef.style), height: '100%', + paddingRight: header.column.getCanResize() ? '8px' : '0px', // Add padding for resize handle }} > {renderHeaderCell(header)} + {header.column.getCanResize() && ( +
{ + e.target.style.opacity = '1'; + e.target.style.backgroundColor = '#6b7280'; + }} + onMouseLeave={(e) => { + if (!header.column.getIsResizing()) { + e.target.style.opacity = '0.5'; + e.target.style.backgroundColor = 'transparent'; + } + }} + /> + )} ); })} diff --git a/frontend/src/components/tables/CustomTable/index.jsx b/frontend/src/components/tables/CustomTable/index.jsx index 3be2dc2f..ec5182c9 100644 --- a/frontend/src/components/tables/CustomTable/index.jsx +++ b/frontend/src/components/tables/CustomTable/index.jsx @@ -18,6 +18,8 @@ const useTable = ({ onRowSelectionChange = null, getExpandedRowHeight = null, state = [], + columnSizing, + setColumnSizing, ...options }) => { const [selectedTableIds, setSelectedTableIds] = useState([]); @@ -85,15 +87,21 @@ const useTable = ({ const table = useReactTable({ defaultColumn: { - size: undefined, minSize: 0, + maxSize: Number.MAX_SAFE_INTEGER, + size: 150, }, ...options, state: { - data: options.data, + ...options.state, selectedTableIds, + ...(columnSizing && { columnSizing }), }, + onStateChange: options.onStateChange, + ...(setColumnSizing && { onColumnSizingChange: setColumnSizing }), getCoreRowModel: options.getCoreRowModel ?? getCoreRowModel(), + enableColumnResizing: true, + columnResizeMode: 'onChange', }); const selectedTableIdsSet = useMemo( diff --git a/frontend/src/components/tables/EPGsTable.jsx b/frontend/src/components/tables/EPGsTable.jsx index 9c40a14a..7cac7d7d 100644 --- a/frontend/src/components/tables/EPGsTable.jsx +++ b/frontend/src/components/tables/EPGsTable.jsx @@ -187,14 +187,15 @@ const EPGsTable = () => { size: 200, }, { - header: 'Source Type', + header: 'Type', accessorKey: 'source_type', - size: 150, + size: 100, }, { header: 'URL / API Key / File Path', accessorKey: 'url', enableSorting: false, + minSize: 250, cell: ({ cell, row }) => { const value = cell.getValue() || @@ -220,7 +221,7 @@ const EPGsTable = () => { { header: 'Status', accessorKey: 'status', - size: 150, + size: 100, cell: ({ row }) => { const data = row.original; @@ -236,6 +237,8 @@ const EPGsTable = () => { header: 'Status Message', accessorKey: 'last_message', enableSorting: false, + minSize: 250, + grow: true, cell: ({ row }) => { const data = row.original; diff --git a/frontend/src/components/tables/LogosTable.jsx b/frontend/src/components/tables/LogosTable.jsx index 46c29e4c..0c4f32b4 100644 --- a/frontend/src/components/tables/LogosTable.jsx +++ b/frontend/src/components/tables/LogosTable.jsx @@ -394,7 +394,7 @@ const LogosTable = () => { { header: 'Name', accessorKey: 'name', - size: 200, + size: 250, cell: ({ getValue }) => ( {getValue()} @@ -479,6 +479,7 @@ const LogosTable = () => { { header: 'URL', accessorKey: 'url', + grow: true, cell: ({ getValue }) => ( { sortable: true, }, { - header: 'Account Type', + header: 'Type', accessorKey: 'account_type', sortable: true, - size: 150, + size: 100, cell: ({ cell }) => { const value = cell.getValue(); return value === 'XC' ? 'XC' : 'M3U'; @@ -443,6 +443,7 @@ const M3UTable = () => { { header: 'URL / File', accessorKey: 'server_url', + size: 250, cell: ({ cell, row }) => { const value = cell.getValue() || row.original.file_path || ''; return ( @@ -461,16 +462,10 @@ const M3UTable = () => { ); }, }, - { - header: 'Max Streams', - accessorKey: 'max_streams', - sortable: true, - size: 150, - }, { header: 'Status', accessorKey: 'status', - size: 150, + size: 100, cell: ({ cell }) => { const value = cell.getValue(); if (!value) return null; @@ -486,6 +481,8 @@ const M3UTable = () => { { header: 'Status Message', accessorKey: 'last_message', + grow: true, + minSize: 250, cell: ({ cell, row }) => { const value = cell.getValue(); const data = row.original; @@ -569,6 +566,12 @@ const M3UTable = () => { ); }, }, + { + header: 'Max Streams', + accessorKey: 'max_streams', + sortable: true, + size: 125, + }, { header: 'Updated', accessorKey: 'updated_at', diff --git a/frontend/src/components/tables/StreamProfilesTable.jsx b/frontend/src/components/tables/StreamProfilesTable.jsx index 76be0462..da2acd30 100644 --- a/frontend/src/components/tables/StreamProfilesTable.jsx +++ b/frontend/src/components/tables/StreamProfilesTable.jsx @@ -72,7 +72,7 @@ const StreamProfiles = () => { { header: 'Name', accessorKey: 'name', - size: 150, + size: 175, cell: ({ cell }) => (
{ { header: 'Command', accessorKey: 'command', - size: 150, + size: 100, cell: ({ cell }) => (
{ { header: 'Parameters', accessorKey: 'parameters', - // size: 200, + grow: true, cell: ({ cell }) => (
{ const [tableSize, _] = useLocalStorage('table-size', 'default'); const channelSelectionStreams = useChannelsTableStore( @@ -62,23 +68,6 @@ const StreamRowActions = ({ ); const fetchLogos = useLogosStore((s) => s.fetchLogos); - const createChannelFromStream = async () => { - const selectedChannelProfileId = - useChannelsStore.getState().selectedProfileId; - - await API.createChannelFromStream({ - name: row.original.name, - channel_number: null, - stream_id: row.original.id, - // Only pass channel_profile_ids if a specific profile is selected (not "All") - ...(selectedChannelProfileId !== '0' && { - channel_profile_ids: selectedChannelProfileId, - }), - }); - await API.requeryChannels(); - fetchLogos(); - }; - const addStreamToChannel = async () => { await API.updateChannel({ id: selectedChannelIds[0], @@ -140,7 +129,7 @@ const StreamRowActions = ({ size={iconSize} color={theme.tailwind.green[5]} variant="transparent" - onClick={createChannelFromStream} + onClick={() => createChannelFromStream(row.original)} > @@ -178,7 +167,7 @@ const StreamRowActions = ({ ); }; -const StreamsTable = ({}) => { +const StreamsTable = () => { const theme = useMantineTheme(); /** @@ -196,6 +185,21 @@ const StreamsTable = ({}) => { const [isLoading, setIsLoading] = useState(true); const [sorting, setSorting] = useState([{ id: 'name', desc: '' }]); const [selectedStreamIds, setSelectedStreamIds] = useState([]); + + // Channel numbering modal state + const [channelNumberingModalOpen, setChannelNumberingModalOpen] = + useState(false); + const [numberingMode, setNumberingMode] = useState('provider'); // 'provider', 'auto', or 'custom' + const [customStartNumber, setCustomStartNumber] = useState(1); + const [rememberChoice, setRememberChoice] = useState(false); + + // Single channel numbering modal state + const [singleChannelModalOpen, setSingleChannelModalOpen] = useState(false); + const [singleChannelMode, setSingleChannelMode] = useState('provider'); // 'provider', 'auto', or 'specific' + const [specificChannelNumber, setSpecificChannelNumber] = useState(1); + const [rememberSingleChoice, setRememberSingleChoice] = useState(false); + const [currentStreamForChannel, setCurrentStreamForChannel] = useState(null); + // const [allRowsSelected, setAllRowsSelected] = useState(false); // Add local storage for page size @@ -212,6 +216,10 @@ const StreamsTable = ({}) => { channel_group: '', m3u_account: '', }); + const [columnSizing, setColumnSizing] = useLocalStorage( + 'streams-table-column-sizing', + {} + ); const debouncedFilters = useDebounce(filters, 500, () => { // Reset to first page whenever filters change to avoid "Invalid page" errors setPagination((prev) => ({ @@ -244,6 +252,10 @@ const StreamsTable = ({}) => { const showVideo = useVideoStore((s) => s.showVideo); const [tableSize, _] = useLocalStorage('table-size', 'default'); + // Warnings store for "remember choice" functionality + const suppressWarning = useWarningsStore((s) => s.suppressWarning); + const isWarningSuppressed = useWarningsStore((s) => s.isWarningSuppressed); + const handleSelectClick = (e) => { e.stopPropagation(); e.preventDefault(); @@ -256,15 +268,17 @@ const StreamsTable = ({}) => { () => [ { id: 'actions', - size: tableSize == 'compact' ? 60 : 80, + size: columnSizing.actions || (tableSize == 'compact' ? 60 : 80), }, { id: 'select', - size: 30, + size: columnSizing.select || 30, }, { header: 'Name', accessorKey: 'name', + grow: true, + size: columnSizing.name || 200, cell: ({ getValue }) => ( { channelGroups[row.channel_group] ? channelGroups[row.channel_group].name : '', + size: columnSizing.group || 150, cell: ({ getValue }) => ( { }, { id: 'm3u', - size: 150, + size: columnSizing.m3u || 150, accessorFn: (row) => playlists.find((playlist) => playlist.id === row.m3u_account)?.name, cell: ({ getValue }) => ( @@ -315,7 +330,7 @@ const StreamsTable = ({}) => { ), }, ], - [channelGroups, playlists] + [channelGroups, playlists, columnSizing, tableSize] ); /** @@ -411,50 +426,84 @@ const StreamsTable = ({}) => { fetchChannelGroups, ]); - // Bulk creation: create channels from selected streams in one API call + // Bulk creation: create channels from selected streams asynchronously const createChannelsFromStreams = async () => { - setIsLoading(true); + if (selectedStreamIds.length === 0) return; + + // Check if user has suppressed the channel numbering dialog + const actionKey = 'channel-numbering-choice'; + if (isWarningSuppressed(actionKey)) { + // Use the remembered settings or default to 'provider' mode + const savedMode = + localStorage.getItem('channel-numbering-mode') || 'provider'; + const savedStartNumber = + localStorage.getItem('channel-numbering-start') || '1'; + + const startingChannelNumberValue = + savedMode === 'provider' + ? null + : savedMode === 'auto' + ? 0 + : Number(savedStartNumber); + + await executeChannelCreation(startingChannelNumberValue); + } else { + // Show the modal to let user choose + setChannelNumberingModalOpen(true); + } + }; + + // Separate function to actually execute the channel creation + const executeChannelCreation = async (startingChannelNumberValue) => { try { const selectedChannelProfileId = useChannelsStore.getState().selectedProfileId; - // Try to fetch the actual stream data for selected streams - let streamsData = []; - try { - streamsData = await API.getStreamsByIds(selectedStreamIds); - } catch (error) { - console.warn('Could not fetch stream details, using IDs only:', error); - } + // Use the async API for all bulk operations + const response = await API.createChannelsFromStreamsAsync( + selectedStreamIds, + selectedChannelProfileId !== '0' ? [selectedChannelProfileId] : null, + startingChannelNumberValue + ); - const streamData = selectedStreamIds.map((streamId) => { - const stream = streamsData.find((s) => s.id === streamId); - return { - stream_id: streamId, - name: stream?.name || `Stream ${streamId}`, - ...(selectedChannelProfileId !== '0' && { - channel_profile_ids: selectedChannelProfileId, - }), - }; - }); + console.log( + `Bulk creation task started: ${response.task_id} for ${response.stream_count} streams` + ); - await API.createChannelsFromStreams(streamData); - await API.requeryChannels(); - - // Refresh channel profiles to update the membership information - await useChannelsStore.getState().fetchChannelProfiles(); - - fetchLogos(); - - // Clear selection and refresh data + // Clear selection since the task has started setSelectedStreamIds([]); - await fetchData(); } catch (error) { - console.error('Error creating channels from streams:', error); - } finally { - setIsLoading(false); + console.error('Error starting bulk channel creation:', error); + // Error notifications will be handled by WebSocket } }; + // Handle confirming the channel numbering modal + const handleChannelNumberingConfirm = async () => { + // Save the choice if user wants to remember it + if (rememberChoice) { + suppressWarning('channel-numbering-choice'); + localStorage.setItem('channel-numbering-mode', numberingMode); + if (numberingMode === 'custom') { + localStorage.setItem( + 'channel-numbering-start', + customStartNumber.toString() + ); + } + } + + // Convert mode to API value + const startingChannelNumberValue = + numberingMode === 'provider' + ? null + : numberingMode === 'auto' + ? 0 + : Number(customStartNumber); + + setChannelNumberingModalOpen(false); + await executeChannelCreation(startingChannelNumberValue); + }; + const editStream = async (stream = null) => { setStream(stream); setModalOpen(true); @@ -476,6 +525,80 @@ const StreamsTable = ({}) => { fetchData(); }; + // Single channel creation functions + const createChannelFromStream = async (stream) => { + // Check if user has suppressed the single channel numbering dialog + const actionKey = 'single-channel-numbering-choice'; + if (isWarningSuppressed(actionKey)) { + // Use the remembered settings or default to 'provider' mode + const savedMode = + localStorage.getItem('single-channel-numbering-mode') || 'provider'; + const savedChannelNumber = + localStorage.getItem('single-channel-numbering-specific') || '1'; + + const channelNumberValue = + savedMode === 'provider' + ? null + : savedMode === 'auto' + ? 0 + : Number(savedChannelNumber); + + await executeSingleChannelCreation(stream, channelNumberValue); + } else { + // Show the modal to let user choose + setCurrentStreamForChannel(stream); + setSingleChannelModalOpen(true); + } + }; + + // Separate function to actually execute single channel creation + const executeSingleChannelCreation = async (stream, channelNumber = null) => { + const selectedChannelProfileId = + useChannelsStore.getState().selectedProfileId; + + await API.createChannelFromStream({ + name: stream.name, + channel_number: channelNumber, + stream_id: stream.id, + // Only pass channel_profile_ids if a specific profile is selected (not "All") + ...(selectedChannelProfileId !== '0' && { + channel_profile_ids: selectedChannelProfileId, + }), + }); + await API.requeryChannels(); + const fetchLogos = useChannelsStore.getState().fetchLogos; + fetchLogos(); + }; + + // Handle confirming the single channel numbering modal + const handleSingleChannelNumberingConfirm = async () => { + // Save the choice if user wants to remember it + if (rememberSingleChoice) { + suppressWarning('single-channel-numbering-choice'); + localStorage.setItem('single-channel-numbering-mode', singleChannelMode); + if (singleChannelMode === 'specific') { + localStorage.setItem( + 'single-channel-numbering-specific', + specificChannelNumber.toString() + ); + } + } + + // Convert mode to API value + const channelNumberValue = + singleChannelMode === 'provider' + ? null + : singleChannelMode === 'auto' + ? 0 + : Number(specificChannelNumber); + + setSingleChannelModalOpen(false); + await executeSingleChannelCreation( + currentStreamForChannel, + channelNumberValue + ); + }; + const addStreamsToChannel = async () => { await API.updateChannel({ id: selectedChannelIds[0], @@ -631,6 +754,7 @@ const StreamsTable = ({}) => { deleteStream={deleteStream} handleWatchStream={handleWatchStream} selectedChannelIds={selectedChannelIds} + createChannelFromStream={createChannelFromStream} /> ); } @@ -652,6 +776,8 @@ const StreamsTable = ({}) => { filters, pagination, sorting, + columnSizing, + setColumnSizing, onRowSelectionChange: onRowSelectionChange, manualPagination: true, manualSorting: true, @@ -768,7 +894,7 @@ const StreamsTable = ({}) => { p={5} disabled={selectedStreamIds.length == 0} > - Create Channels + {`Create Channels (${selectedStreamIds.length})`} + + + + + + {/* Single Channel Numbering Modal */} + setSingleChannelModalOpen(false)} + title="Channel Number Assignment" + size="md" + centered + > + + + Choose how to assign the channel number for " + {currentStreamForChannel?.name}": + + + + + + + + + + + {singleChannelMode === 'specific' && ( + + )} + + + setRememberSingleChoice(event.currentTarget.checked) + } + label="Remember this choice and don't ask again" + /> + + + + + + + ); }; diff --git a/frontend/src/components/tables/UserAgentsTable.jsx b/frontend/src/components/tables/UserAgentsTable.jsx index 265ed125..48573926 100644 --- a/frontend/src/components/tables/UserAgentsTable.jsx +++ b/frontend/src/components/tables/UserAgentsTable.jsx @@ -61,11 +61,13 @@ const UserAgentsTable = () => { { header: 'Name', accessorKey: 'name', + size: 125, }, { header: 'User-Agent', accessorKey: 'user_agent', enableSorting: false, + grow: true, cell: ({ cell }) => (
{ { header: 'Email', accessorKey: 'email', + grow: true, cell: ({ getValue }) => ( { * (unused + channel-used, excluding VOD-only logos) */ export const useChannelLogoSelection = () => { + const [isInitialized, setIsInitialized] = useState(false); const channelLogos = useLogosStore((s) => s.channelLogos); const hasLoadedChannelLogos = useLogosStore((s) => s.hasLoadedChannelLogos); const backgroundLoading = useLogosStore((s) => s.backgroundLoading); @@ -56,7 +57,7 @@ export const useChannelLogoSelection = () => { return; } - if (hasLoadedChannelLogos && hasLogos) { + if ((hasLoadedChannelLogos && hasLogos) || isInitialized) { return; } @@ -71,6 +72,7 @@ export const useChannelLogoSelection = () => { hasLoadedChannelLogos, hasLogos, fetchChannelAssignableLogos, + isInitialized, ]); return { diff --git a/frontend/src/pages/Channels.jsx b/frontend/src/pages/Channels.jsx index c265e3ef..7663276d 100644 --- a/frontend/src/pages/Channels.jsx +++ b/frontend/src/pages/Channels.jsx @@ -5,9 +5,22 @@ import { Box } from '@mantine/core'; import { Allotment } from 'allotment'; import { USER_LEVELS } from '../constants'; import useAuthStore from '../store/auth'; +import useLocalStorage from '../hooks/useLocalStorage'; const ChannelsPage = () => { const authUser = useAuthStore((s) => s.user); + const [allotmentSizes, setAllotmentSizes] = useLocalStorage( + 'channels-splitter-sizes', + [50, 50] + ); + + const handleSplitChange = (sizes) => { + setAllotmentSizes(sizes); + }; + + const handleResize = (sizes) => { + setAllotmentSizes(sizes); + }; if (!authUser.id) { return <>; @@ -30,10 +43,12 @@ const ChannelsPage = () => { }} >
diff --git a/frontend/src/pages/VODs.jsx b/frontend/src/pages/VODs.jsx index 16d43a9a..3c9e2b0f 100644 --- a/frontend/src/pages/VODs.jsx +++ b/frontend/src/pages/VODs.jsx @@ -264,8 +264,7 @@ const useCardColumns = () => { }; const VODsPage = () => { - const movies = useVODStore((s) => s.movies); - const series = useVODStore((s) => s.series); + const currentPageContent = useVODStore((s) => s.currentPageContent); // Direct subscription const allCategories = useVODStore((s) => s.categories); const filters = useVODStore((s) => s.filters); const currentPage = useVODStore((s) => s.currentPage); @@ -288,8 +287,7 @@ const VODsPage = () => { setPageSize(Number(value)); localStorage.setItem('vodsPageSize', value); }; - const fetchMovies = useVODStore((s) => s.fetchMovies); - const fetchSeries = useVODStore((s) => s.fetchSeries); + const fetchContent = useVODStore((s) => s.fetchContent); const fetchCategories = useVODStore((s) => s.fetchCategories); // const showVideo = useVideoStore((s) => s.showVideo); - removed as unused @@ -307,36 +305,10 @@ const VODsPage = () => { // Helper function to get display data based on current filters const getDisplayData = () => { - if (filters.type === 'series') { - return Object.values(series).map((item) => ({ - ...item, - _vodType: 'series', - })); - } else if (filters.type === 'movies') { - return Object.values(movies).map((item) => ({ - ...item, - _vodType: 'movie', - })); - } else { - // 'all' - combine movies and series, tagging each with its type, then sort alphabetically by name/title - const combined = [ - ...Object.values(movies).map((item) => ({ - ...item, - _vodType: 'movie', - })), - ...Object.values(series).map((item) => ({ - ...item, - _vodType: 'series', - })), - ]; - return combined.sort((a, b) => { - const nameA = (a.name || a.title || '').toLowerCase(); - const nameB = (b.name || b.title || '').toLowerCase(); - if (nameA < nameB) return -1; - if (nameA > nameB) return 1; - return 0; - }); - } + return (currentPageContent || []).map((item) => ({ + ...item, + _vodType: item.contentType === 'movie' ? 'movie' : 'series', + })); }; useEffect(() => { @@ -360,17 +332,8 @@ const VODsPage = () => { }, [fetchCategories]); useEffect(() => { - if (filters.type === 'series') { - fetchSeries().finally(() => setInitialLoad(false)); - } else if (filters.type === 'movies') { - fetchMovies().finally(() => setInitialLoad(false)); - } else { - // 'all': fetch both movies and series - Promise.all([fetchMovies(), fetchSeries()]).finally(() => - setInitialLoad(false) - ); - } - }, [filters, currentPage, pageSize, fetchMovies, fetchSeries]); + fetchContent().finally(() => setInitialLoad(false)); + }, [filters, currentPage, pageSize, fetchContent]); const handleVODCardClick = (vod) => { setSelectedVOD(vod); @@ -464,46 +427,25 @@ const VODsPage = () => { ) : ( <> - {filters.type === 'series' ? ( - - {Object.values(series).map((seriesItem) => ( - - - - ))} - - ) : ( - - {getDisplayData().map((item) => ( - - {item._vodType === 'series' ? ( - - ) : ( - - )} - - ))} - - )} + + {getDisplayData().map((item) => ( + + {item.contentType === 'series' ? ( + + ) : ( + + )} + + ))} + {/* Pagination */} {totalPages > 1 && ( diff --git a/frontend/src/store/logos.jsx b/frontend/src/store/logos.jsx index 4a0b945c..eb2a7597 100644 --- a/frontend/src/store/logos.jsx +++ b/frontend/src/store/logos.jsx @@ -3,7 +3,7 @@ import api from '../api'; const useLogosStore = create((set, get) => ({ logos: {}, - channelLogos: {}, // Separate state for channel-assignable logos + channelLogos: {}, // Keep this for simplicity, but we'll be more careful about when we populate it isLoading: false, backgroundLoading: false, hasLoadedAll: false, // Track if we've loaded all logos @@ -21,12 +21,29 @@ const useLogosStore = create((set, get) => ({ }, addLogo: (newLogo) => - set((state) => ({ - logos: { + set((state) => { + // Add to main logos store always + const newLogos = { ...state.logos, [newLogo.id]: { ...newLogo }, - }, - })), + }; + + // Add to channelLogos if the user has loaded channel-assignable logos + // This means they're using channel forms and the new logo should be available there + // Newly created logos are channel-assignable (they start unused) + let newChannelLogos = state.channelLogos; + if (state.hasLoadedChannelLogos) { + newChannelLogos = { + ...state.channelLogos, + [newLogo.id]: { ...newLogo }, + }; + } + + return { + logos: newLogos, + channelLogos: newChannelLogos, + }; + }), updateLogo: (logo) => set((state) => ({ @@ -34,13 +51,25 @@ const useLogosStore = create((set, get) => ({ ...state.logos, [logo.id]: { ...logo }, }, + // Update in channelLogos if it exists there + channelLogos: state.channelLogos[logo.id] + ? { + ...state.channelLogos, + [logo.id]: { ...logo }, + } + : state.channelLogos, })), removeLogo: (logoId) => set((state) => { const newLogos = { ...state.logos }; + const newChannelLogos = { ...state.channelLogos }; delete newLogos[logoId]; - return { logos: newLogos }; + delete newChannelLogos[logoId]; + return { + logos: newLogos, + channelLogos: newChannelLogos, + }; }), // Smart loading methods @@ -155,8 +184,15 @@ const useLogosStore = create((set, get) => ({ console.log(`Fetched ${logos.length} channel-assignable logos`); - // Store in separate channelLogos state + // Store in both places, but this is intentional and only when specifically requested set({ + logos: { + ...get().logos, // Keep existing logos + ...logos.reduce((acc, logo) => { + acc[logo.id] = { ...logo }; + return acc; + }, {}), + }, channelLogos: logos.reduce((acc, logo) => { acc[logo.id] = { ...logo }; return acc; diff --git a/frontend/src/store/useVODStore.jsx b/frontend/src/store/useVODStore.jsx index dd3f5b8a..b0aecd61 100644 --- a/frontend/src/store/useVODStore.jsx +++ b/frontend/src/store/useVODStore.jsx @@ -2,8 +2,8 @@ import { create } from 'zustand'; import api from '../api'; const useVODStore = create((set, get) => ({ - movies: {}, - series: {}, + content: {}, // Store for individual content details (when fetching movie/series details) + currentPageContent: [], // Store the current page's results episodes: {}, categories: {}, loading: false, @@ -34,12 +34,12 @@ const useVODStore = create((set, get) => ({ currentPage: 1, // Reset to first page when page size changes })), - fetchMovies: async () => { + fetchContent: async () => { try { set({ loading: true, error: null }); const state = get(); - const params = new URLSearchParams(); + const params = new URLSearchParams(); params.append('page', state.currentPage); params.append('page_size', state.pageSize); @@ -51,60 +51,62 @@ const useVODStore = create((set, get) => ({ params.append('category', state.filters.category); } - const response = await api.getMovies(params); + let allResults = []; + let totalCount = 0; - // Handle both paginated and non-paginated responses - const results = response.results || response; - const count = response.count || results.length; + if (state.filters.type === 'movies') { + // Fetch only movies + const response = await api.getMovies(params); + const results = response.results || response; + allResults = results.map((item) => ({ ...item, contentType: 'movie' })); + totalCount = response.count || results.length; + } else if (state.filters.type === 'series') { + // Fetch only series + const response = await api.getSeries(params); + const results = response.results || response; + allResults = results.map((item) => ({ + ...item, + contentType: 'series', + })); + totalCount = response.count || results.length; + } else { + // Use the new unified backend endpoint for 'all' view + const response = await api.getAllContent(params); + console.log('getAllContent response:', response); + console.log('response type:', typeof response); + console.log( + 'response keys:', + response ? Object.keys(response) : 'no response' + ); + const results = response.results || response; + console.log('results:', results); + console.log('results type:', typeof results); + console.log('results is array:', Array.isArray(results)); + + // Check if results is actually an array before calling map + if (!Array.isArray(results)) { + console.error('Results is not an array:', results); + throw new Error('Invalid response format - results is not an array'); + } + + // The backend already provides content_type and proper sorting/pagination + allResults = results.map((item) => ({ + ...item, + contentType: item.content_type, // Backend provides this field + })); + totalCount = response.count || results.length; + } + + // Store the current page results directly (don't accumulate all pages) set({ - movies: results.reduce((acc, movie) => { - acc[movie.id] = movie; - return acc; - }, {}), - totalCount: count, + currentPageContent: allResults, // This is the paginated data for current page + totalCount, loading: false, }); } catch (error) { - console.error('Failed to fetch movies:', error); - set({ error: 'Failed to load movies.', loading: false }); - } - }, - - fetchSeries: async () => { - set({ loading: true, error: null }); - try { - const state = get(); - const params = new URLSearchParams(); - - params.append('page', state.currentPage); - params.append('page_size', state.pageSize); - - if (state.filters.search) { - params.append('search', state.filters.search); - } - - if (state.filters.category) { - params.append('category', state.filters.category); - } - - const response = await api.getSeries(params); - - // Handle both paginated and non-paginated responses - const results = response.results || response; - const count = response.count || results.length; - - set({ - series: results.reduce((acc, series) => { - acc[series.id] = series; - return acc; - }, {}), - totalCount: count, - loading: false, - }); - } catch (error) { - console.error('Failed to fetch series:', error); - set({ error: 'Failed to load series.', loading: false }); + console.error('Failed to fetch content:', error); + set({ error: 'Failed to load content.', loading: false }); } }, @@ -158,9 +160,12 @@ const useVODStore = create((set, get) => ({ }; console.log('Fetched Movie Details:', movieDetails); set((state) => ({ - movies: { - ...state.movies, - [movieDetails.id]: movieDetails, + content: { + ...state.content, + [`movie_${movieDetails.id}`]: { + ...movieDetails, + contentType: 'movie', + }, }, loading: false, })); @@ -261,36 +266,48 @@ const useVODStore = create((set, get) => ({ addMovie: (movie) => set((state) => ({ - movies: { ...state.movies, [movie.id]: movie }, + content: { + ...state.content, + [`movie_${movie.id}`]: { ...movie, contentType: 'movie' }, + }, })), updateMovie: (movie) => set((state) => ({ - movies: { ...state.movies, [movie.id]: movie }, + content: { + ...state.content, + [`movie_${movie.id}`]: { ...movie, contentType: 'movie' }, + }, })), removeMovie: (movieId) => set((state) => { - const updatedMovies = { ...state.movies }; - delete updatedMovies[movieId]; - return { movies: updatedMovies }; + const updatedContent = { ...state.content }; + delete updatedContent[`movie_${movieId}`]; + return { content: updatedContent }; }), addSeries: (series) => set((state) => ({ - series: { ...state.series, [series.id]: series }, + content: { + ...state.content, + [`series_${series.id}`]: { ...series, contentType: 'series' }, + }, })), updateSeries: (series) => set((state) => ({ - series: { ...state.series, [series.id]: series }, + content: { + ...state.content, + [`series_${series.id}`]: { ...series, contentType: 'series' }, + }, })), removeSeries: (seriesId) => set((state) => { - const updatedSeries = { ...state.series }; - delete updatedSeries[seriesId]; - return { series: updatedSeries }; + const updatedContent = { ...state.content }; + delete updatedContent[`series_${seriesId}`]; + return { content: updatedContent }; }), fetchSeriesInfo: async (seriesId) => { @@ -369,9 +386,9 @@ const useVODStore = create((set, get) => ({ } set((state) => ({ - series: { - ...state.series, - [seriesInfo.id]: seriesInfo, + content: { + ...state.content, + [`series_${seriesInfo.id}`]: { ...seriesInfo, contentType: 'series' }, }, loading: false, })); @@ -387,6 +404,29 @@ const useVODStore = create((set, get) => ({ throw error; } }, + + // Helper methods for getting filtered content + getFilteredContent: () => { + const state = get(); + // Return the current page content directly - backend handles all filtering/pagination + return state.currentPageContent; + }, + + getMovies: () => { + const state = get(); + return Object.values(state.content).filter( + (item) => item.contentType === 'movie' + ); + }, + + getSeries: () => { + const state = get(); + return Object.values(state.content).filter( + (item) => item.contentType === 'series' + ); + }, + + clearContent: () => set({ content: {}, totalCount: 0 }), })); export default useVODStore; diff --git a/scripts/epg_match.py b/scripts/epg_match.py deleted file mode 100644 index 890ffe3a..00000000 --- a/scripts/epg_match.py +++ /dev/null @@ -1,182 +0,0 @@ -# ml_model.py - -import sys -import json -import re -import os -import logging - -from rapidfuzz import fuzz -from sentence_transformers import util -from sentence_transformers import SentenceTransformer as st - -# Set up logger -logger = logging.getLogger(__name__) - -# Load the sentence-transformers model once at the module level -SENTENCE_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2" -MODEL_PATH = os.path.join("/app", "models", "all-MiniLM-L6-v2") - -# Thresholds -BEST_FUZZY_THRESHOLD = 85 -LOWER_FUZZY_THRESHOLD = 40 -EMBED_SIM_THRESHOLD = 0.65 - -def process_data(input_data): - os.makedirs(MODEL_PATH, exist_ok=True) - - # If not present locally, download: - if not os.path.exists(os.path.join(MODEL_PATH, "config.json")): - logger.info(f"Local model not found in {MODEL_PATH}; downloading from {SENTENCE_MODEL_NAME}...") - st_model = st(SENTENCE_MODEL_NAME, cache_folder=MODEL_PATH) - else: - logger.info(f"Loading local model from {MODEL_PATH}") - st_model = st(MODEL_PATH) - - channels = input_data["channels"] - epg_data = input_data["epg_data"] - region_code = input_data.get("region_code", None) - - epg_embeddings = None - if any(row["norm_name"] for row in epg_data): - epg_embeddings = st_model.encode( - [row["norm_name"] for row in epg_data], - convert_to_tensor=True - ) - - channels_to_update = [] - matched_channels = [] - - for chan in channels: - normalized_tvg_id = chan.get("tvg_id", "") - fallback_name = chan["tvg_id"].strip() if chan["tvg_id"] else chan["name"] - - # Exact TVG ID match (direct match) - epg_by_tvg_id = next((epg for epg in epg_data if epg["tvg_id"] == normalized_tvg_id), None) - if normalized_tvg_id and epg_by_tvg_id: - chan["epg_data_id"] = epg_by_tvg_id["id"] - channels_to_update.append(chan) - - # Add to matched_channels list so it's counted in the total - matched_channels.append((chan['id'], fallback_name, epg_by_tvg_id["tvg_id"])) - - logger.info(f"Channel {chan['id']} '{fallback_name}' => EPG found by tvg_id={epg_by_tvg_id['tvg_id']}") - continue - - # If channel has a tvg_id that doesn't exist in EPGData, do direct check. - # I don't THINK this should happen now that we assign EPG on channel creation. - if chan["tvg_id"]: - epg_match = [epg["id"] for epg in epg_data if epg["tvg_id"] == chan["tvg_id"]] - if epg_match: - chan["epg_data_id"] = epg_match[0] - logger.info(f"Channel {chan['id']} '{chan['name']}' => EPG found by tvg_id={chan['tvg_id']}") - channels_to_update.append(chan) - continue - - # C) Perform name-based fuzzy matching - if not chan["norm_chan"]: - logger.debug(f"Channel {chan['id']} '{chan['name']}' => empty after normalization, skipping") - continue - - best_score = 0 - best_epg = None - for row in epg_data: - if not row["norm_name"]: - continue - - base_score = fuzz.ratio(chan["norm_chan"], row["norm_name"]) - bonus = 0 - # Region-based bonus/penalty - combined_text = row["tvg_id"].lower() + " " + row["name"].lower() - dot_regions = re.findall(r'\.([a-z]{2})', combined_text) - if region_code: - if dot_regions: - if region_code in dot_regions: - bonus = 30 # bigger bonus if .us or .ca matches - else: - bonus = -15 - elif region_code in combined_text: - bonus = 15 - score = base_score + bonus - - logger.debug( - f"Channel {chan['id']} '{fallback_name}' => EPG row {row['id']}: " - f"name='{row['name']}', norm_name='{row['norm_name']}', " - f"combined_text='{combined_text}', dot_regions={dot_regions}, " - f"base_score={base_score}, bonus={bonus}, total_score={score}" - ) - - if score > best_score: - best_score = score - best_epg = row - - # If no best match was found, skip - if not best_epg: - logger.debug(f"Channel {chan['id']} '{fallback_name}' => no EPG match at all.") - continue - - # If best_score is above BEST_FUZZY_THRESHOLD => direct accept - if best_score >= BEST_FUZZY_THRESHOLD: - chan["epg_data_id"] = best_epg["id"] - channels_to_update.append(chan) - - matched_channels.append((chan['id'], fallback_name, best_epg["tvg_id"])) - logger.info( - f"Channel {chan['id']} '{fallback_name}' => matched tvg_id={best_epg['tvg_id']} " - f"(score={best_score})" - ) - - # If best_score is in the “middle range,” do embedding check - elif best_score >= LOWER_FUZZY_THRESHOLD and epg_embeddings is not None: - chan_embedding = st_model.encode(chan["norm_chan"], convert_to_tensor=True) - sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] - top_index = int(sim_scores.argmax()) - top_value = float(sim_scores[top_index]) - if top_value >= EMBED_SIM_THRESHOLD: - matched_epg = epg_data[top_index] - chan["epg_data_id"] = matched_epg["id"] - channels_to_update.append(chan) - - matched_channels.append((chan['id'], fallback_name, matched_epg["tvg_id"])) - logger.info( - f"Channel {chan['id']} '{fallback_name}' => matched EPG tvg_id={matched_epg['tvg_id']} " - f"(fuzzy={best_score}, cos-sim={top_value:.2f})" - ) - else: - logger.info( - f"Channel {chan['id']} '{fallback_name}' => fuzzy={best_score}, " - f"cos-sim={top_value:.2f} < {EMBED_SIM_THRESHOLD}, skipping" - ) - else: - # No good match found - fuzzy score is too low - logger.info( - f"Channel {chan['id']} '{fallback_name}' => best fuzzy match score={best_score} < {LOWER_FUZZY_THRESHOLD}, skipping" - ) - - return { - "channels_to_update": channels_to_update, - "matched_channels": matched_channels - } - -def main(): - # Configure logging - logging_level = os.environ.get('DISPATCHARR_LOG_LEVEL', 'INFO') - logging.basicConfig( - level=getattr(logging, logging_level), - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - stream=sys.stderr - ) - - # Read input data from a file - input_file_path = sys.argv[1] - with open(input_file_path, 'r') as f: - input_data = json.load(f) - - # Process data with the ML model (or your logic) - result = process_data(input_data) - - # Output result to stdout - print(json.dumps(result)) - -if __name__ == "__main__": - main()