diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 05462d0f..a607dc07 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -136,6 +136,9 @@ class M3UAccountSerializer(serializers.ModelSerializer): validators=[validate_flexible_url], ) enable_vod = serializers.BooleanField(required=False, write_only=True) + auto_enable_new_groups_live = serializers.BooleanField(required=False, write_only=True) + auto_enable_new_groups_vod = serializers.BooleanField(required=False, write_only=True) + auto_enable_new_groups_series = serializers.BooleanField(required=False, write_only=True) class Meta: model = M3UAccount @@ -164,6 +167,9 @@ class M3UAccountSerializer(serializers.ModelSerializer): "status", "last_message", "enable_vod", + "auto_enable_new_groups_live", + "auto_enable_new_groups_vod", + "auto_enable_new_groups_series", ] extra_kwargs = { "password": { @@ -175,23 +181,36 @@ class M3UAccountSerializer(serializers.ModelSerializer): def to_representation(self, instance): data = super().to_representation(instance) - # Parse custom_properties to get VOD preference + # Parse custom_properties to get VOD preference and auto_enable_new_groups settings custom_props = instance.custom_properties or {} data["enable_vod"] = custom_props.get("enable_vod", False) + data["auto_enable_new_groups_live"] = custom_props.get("auto_enable_new_groups_live", True) + data["auto_enable_new_groups_vod"] = custom_props.get("auto_enable_new_groups_vod", True) + data["auto_enable_new_groups_series"] = custom_props.get("auto_enable_new_groups_series", True) return data def update(self, instance, validated_data): - # Handle enable_vod preference + # Handle enable_vod preference and auto_enable_new_groups settings enable_vod = validated_data.pop("enable_vod", None) + auto_enable_new_groups_live = validated_data.pop("auto_enable_new_groups_live", None) + auto_enable_new_groups_vod = validated_data.pop("auto_enable_new_groups_vod", None) + auto_enable_new_groups_series = validated_data.pop("auto_enable_new_groups_series", None) + # Get existing custom_properties + custom_props = instance.custom_properties or {} + + # Update preferences if enable_vod is not None: - # Get existing custom_properties - custom_props = instance.custom_properties or {} - - # Update VOD preference custom_props["enable_vod"] = enable_vod - validated_data["custom_properties"] = custom_props + if auto_enable_new_groups_live is not None: + custom_props["auto_enable_new_groups_live"] = auto_enable_new_groups_live + if auto_enable_new_groups_vod is not None: + custom_props["auto_enable_new_groups_vod"] = auto_enable_new_groups_vod + if auto_enable_new_groups_series is not None: + custom_props["auto_enable_new_groups_series"] = auto_enable_new_groups_series + + validated_data["custom_properties"] = custom_props # Pop out channel group memberships so we can handle them manually channel_group_data = validated_data.pop("channel_group", []) @@ -225,14 +244,20 @@ class M3UAccountSerializer(serializers.ModelSerializer): return instance def create(self, validated_data): - # Handle enable_vod preference during creation + # Handle enable_vod preference and auto_enable_new_groups settings during creation enable_vod = validated_data.pop("enable_vod", False) + auto_enable_new_groups_live = validated_data.pop("auto_enable_new_groups_live", True) + auto_enable_new_groups_vod = validated_data.pop("auto_enable_new_groups_vod", True) + auto_enable_new_groups_series = validated_data.pop("auto_enable_new_groups_series", True) # Parse existing custom_properties or create new custom_props = validated_data.get("custom_properties", {}) - # Set VOD preference + # Set preferences (default to True for auto_enable_new_groups) custom_props["enable_vod"] = enable_vod + custom_props["auto_enable_new_groups_live"] = auto_enable_new_groups_live + custom_props["auto_enable_new_groups_vod"] = auto_enable_new_groups_vod + custom_props["auto_enable_new_groups_series"] = auto_enable_new_groups_series validated_data["custom_properties"] = custom_props return super().create(validated_data) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 0ba595c5..593b2704 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -488,25 +488,29 @@ def process_groups(account, groups): } logger.info(f"Currently {len(existing_groups)} existing groups") - group_objs = [] + # Check if we should auto-enable new groups based on account settings + account_custom_props = account.custom_properties or {} + auto_enable_new_groups_live = account_custom_props.get("auto_enable_new_groups_live", True) + + # Separate existing groups from groups that need to be created + existing_group_objs = [] groups_to_create = [] + for group_name, custom_props in groups.items(): - logger.debug(f"Handling group for M3U account {account.id}: {group_name}") - - if group_name not in existing_groups: - groups_to_create.append( - ChannelGroup( - name=group_name, - ) - ) + if group_name in existing_groups: + existing_group_objs.append(existing_groups[group_name]) else: - group_objs.append(existing_groups[group_name]) + groups_to_create.append(ChannelGroup(name=group_name)) + # Create new groups and fetch them back with IDs + newly_created_group_objs = [] if groups_to_create: - logger.debug(f"Creating {len(groups_to_create)} groups") - created = ChannelGroup.bulk_create_and_fetch(groups_to_create) - logger.debug(f"Created {len(created)} groups") - group_objs.extend(created) + logger.info(f"Creating {len(groups_to_create)} new groups for account {account.id}") + newly_created_group_objs = list(ChannelGroup.bulk_create_and_fetch(groups_to_create)) + logger.debug(f"Successfully created {len(newly_created_group_objs)} new groups") + + # Combine all groups + all_group_objs = existing_group_objs + newly_created_group_objs # Get existing relationships for this account existing_relationships = { @@ -536,7 +540,7 @@ def process_groups(account, groups): relations_to_delete.append(rel) logger.debug(f"Marking relationship for deletion: group '{group_name}' no longer exists in source for account {account.id}") - for group in group_objs: + for group in all_group_objs: custom_props = groups.get(group.name, {}) if group.name in existing_relationships: @@ -566,35 +570,17 @@ def process_groups(account, groups): else: logger.debug(f"xc_id unchanged for group '{group.name}' - account {account.id}") else: - # Create new relationship - but check if there's an existing relationship that might have user settings - # This can happen if the group was temporarily removed and is now back - try: - potential_existing = ChannelGroupM3UAccount.objects.filter( - m3u_account=account, - channel_group=group - ).first() + # Create new relationship - this group is new to this M3U account + # Use the auto_enable setting to determine if it should start enabled + if not auto_enable_new_groups_live: + logger.info(f"Group '{group.name}' is new to account {account.id} - creating relationship but DISABLED (auto_enable_new_groups_live=False)") - if potential_existing: - # Merge with existing custom properties to preserve user settings - existing_custom_props = potential_existing.custom_properties or {} - - # Merge new properties with existing ones - merged_custom_props = existing_custom_props.copy() - merged_custom_props.update(custom_props) - custom_props = merged_custom_props - logger.debug(f"Merged custom properties for existing relationship: group '{group.name}' - account {account.id}") - except Exception as e: - logger.debug(f"Could not check for existing relationship: {str(e)}") - # Fall back to using just the new custom properties - pass - - # Create new relationship relations_to_create.append( ChannelGroupM3UAccount( channel_group=group, m3u_account=account, custom_properties=custom_props, - enabled=True, # Default to enabled + enabled=auto_enable_new_groups_live, ) ) diff --git a/apps/output/views.py b/apps/output/views.py index 6eee7ccc..15036710 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -342,9 +342,9 @@ def generate_epg(request, profile_name=None, user=None): channels = Channel.objects.filter( channelprofilemembership__channel_profile=channel_profile, channelprofilemembership__enabled=True, - ) + ).order_by("channel_number") else: - channels = Channel.objects.all() + channels = Channel.objects.all().order_by("channel_number") # Check if the request wants to use direct logo URLs instead of cache use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' @@ -455,241 +455,256 @@ def generate_epg(request, profile_name=None, user=None): else: # For real EPG data - filter only if days parameter was specified if num_days > 0: - programs = channel.epg_data.programs.filter( + programs_qs = channel.epg_data.programs.filter( start_time__gte=now, start_time__lt=cutoff_date - ) + ).order_by('id') # Explicit ordering for consistent chunking else: # Return all programs if days=0 or not specified - programs = channel.epg_data.programs.all() + programs_qs = channel.epg_data.programs.all().order_by('id') - # Process programs in chunks to avoid memory issues + # Process programs in chunks to avoid cursor timeout issues program_batch = [] - batch_size = 100 + batch_size = 250 + chunk_size = 1000 # Fetch 1000 programs at a time from DB - for prog in programs.iterator(): # Use iterator to avoid loading all at once - start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") - stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") + # Fetch chunks until no more results (avoids count() query) + offset = 0 + while True: + # Fetch a chunk of programs - this closes the cursor after fetching + program_chunk = list(programs_qs[offset:offset + chunk_size]) - program_xml = [f' '] - program_xml.append(f' {html.escape(prog.title)}') + # Break if no more programs + if not program_chunk: + break - # Add subtitle if available - if prog.sub_title: - program_xml.append(f" {html.escape(prog.sub_title)}") + # Process each program in the chunk + for prog in program_chunk: + start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") + stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") - # Add description if available - if prog.description: - program_xml.append(f" {html.escape(prog.description)}") + program_xml = [f' '] + program_xml.append(f' {html.escape(prog.title)}') - # Process custom properties if available - if prog.custom_properties: - custom_data = prog.custom_properties or {} + # Add subtitle if available + if prog.sub_title: + program_xml.append(f" {html.escape(prog.sub_title)}") - # Add categories if available - if "categories" in custom_data and custom_data["categories"]: - for category in custom_data["categories"]: - program_xml.append(f" {html.escape(category)}") + # Add description if available + if prog.description: + program_xml.append(f" {html.escape(prog.description)}") - # Add keywords if available - if "keywords" in custom_data and custom_data["keywords"]: - for keyword in custom_data["keywords"]: - program_xml.append(f" {html.escape(keyword)}") + # Process custom properties if available + if prog.custom_properties: + custom_data = prog.custom_properties or {} - # Handle episode numbering - multiple formats supported - # Prioritize onscreen_episode over standalone episode for onscreen system - if "onscreen_episode" in custom_data: - program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') - elif "episode" in custom_data: - program_xml.append(f' E{custom_data["episode"]}') + # Add categories if available + if "categories" in custom_data and custom_data["categories"]: + for category in custom_data["categories"]: + program_xml.append(f" {html.escape(category)}") - # Handle dd_progid format - if 'dd_progid' in custom_data: - program_xml.append(f' {html.escape(custom_data["dd_progid"])}') + # Add keywords if available + if "keywords" in custom_data and custom_data["keywords"]: + for keyword in custom_data["keywords"]: + program_xml.append(f" {html.escape(keyword)}") - # Handle external database IDs - for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: - if f'{system}_id' in custom_data: - program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') + # Handle episode numbering - multiple formats supported + # Prioritize onscreen_episode over standalone episode for onscreen system + if "onscreen_episode" in custom_data: + program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') + elif "episode" in custom_data: + program_xml.append(f' E{custom_data["episode"]}') - # Add season and episode numbers in xmltv_ns format if available - if "season" in custom_data and "episode" in custom_data: - season = ( - int(custom_data["season"]) - 1 - if str(custom_data["season"]).isdigit() - else 0 - ) - episode = ( - int(custom_data["episode"]) - 1 - if str(custom_data["episode"]).isdigit() - else 0 - ) - program_xml.append(f' {season}.{episode}.') + # Handle dd_progid format + if 'dd_progid' in custom_data: + program_xml.append(f' {html.escape(custom_data["dd_progid"])}') - # Add language information - if "language" in custom_data: - program_xml.append(f' {html.escape(custom_data["language"])}') + # Handle external database IDs + for system in ['thetvdb.com', 'themoviedb.org', 'imdb.com']: + if f'{system}_id' in custom_data: + program_xml.append(f' {html.escape(custom_data[f"{system}_id"])}') - if "original_language" in custom_data: - program_xml.append(f' {html.escape(custom_data["original_language"])}') + # Add season and episode numbers in xmltv_ns format if available + if "season" in custom_data and "episode" in custom_data: + season = ( + int(custom_data["season"]) - 1 + if str(custom_data["season"]).isdigit() + else 0 + ) + episode = ( + int(custom_data["episode"]) - 1 + if str(custom_data["episode"]).isdigit() + else 0 + ) + program_xml.append(f' {season}.{episode}.') - # Add length information - if "length" in custom_data and isinstance(custom_data["length"], dict): - length_value = custom_data["length"].get("value", "") - length_units = custom_data["length"].get("units", "minutes") - program_xml.append(f' {html.escape(str(length_value))}') + # Add language information + if "language" in custom_data: + program_xml.append(f' {html.escape(custom_data["language"])}') - # Add video information - if "video" in custom_data and isinstance(custom_data["video"], dict): - program_xml.append(" ") + if "original_language" in custom_data: + program_xml.append(f' {html.escape(custom_data["original_language"])}') - # Add audio information - if "audio" in custom_data and isinstance(custom_data["audio"], dict): - program_xml.append(" ") + # Add length information + if "length" in custom_data and isinstance(custom_data["length"], dict): + length_value = custom_data["length"].get("value", "") + length_units = custom_data["length"].get("units", "minutes") + program_xml.append(f' {html.escape(str(length_value))}') - # Add subtitles information - if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): - for subtitle in custom_data["subtitles"]: - if isinstance(subtitle, dict): - subtitle_type = subtitle.get("type", "") - type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" - program_xml.append(f" ") - if "language" in subtitle: - program_xml.append(f" {html.escape(subtitle['language'])}") - program_xml.append(" ") + # Add video information + if "video" in custom_data and isinstance(custom_data["video"], dict): + program_xml.append(" ") - # Add rating if available - if "rating" in custom_data: - rating_system = custom_data.get("rating_system", "TV Parental Guidelines") - program_xml.append(f' ') - program_xml.append(f' {html.escape(custom_data["rating"])}') - program_xml.append(f" ") + # Add audio information + if "audio" in custom_data and isinstance(custom_data["audio"], dict): + program_xml.append(" ") - # Add star ratings - if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): - for star_rating in custom_data["star_ratings"]: - if isinstance(star_rating, dict) and "value" in star_rating: - system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" - program_xml.append(f" ") - program_xml.append(f" {html.escape(star_rating['value'])}") - program_xml.append(" ") + # Add subtitles information + if "subtitles" in custom_data and isinstance(custom_data["subtitles"], list): + for subtitle in custom_data["subtitles"]: + if isinstance(subtitle, dict): + subtitle_type = subtitle.get("type", "") + type_attr = f' type="{html.escape(subtitle_type)}"' if subtitle_type else "" + program_xml.append(f" ") + if "language" in subtitle: + program_xml.append(f" {html.escape(subtitle['language'])}") + program_xml.append(" ") - # Add reviews - if "reviews" in custom_data and isinstance(custom_data["reviews"], list): - for review in custom_data["reviews"]: - if isinstance(review, dict) and "content" in review: - review_type = review.get("type", "text") - attrs = [f'type="{html.escape(review_type)}"'] - if "source" in review: - attrs.append(f'source="{html.escape(review["source"])}"') - if "reviewer" in review: - attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') - attr_str = " ".join(attrs) - program_xml.append(f' {html.escape(review["content"])}') + # Add rating if available + if "rating" in custom_data: + rating_system = custom_data.get("rating_system", "TV Parental Guidelines") + program_xml.append(f' ') + program_xml.append(f' {html.escape(custom_data["rating"])}') + program_xml.append(f" ") - # Add images - if "images" in custom_data and isinstance(custom_data["images"], list): - for image in custom_data["images"]: - if isinstance(image, dict) and "url" in image: - attrs = [] - for attr in ['type', 'size', 'orient', 'system']: - if attr in image: - attrs.append(f'{attr}="{html.escape(image[attr])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f' {html.escape(image["url"])}') + # Add star ratings + if "star_ratings" in custom_data and isinstance(custom_data["star_ratings"], list): + for star_rating in custom_data["star_ratings"]: + if isinstance(star_rating, dict) and "value" in star_rating: + system_attr = f' system="{html.escape(star_rating["system"])}"' if "system" in star_rating else "" + program_xml.append(f" ") + program_xml.append(f" {html.escape(star_rating['value'])}") + program_xml.append(" ") - # Add enhanced credits handling - if "credits" in custom_data: - program_xml.append(" ") - credits = custom_data["credits"] + # Add reviews + if "reviews" in custom_data and isinstance(custom_data["reviews"], list): + for review in custom_data["reviews"]: + if isinstance(review, dict) and "content" in review: + review_type = review.get("type", "text") + attrs = [f'type="{html.escape(review_type)}"'] + if "source" in review: + attrs.append(f'source="{html.escape(review["source"])}"') + if "reviewer" in review: + attrs.append(f'reviewer="{html.escape(review["reviewer"])}"') + attr_str = " ".join(attrs) + program_xml.append(f' {html.escape(review["content"])}') - # Handle different credit types - for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: - if role in credits: - people = credits[role] - if isinstance(people, list): - for person in people: - program_xml.append(f" <{role}>{html.escape(person)}") - else: - program_xml.append(f" <{role}>{html.escape(people)}") + # Add images + if "images" in custom_data and isinstance(custom_data["images"], list): + for image in custom_data["images"]: + if isinstance(image, dict) and "url" in image: + attrs = [] + for attr in ['type', 'size', 'orient', 'system']: + if attr in image: + attrs.append(f'{attr}="{html.escape(image[attr])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f' {html.escape(image["url"])}') - # Handle actors separately to include role and guest attributes - if "actor" in credits: - actors = credits["actor"] - if isinstance(actors, list): - for actor in actors: - if isinstance(actor, dict): - name = actor.get("name", "") - role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" - guest_attr = ' guest="yes"' if actor.get("guest") else "" - program_xml.append(f" {html.escape(name)}") + # Add enhanced credits handling + if "credits" in custom_data: + program_xml.append(" ") + credits = custom_data["credits"] + + # Handle different credit types + for role in ['director', 'writer', 'adapter', 'producer', 'composer', 'editor', 'presenter', 'commentator', 'guest']: + if role in credits: + people = credits[role] + if isinstance(people, list): + for person in people: + program_xml.append(f" <{role}>{html.escape(person)}") else: - program_xml.append(f" {html.escape(actor)}") + program_xml.append(f" <{role}>{html.escape(people)}") + + # Handle actors separately to include role and guest attributes + if "actor" in credits: + actors = credits["actor"] + if isinstance(actors, list): + for actor in actors: + if isinstance(actor, dict): + name = actor.get("name", "") + role_attr = f' role="{html.escape(actor["role"])}"' if "role" in actor else "" + guest_attr = ' guest="yes"' if actor.get("guest") else "" + program_xml.append(f" {html.escape(name)}") + else: + program_xml.append(f" {html.escape(actor)}") + else: + program_xml.append(f" {html.escape(actors)}") + + program_xml.append(" ") + + # Add program date if available (full date, not just year) + if "date" in custom_data: + program_xml.append(f' {html.escape(custom_data["date"])}') + + # Add country if available + if "country" in custom_data: + program_xml.append(f' {html.escape(custom_data["country"])}') + + # Add icon if available + if "icon" in custom_data: + program_xml.append(f' ') + + # Add special flags as proper tags with enhanced handling + if custom_data.get("previously_shown", False): + prev_shown_details = custom_data.get("previously_shown_details", {}) + attrs = [] + if "start" in prev_shown_details: + attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') + if "channel" in prev_shown_details: + attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') + attr_str = " " + " ".join(attrs) if attrs else "" + program_xml.append(f" ") + + if custom_data.get("premiere", False): + premiere_text = custom_data.get("premiere_text", "") + if premiere_text: + program_xml.append(f" {html.escape(premiere_text)}") else: - program_xml.append(f" {html.escape(actors)}") + program_xml.append(" ") - program_xml.append(" ") + if custom_data.get("last_chance", False): + last_chance_text = custom_data.get("last_chance_text", "") + if last_chance_text: + program_xml.append(f" {html.escape(last_chance_text)}") + else: + program_xml.append(" ") - # Add program date if available (full date, not just year) - if "date" in custom_data: - program_xml.append(f' {html.escape(custom_data["date"])}') + if custom_data.get("new", False): + program_xml.append(" ") - # Add country if available - if "country" in custom_data: - program_xml.append(f' {html.escape(custom_data["country"])}') + if custom_data.get('live', False): + program_xml.append(' ') - # Add icon if available - if "icon" in custom_data: - program_xml.append(f' ') + program_xml.append(" ") - # Add special flags as proper tags with enhanced handling - if custom_data.get("previously_shown", False): - prev_shown_details = custom_data.get("previously_shown_details", {}) - attrs = [] - if "start" in prev_shown_details: - attrs.append(f'start="{html.escape(prev_shown_details["start"])}"') - if "channel" in prev_shown_details: - attrs.append(f'channel="{html.escape(prev_shown_details["channel"])}"') - attr_str = " " + " ".join(attrs) if attrs else "" - program_xml.append(f" ") + # Add to batch + program_batch.extend(program_xml) - if custom_data.get("premiere", False): - premiere_text = custom_data.get("premiere_text", "") - if premiere_text: - program_xml.append(f" {html.escape(premiere_text)}") - else: - program_xml.append(" ") + # Send batch when full or send keep-alive + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] - if custom_data.get("last_chance", False): - last_chance_text = custom_data.get("last_chance_text", "") - if last_chance_text: - program_xml.append(f" {html.escape(last_chance_text)}") - else: - program_xml.append(" ") - - if custom_data.get("new", False): - program_xml.append(" ") - - if custom_data.get('live', False): - program_xml.append(' ') - - program_xml.append(" ") - - # Add to batch - program_batch.extend(program_xml) - - # Send batch when full or send keep-alive - if len(program_batch) >= batch_size: - yield '\n'.join(program_batch) + '\n' - program_batch = [] # Send keep-alive every batch + # Move to next chunk + offset += chunk_size # Send remaining programs in batch if program_batch: diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index da5daaa7..0d638c1a 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -131,6 +131,8 @@ class ProxyServer: max_retries = 10 base_retry_delay = 1 # Start with 1 second delay max_retry_delay = 30 # Cap at 30 seconds + pubsub_client = None + pubsub = None while True: try: @@ -339,20 +341,27 @@ class ProxyServer: logger.error(f"Error in event listener: {e}. Retrying in {final_delay:.1f}s (attempt {retry_count})") gevent.sleep(final_delay) # REPLACE: time.sleep(final_delay) - # Try to clean up the old connection - try: - if 'pubsub' in locals(): - pubsub.close() - if 'pubsub_client' in locals(): - pubsub_client.close() - except: - pass - except Exception as e: logger.error(f"Error in event listener: {e}") # Add a short delay to prevent rapid retries on persistent errors gevent.sleep(5) # REPLACE: time.sleep(5) + finally: + # Always clean up PubSub connections in all error paths + try: + if pubsub: + pubsub.close() + pubsub = None + except Exception as e: + logger.debug(f"Error closing pubsub: {e}") + + try: + if pubsub_client: + pubsub_client.close() + pubsub_client = None + except Exception as e: + logger.debug(f"Error closing pubsub_client: {e}") + thread = threading.Thread(target=event_listener, daemon=True) thread.name = "redis-event-listener" thread.start() @@ -596,7 +605,7 @@ class ProxyServer: if channel_user_agent: metadata["user_agent"] = channel_user_agent - # CRITICAL FIX: Make sure stream_id is always set in metadata and properly logged + # Make sure stream_id is always set in metadata and properly logged if channel_stream_id: metadata["stream_id"] = str(channel_stream_id) logger.info(f"Storing stream_id {channel_stream_id} in metadata for channel {channel_id}") diff --git a/apps/proxy/ts_proxy/services/channel_service.py b/apps/proxy/ts_proxy/services/channel_service.py index 932479ea..ac8f3a10 100644 --- a/apps/proxy/ts_proxy/services/channel_service.py +++ b/apps/proxy/ts_proxy/services/channel_service.py @@ -600,25 +600,25 @@ class ChannelService: try: from apps.channels.models import Stream from django.utils import timezone - + stream = Stream.objects.get(id=stream_id) - + # Get existing stats or create new dict current_stats = stream.stream_stats or {} - + # Update with new stats for key, value in stats.items(): if value is not None: current_stats[key] = value - + # Save updated stats and timestamp stream.stream_stats = current_stats stream.stream_stats_updated_at = timezone.now() stream.save(update_fields=['stream_stats', 'stream_stats_updated_at']) - + logger.debug(f"Updated stream stats in database for stream {stream_id}: {stats}") return True - + except Exception as e: logger.error(f"Error updating stream stats in database for stream {stream_id}: {e}") return False @@ -678,7 +678,7 @@ class ChannelService: switch_request = { "event": EventType.STREAM_SWITCH, - "channel_id": channel_id, + "channel_id": str(channel_id), "url": new_url, "user_agent": user_agent, "stream_id": stream_id, @@ -703,7 +703,7 @@ class ChannelService: stop_request = { "event": EventType.CHANNEL_STOP, - "channel_id": channel_id, + "channel_id": str(channel_id), "requester_worker_id": proxy_server.worker_id, "timestamp": time.time() } @@ -726,7 +726,7 @@ class ChannelService: stop_request = { "event": EventType.CLIENT_STOP, - "channel_id": channel_id, + "channel_id": str(channel_id), "client_id": client_id, "requester_worker_id": proxy_server.worker_id, "timestamp": time.time() diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index e80d4527..be6a9c4e 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -1219,6 +1219,30 @@ class StreamManager: except Exception as e: logger.error(f"Final kill attempt failed for channel {self.channel_id}: {e}") + # Explicitly close all subprocess pipes to prevent file descriptor leaks + try: + if self.transcode_process.stdin: + self.transcode_process.stdin.close() + if self.transcode_process.stdout: + self.transcode_process.stdout.close() + if self.transcode_process.stderr: + self.transcode_process.stderr.close() + logger.debug(f"Closed all subprocess pipes for channel {self.channel_id}") + except Exception as e: + logger.debug(f"Error closing subprocess pipes for channel {self.channel_id}: {e}") + + # Join stderr reader thread to ensure it's fully terminated + if hasattr(self, 'stderr_reader_thread') and self.stderr_reader_thread and self.stderr_reader_thread.is_alive(): + try: + logger.debug(f"Waiting for stderr reader thread to terminate for channel {self.channel_id}") + self.stderr_reader_thread.join(timeout=2.0) + if self.stderr_reader_thread.is_alive(): + logger.warning(f"Stderr reader thread did not terminate within timeout for channel {self.channel_id}") + except Exception as e: + logger.debug(f"Error joining stderr reader thread for channel {self.channel_id}: {e}") + finally: + self.stderr_reader_thread = None + self.transcode_process = None self.transcode_process_active = False # Reset the flag diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index e31d0418..bb07d697 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -491,7 +491,7 @@ def stream_xc(request, username, password, channel_id): channel = get_object_or_404(Channel, id=channel_id) # @TODO: we've got the file 'type' via extension, support this when we support multiple outputs - return stream_ts(request._request, channel.uuid) + return stream_ts(request._request, str(channel.uuid)) @csrf_exempt diff --git a/apps/vod/tasks.py b/apps/vod/tasks.py index 1a2e51ca..bc8ad80f 100644 --- a/apps/vod/tasks.py +++ b/apps/vod/tasks.py @@ -187,16 +187,28 @@ def batch_create_categories(categories_data, category_type, account): logger.debug(f"Found {len(existing_categories)} existing categories") + # Check if we should auto-enable new categories based on account settings + account_custom_props = account.custom_properties or {} + if category_type == 'movie': + auto_enable_new = account_custom_props.get("auto_enable_new_groups_vod", True) + else: # series + auto_enable_new = account_custom_props.get("auto_enable_new_groups_series", True) + # Create missing categories in batch new_categories = [] + for name in category_names: if name not in existing_categories: + # Always create new categories new_categories.append(VODCategory(name=name, category_type=category_type)) else: + # Existing category - create relationship with enabled based on auto_enable setting + # (category exists globally but is new to this account) relations_to_create.append(M3UVODCategoryRelation( category=existing_categories[name], m3u_account=account, custom_properties={}, + enabled=auto_enable_new, )) logger.debug(f"{len(new_categories)} new categories found") @@ -204,24 +216,68 @@ def batch_create_categories(categories_data, category_type, account): if new_categories: logger.debug("Creating new categories...") - created_categories = VODCategory.bulk_create_and_fetch(new_categories, ignore_conflicts=True) + created_categories = list(VODCategory.bulk_create_and_fetch(new_categories, ignore_conflicts=True)) + + # Create relations for newly created categories with enabled based on auto_enable setting + for cat in created_categories: + if not auto_enable_new: + logger.info(f"New {category_type} category '{cat.name}' created but DISABLED - auto_enable_new_groups is disabled for account {account.id}") + + relations_to_create.append( + M3UVODCategoryRelation( + category=cat, + m3u_account=account, + custom_properties={}, + enabled=auto_enable_new, + ) + ) + # Convert to dictionary for easy lookup newly_created = {cat.name: cat for cat in created_categories} - - relations_to_create += [ - M3UVODCategoryRelation( - category=cat, - m3u_account=account, - custom_properties={}, - ) for cat in newly_created.values() - ] - existing_categories.update(newly_created) # Create missing relations logger.debug("Updating category account relations...") M3UVODCategoryRelation.objects.bulk_create(relations_to_create, ignore_conflicts=True) + # Delete orphaned category relationships (categories no longer in the M3U source) + current_category_ids = set(existing_categories[name].id for name in category_names) + existing_relations = M3UVODCategoryRelation.objects.filter( + m3u_account=account, + category__category_type=category_type + ).select_related('category') + + relations_to_delete = [ + rel for rel in existing_relations + if rel.category_id not in current_category_ids + ] + + if relations_to_delete: + M3UVODCategoryRelation.objects.filter( + id__in=[rel.id for rel in relations_to_delete] + ).delete() + logger.info(f"Deleted {len(relations_to_delete)} orphaned {category_type} category relationships for account {account.id}: {[rel.category.name for rel in relations_to_delete]}") + + # Check if any of the deleted relationships left categories with no remaining associations + orphaned_category_ids = [] + for rel in relations_to_delete: + category = rel.category + + # Check if this category has any remaining M3U account relationships + remaining_relationships = M3UVODCategoryRelation.objects.filter( + category=category + ).exists() + + # If no relationships remain, it's safe to delete the category + if not remaining_relationships: + orphaned_category_ids.append(category.id) + logger.debug(f"Category '{category.name}' has no remaining associations and will be deleted") + + # Delete orphaned categories + if orphaned_category_ids: + VODCategory.objects.filter(id__in=orphaned_category_ids).delete() + logger.info(f"Deleted {len(orphaned_category_ids)} orphaned {category_type} categories with no remaining associations") + # 🔑 Fetch all relations for this account, for all categories # relations = { rel.id: rel for rel in M3UVODCategoryRelation.objects # .filter(category__in=existing_categories.values(), m3u_account=account) diff --git a/debian_install.sh b/debian_install.sh index 7f35075b..bda506b1 100755 --- a/debian_install.sh +++ b/debian_install.sh @@ -68,15 +68,16 @@ install_packages() { echo ">>> Installing system packages..." apt-get update declare -a packages=( - git curl wget build-essential gcc libpcre3-dev libpq-dev + git curl wget build-essential gcc libpq-dev python3-dev python3-venv python3-pip nginx redis-server postgresql postgresql-contrib ffmpeg procps streamlink + sudo ) apt-get install -y --no-install-recommends "${packages[@]}" if ! command -v node >/dev/null 2>&1; then echo ">>> Installing Node.js..." - curl -sL https://deb.nodesource.com/setup_23.x | bash - + curl -sL https://deb.nodesource.com/setup_24.x | bash - apt-get install -y nodejs fi @@ -186,7 +187,32 @@ EOSU } ############################################################################## -# 8) Django Migrations & Static +# 8) Create Directories +############################################################################## + +create_directories() { + mkdir -p /data/logos + mkdir -p /data/recordings + mkdir -p /data/uploads/m3us + mkdir -p /data/uploads/epgs + mkdir -p /data/m3us + mkdir -p /data/epgs + mkdir -p /data/plugins + mkdir -p /data/db + + # Needs to own ALL of /data except db + chown -R $DISPATCH_USER:$DISPATCH_GROUP /data + chown -R postgres:postgres /data/db + chmod +x /data + + mkdir -p "$APP_DIR"/logo_cache + mkdir -p "$APP_DIR"/media + chown -R $DISPATCH_USER:$DISPATCH_GROUP "$APP_DIR"/logo_cache + chown -R $DISPATCH_USER:$DISPATCH_GROUP "$APP_DIR"/media +} + +############################################################################## +# 9) Django Migrations & Static ############################################################################## django_migrate_collectstatic() { @@ -204,7 +230,7 @@ EOSU } ############################################################################## -# 9) Configure Services & Nginx +# 10) Configure Services & Nginx ############################################################################## configure_services() { @@ -360,7 +386,7 @@ EOF } ############################################################################## -# 10) Start Services +# 11) Start Services ############################################################################## start_services() { @@ -371,7 +397,7 @@ start_services() { } ############################################################################## -# 11) Summary +# 12) Summary ############################################################################## show_summary() { @@ -408,10 +434,11 @@ main() { clone_dispatcharr_repo setup_python_env build_frontend + create_directories django_migrate_collectstatic configure_services start_services show_summary } -main "$@" +main "$@" \ No newline at end of file diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index 4a473afe..c5ac5f83 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -33,7 +33,13 @@ const OptionWithTooltip = forwardRef( ) ); -const LiveGroupFilter = ({ playlist, groupStates, setGroupStates }) => { +const LiveGroupFilter = ({ + playlist, + groupStates, + setGroupStates, + autoEnableNewGroupsLive, + setAutoEnableNewGroupsLive, +}) => { const channelGroups = useChannelsStore((s) => s.channelGroups); const profiles = useChannelsStore((s) => s.profiles); const streamProfiles = useStreamProfilesStore((s) => s.profiles); @@ -159,6 +165,16 @@ const LiveGroupFilter = ({ playlist, groupStates, setGroupStates }) => { + + setAutoEnableNewGroupsLive(event.currentTarget.checked) + } + size="sm" + description="When disabled, new groups from the M3U source will be created but disabled by default. You can enable them manually later." + /> + { const [isLoading, setIsLoading] = useState(false); const [movieCategoryStates, setMovieCategoryStates] = useState([]); const [seriesCategoryStates, setSeriesCategoryStates] = useState([]); + const [autoEnableNewGroupsLive, setAutoEnableNewGroupsLive] = useState(true); + const [autoEnableNewGroupsVod, setAutoEnableNewGroupsVod] = useState(true); + const [autoEnableNewGroupsSeries, setAutoEnableNewGroupsSeries] = + useState(true); + + useEffect(() => { + if (!playlist) return; + + // Initialize account-level settings + setAutoEnableNewGroupsLive(playlist.auto_enable_new_groups_live ?? true); + setAutoEnableNewGroupsVod(playlist.auto_enable_new_groups_vod ?? true); + setAutoEnableNewGroupsSeries( + playlist.auto_enable_new_groups_series ?? true + ); + }, [playlist]); useEffect(() => { if (Object.keys(channelGroups).length === 0) { @@ -116,6 +131,14 @@ const M3UGroupFilter = ({ playlist = null, isOpen, onClose }) => { })) .filter((state) => state.enabled !== state.original_enabled); + // Update account-level settings via the proper account endpoint + await API.updatePlaylist({ + id: playlist.id, + auto_enable_new_groups_live: autoEnableNewGroupsLive, + auto_enable_new_groups_vod: autoEnableNewGroupsVod, + auto_enable_new_groups_series: autoEnableNewGroupsSeries, + }); + // Update group settings via API endpoint await API.updateM3UGroupSettings( playlist.id, @@ -176,6 +199,8 @@ const M3UGroupFilter = ({ playlist = null, isOpen, onClose }) => { playlist={playlist} groupStates={groupStates} setGroupStates={setGroupStates} + autoEnableNewGroupsLive={autoEnableNewGroupsLive} + setAutoEnableNewGroupsLive={setAutoEnableNewGroupsLive} /> @@ -185,6 +210,8 @@ const M3UGroupFilter = ({ playlist = null, isOpen, onClose }) => { categoryStates={movieCategoryStates} setCategoryStates={setMovieCategoryStates} type="movie" + autoEnableNewGroups={autoEnableNewGroupsVod} + setAutoEnableNewGroups={setAutoEnableNewGroupsVod} /> @@ -194,6 +221,8 @@ const M3UGroupFilter = ({ playlist = null, isOpen, onClose }) => { categoryStates={seriesCategoryStates} setCategoryStates={setSeriesCategoryStates} type="series" + autoEnableNewGroups={autoEnableNewGroupsSeries} + setAutoEnableNewGroups={setAutoEnableNewGroupsSeries} /> diff --git a/frontend/src/components/forms/VODCategoryFilter.jsx b/frontend/src/components/forms/VODCategoryFilter.jsx index 7b922f06..a6dccdd2 100644 --- a/frontend/src/components/forms/VODCategoryFilter.jsx +++ b/frontend/src/components/forms/VODCategoryFilter.jsx @@ -10,6 +10,7 @@ import { Text, Divider, Box, + Checkbox, } from '@mantine/core'; import { CircleCheck, CircleX } from 'lucide-react'; import useVODStore from '../../store/useVODStore'; @@ -19,6 +20,8 @@ const VODCategoryFilter = ({ categoryStates, setCategoryStates, type, + autoEnableNewGroups, + setAutoEnableNewGroups, }) => { const categories = useVODStore((s) => s.categories); const [filter, setFilter] = useState(''); @@ -85,6 +88,16 @@ const VODCategoryFilter = ({ return ( + + setAutoEnableNewGroups(event.currentTarget.checked) + } + size="sm" + description="When disabled, new categories from the provider will be created but disabled by default. You can enable them manually later." + /> +