From f2a238915af07fedcb20aa27b174175c6d298ee1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Jun 2025 14:54:47 -0500 Subject: [PATCH] Use streaming response during EPG generation to avoid clients timing out. Closes #179 --- apps/output/views.py | 484 ++++++++++++++++++++++--------------------- 1 file changed, 244 insertions(+), 240 deletions(-) diff --git a/apps/output/views.py b/apps/output/views.py index 4be0f811..4ef9f4f2 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -1,5 +1,5 @@ import ipaddress -from django.http import HttpResponse, JsonResponse, Http404, HttpResponseForbidden +from django.http import HttpResponse, JsonResponse, Http404, HttpResponseForbidden, StreamingHttpResponse from rest_framework.response import Response from django.urls import reverse from apps.channels.models import Channel, ChannelProfile, ChannelGroup @@ -12,11 +12,10 @@ from dispatcharr.utils import network_access_allowed from django.utils import timezone from django.shortcuts import get_object_or_404 from datetime import datetime, timedelta -import re import html # Add this import for XML escaping +import json # Add this import for JSON parsing +import time # Add this import for keep-alive delays from tzlocal import get_localzone -import time -import json from urllib.parse import urlparse import base64 @@ -278,118 +277,76 @@ def generate_dummy_epg( def generate_epg(request, profile_name=None, user=None): """ - Dynamically generate an XMLTV (EPG) file using the new EPGData/ProgramData models. + Dynamically generate an XMLTV (EPG) file using streaming response to handle keep-alives. Since the EPG data is stored independently of Channels, we group programmes by their associated EPGData record. - This version filters data based on the 'days' parameter. + This version filters data based on the 'days' parameter and sends keep-alives during processing. """ - xml_lines = [] - xml_lines.append('') - xml_lines.append( - '' - ) + def epg_generator(): + """Generator function that yields EPG data with keep-alives during processing""" # Send initial HTTP headers as comments (these will be ignored by XML parsers but keep connection alive) - if user is not None: - if user.user_level == 0: - filters = { - "channelprofilemembership__enabled": True, - "user_level__lte": user.user_level, - } + xml_lines = [] + xml_lines.append('') + xml_lines.append( + '' + ) - if user.channel_profiles.count() != 0: - channel_profiles = user.channel_profiles.all() - filters["channelprofilemembership__channel_profile__in"] = ( - channel_profiles + # Get channels based on user/profile + if user is not None: + if user.user_level == 0: + filters = { + "channelprofilemembership__enabled": True, + "user_level__lte": user.user_level, + } + + if user.channel_profiles.count() != 0: + channel_profiles = user.channel_profiles.all() + filters["channelprofilemembership__channel_profile__in"] = ( + channel_profiles + ) + + channels = Channel.objects.filter(**filters).order_by("channel_number") + else: + channels = Channel.objects.filter(user_level__lte=user.user_level).order_by( + "channel_number" ) - - channels = Channel.objects.filter(**filters).order_by("channel_number") else: - channels = Channel.objects.filter(user_level__lte=user.user_level).order_by( - "channel_number" - ) - else: - if profile_name is not None: - channel_profile = ChannelProfile.objects.get(name=profile_name) - channels = Channel.objects.filter( - channelprofilemembership__channel_profile=channel_profile, - channelprofilemembership__enabled=True, - ) - else: - channels = Channel.objects.all() - - # Check if the request wants to use direct logo URLs instead of cache - use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' - - # Get the source to use for tvg-id value - # Options: 'channel_number' (default), 'tvg_id', 'gracenote' - tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() - - # Get the number of days for EPG data - try: - # Default to 0 days (everything) for real EPG if not specified - days_param = request.GET.get('days', '0') - num_days = int(days_param) - # Set reasonable limits - num_days = max(0, min(num_days, 365)) # Between 0 and 365 days - except ValueError: - num_days = 0 # Default to all data if invalid value - - # For dummy EPG, use either the specified value or default to 3 days - dummy_days = num_days if num_days > 0 else 3 - - # Calculate cutoff date for EPG data filtering (only if days > 0) - now = timezone.now() - cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None - - # Retrieve all active channels - for channel in channels: - # Format channel number as integer if it has no decimal component - same as M3U generation - if channel.channel_number is not None: - if channel.channel_number == int(channel.channel_number): - formatted_channel_number = int(channel.channel_number) + if profile_name is not None: + channel_profile = ChannelProfile.objects.get(name=profile_name) + channels = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True, + ) else: - formatted_channel_number = channel.channel_number - else: - formatted_channel_number = "" + channels = Channel.objects.all() - # Determine the channel ID based on the selected source - if tvg_id_source == 'tvg_id' and channel.tvg_id: - channel_id = channel.tvg_id - elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: - channel_id = channel.tvc_guide_stationid - else: - # Default to channel number (original behavior) - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) + # Check if the request wants to use direct logo URLs instead of cache + use_cached_logos = request.GET.get('cachedlogos', 'true').lower() != 'false' - # Add channel logo if available - tvg_logo = "" - if channel.logo: - if use_cached_logos: - # Use cached logo as before - tvg_logo = request.build_absolute_uri(reverse('api:channels:logo-cache', args=[channel.logo.id])) - else: - # Try to find direct logo URL from channel's streams - direct_logo = channel.logo.url if channel.logo.url.startswith(('http://', 'https://')) else None - # If direct logo found, use it; otherwise fall back to cached version - if direct_logo: - tvg_logo = direct_logo - else: - tvg_logo = request.build_absolute_uri(reverse('api:channels:logo-cache', args=[channel.logo.id])) - display_name = channel.epg_data.name if channel.epg_data else channel.name - xml_lines.append(f' ') - xml_lines.append(f' {html.escape(display_name)}') - xml_lines.append(f' ') + # Get the source to use for tvg-id value + # Options: 'channel_number' (default), 'tvg_id', 'gracenote' + tvg_id_source = request.GET.get('tvg_id_source', 'channel_number').lower() - xml_lines.append(" ") + # Get the number of days for EPG data + try: + # Default to 0 days (everything) for real EPG if not specified + days_param = request.GET.get('days', '0') + num_days = int(days_param) + # Set reasonable limits + num_days = max(0, min(num_days, 365)) # Between 0 and 365 days + except ValueError: + num_days = 0 # Default to all data if invalid value - for channel in channels: - # Use the same channel ID determination for program entries - if tvg_id_source == 'tvg_id' and channel.tvg_id: - channel_id = channel.tvg_id - elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: - channel_id = channel.tvc_guide_stationid - else: - # Get formatted channel number + # For dummy EPG, use either the specified value or default to 3 days + dummy_days = num_days if num_days > 0 else 3 + + # Calculate cutoff date for EPG data filtering (only if days > 0) + now = timezone.now() + cutoff_date = now + timedelta(days=num_days) if num_days > 0 else None + + # Process channels for the section + for channel in channels: + # Format channel number as integer if it has no decimal component - same as M3U generation if channel.channel_number is not None: if channel.channel_number == int(channel.channel_number): formatted_channel_number = int(channel.channel_number) @@ -397,167 +354,214 @@ def generate_epg(request, profile_name=None, user=None): formatted_channel_number = channel.channel_number else: formatted_channel_number = "" - # Default to channel number - channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - display_name = channel.epg_data.name if channel.epg_data else channel.name - if not channel.epg_data: - # Use the enhanced dummy EPG generation function with defaults - program_length_hours = 4 # Default to 4-hour program blocks - generate_dummy_epg( - channel_id, - display_name, - xml_lines, - num_days=dummy_days, # Use dummy_days (3 days by default) - program_length_hours=program_length_hours - ) - else: - # For real EPG data - filter only if days parameter was specified - if num_days > 0: - programs = channel.epg_data.programs.filter( - start_time__gte=now, - start_time__lt=cutoff_date - ) + # Determine the channel ID based on the selected source + if tvg_id_source == 'tvg_id' and channel.tvg_id: + channel_id = channel.tvg_id + elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: + channel_id = channel.tvc_guide_stationid else: - # Return all programs if days=0 or not specified - programs = channel.epg_data.programs.all() + # Default to channel number (original behavior) + channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) - for prog in programs: - start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") - stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") - xml_lines.append(f' ') - xml_lines.append(f' {html.escape(prog.title)}') + # Add channel logo if available + tvg_logo = "" + if channel.logo: + if use_cached_logos: + # Use cached logo as before + tvg_logo = request.build_absolute_uri(reverse('api:channels:logo-cache', args=[channel.logo.id])) + else: + # Try to find direct logo URL from channel's streams + direct_logo = channel.logo.url if channel.logo.url.startswith(('http://', 'https://')) else None + # If direct logo found, use it; otherwise fall back to cached version + if direct_logo: + tvg_logo = direct_logo + else: + tvg_logo = request.build_absolute_uri(reverse('api:channels:logo-cache', args=[channel.logo.id])) - # Add subtitle if available - if prog.sub_title: - xml_lines.append( - f" {html.escape(prog.sub_title)}" + display_name = channel.epg_data.name if channel.epg_data else channel.name + xml_lines.append(f' ') + xml_lines.append(f' {html.escape(display_name)}') + xml_lines.append(f' ') + xml_lines.append(" ") + + # Send all channel definitions + yield '\n'.join(xml_lines) + '\n' + xml_lines = [] # Clear to save memory + + # Process programs for each channel + for channel in channels: + + # Use the same channel ID determination for program entries + if tvg_id_source == 'tvg_id' and channel.tvg_id: + channel_id = channel.tvg_id + elif tvg_id_source == 'gracenote' and channel.tvc_guide_stationid: + channel_id = channel.tvc_guide_stationid + else: + # Get formatted channel number + if channel.channel_number is not None: + if channel.channel_number == int(channel.channel_number): + formatted_channel_number = int(channel.channel_number) + else: + formatted_channel_number = channel.channel_number + else: + formatted_channel_number = "" + # Default to channel number + channel_id = str(formatted_channel_number) if formatted_channel_number != "" else str(channel.id) + + display_name = channel.epg_data.name if channel.epg_data else channel.name + + if not channel.epg_data: + # Use the enhanced dummy EPG generation function with defaults + program_length_hours = 4 # Default to 4-hour program blocks + dummy_programs = generate_dummy_programs(channel_id, display_name, num_days=dummy_days, program_length_hours=program_length_hours) + + for program in dummy_programs: + # Format times in XMLTV format + start_str = program['start_time'].strftime("%Y%m%d%H%M%S %z") + stop_str = program['end_time'].strftime("%Y%m%d%H%M%S %z") + + # Create program entry with escaped channel name + yield f' \n' + yield f" {html.escape(program['title'])}\n" + yield f" {html.escape(program['description'])}\n" + yield f" \n" + + else: + # For real EPG data - filter only if days parameter was specified + if num_days > 0: + programs = channel.epg_data.programs.filter( + start_time__gte=now, + start_time__lt=cutoff_date ) + else: + # Return all programs if days=0 or not specified + programs = channel.epg_data.programs.all() - # Add description if available - if prog.description: - xml_lines.append( - f" {html.escape(prog.description)}" - ) + # Process programs in chunks to avoid memory issues + program_batch = [] + batch_size = 100 - # Process custom properties if available - if prog.custom_properties: - try: - import json + for prog in programs.iterator(): # Use iterator to avoid loading all at once + start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") + stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") - custom_data = json.loads(prog.custom_properties) + program_xml = [f' '] + program_xml.append(f' {html.escape(prog.title)}') - # Add categories if available - if "categories" in custom_data and custom_data["categories"]: - for category in custom_data["categories"]: - xml_lines.append( - f" {html.escape(category)}" + # Add subtitle if available + if prog.sub_title: + program_xml.append(f" {html.escape(prog.sub_title)}") + + # Add description if available + if prog.description: + program_xml.append(f" {html.escape(prog.description)}") + + # Process custom properties if available + if prog.custom_properties: + try: + custom_data = json.loads(prog.custom_properties) + + # Add categories if available + if "categories" in custom_data and custom_data["categories"]: + for category in custom_data["categories"]: + program_xml.append(f" {html.escape(category)}") + + # Handle episode numbering - multiple formats supported + # Standard episode number if available + if "episode" in custom_data: + program_xml.append(f' E{custom_data["episode"]}') + + # Handle onscreen episode format (like S06E128) + if "onscreen_episode" in custom_data: + program_xml.append(f' {html.escape(custom_data["onscreen_episode"])}') + + # Handle dd_progid format + if 'dd_progid' in custom_data: + program_xml.append(f' {html.escape(custom_data["dd_progid"])}') + + # Add season and episode numbers in xmltv_ns format if available + if "season" in custom_data and "episode" in custom_data: + season = ( + int(custom_data["season"]) - 1 + if str(custom_data["season"]).isdigit() + else 0 ) + episode = ( + int(custom_data["episode"]) - 1 + if str(custom_data["episode"]).isdigit() + else 0 + ) + program_xml.append(f' {season}.{episode}.') - # Handle episode numbering - multiple formats supported - # Standard episode number if available - if "episode" in custom_data: - xml_lines.append( - f' E{custom_data["episode"]}' - ) + # Add rating if available + if "rating" in custom_data: + rating_system = custom_data.get("rating_system", "TV Parental Guidelines") + program_xml.append(f' ') + program_xml.append(f' {html.escape(custom_data["rating"])}') + program_xml.append(f" ") - # Handle onscreen episode format (like S06E128) - if "onscreen_episode" in custom_data: - xml_lines.append( - f' {html.escape(custom_data["onscreen_episode"])}' - ) + # Add actors/directors/writers if available + if "credits" in custom_data: + program_xml.append(f" ") + for role, people in custom_data["credits"].items(): + if isinstance(people, list): + for person in people: + program_xml.append(f" <{role}>{html.escape(person)}") + else: + program_xml.append(f" <{role}>{html.escape(people)}") + program_xml.append(f" ") - # Handle dd_progid format - if 'dd_progid' in custom_data: - xml_lines.append(f' {html.escape(custom_data["dd_progid"])}') + # Add program date/year if available + if "year" in custom_data: + program_xml.append(f' {html.escape(custom_data["year"])}') - # Add season and episode numbers in xmltv_ns format if available - if "season" in custom_data and "episode" in custom_data: - season = ( - int(custom_data["season"]) - 1 - if str(custom_data["season"]).isdigit() - else 0 - ) - episode = ( - int(custom_data["episode"]) - 1 - if str(custom_data["episode"]).isdigit() - else 0 - ) - xml_lines.append( - f' {season}.{episode}.' - ) + # Add country if available + if "country" in custom_data: + program_xml.append(f' {html.escape(custom_data["country"])}') - # Add rating if available - if "rating" in custom_data: - rating_system = custom_data.get( - "rating_system", "TV Parental Guidelines" - ) - xml_lines.append( - f' ' - ) - xml_lines.append( - f' {html.escape(custom_data["rating"])}' - ) - xml_lines.append(f" ") + # Add icon if available + if "icon" in custom_data: + program_xml.append(f' ') - # Add actors/directors/writers if available - if "credits" in custom_data: - xml_lines.append(f" ") - for role, people in custom_data["credits"].items(): - if isinstance(people, list): - for person in people: - xml_lines.append( - f" <{role}>{html.escape(person)}" - ) - else: - xml_lines.append( - f" <{role}>{html.escape(people)}" - ) - xml_lines.append(f" ") + # Add special flags as proper tags + if custom_data.get("previously_shown", False): + program_xml.append(f" ") - # Add program date/year if available - if "year" in custom_data: - xml_lines.append( - f' {html.escape(custom_data["year"])}' - ) + if custom_data.get("premiere", False): + program_xml.append(f" ") - # Add country if available - if "country" in custom_data: - xml_lines.append( - f' {html.escape(custom_data["country"])}' - ) + if custom_data.get("new", False): + program_xml.append(f" ") - # Add icon if available - if "icon" in custom_data: - xml_lines.append( - f' ' - ) + if custom_data.get('live', False): + program_xml.append(f' ') - # Add special flags as proper tags - if custom_data.get("previously_shown", False): - xml_lines.append(f" ") + except Exception as e: + program_xml.append(f" ") - if custom_data.get("premiere", False): - xml_lines.append(f" ") + program_xml.append(" ") - if custom_data.get("new", False): - xml_lines.append(f" ") + # Add to batch + program_batch.extend(program_xml) - if custom_data.get('live', False): - xml_lines.append(f' ') + # Send batch when full or send keep-alive + if len(program_batch) >= batch_size: + yield '\n'.join(program_batch) + '\n' + program_batch = [] # Send keep-alive every batch - except Exception as e: - xml_lines.append( - f" " - ) + # Send remaining programs in batch + if program_batch: + yield '\n'.join(program_batch) + '\n' - xml_lines.append(" ") - - xml_lines.append("") - xml_content = "\n".join(xml_lines) - - response = HttpResponse(xml_content, content_type="application/xml") - response["Content-Disposition"] = 'attachment; filename="epg.xml"' + # Send final closing tag and completion message + yield "\n" # Return streaming response + response = StreamingHttpResponse( + streaming_content=epg_generator(), + content_type="application/xml" + ) + response["Content-Disposition"] = 'attachment; filename="Dispatcharr.xml"' + response["Cache-Control"] = "no-cache" return response