From 0cf2c74b40189ac4014ec9292f2d5cf0be1df66e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 14 Aug 2025 12:51:39 -0500 Subject: [PATCH] Change from individual database calls to batch. --- apps/vod/tasks.py | 1232 +++++++++++++++++++++++++++++++-------------- 1 file changed, 846 insertions(+), 386 deletions(-) diff --git a/apps/vod/tasks.py b/apps/vod/tasks.py index 55e4461e..2bd5d9d4 100644 --- a/apps/vod/tasks.py +++ b/apps/vod/tasks.py @@ -1,6 +1,7 @@ from celery import shared_task from django.utils import timezone from django.db import transaction +from django.db.models import Q from apps.m3u.models import M3UAccount from core.xtream_codes import Client as XtreamCodesClient from .models import ( @@ -18,7 +19,7 @@ logger = logging.getLogger(__name__) @shared_task def refresh_vod_content(account_id): - """Refresh VOD content for an M3U account""" + """Refresh VOD content for an M3U account with batch processing for improved performance""" try: account = M3UAccount.objects.get(id=account_id, is_active=True) @@ -26,7 +27,8 @@ def refresh_vod_content(account_id): logger.warning(f"VOD refresh called for non-XC account {account_id}") return "VOD refresh only available for XtreamCodes accounts" - logger.info(f"Starting VOD refresh for account {account.name}") + logger.info(f"Starting batch VOD refresh for account {account.name}") + start_time = timezone.now() with XtreamCodesClient( account.server_url, @@ -35,14 +37,17 @@ def refresh_vod_content(account_id): account.get_user_agent().user_agent ) as client: - # Refresh movies + # Refresh movies with batch processing refresh_movies(client, account) - # Refresh series + # Refresh series with batch processing refresh_series(client, account) - logger.info(f"VOD refresh completed for account {account.name}") - return f"VOD refresh completed for account {account.name}" + end_time = timezone.now() + duration = (end_time - start_time).total_seconds() + + logger.info(f"Batch VOD refresh completed for account {account.name} in {duration:.2f} seconds") + return f"Batch VOD refresh completed for account {account.name} in {duration:.2f} seconds" except Exception as e: logger.error(f"Error refreshing VOD for account {account_id}: {str(e)}") @@ -53,208 +58,671 @@ def refresh_movies(client, account): """Refresh movie content - only basic list, no detailed calls""" logger.info(f"Refreshing movies for account {account.name}") - # Get movie categories + # Get movie categories and pre-create them in batch categories = client.get_vod_categories() + category_map = batch_create_categories(categories, 'movie') + # Collect all movie data first + all_movies_data = [] for category_data in categories: - category_name = category_data.get('category_name', 'Unknown') category_id = category_data.get('category_id') - - # Get or create category - category, created = VODCategory.objects.get_or_create( - name=category_name, - category_type='movie', - defaults={'name': category_name} - ) + category_name = category_data.get('category_name', 'Unknown') + category = category_map.get(category_name) # Get movies in this category - only basic list movies = client.get_vod_streams(category_id) for movie_data in movies: - process_movie_basic(client, account, movie_data, category) + # Store category ID instead of object to avoid JSON serialization issues + movie_data['_category_id'] = category.id if category else None + movie_data['_category_name'] = category_name + all_movies_data.append(movie_data) + + # Process all movies in batch + batch_process_movies(client, account, all_movies_data) def refresh_series(client, account): """Refresh series content - only basic list, no detailed calls""" logger.info(f"Refreshing series for account {account.name}") - # Get series categories + # Get series categories and pre-create them in batch categories = client.get_series_categories() + category_map = batch_create_categories(categories, 'series') + # Collect all series data first + all_series_data = [] for category_data in categories: - category_name = category_data.get('category_name', 'Unknown') category_id = category_data.get('category_id') - - # Get or create category - category, created = VODCategory.objects.get_or_create( - name=category_name, - category_type='series', - defaults={'name': category_name} - ) + category_name = category_data.get('category_name', 'Unknown') + category = category_map.get(category_name) # Get series in this category - only basic list series_list = client.get_series(category_id) for series_data in series_list: - process_series_basic(client, account, series_data, category) + # Store category ID instead of object to avoid JSON serialization issues + series_data['_category_id'] = category.id if category else None + series_data['_category_name'] = category_name + all_series_data.append(series_data) + + # Process all series in batch + batch_process_series(client, account, all_series_data) -def process_movie_basic(client, account, movie_data, category): - """Process a single movie - basic info only, no detailed API call""" - try: - stream_id = movie_data.get('stream_id') - name = movie_data.get('name', 'Unknown') +# Batch processing functions for improved efficiency - # Extract all available metadata from the basic data - year = extract_year_from_data(movie_data, 'name') +def batch_create_categories(categories_data, category_type): + """Create categories in batch and return a mapping""" + category_names = [cat.get('category_name', 'Unknown') for cat in categories_data] - # Extract TMDB and IMDB IDs if available in basic data - tmdb_id = movie_data.get('tmdb_id') or movie_data.get('tmdb') - imdb_id = movie_data.get('imdb_id') or movie_data.get('imdb') + # Get existing categories + existing_categories = { + cat.name: cat for cat in VODCategory.objects.filter( + name__in=category_names, + category_type=category_type + ) + } - # Extract additional metadata that might be available in basic data - description = movie_data.get('description') or movie_data.get('plot') or '' - rating = movie_data.get('rating') or movie_data.get('vote_average') or '' - genre = movie_data.get('genre') or movie_data.get('category_name') or '' + # Create missing categories in batch + new_categories = [] + for name in category_names: + if name not in existing_categories: + new_categories.append(VODCategory(name=name, category_type=category_type)) - # Extract trailer - trailer = movie_data.get('trailer') or movie_data.get('youtube_trailer') or '' - - duration_secs = None - - # Try to extract duration from various possible fields - if movie_data.get('duration_secs'): - duration_secs = int(movie_data.get('duration_secs')) - elif movie_data.get('duration'): - # Handle duration that might be in different formats - duration_str = str(movie_data.get('duration')) - if duration_str.isdigit(): - duration_secs = int(duration_str) * 60 # Assume minutes if just a number - else: - # Try to parse time format like "01:30:00" - try: - time_parts = duration_str.split(':') - if len(time_parts) == 3: - hours, minutes, seconds = map(int, time_parts) - duration_secs = (hours * 3600) + (minutes * 60) + seconds - elif len(time_parts) == 2: - minutes, seconds = map(int, time_parts) - duration_secs = minutes * 60 + seconds - except (ValueError, AttributeError): - pass - - # Build info dict with all extracted data - info = { - 'plot': description, - 'rating': rating, - 'genre': genre, - 'duration_secs': duration_secs, - 'trailer': trailer, + if new_categories: + VODCategory.objects.bulk_create(new_categories, ignore_conflicts=True) + # Fetch the newly created categories + newly_created = { + cat.name: cat for cat in VODCategory.objects.filter( + name__in=[cat.name for cat in new_categories], + category_type=category_type + ) } + existing_categories.update(newly_created) - # Use find_or_create_movie to handle duplicates properly - movie = find_or_create_movie( - name=name, - year=year, - tmdb_id=tmdb_id, - imdb_id=imdb_id, - info=info - ) + return existing_categories - # Handle logo from basic data if available - if movie_data.get('stream_icon'): - logo, _ = Logo.objects.get_or_create( - url=movie_data['stream_icon'], - defaults={'name': name} - ) - if not movie.logo: - movie.logo = logo - movie.save(update_fields=['logo']) - # Create or update relation - stream_url = client.get_vod_stream_url(stream_id) +def batch_process_movies(client, account, movies_data): + """Process movies in batches for better performance""" + if not movies_data: + return - relation, created = M3UMovieRelation.objects.update_or_create( + logger.info(f"Batch processing {len(movies_data)} movies for account {account.name}") + + # Extract unique identifiers for existing lookups + movie_names = [movie.get('name', 'Unknown') for movie in movies_data] + tmdb_ids = [movie.get('tmdb_id') or movie.get('tmdb') for movie in movies_data if movie.get('tmdb_id') or movie.get('tmdb')] + imdb_ids = [movie.get('imdb_id') or movie.get('imdb') for movie in movies_data if movie.get('imdb_id') or movie.get('imdb')] + stream_ids = [str(movie.get('stream_id')) for movie in movies_data] + + # Pre-fetch categories by ID + category_ids = [movie.get('_category_id') for movie in movies_data if movie.get('_category_id')] + categories_by_id = { + cat.id: cat for cat in VODCategory.objects.filter(id__in=category_ids) + } if category_ids else {} + + # Pre-fetch existing movies to avoid N+1 queries and duplicates + existing_movies_by_tmdb = {} + existing_movies_by_imdb = {} + existing_movies_by_name_year = {} + existing_movies_by_name_year_tmdb = {} + existing_movies_by_name_year_imdb = {} + + # Create comprehensive lookups to handle all unique constraint combinations + if tmdb_ids: + for movie in Movie.objects.filter(tmdb_id__in=tmdb_ids): + existing_movies_by_tmdb[movie.tmdb_id] = movie + # Also index by name+year+tmdb_id combination + if movie.name and movie.tmdb_id: + key = f"{movie.name}_{movie.year or 'None'}_{movie.tmdb_id}" + existing_movies_by_name_year_tmdb[key] = movie + + if imdb_ids: + for movie in Movie.objects.filter(imdb_id__in=imdb_ids): + existing_movies_by_imdb[movie.imdb_id] = movie + # Also index by name+year+imdb_id combination + if movie.name and movie.imdb_id: + key = f"{movie.name}_{movie.year or 'None'}_{movie.imdb_id}" + existing_movies_by_name_year_imdb[key] = movie + + # Get all movies with matching names to check for name+year combinations + for movie in Movie.objects.filter(name__in=movie_names): + name_year_key = f"{movie.name}_{movie.year or 'None'}" + existing_movies_by_name_year[name_year_key] = movie + + # Also add to tmdb/imdb specific lookups if they have those IDs + if movie.tmdb_id: + tmdb_key = f"{movie.name}_{movie.year or 'None'}_{movie.tmdb_id}" + existing_movies_by_name_year_tmdb[tmdb_key] = movie + if movie.imdb_id: + imdb_key = f"{movie.name}_{movie.year or 'None'}_{movie.imdb_id}" + existing_movies_by_name_year_imdb[imdb_key] = movie + + # Pre-fetch existing relations + existing_relations = { + rel.stream_id: rel for rel in M3UMovieRelation.objects.filter( m3u_account=account, - stream_id=str(stream_id), - defaults={ - 'movie': movie, - 'category': category, - 'url': stream_url, - 'container_extension': movie_data.get('container_extension', 'mp4'), - 'custom_properties': { + stream_id__in=stream_ids + ).select_related('movie') + } + + # Pre-fetch existing logos + logo_urls = [movie.get('stream_icon') for movie in movies_data if movie.get('stream_icon')] + existing_logos = { + logo.url: logo for logo in Logo.objects.filter(url__in=logo_urls) + } + + # Process movies in batches + movies_to_create = [] + movies_to_update = [] + relations_to_create = [] + relations_to_update = [] + logos_to_create = [] + + # Track movies being created in this batch to prevent duplicates within the batch + batch_movies_by_tmdb_key = {} + batch_movies_by_imdb_key = {} + batch_movies_by_name_year = {} + + for movie_data in movies_data: + try: + stream_id = str(movie_data.get('stream_id')) + name = movie_data.get('name', 'Unknown') + category_id = movie_data.get('_category_id') + category = categories_by_id.get(category_id) if category_id else None + + # Extract metadata + year = extract_year_from_data(movie_data, 'name') + tmdb_id = movie_data.get('tmdb_id') or movie_data.get('tmdb') + imdb_id = movie_data.get('imdb_id') or movie_data.get('imdb') + + # Find existing movie using comprehensive lookup + movie = None + + # First, check if we're already creating this movie in the current batch + if tmdb_id and name: + tmdb_key = f"{name}_{year or 'None'}_{tmdb_id}" + if tmdb_key in batch_movies_by_tmdb_key: + movie = batch_movies_by_tmdb_key[tmdb_key] + elif tmdb_key in existing_movies_by_name_year_tmdb: + movie = existing_movies_by_name_year_tmdb[tmdb_key] + + if not movie and imdb_id and name: + imdb_key = f"{name}_{year or 'None'}_{imdb_id}" + if imdb_key in batch_movies_by_imdb_key: + movie = batch_movies_by_imdb_key[imdb_key] + elif imdb_key in existing_movies_by_name_year_imdb: + movie = existing_movies_by_name_year_imdb[imdb_key] + + # Check batch tracking for name+year combinations + if not movie: + name_year_key = f"{name}_{year or 'None'}" + if name_year_key in batch_movies_by_name_year: + movie = batch_movies_by_name_year[name_year_key] + + # Fallback to existing database lookups + if not movie and tmdb_id and tmdb_id in existing_movies_by_tmdb: + movie = existing_movies_by_tmdb[tmdb_id] + elif not movie and imdb_id and imdb_id in existing_movies_by_imdb: + movie = existing_movies_by_imdb[imdb_id] + + # Final fallback to name+year lookup in database + if not movie: + name_year_key = f"{name}_{year or 'None'}" + if name_year_key in existing_movies_by_name_year: + movie = existing_movies_by_name_year[name_year_key] + + # Prepare movie data + description = movie_data.get('description') or movie_data.get('plot') or '' + rating = movie_data.get('rating') or movie_data.get('vote_average') or '' + genre = movie_data.get('genre') or movie_data.get('category_name') or '' + duration_secs = extract_duration_from_data(movie_data) + trailer = movie_data.get('trailer') or movie_data.get('youtube_trailer') or '' + + info = { + 'plot': description, + 'rating': rating, + 'genre': genre, + 'duration_secs': duration_secs, + 'trailer': trailer, + } + + # Handle logo + logo_url = None + if movie_data.get('stream_icon'): + logo_url = movie_data['stream_icon'] + if logo_url not in existing_logos: + # Queue for batch creation + logo = Logo(url=logo_url, name=name) + logos_to_create.append(logo) + existing_logos[logo_url] = logo # Temporary placeholder + + if movie: + # Update existing movie if needed + updated = False + if description and description != movie.description: + movie.description = description + updated = True + if rating and rating != movie.rating: + movie.rating = rating + updated = True + if genre and genre != movie.genre: + movie.genre = genre + updated = True + if year and year != movie.year: + movie.year = year + updated = True + if tmdb_id and tmdb_id != movie.tmdb_id: + movie.tmdb_id = tmdb_id + updated = True + if imdb_id and imdb_id != movie.imdb_id: + movie.imdb_id = imdb_id + updated = True + if duration_secs and duration_secs != movie.duration_secs: + movie.duration_secs = duration_secs + updated = True + + # Update custom properties + custom_props = movie.custom_properties or {} + if trailer and trailer != custom_props.get('trailer'): + custom_props['trailer'] = trailer + movie.custom_properties = custom_props + updated = True + + # Update logo if we have one and it's different + if logo_url and logo_url in existing_logos: + new_logo = existing_logos[logo_url] + if movie.logo != new_logo: + movie.logo = new_logo + updated = True + + if updated: + movies_to_update.append(movie) + else: + # Create new movie + custom_props = {'trailer': trailer} if trailer else None + movie = Movie( + name=name, + year=year, + tmdb_id=tmdb_id, + imdb_id=imdb_id, + description=description, + rating=rating, + genre=genre, + duration_secs=duration_secs, + custom_properties=custom_props + ) + # Store logo URL temporarily for later assignment + if logo_url: + movie._logo_url = logo_url + movies_to_create.append(movie) + + # Add to batch tracking to prevent duplicates within the same batch + if tmdb_id and name: + tmdb_key = f"{name}_{year or 'None'}_{tmdb_id}" + batch_movies_by_tmdb_key[tmdb_key] = movie + if imdb_id and name: + imdb_key = f"{name}_{year or 'None'}_{imdb_id}" + batch_movies_by_imdb_key[imdb_key] = movie + # Always track by name+year as fallback + name_year_key = f"{name}_{year or 'None'}" + batch_movies_by_name_year[name_year_key] = movie + + # Handle relation + stream_url = client.get_vod_stream_url(stream_id) + + if stream_id in existing_relations: + # Update existing relation + relation = existing_relations[stream_id] + relation.movie = movie + relation.category = category + relation.url = stream_url + relation.container_extension = movie_data.get('container_extension', 'mp4') + relation.custom_properties = { 'basic_data': movie_data, - 'detailed_fetched': False # Flag to indicate detailed data not fetched + 'detailed_fetched': False } + relations_to_update.append(relation) + else: + # Create new relation + relation = M3UMovieRelation( + m3u_account=account, + movie=movie, + category=category, + stream_id=stream_id, + url=stream_url, + container_extension=movie_data.get('container_extension', 'mp4'), + custom_properties={ + 'basic_data': movie_data, + 'detailed_fetched': False + } + ) + relations_to_create.append(relation) + + except Exception as e: + logger.error(f"Error preparing movie {movie_data.get('name', 'Unknown')}: {str(e)}") + + # Execute batch operations + with transaction.atomic(): + # Create logos first and fetch them back + if logos_to_create: + Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True) + # Refresh existing_logos with newly created ones + logo_urls_created = [logo.url for logo in logos_to_create] + newly_created_logos = { + logo.url: logo for logo in Logo.objects.filter(url__in=logo_urls_created) } - ) + existing_logos.update(newly_created_logos) - if created: - logger.debug(f"Created new movie relation: {name}") - else: - logger.debug(f"Updated movie relation: {name}") + # Now assign correct logos to movies/series before creating them + for movie in movies_to_create: + if hasattr(movie, '_logo_url') and movie._logo_url in existing_logos: + movie.logo = existing_logos[movie._logo_url] + delattr(movie, '_logo_url') # Clean up temporary attribute - except Exception as e: - logger.error(f"Error processing movie {movie_data.get('name', 'Unknown')}: {str(e)}") + # Create new movies + if movies_to_create: + Movie.objects.bulk_create(movies_to_create) + logger.info(f"Created {len(movies_to_create)} new movies") # Update existing movies + if movies_to_update: + Movie.objects.bulk_update(movies_to_update, [ + 'description', 'rating', 'genre', 'year', 'tmdb_id', 'imdb_id', + 'duration_secs', 'custom_properties', 'logo' + ]) + + # Create new relations + if relations_to_create: + M3UMovieRelation.objects.bulk_create(relations_to_create) + + # Update existing relations + if relations_to_update: + M3UMovieRelation.objects.bulk_update(relations_to_update, [ + 'movie', 'category', 'url', 'container_extension', 'custom_properties' + ]) + + logger.info(f"Batch processed: {len(movies_to_create)} new movies, {len(movies_to_update)} updated movies, " + f"{len(relations_to_create)} new relations, {len(relations_to_update)} updated relations") -def process_series_basic(client, account, series_data, category): - """Process a single series - basic info only, no detailed API call""" - try: - series_id = series_data.get('series_id') - name = series_data.get('name', 'Unknown') +def batch_process_series(client, account, series_data_list): + """Process series in batches for better performance""" + if not series_data_list: + return - # Extract all available metadata from the basic data - year = extract_year(series_data.get('releaseDate', '')) # Use releaseDate from API - if not year and series_data.get('release_date'): - year = extract_year(series_data.get('release_date')) + logger.info(f"Batch processing {len(series_data_list)} series for account {account.name}") - # Extract TMDB and IMDB IDs if available in basic data - tmdb_id = series_data.get('tmdb') or series_data.get('tmdb_id') - imdb_id = series_data.get('imdb') or series_data.get('imdb_id') + # Extract unique identifiers for existing lookups + series_names = [series.get('name', 'Unknown') for series in series_data_list] + tmdb_ids = [series.get('tmdb') or series.get('tmdb_id') for series in series_data_list if series.get('tmdb') or series.get('tmdb_id')] + imdb_ids = [series.get('imdb') or series.get('imdb_id') for series in series_data_list if series.get('imdb') or series.get('imdb_id')] + series_ids = [str(series.get('series_id')) for series in series_data_list] - # Use find_or_create_series to handle duplicates properly - logger.debug(f"Processing series: {name} ({year})") - series = find_or_create_series( - name=name, - year=year, - tmdb_id=tmdb_id, - imdb_id=imdb_id, - info=series_data - ) - # Handle logo from basic data if available - if series_data.get('cover'): - logo, _ = Logo.objects.get_or_create( - url=series_data['cover'], - defaults={'name': name} - ) - if not series.logo: - series.logo = logo - series.save(update_fields=['logo']) + # Pre-fetch categories by ID + category_ids = [series.get('_category_id') for series in series_data_list if series.get('_category_id')] + categories_by_id = { + cat.id: cat for cat in VODCategory.objects.filter(id__in=category_ids) + } if category_ids else {} - # Create or update series relation - series_relation, created = M3USeriesRelation.objects.update_or_create( + # Pre-fetch existing series to avoid N+1 queries + existing_series_by_tmdb = {} + existing_series_by_imdb = {} + existing_series_by_name = {} + + if tmdb_ids: + for series in Series.objects.filter(tmdb_id__in=tmdb_ids): + existing_series_by_tmdb[series.tmdb_id] = series + + if imdb_ids: + for series in Series.objects.filter(imdb_id__in=imdb_ids): + existing_series_by_imdb[series.imdb_id] = series + + for series in Series.objects.filter(name__in=series_names): + key = f"{series.name}_{series.year or 'None'}" + existing_series_by_name[key] = series + + # Pre-fetch existing relations + existing_relations = { + rel.external_series_id: rel for rel in M3USeriesRelation.objects.filter( m3u_account=account, - external_series_id=str(series_id), - defaults={ - 'series': series, - 'category': category, - 'custom_properties': { + external_series_id__in=series_ids + ).select_related('series') + } + + # Pre-fetch existing logos + logo_urls = [series.get('cover') for series in series_data_list if series.get('cover')] + existing_logos = { + logo.url: logo for logo in Logo.objects.filter(url__in=logo_urls) + } + + # Process series in batches + series_to_create = [] + series_to_update = [] + relations_to_create = [] + relations_to_update = [] + logos_to_create = [] + + # Track series being created in this batch to prevent duplicates within the batch + batch_series_by_tmdb_key = {} + batch_series_by_imdb_key = {} + batch_series_by_name_year = {} + + for series_data in series_data_list: + try: + series_id = str(series_data.get('series_id')) + name = series_data.get('name', 'Unknown') + category_id = series_data.get('_category_id') + category = categories_by_id.get(category_id) if category_id else None + + # Extract metadata + year = extract_year(series_data.get('releaseDate', '')) + if not year and series_data.get('release_date'): + year = extract_year(series_data.get('release_date')) + + tmdb_id = series_data.get('tmdb') or series_data.get('tmdb_id') + imdb_id = series_data.get('imdb') or series_data.get('imdb_id') + + # Find existing series - check batch first, then database + series = None + + # First, check if we're already creating this series in the current batch + if tmdb_id and name: + tmdb_key = f"{name}_{year or 'None'}_{tmdb_id}" + if tmdb_key in batch_series_by_tmdb_key: + series = batch_series_by_tmdb_key[tmdb_key] + + if not series and imdb_id and name: + imdb_key = f"{name}_{year or 'None'}_{imdb_id}" + if imdb_key in batch_series_by_imdb_key: + series = batch_series_by_imdb_key[imdb_key] + + if not series: + name_year_key = f"{name}_{year or 'None'}" + if name_year_key in batch_series_by_name_year: + series = batch_series_by_name_year[name_year_key] + + # Fallback to database lookups + if not series and tmdb_id and tmdb_id in existing_series_by_tmdb: + series = existing_series_by_tmdb[tmdb_id] + elif not series and imdb_id and imdb_id in existing_series_by_imdb: + series = existing_series_by_imdb[imdb_id] + elif not series: + name_year_key = f"{name}_{year or 'None'}" + if name_year_key in existing_series_by_name: + series = existing_series_by_name[name_year_key] + + # Handle logo + logo_url = None + if series_data.get('cover'): + logo_url = series_data['cover'] + if logo_url not in existing_logos: + # Queue for batch creation + logo = Logo(url=logo_url, name=name) + logos_to_create.append(logo) + existing_logos[logo_url] = logo # Temporary placeholder + + if series: + # Update existing series if needed + updated = False + description = series_data.get('plot', '') + rating = series_data.get('rating', '') + genre = series_data.get('genre', '') + + if description and description != series.description: + series.description = description + updated = True + if rating and rating != series.rating: + series.rating = rating + updated = True + if genre and genre != series.genre: + series.genre = genre + updated = True + if year and year != series.year: + series.year = year + updated = True + if tmdb_id and tmdb_id != series.tmdb_id: + series.tmdb_id = tmdb_id + updated = True + if imdb_id and imdb_id != series.imdb_id: + series.imdb_id = imdb_id + updated = True + + # Update logo if we have one and it's different + if logo_url and logo_url in existing_logos: + new_logo = existing_logos[logo_url] + if series.logo != new_logo: + series.logo = new_logo + updated = True + + if updated: + series_to_update.append(series) + else: + # Create new series + series = Series( + name=name, + year=year, + tmdb_id=tmdb_id, + imdb_id=imdb_id, + description=series_data.get('plot', ''), + rating=series_data.get('rating', ''), + genre=series_data.get('genre', '') + ) + # Store logo URL temporarily for later assignment + if logo_url: + series._logo_url = logo_url + series_to_create.append(series) + + # Add to batch tracking to prevent duplicates within the same batch + if tmdb_id and name: + tmdb_key = f"{name}_{year or 'None'}_{tmdb_id}" + batch_series_by_tmdb_key[tmdb_key] = series + if imdb_id and name: + imdb_key = f"{name}_{year or 'None'}_{imdb_id}" + batch_series_by_imdb_key[imdb_key] = series + # Always track by name+year as fallback + name_year_key = f"{name}_{year or 'None'}" + batch_series_by_name_year[name_year_key] = series + + # Handle relation + if series_id in existing_relations: + # Update existing relation + relation = existing_relations[series_id] + relation.series = series + relation.category = category + relation.custom_properties = { 'basic_data': series_data, - 'detailed_fetched': False, # Flag to indicate detailed data not fetched - 'episodes_fetched': False # Flag to indicate episodes not fetched - }, - 'last_episode_refresh': None # Set to None since we haven't fetched episodes + 'detailed_fetched': False, + 'episodes_fetched': False + } + relation.last_episode_refresh = None + relations_to_update.append(relation) + else: + # Create new relation + relation = M3USeriesRelation( + m3u_account=account, + series=series, + category=category, + external_series_id=series_id, + custom_properties={ + 'basic_data': series_data, + 'detailed_fetched': False, + 'episodes_fetched': False + }, + last_episode_refresh=None + ) + relations_to_create.append(relation) + + except Exception as e: + logger.error(f"Error preparing series {series_data.get('name', 'Unknown')}: {str(e)}") + + # Execute batch operations + with transaction.atomic(): + # Create logos first and fetch them back + if logos_to_create: + Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True) + # Refresh existing_logos with newly created ones + logo_urls_created = [logo.url for logo in logos_to_create] + newly_created_logos = { + logo.url: logo for logo in Logo.objects.filter(url__in=logo_urls_created) } - ) + existing_logos.update(newly_created_logos) - if created: - logger.debug(f"Created new series relation: {name}") + # Now assign correct logos to series before creating them + for series in series_to_create: + if hasattr(series, '_logo_url') and series._logo_url in existing_logos: + series.logo = existing_logos[series._logo_url] + delattr(series, '_logo_url') # Clean up temporary attribute + + # Create new series + if series_to_create: + Series.objects.bulk_create(series_to_create) + logger.info(f"Created {len(series_to_create)} new series") # Update existing series + if series_to_update: + Series.objects.bulk_update(series_to_update, [ + 'description', 'rating', 'genre', 'year', 'tmdb_id', 'imdb_id', 'logo' + ]) + + # Create new relations + if relations_to_create: + M3USeriesRelation.objects.bulk_create(relations_to_create) + + # Update existing relations + if relations_to_update: + M3USeriesRelation.objects.bulk_update(relations_to_update, [ + 'series', 'category', 'custom_properties', 'last_episode_refresh' + ]) + + logger.info(f"Batch processed: {len(series_to_create)} new series, {len(series_to_update)} updated series, " + f"{len(relations_to_create)} new relations, {len(relations_to_update)} updated relations") + + +def extract_duration_from_data(movie_data): + """Extract duration in seconds from movie data""" + duration_secs = None + + # Try to extract duration from various possible fields + if movie_data.get('duration_secs'): + duration_secs = int(movie_data.get('duration_secs')) + elif movie_data.get('duration'): + # Handle duration that might be in different formats + duration_str = str(movie_data.get('duration')) + if duration_str.isdigit(): + duration_secs = int(duration_str) * 60 # Assume minutes if just a number else: - logger.debug(f"Updated series relation: {name}") + # Try to parse time format like "01:30:00" + try: + time_parts = duration_str.split(':') + if len(time_parts) == 3: + hours, minutes, seconds = map(int, time_parts) + duration_secs = (hours * 3600) + (minutes * 60) + seconds + elif len(time_parts) == 2: + minutes, seconds = map(int, time_parts) + duration_secs = minutes * 60 + seconds + except (ValueError, AttributeError): + pass - except Exception as e: - logger.error(f"Error processing series {series_data.get('name', 'Unknown')}: {str(e)}") + return duration_secs # Remove the detailed processing functions since they're no longer used during refresh @@ -292,9 +760,8 @@ def refresh_series_episodes(account, series, external_series_id, episodes_data=N m3u_relations__m3u_account=account ).delete() - for season_num, season_episodes in episodes_data.items(): - for episode_data in season_episodes: - process_episode(account, series, episode_data, int(season_num)) + # Process all episodes in batch + batch_process_episodes(account, series, episodes_data) # Update the series relation to mark episodes as fetched series_relation = M3USeriesRelation.objects.filter( @@ -314,246 +781,182 @@ def refresh_series_episodes(account, series, external_series_id, episodes_data=N logger.error(f"Error refreshing episodes for series {series.name}: {str(e)}") -def process_episode(account, series, episode_data, season_number): - """Process a single episode""" - try: - episode_id = episode_data.get('id') - episode_name = episode_data.get('title', 'Unknown Episode') - episode_number = episode_data.get('episode_num', 0) - info = episode_data.get('info', {}) - # Aggregate all relevant info into custom_props - custom_props = {} - if info: - if info.get('crew'): - custom_props['crew'] = info.get('crew') - if info.get('movie_image'): - custom_props['movie_image'] = info.get('movie_image') - if info.get('backdrop_path'): - custom_props['backdrop_path'] = info.get('backdrop_path') - # Add any other relevant fields as needed - description = info.get('plot') or info.get('overview', '') if info else '' - rating = info.get('rating', '') if info else '' - air_date = extract_date_from_data(info) if info else None +def batch_process_episodes(account, series, episodes_data): + """Process episodes in batches for better performance""" + if not episodes_data: + return - episode, created = Episode.objects.update_or_create( - series=series, - season_number=season_number, - episode_number=episode_number, - defaults={ - 'name': episode_name, - 'description': description, - 'rating': rating, - 'air_date': air_date, - 'duration_secs': info.get('duration_secs') if info else None, - 'tmdb_id': info.get('tmdb_id') if info else None, - 'imdb_id': info.get('imdb_id') if info else None, - 'custom_properties': custom_props if custom_props else None, - } - ) + # Flatten episodes data + all_episodes_data = [] + for season_num, season_episodes in episodes_data.items(): + for episode_data in season_episodes: + episode_data['_season_number'] = int(season_num) + all_episodes_data.append(episode_data) - # Create or update episode relation - relation, created = M3UEpisodeRelation.objects.update_or_create( + if not all_episodes_data: + return + + logger.info(f"Batch processing {len(all_episodes_data)} episodes for series {series.name}") + + # Extract episode identifiers + episode_keys = [] + episode_ids = [] + for episode_data in all_episodes_data: + season_num = episode_data['_season_number'] + episode_num = episode_data.get('episode_num', 0) + episode_keys.append((series.id, season_num, episode_num)) + episode_ids.append(str(episode_data.get('id'))) + + # Pre-fetch existing episodes + existing_episodes = {} + for episode in Episode.objects.filter(series=series): + key = (episode.series_id, episode.season_number, episode.episode_number) + existing_episodes[key] = episode + + # Pre-fetch existing episode relations + existing_relations = { + rel.stream_id: rel for rel in M3UEpisodeRelation.objects.filter( m3u_account=account, - episode=episode, - defaults={ - 'stream_id': str(episode_id), - 'container_extension': episode_data.get('container_extension', 'mp4'), - 'custom_properties': { + stream_id__in=episode_ids + ).select_related('episode') + } + + # Prepare batch operations + episodes_to_create = [] + episodes_to_update = [] + relations_to_create = [] + relations_to_update = [] + + for episode_data in all_episodes_data: + try: + episode_id = str(episode_data.get('id')) + episode_name = episode_data.get('title', 'Unknown Episode') + season_number = episode_data['_season_number'] + episode_number = episode_data.get('episode_num', 0) + info = episode_data.get('info', {}) + + # Extract episode metadata + description = info.get('plot') or info.get('overview', '') if info else '' + rating = info.get('rating', '') if info else '' + air_date = extract_date_from_data(info) if info else None + duration_secs = info.get('duration_secs') if info else None + tmdb_id = info.get('tmdb_id') if info else None + imdb_id = info.get('imdb_id') if info else None + + # Prepare custom properties + custom_props = {} + if info: + if info.get('crew'): + custom_props['crew'] = info.get('crew') + if info.get('movie_image'): + custom_props['movie_image'] = info.get('movie_image') + if info.get('backdrop_path'): + custom_props['backdrop_path'] = info.get('backdrop_path') + + # Find existing episode + episode_key = (series.id, season_number, episode_number) + episode = existing_episodes.get(episode_key) + + if episode: + # Update existing episode + updated = False + if episode_name != episode.name: + episode.name = episode_name + updated = True + if description != episode.description: + episode.description = description + updated = True + if rating != episode.rating: + episode.rating = rating + updated = True + if air_date != episode.air_date: + episode.air_date = air_date + updated = True + if duration_secs != episode.duration_secs: + episode.duration_secs = duration_secs + updated = True + if tmdb_id != episode.tmdb_id: + episode.tmdb_id = tmdb_id + updated = True + if imdb_id != episode.imdb_id: + episode.imdb_id = imdb_id + updated = True + if custom_props != episode.custom_properties: + episode.custom_properties = custom_props if custom_props else None + updated = True + + if updated: + episodes_to_update.append(episode) + else: + # Create new episode + episode = Episode( + series=series, + name=episode_name, + description=description, + air_date=air_date, + rating=rating, + duration_secs=duration_secs, + season_number=season_number, + episode_number=episode_number, + tmdb_id=tmdb_id, + imdb_id=imdb_id, + custom_properties=custom_props if custom_props else None + ) + episodes_to_create.append(episode) + + # Handle episode relation + if episode_id in existing_relations: + # Update existing relation + relation = existing_relations[episode_id] + relation.episode = episode + relation.container_extension = episode_data.get('container_extension', 'mp4') + relation.custom_properties = { 'info': episode_data, 'season_number': season_number } - } - ) + relations_to_update.append(relation) + else: + # Create new relation + relation = M3UEpisodeRelation( + m3u_account=account, + episode=episode, + stream_id=episode_id, + container_extension=episode_data.get('container_extension', 'mp4'), + custom_properties={ + 'info': episode_data, + 'season_number': season_number + } + ) + relations_to_create.append(relation) - if created: - logger.debug(f"Created new episode: {episode_name} S{season_number}E{episode_number}") - else: - logger.debug(f"Updated episode: {episode_name} S{season_number}E{episode_number}") + except Exception as e: + logger.error(f"Error preparing episode {episode_data.get('title', 'Unknown')}: {str(e)}") - except Exception as e: - logger.error(f"Error processing episode {episode_data.get('title', 'Unknown')}: {str(e)}") + # Execute batch operations + with transaction.atomic(): + # Create new episodes + if episodes_to_create: + Episode.objects.bulk_create(episodes_to_create) + # Update existing episodes + if episodes_to_update: + Episode.objects.bulk_update(episodes_to_update, [ + 'name', 'description', 'air_date', 'rating', 'duration_secs', + 'tmdb_id', 'imdb_id', 'custom_properties' + ]) -def find_or_create_movie(name, year, tmdb_id, imdb_id, info): - """Find existing movie or create new one based on metadata""" - movie = None + # Create new episode relations + if relations_to_create: + M3UEpisodeRelation.objects.bulk_create(relations_to_create) - # Try to find by TMDB ID first - if tmdb_id: - movie = Movie.objects.filter(tmdb_id=tmdb_id).first() + # Update existing episode relations + if relations_to_update: + M3UEpisodeRelation.objects.bulk_update(relations_to_update, [ + 'episode', 'container_extension', 'custom_properties' + ]) - # Try to find by IMDB ID if not found by TMDB - if not movie and imdb_id: - movie = Movie.objects.filter(imdb_id=imdb_id).first() - - # Try to find by name and year if not found by IDs - if not movie and year: - movie = Movie.objects.filter(name=name, year=year).first() - - # Try to find by name only if still not found - if not movie: - movie = Movie.objects.filter(name=name).first() - - # If we found an existing movie, update it - if movie: - updated = False - if (info.get('plot') or info.get('description')) and (info.get('plot') or info.get('description')) != movie.description: - movie.description = info.get('plot') or info.get('description') - updated = True - if info.get('rating') and info.get('rating') != movie.rating: - movie.rating = info.get('rating') - updated = True - if info.get('genre') and info.get('genre') != movie.genre: - movie.genre = info.get('genre') - updated = True - if year and year != movie.year: - movie.year = year - updated = True - if tmdb_id and tmdb_id != movie.tmdb_id: - movie.tmdb_id = tmdb_id - updated = True - if imdb_id and imdb_id != movie.imdb_id: - movie.imdb_id = imdb_id - updated = True - - duration_secs = info.get('duration_secs') - if duration_secs and duration_secs != movie.duration_secs: - movie.duration_secs = duration_secs - updated = True - - # Update custom_properties with trailer and other metadata - custom_props = movie.custom_properties or {} - custom_props_updated = False - if info.get('trailer') and info.get('trailer') != custom_props.get('trailer'): - custom_props['trailer'] = info.get('trailer') - custom_props_updated = True - if info.get('youtube_trailer') and info.get('youtube_trailer') != custom_props.get('youtube_trailer'): - custom_props['youtube_trailer'] = info.get('youtube_trailer') - custom_props_updated = True - - if custom_props_updated: - movie.custom_properties = custom_props - updated = True - - if updated: - movie.save() - return movie - - # Create new movie if not found - custom_props = {} - if info.get('trailer'): - custom_props['trailer'] = info.get('trailer') - if info.get('youtube_trailer'): - custom_props['youtube_trailer'] = info.get('youtube_trailer') - - return Movie.objects.create( - name=name, - year=year, - tmdb_id=tmdb_id, - imdb_id=imdb_id, - description=info.get('plot') or info.get('description', ''), - rating=info.get('rating', ''), - genre=info.get('genre', ''), - duration_secs=info.get('duration_secs'), - custom_properties=custom_props if custom_props else None - ) - - -def find_or_create_series(name, year, tmdb_id, imdb_id, info): - """Find existing series or create new one based on metadata""" - series = None - updated = False - # Try to find by TMDB ID first - if tmdb_id: - series = Series.objects.filter(tmdb_id=tmdb_id).first() - - # Try to find by IMDB ID if not found by TMDB - if not series and imdb_id: - series = Series.objects.filter(imdb_id=imdb_id).first() - - # Try to find by name and year if not found by IDs - if not series and year: - series = Series.objects.filter(name=name, year=year).first() - - # Try to find by name only if still not found - if not series: - series = Series.objects.filter(name=name).first() - - # If still not found, create a new series - if not series: - series = Series.objects.create( - name=name, - year=year, - tmdb_id=tmdb_id, - imdb_id=imdb_id, - description=info.get('plot', ''), - rating=info.get('rating', ''), - genre=info.get('genre', ''), - ) - - # Update series metadata - if info.get('plot') and info.get('plot') != series.description: - series.description = info.get('plot') - updated = True - if info.get('rating') and info.get('rating') != series.rating: - series.rating = info.get('rating') - updated = True - if info.get('genre') and info.get('genre') != series.genre: - series.genre = info.get('genre') - updated = True - if year and year != series.year: - series.year = year - updated = True - if tmdb_id and tmdb_id != series.tmdb_id: - series.tmdb_id = tmdb_id - updated = True - if imdb_id and imdb_id != series.imdb_id: - series.imdb_id = imdb_id - updated = True - - # Update custom_properties with trailer and other metadata - custom_props = series.custom_properties or {} - custom_props_updated = False - if info.get('trailer') and info.get('trailer') != custom_props.get('youtube_trailer'): - custom_props['youtube_trailer'] = info.get('trailer') - custom_props_updated = True - if info.get('youtube_trailer') and info.get('youtube_trailer') != custom_props.get('youtube_trailer'): - custom_props['youtube_trailer'] = info.get('youtube_trailer') - custom_props_updated = True - if info.get('backdrop_path') and info.get('backdrop_path') != custom_props.get('backdrop_path'): - custom_props['backdrop_path'] = info.get('backdrop_path') - custom_props_updated = True - if info.get('episode_run_time') and info.get('episode_run_time') != custom_props.get('episode_run_time'): - custom_props['episode_run_time'] = info.get('episode_run_time') - custom_props_updated = True - if info.get('cast') and info.get('cast') != custom_props.get('cast'): - custom_props['cast'] = info.get('cast') - custom_props_updated = True - if info.get('director') and info.get('director') != custom_props.get('director'): - custom_props['director'] = info.get('director') - custom_props_updated = True - if ( - (info.get('release_date') and info.get('release_date') != custom_props.get('release_date')) or - (info.get('releaseDate') and info.get('releaseDate') != custom_props.get('release_date')) or - (info.get('releasedate') and info.get('releasedate') != custom_props.get('release_date')) - ): - # Prefer release_date, then releaseDate, then releasedate - release_date_val = ( - info.get('release_date') or - info.get('releaseDate') or - info.get('releasedate') - ) - custom_props['release_date'] = release_date_val - custom_props_updated = True - if not year and custom_props.get('release_date'): - year = extract_year(custom_props.get('release_date')) - updated = True - if custom_props_updated: - series.custom_properties = custom_props - updated = True - if updated: - series.save() - return series + logger.info(f"Batch processed episodes: {len(episodes_to_create)} new, {len(episodes_to_update)} updated, " + f"{len(relations_to_create)} new relations, {len(relations_to_update)} updated relations") def extract_year(date_string): @@ -565,6 +968,7 @@ def extract_year(date_string): except (ValueError, IndexError): return None + def extract_year_from_title(title): """Extract year from movie title if present""" if not title: @@ -628,6 +1032,62 @@ def extract_year_from_data(data, title_key='name'): +@shared_task +def batch_refresh_series_episodes(account_id, series_ids=None): + """ + Batch refresh episodes for multiple series. + If series_ids is None, refresh all series that haven't been refreshed recently. + """ + try: + account = M3UAccount.objects.get(id=account_id, is_active=True) + + if account.account_type != M3UAccount.Types.XC: + logger.warning(f"Episode refresh called for non-XC account {account_id}") + return "Episode refresh only available for XtreamCodes accounts" + + # Determine which series to refresh + if series_ids: + series_relations = M3USeriesRelation.objects.filter( + m3u_account=account, + series__id__in=series_ids + ).select_related('series') + else: + # Refresh series that haven't been refreshed in the last 24 hours + cutoff_time = timezone.now() - timezone.timedelta(hours=24) + series_relations = M3USeriesRelation.objects.filter( + m3u_account=account, + last_episode_refresh__lt=cutoff_time + ).select_related('series') + + logger.info(f"Batch refreshing episodes for {series_relations.count()} series") + + with XtreamCodesClient( + account.server_url, + account.username, + account.password, + account.get_user_agent().user_agent + ) as client: + + refreshed_count = 0 + for relation in series_relations: + try: + refresh_series_episodes( + account, + relation.series, + relation.external_series_id + ) + refreshed_count += 1 + except Exception as e: + logger.error(f"Error refreshing episodes for series {relation.series.name}: {str(e)}") + + logger.info(f"Batch episode refresh completed for {refreshed_count} series") + return f"Batch episode refresh completed for {refreshed_count} series" + + except Exception as e: + logger.error(f"Error in batch episode refresh for account {account_id}: {str(e)}") + return f"Batch episode refresh failed: {str(e)}" + + @shared_task def cleanup_orphaned_vod_content(): """Clean up VOD content that has no M3U relations"""