diff --git a/CHANGELOG.md b/CHANGELOG.md index b0ccfb58..adafe9f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **`get_vod_streams` XC API response time reduced significantly for large libraries.** The endpoint previously queried `Movie` with a `Prefetch` that loaded all active relations per movie (one row per account that carries the title), then discarded all but the highest-priority one in Python. With 48k movies across multiple active accounts this fetched a multiple of 48k rows. The query now uses a single `DISTINCT ON (movie_id)` pass over `M3UMovieRelation` ordered by account priority, returning exactly one row per movie. Logo URL construction (`reverse()` + `build_absolute_uri_with_port()`) is also computed once and reused via prefix/suffix string concatenation, matching the existing pattern in `get_live_streams`. Observed improvement: 18s → 12s for a 48k-movie library. - **Database connections are now managed by a persistent per-worker pool.** Previously each request opened a fresh TCP connection to PostgreSQL, paid a full authentication handshake, and closed the connection at request end. `django-db-geventpool` now maintains a pool of warm connections per uWSGI worker; requests borrow a connection and return it when done, eliminating connection-setup overhead on every request. Pool size is bounded (`MAX_CONNS=8` per worker, `REUSE_CONNS=3` warm connections kept idle) to stay comfortably within PostgreSQL's default `max_connections=100` across all uWSGI workers, Celery workers, and Daphne. — Thanks [@JCBird1012](https://github.com/JCBird1012) - **Reduced Redis round-trips on the Stats page channel status endpoint.** `get_basic_channel_info` was making up to 6 individual `HGET` calls per connected client plus a redundant `HGET` for `TOTAL_BYTES` (already present in the preceding `HGETALL` result). Client metadata is now fetched with a single `HMGET` per client, and `TOTAL_BYTES` is read from the already-fetched hash. Under load with many active streams this significantly reduces the time each uWSGI worker holds the GIL servicing the stats endpoint, reducing the chance of concurrent requests from other pages timing out with a 503. The same `HGET`-to-`HMGET` consolidation was applied to `stream_ts` and `get_user_active_connections`. The Stats page frontend was also fixed to fire the initial fetch only once on mount (previously two `useEffect` hooks both triggered an immediate fetch on load). — Thanks [@JCBird1012](https://github.com/JCBird1012) diff --git a/apps/output/views.py b/apps/output/views.py index 4d928ade..44c44a86 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -2549,65 +2549,79 @@ def xc_get_vod_categories(user): def xc_get_vod_streams(request, user, category_id=None): """Get VOD streams (movies) for XtreamCodes API""" - from apps.vod.models import Movie, M3UMovieRelation - from django.db.models import Prefetch + from apps.vod.models import M3UMovieRelation + from django.db import connection + + rel_filters = {"m3u_account__is_active": True} + if category_id: + rel_filters["category_id"] = category_id + + base_qs = ( + M3UMovieRelation.objects + .filter(**rel_filters) + .select_related('movie', 'movie__logo', 'category') + ) + + if connection.vendor == 'postgresql': + # DISTINCT ON returns one row per movie (highest-priority active relation) + # in a single query. ORDER BY must lead with the DISTINCT field. + relations = list( + base_qs.order_by('movie_id', '-m3u_account__priority', 'id') + .distinct('movie_id') + ) + else: + # SQLite fallback: fetch all matching relations, deduplicate in Python. + seen: dict = {} + for rel in base_qs.order_by('-m3u_account__priority', 'id'): + if rel.movie_id not in seen: + seen[rel.movie_id] = rel + relations = list(seen.values()) + + # Precompute logo URL prefix/suffix once (mirrors _xc_live_streams_setup) + # so each row only needs a string concat instead of reverse() + URI build. + _base_url = build_absolute_uri_with_port(request, "") + _sample_logo_path = reverse("api:vod:vodlogo-cache", args=[0]) + _logo_prefix_raw, _, _logo_suffix_raw = _sample_logo_path.partition("/0/") + _logo_url_prefix = _base_url + _logo_prefix_raw + "/" + _logo_url_suffix = "/" + _logo_suffix_raw + + # Sort by name (DISTINCT ON forces ORDER BY movie_id; SQLite path is unsorted). + relations.sort(key=lambda r: (r.movie.name or "").lower()) streams = [] + append = streams.append + for num, relation in enumerate(relations, 1): + movie = relation.movie + custom_props = movie.custom_properties or {} + category = relation.category + category_id_str = str(category.id) if category else "0" + category_id_list = [category.id] if category else [] + rating = movie.rating + logo = movie.logo - # All authenticated users get access to VOD from all active M3U accounts - filters = {"m3u_relations__m3u_account__is_active": True} - - if category_id: - filters["m3u_relations__category_id"] = category_id - - # Optimize with prefetch_related to eliminate N+1 queries - # This loads all relations in a single query instead of one per movie - movies = Movie.objects.filter(**filters).select_related('logo').prefetch_related( - Prefetch( - 'm3u_relations', - queryset=M3UMovieRelation.objects.filter( - m3u_account__is_active=True - ).select_related('m3u_account', 'category').order_by('-m3u_account__priority', 'id'), - to_attr='active_relations' - ) - ).distinct() - - for movie in movies: - # Get the first (highest priority) relation from prefetched data - # This avoids the N+1 query problem entirely - if hasattr(movie, 'active_relations') and movie.active_relations: - relation = movie.active_relations[0] - else: - # Fallback - should rarely be needed with proper prefetching - continue - - streams.append({ - "num": movie.id, + append({ + "num": num, "name": movie.name, "stream_type": "movie", "stream_id": movie.id, "stream_icon": ( - None if not movie.logo - else build_absolute_uri_with_port( - request, - reverse("api:vod:vodlogo-cache", args=[movie.logo.id]) - ) + f"{_logo_url_prefix}{logo.id}{_logo_url_suffix}" if logo else None ), - "rating": movie.rating or "0", - "rating_5based": round(float(movie.rating or 0) / 2, 2) if movie.rating else 0, + "rating": rating or "0", + "rating_5based": round(float(rating or 0) / 2, 2) if rating else 0, "added": str(int(movie.created_at.timestamp())), "is_adult": 0, "tmdb_id": movie.tmdb_id or "", "imdb_id": movie.imdb_id or "", - "trailer": (movie.custom_properties or {}).get('youtube_trailer') or "", + "trailer": custom_props.get('youtube_trailer') or "", "plot": movie.description or "", "genre": movie.genre or "", "year": movie.year or "", - "director": (movie.custom_properties or {}).get('director', ''), - "cast": (movie.custom_properties or {}).get('actors', ''), - "release_date": (movie.custom_properties or {}).get('release_date', ''), - "category_id": str(relation.category.id) if relation.category else "0", - "category_ids": [int(relation.category.id)] if relation.category else [], + "director": custom_props.get('director', ''), + "cast": custom_props.get('actors', ''), + "release_date": custom_props.get('release_date', ''), + "category_id": category_id_str, + "category_ids": category_id_list, "container_extension": relation.container_extension or "mp4", "custom_sid": None, "direct_source": "",