From ff90771a3fd3755b7552714f8411aad9096d4fe6 Mon Sep 17 00:00:00 2001 From: Dispatcharr Date: Sat, 20 Dec 2025 19:48:35 -0600 Subject: [PATCH] Media Server Added media center capabilities --- CHANGELOG.md | 11 + apps/api/urls.py | 1 + apps/media_library/__init__.py | 0 apps/media_library/admin.py | 22 + apps/media_library/api_urls.py | 22 + apps/media_library/api_views.py | 455 ++++++ apps/media_library/apps.py | 11 + apps/media_library/classification.py | 332 +++++ apps/media_library/metadata.py | 903 ++++++++++++ apps/media_library/migrations/0001_initial.py | 408 ++++++ apps/media_library/migrations/__init__.py | 0 apps/media_library/models.py | 327 +++++ apps/media_library/scanner.py | 373 +++++ apps/media_library/serializers.py | 305 ++++ apps/media_library/signals.py | 86 ++ apps/media_library/tasks.py | 255 ++++ apps/media_library/utils.py | 32 + apps/media_library/vod.py | 416 ++++++ apps/proxy/vod_proxy/views.py | 128 +- dispatcharr/settings.py | 1 + frontend/src/App.jsx | 2 + frontend/src/api.js | 291 ++++ frontend/src/components/FloatingVideo.jsx | 146 +- frontend/src/components/Sidebar.jsx | 12 +- .../components/library/AlphabetSidebar.jsx | 129 ++ .../src/components/library/LibraryCard.jsx | 152 ++ .../components/library/LibraryFormModal.jsx | 436 ++++++ .../components/library/LibraryScanDrawer.jsx | 548 +++++++ frontend/src/components/library/MediaCard.jsx | 374 +++++ .../src/components/library/MediaCarousel.jsx | 105 ++ .../components/library/MediaDetailModal.jsx | 1300 +++++++++++++++++ .../src/components/library/MediaEditModal.jsx | 305 ++++ frontend/src/components/library/MediaGrid.jsx | 164 +++ frontend/src/pages/Libraries.jsx | 203 +++ frontend/src/pages/Library.jsx | 806 ++++++++++ frontend/src/pages/Settings.jsx | 204 +++ frontend/src/store/library.jsx | 132 ++ frontend/src/store/mediaLibrary.jsx | 231 +++ requirements.txt | 2 + 39 files changed, 9616 insertions(+), 14 deletions(-) create mode 100644 apps/media_library/__init__.py create mode 100644 apps/media_library/admin.py create mode 100644 apps/media_library/api_urls.py create mode 100644 apps/media_library/api_views.py create mode 100644 apps/media_library/apps.py create mode 100644 apps/media_library/classification.py create mode 100644 apps/media_library/metadata.py create mode 100644 apps/media_library/migrations/0001_initial.py create mode 100644 apps/media_library/migrations/__init__.py create mode 100644 apps/media_library/models.py create mode 100644 apps/media_library/scanner.py create mode 100644 apps/media_library/serializers.py create mode 100644 apps/media_library/signals.py create mode 100644 apps/media_library/tasks.py create mode 100644 apps/media_library/utils.py create mode 100644 apps/media_library/vod.py create mode 100644 frontend/src/components/library/AlphabetSidebar.jsx create mode 100644 frontend/src/components/library/LibraryCard.jsx create mode 100644 frontend/src/components/library/LibraryFormModal.jsx create mode 100644 frontend/src/components/library/LibraryScanDrawer.jsx create mode 100644 frontend/src/components/library/MediaCard.jsx create mode 100644 frontend/src/components/library/MediaCarousel.jsx create mode 100644 frontend/src/components/library/MediaDetailModal.jsx create mode 100644 frontend/src/components/library/MediaEditModal.jsx create mode 100644 frontend/src/components/library/MediaGrid.jsx create mode 100644 frontend/src/pages/Libraries.jsx create mode 100644 frontend/src/pages/Library.jsx create mode 100644 frontend/src/store/library.jsx create mode 100644 frontend/src/store/mediaLibrary.jsx diff --git a/CHANGELOG.md b/CHANGELOG.md index adb9c748..489c5d48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Media Library app with library management, scanning pipeline, watch progress, and metadata sync +- Libraries management page and Media Library browsing page in the frontend, plus sidebar navigation +- Media Library API endpoints for libraries, scans, directory browse, items, episodes, and stream URLs +- VOD integration for Media Library items, including local-file playback through the VOD proxy + +### Changed + +- VOD proxy supports local file streaming and optional inclusion of inactive accounts for library playback + ### Fixed - XtreamCodes EPG `has_archive` field now returns integer `0` instead of string `"0"` for proper JSON type consistency diff --git a/apps/api/urls.py b/apps/api/urls.py index 4c92c70a..31120b3b 100644 --- a/apps/api/urls.py +++ b/apps/api/urls.py @@ -28,6 +28,7 @@ urlpatterns = [ path('plugins/', include(('apps.plugins.api_urls', 'plugins'), namespace='plugins')), path('vod/', include(('apps.vod.api_urls', 'vod'), namespace='vod')), path('backups/', include(('apps.backups.api_urls', 'backups'), namespace='backups')), + path('media-library/', include(('apps.media_library.api_urls', 'media_library'), namespace='media_library')), # path('output/', include(('apps.output.api_urls', 'output'), namespace='output')), #path('player/', include(('apps.player.api_urls', 'player'), namespace='player')), #path('settings/', include(('apps.settings.api_urls', 'settings'), namespace='settings')), diff --git a/apps/media_library/__init__.py b/apps/media_library/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/media_library/admin.py b/apps/media_library/admin.py new file mode 100644 index 00000000..988fa37c --- /dev/null +++ b/apps/media_library/admin.py @@ -0,0 +1,22 @@ +from django.contrib import admin + +from apps.media_library.models import ( + ArtworkAsset, + Library, + LibraryLocation, + LibraryScan, + MediaFile, + MediaItem, + MediaItemVODLink, + WatchProgress, +) + + +admin.site.register(Library) +admin.site.register(LibraryLocation) +admin.site.register(MediaItem) +admin.site.register(MediaFile) +admin.site.register(MediaItemVODLink) +admin.site.register(ArtworkAsset) +admin.site.register(WatchProgress) +admin.site.register(LibraryScan) diff --git a/apps/media_library/api_urls.py b/apps/media_library/api_urls.py new file mode 100644 index 00000000..4b371f8c --- /dev/null +++ b/apps/media_library/api_urls.py @@ -0,0 +1,22 @@ +from django.urls import path +from rest_framework.routers import DefaultRouter + +from apps.media_library.api_views import ( + LibraryScanViewSet, + LibraryViewSet, + MediaItemViewSet, + browse_library_path, +) + +app_name = "media_library" + +router = DefaultRouter() +router.register(r"libraries", LibraryViewSet, basename="library") +router.register(r"scans", LibraryScanViewSet, basename="library-scan") +router.register(r"items", MediaItemViewSet, basename="media-item") + +urlpatterns = [ + path("browse/", browse_library_path, name="browse"), +] + +urlpatterns += router.urls diff --git a/apps/media_library/api_views.py b/apps/media_library/api_views.py new file mode 100644 index 00000000..32ae6cdd --- /dev/null +++ b/apps/media_library/api_views.py @@ -0,0 +1,455 @@ +import os +from django.utils import timezone +from django.urls import reverse +from rest_framework import status, viewsets +from rest_framework.decorators import action, api_view, permission_classes +from rest_framework.filters import OrderingFilter, SearchFilter +from rest_framework.pagination import PageNumberPagination +from rest_framework.response import Response + +from apps.accounts.permissions import Authenticated, IsAdmin, permission_classes_by_action +from apps.media_library.models import Library, LibraryScan, MediaFile, MediaItem, WatchProgress +from apps.media_library.serializers import ( + LibraryScanSerializer, + LibrarySerializer, + MediaItemDetailSerializer, + MediaItemSerializer, + MediaItemUpdateSerializer, +) +from apps.media_library.tasks import refresh_media_item_metadata, scan_library +from apps.media_library.vod import sync_library_vod_account_state, sync_vod_for_media_item + + +class MediaLibraryPagination(PageNumberPagination): + page_size = 200 + page_size_query_param = "limit" + max_page_size = 1000 + + def get_page_size(self, request): + if "limit" not in request.query_params and "page" not in request.query_params: + return None + return super().get_page_size(request) + + +class LibraryViewSet(viewsets.ModelViewSet): + queryset = Library.objects.prefetch_related("locations") + serializer_class = LibrarySerializer + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + def perform_create(self, serializer): + library = serializer.save() + sync_library_vod_account_state(library) + + def perform_update(self, serializer): + library = serializer.save() + sync_library_vod_account_state(library) + + @action(detail=True, methods=["post"], url_path="scan") + def start_scan(self, request, pk=None): + library = self.get_object() + full = bool(request.data.get("full", False)) + scan = LibraryScan.objects.create( + library=library, + scan_type=LibraryScan.SCAN_FULL if full else LibraryScan.SCAN_QUICK, + status=LibraryScan.STATUS_QUEUED, + summary="Full scan" if full else "Quick scan", + stages={}, + ) + task = scan_library.delay(library.id, full=full, scan_id=scan.id) + scan.task_id = task.id + scan.save(update_fields=["task_id", "updated_at"]) + return Response(LibraryScanSerializer(scan).data, status=status.HTTP_201_CREATED) + + +class LibraryScanViewSet(viewsets.ModelViewSet): + serializer_class = LibraryScanSerializer + pagination_class = MediaLibraryPagination + + def get_queryset(self): + queryset = LibraryScan.objects.all() + library_id = self.request.query_params.get("library") + if library_id: + queryset = queryset.filter(library_id=library_id) + return queryset + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + @action(detail=True, methods=["post"], url_path="cancel") + def cancel_scan(self, request, pk=None): + scan = self.get_object() + if scan.status not in { + LibraryScan.STATUS_PENDING, + LibraryScan.STATUS_QUEUED, + LibraryScan.STATUS_RUNNING, + }: + return Response( + {"detail": "Scan is not running."}, + status=status.HTTP_400_BAD_REQUEST, + ) + scan.status = LibraryScan.STATUS_CANCELLED + scan.save(update_fields=["status", "updated_at"]) + return Response(LibraryScanSerializer(scan).data) + + @action(detail=False, methods=["delete"], url_path="purge") + def purge_scans(self, request): + library_id = request.query_params.get("library") + queryset = LibraryScan.objects.filter( + status__in=[ + LibraryScan.STATUS_COMPLETED, + LibraryScan.STATUS_FAILED, + LibraryScan.STATUS_CANCELLED, + ] + ) + if library_id: + queryset = queryset.filter(library_id=library_id) + deleted, _ = queryset.delete() + return Response({"deleted": deleted}) + + def destroy(self, request, *args, **kwargs): + scan = self.get_object() + if scan.status not in {LibraryScan.STATUS_PENDING, LibraryScan.STATUS_QUEUED}: + return Response( + {"detail": "Only queued scans can be deleted."}, + status=status.HTTP_400_BAD_REQUEST, + ) + return super().destroy(request, *args, **kwargs) + + +class MediaItemViewSet(viewsets.ModelViewSet): + serializer_class = MediaItemSerializer + pagination_class = MediaLibraryPagination + filter_backends = [SearchFilter, OrderingFilter] + search_fields = ["title", "sort_title", "normalized_title"] + ordering_fields = [ + "updated_at", + "first_imported_at", + "release_year", + "title", + "sort_title", + ] + ordering = ["-updated_at"] + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + def get_queryset(self): + queryset = MediaItem.objects.all() + library_ids = self.request.query_params.getlist("library") + if not library_ids: + libraries = self.request.query_params.get("libraries") or self.request.query_params.get("library_ids") + if libraries: + library_ids = [entry for entry in libraries.split(",") if entry] + if library_ids: + queryset = queryset.filter(library_id__in=library_ids) + item_type = self.request.query_params.get("type") + if item_type: + queryset = queryset.filter(item_type=item_type) + return queryset + + def get_serializer_class(self): + if self.action == "retrieve": + return MediaItemDetailSerializer + if self.action in {"update", "partial_update"}: + return MediaItemUpdateSerializer + return MediaItemSerializer + + def perform_update(self, serializer): + media_item = serializer.save() + try: + sync_vod_for_media_item(media_item) + except Exception: + pass + + @action(detail=True, methods=["get"], url_path="episodes") + def episodes(self, request, pk=None): + item = self.get_object() + if item.item_type != MediaItem.TYPE_SHOW: + return Response( + {"detail": "Episodes are only available for series."}, + status=status.HTTP_400_BAD_REQUEST, + ) + episodes = ( + MediaItem.objects.filter(parent=item, item_type=MediaItem.TYPE_EPISODE) + .order_by("season_number", "episode_number", "id") + ) + serializer = MediaItemSerializer(episodes, many=True, context={"request": request}) + return Response(serializer.data) + + @action(detail=True, methods=["post"], url_path="refresh-metadata") + def refresh_metadata(self, request, pk=None): + item = self.get_object() + refresh_media_item_metadata.delay(item.id) + return Response({"queued": True}) + + @action(detail=True, methods=["post"], url_path="stream") + def stream(self, request, pk=None): + item = self.get_object() + if item.item_type not in {MediaItem.TYPE_MOVIE, MediaItem.TYPE_EPISODE}: + return Response( + {"detail": "Streaming is only available for movies and episodes."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + file_id = request.data.get("fileId") or request.data.get("file_id") + media_file = None + if file_id: + media_file = MediaFile.objects.filter(id=file_id, media_item=item).first() + if not media_file: + media_file = item.files.filter(is_primary=True).first() or item.files.first() + if not media_file: + return Response( + {"detail": "No media file is linked to this item."}, + status=status.HTTP_404_NOT_FOUND, + ) + + try: + sync_vod_for_media_item(item) + item.refresh_from_db() + except Exception: + pass + + link = getattr(item, "vod_link", None) + vod_uuid = None + vod_type = None + if link: + if item.item_type == MediaItem.TYPE_MOVIE and link.vod_movie_id: + vod_uuid = link.vod_movie.uuid + vod_type = "movie" + elif item.item_type == MediaItem.TYPE_EPISODE and link.vod_episode_id: + vod_uuid = link.vod_episode.uuid + vod_type = "episode" + + if not vod_uuid: + return Response( + {"detail": "Streaming endpoint is not ready yet."}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + + params = [] + if item.library.vod_account_id: + params.append(f"m3u_account_id={item.library.vod_account_id}") + params.append("include_inactive=1") + + query = f"?{'&'.join(params)}" if params else "" + stream_path = reverse( + "proxy:vod_proxy:vod_stream", + kwargs={"content_type": vod_type, "content_id": vod_uuid}, + ) + stream_url = request.build_absolute_uri(f"{stream_path}{query}") + return Response( + { + "url": stream_url, + "stream_url": stream_url, + "start_offset_ms": 0, + "file_id": media_file.id, + "duration_ms": media_file.duration_ms or item.runtime_ms, + "requires_transcode": False, + } + ) + + def _get_duration_ms(self, media_item: MediaItem) -> int: + if media_item.runtime_ms: + return media_item.runtime_ms + file = media_item.files.filter(is_primary=True).first() or media_item.files.first() + if file and file.duration_ms: + return file.duration_ms + return 0 + + @action(detail=True, methods=["post"], url_path="progress") + def update_progress(self, request, pk=None): + item = self.get_object() + if item.item_type not in {MediaItem.TYPE_MOVIE, MediaItem.TYPE_EPISODE}: + return Response( + {"detail": "Progress tracking is only available for movies and episodes."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + def _parse_int(value): + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + position_ms = _parse_int( + request.data.get("position_ms") + or request.data.get("positionMs") + or request.data.get("position") + ) + duration_ms = _parse_int( + request.data.get("duration_ms") + or request.data.get("durationMs") + or request.data.get("duration") + ) + completed_raw = request.data.get("completed") + completed = bool(completed_raw) if completed_raw is not None else False + + if position_ms is None and not completed: + return Response( + {"detail": "position_ms is required to update progress."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if position_ms is None: + position_ms = 0 + if position_ms < 0: + position_ms = 0 + + file_id = request.data.get("file_id") or request.data.get("fileId") + media_file = None + if file_id: + media_file = MediaFile.objects.filter(id=file_id, media_item=item).first() + + if not duration_ms: + if media_file and media_file.duration_ms: + duration_ms = media_file.duration_ms + else: + duration_ms = self._get_duration_ms(item) or None + + if duration_ms and position_ms > duration_ms: + position_ms = duration_ms + + if duration_ms and position_ms / max(duration_ms, 1) >= 0.95: + completed = True + + if completed and duration_ms: + position_ms = duration_ms + + progress, _ = WatchProgress.objects.get_or_create( + user=request.user, media_item=item + ) + progress.position_ms = position_ms + if duration_ms: + progress.duration_ms = duration_ms + if media_file: + progress.file = media_file + progress.completed = completed + progress.last_watched_at = timezone.now() + progress.save() + + serializer = MediaItemSerializer(item, context={"request": request}) + return Response(serializer.data) + + @action(detail=True, methods=["post"], url_path="mark-watched") + def mark_watched(self, request, pk=None): + item = self.get_object() + user = request.user + duration_ms = self._get_duration_ms(item) + progress, _ = WatchProgress.objects.get_or_create(user=user, media_item=item) + progress.position_ms = duration_ms or progress.position_ms or 0 + progress.duration_ms = duration_ms or progress.duration_ms + progress.completed = True + progress.last_watched_at = timezone.now() + progress.save() + serializer = MediaItemSerializer(item, context={"request": request}) + return Response(serializer.data) + + @action(detail=True, methods=["post"], url_path="clear-progress") + def clear_progress(self, request, pk=None): + item = self.get_object() + WatchProgress.objects.filter(user=request.user, media_item=item).delete() + serializer = MediaItemSerializer(item, context={"request": request}) + return Response(serializer.data) + + @action(detail=True, methods=["post"], url_path="series/mark-watched") + def mark_series_watched(self, request, pk=None): + series = self.get_object() + if series.item_type != MediaItem.TYPE_SHOW: + return Response( + {"detail": "Series actions are only available for shows."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + episodes = MediaItem.objects.filter(parent=series, item_type=MediaItem.TYPE_EPISODE) + existing = { + entry.media_item_id: entry + for entry in WatchProgress.objects.filter(user=request.user, media_item__in=episodes) + } + now = timezone.now() + to_create = [] + to_update = [] + + for episode in episodes: + duration_ms = self._get_duration_ms(episode) + if episode.id in existing: + progress = existing[episode.id] + progress.position_ms = duration_ms or progress.position_ms or 0 + progress.duration_ms = duration_ms or progress.duration_ms + progress.completed = True + progress.last_watched_at = now + to_update.append(progress) + else: + to_create.append( + WatchProgress( + user=request.user, + media_item=episode, + position_ms=duration_ms or 0, + duration_ms=duration_ms or None, + completed=True, + last_watched_at=now, + ) + ) + + if to_create: + WatchProgress.objects.bulk_create(to_create, ignore_conflicts=True) + if to_update: + WatchProgress.objects.bulk_update( + to_update, + ["position_ms", "duration_ms", "completed", "last_watched_at", "updated_at"], + ) + + serializer = MediaItemSerializer(series, context={"request": request}) + return Response({"item": serializer.data}) + + @action(detail=True, methods=["post"], url_path="series/clear-progress") + def clear_series_progress(self, request, pk=None): + series = self.get_object() + if series.item_type != MediaItem.TYPE_SHOW: + return Response( + {"detail": "Series actions are only available for shows."}, + status=status.HTTP_400_BAD_REQUEST, + ) + episodes = MediaItem.objects.filter(parent=series, item_type=MediaItem.TYPE_EPISODE) + WatchProgress.objects.filter(user=request.user, media_item__in=episodes).delete() + serializer = MediaItemSerializer(series, context={"request": request}) + return Response({"item": serializer.data}) + + +@api_view(["GET"]) +@permission_classes([IsAdmin]) +def browse_library_path(request): + raw_path = request.query_params.get("path") or "" + if not raw_path: + path = os.path.abspath(os.sep) + else: + path = os.path.abspath(os.path.expanduser(raw_path)) + + if not os.path.exists(path) or not os.path.isdir(path): + return Response({"detail": "Path not found."}, status=status.HTTP_404_NOT_FOUND) + + parent = os.path.dirname(path.rstrip(os.sep)) + if parent == path: + parent = None + + entries = [] + try: + with os.scandir(path) as it: + for entry in it: + if entry.is_dir(): + entries.append({"name": entry.name, "path": entry.path}) + except PermissionError: + return Response({"detail": "Permission denied."}, status=status.HTTP_403_FORBIDDEN) + + entries.sort(key=lambda item: item["name"].lower()) + return Response({"path": path, "parent": parent, "entries": entries}) diff --git a/apps/media_library/apps.py b/apps/media_library/apps.py new file mode 100644 index 00000000..580dd867 --- /dev/null +++ b/apps/media_library/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + + +class MediaLibraryConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.media_library' + verbose_name = 'Media Library' + label = 'media_library' + + def ready(self): + import apps.media_library.signals diff --git a/apps/media_library/classification.py b/apps/media_library/classification.py new file mode 100644 index 00000000..860a53b1 --- /dev/null +++ b/apps/media_library/classification.py @@ -0,0 +1,332 @@ +import os +import re +from typing import Optional + +from guessit import guessit + +from apps.media_library.models import Library, MediaItem +from apps.media_library.utils import ClassificationResult, _json_safe, normalize_title + +SEASON_FOLDER_PATTERN = re.compile( + r"^(?:season|series|s)[\s\._-]*([0-9]{1,3})$", re.IGNORECASE +) +EPISODE_PATTERN = re.compile( + r""" + (?: + s(?P\d{1,3}) + [\.\-_\s]* + e(?P\d{1,4}) + (?:[\.\-_\s]*e?(?P\d{1,4}))? + ) + | + (?P\d{1,4}) + """, + re.IGNORECASE | re.VERBOSE, +) +YEAR_PATTERN = re.compile(r"(? str: + base, _ext = os.path.splitext(file_name) + return base + + +def _first_text(value) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (list, tuple, set)): + for entry in value: + text = _first_text(entry) + if text: + return text + return "" + if isinstance(value, dict): + for key in ("title", "name", "value"): + if key in value: + text = _first_text(value[key]) + if text: + return text + return "" + return str(value) + + +def _ensure_series_name_from_path( + relative_path: str, default: str | None = None +) -> str: + """ + Best effort extraction of a series title from a relative path. + Matches Jellyfin's approach of sanitizing folder names by replacing + dots/underscores with spaces when they separate words. + """ + if not relative_path: + return default or "" + + segments = [segment for segment in relative_path.split(os.sep) if segment] + if not segments: + return default or "" + + series_candidate = None + for segment in segments: + if SEASON_FOLDER_PATTERN.match(segment): + continue + series_candidate = segment + break + + if series_candidate is None: + series_candidate = segments[0] + + sanitized = re.sub( + r"(([^._]{2,})[\._]*)|([\._]([^._]{2,}))", r"\2\4", series_candidate + ).replace("_", " ").replace(".", " ") + sanitized = re.sub(r"\s+", " ", sanitized).strip() + return sanitized or (default or "") + + +def _season_from_segments(segments: list[str]) -> Optional[int]: + for segment in reversed(segments): + match = SEASON_FOLDER_PATTERN.match(segment) + if match: + try: + return int(match.group(1)) + except (TypeError, ValueError): + continue + return None + + +def _extract_year_from_text(text: str | None) -> Optional[int]: + if not text: + return None + candidates = [] + for match in YEAR_PATTERN.finditer(text): + try: + year = int(match.group(1)) + candidates.append(year) + except (TypeError, ValueError): + continue + if not candidates: + return None + return candidates[-1] + + +def _tokenize_release_name(text: str) -> list[str]: + """ + Break a release name into tokens with punctuation stripped so we can + drop common quality/source tags. Periods and underscores are treated + as separators to better match scene-style names. + """ + if not text: + return [] + cleaned = re.sub(r"[\[\]\(\)\{\}]", " ", text) + cleaned = re.sub(r"[._]", " ", cleaned) + cleaned = cleaned.replace("-", " ") + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return [token for token in cleaned.split(" ") if token] + + +def _is_release_junk(token: str) -> bool: + if not token: + return True + if MOVIE_JUNK_TOKEN_PATTERN.match(token): + return True + if re.match(r"^\d+\.\d+$", token): + return True + token_lower = token.lower() + if token_lower in {"yts.mx", "yts.ag", "yts.lt", "yts.mx", "rarbg"}: + return True + return False + + +def _sanitize_movie_title( + title_source: str, base_name: str, year_hint: int | None = None +) -> tuple[str, int | None]: + """ + Extract a clean movie title (and possibly year) from a release-style + name. Removes quality/source tags and stops parsing once a year token + is reached so entries like "6.Underground.2019.1080p..." become + "6 Underground". + """ + year = ( + year_hint + or _extract_year_from_text(title_source) + or _extract_year_from_text(base_name) + ) + tokens = _tokenize_release_name(title_source) + cleaned_tokens: list[str] = [] + for token in tokens: + token_stripped = token.strip(" -_.") + if not token_stripped: + continue + if year and token_stripped.isdigit() and int(token_stripped) == year: + break + if _is_release_junk(token_stripped): + continue + cleaned_tokens.append(token_stripped) + cleaned_title = " ".join(cleaned_tokens).strip() + if not cleaned_title and title_source != base_name: + tokens = _tokenize_release_name(base_name) + cleaned_tokens = [] + for token in tokens: + token_stripped = token.strip(" -_.") + if not token_stripped: + continue + if year and token_stripped.isdigit() and int(token_stripped) == year: + break + if _is_release_junk(token_stripped): + continue + cleaned_tokens.append(token_stripped) + cleaned_title = " ".join(cleaned_tokens).strip() + if not cleaned_title: + cleaned_title = re.sub(r"[._]", " ", base_name) + cleaned_title = re.sub(r"\s+", " ", cleaned_title).strip() + return cleaned_title, year + + +def classify_media_entry( + library: Library, + *, + relative_path: str, + file_name: str, +) -> ClassificationResult: + """ + Wrapper around guessit that injects folder hints similar to Jellyfin's resolver. + """ + base_name = _strip_extension(file_name) + guess_data = {} + guess_target = file_name + if relative_path: + guess_target = os.path.join(relative_path, file_name) + guess_options = {} + if library.library_type == Library.LIBRARY_TYPE_SHOWS: + guess_options = {"type": "episode", "show_type": "series"} + elif library.library_type == Library.LIBRARY_TYPE_MOVIES: + guess_options = {"type": "movie"} + guessed_title = "" + + try: + if guess_options: + guess_data = guessit(guess_target, options=guess_options) + else: + guess_data = guessit(guess_target) + guessed_title = _first_text(guess_data.get("title")) + except Exception: + guess_data = {} + guessed_title = "" + + segments = [segment for segment in relative_path.split(os.sep) if segment] + series_name = _ensure_series_name_from_path( + relative_path, default=guessed_title or base_name + ) + + if library.library_type == Library.LIBRARY_TYPE_SHOWS: + season_number = _safe_number(guess_data.get("season")) + episode_number = _safe_number(guess_data.get("episode")) + episode_title = _first_text(guess_data.get("episode_title")) + + if season_number is None: + season_number = _season_from_segments(segments) + + if episode_number is None: + episode_number = _safe_number(guess_data.get("episode_list")) + + episode_list_raw = guess_data.get("episode_list") or [] + if not episode_list_raw and isinstance( + guess_data.get("episode"), (list, tuple) + ): + episode_list_raw = list(guess_data.get("episode") or []) + episode_list = [] + for item in episode_list_raw: + number = _safe_number(item) + if number is not None and number not in episode_list: + episode_list.append(number) + + if episode_number is None: + match = EPISODE_PATTERN.search(file_name) + if match: + episode_number = _safe_number( + match.group("episode") or match.group("abs_episode") + ) + if season_number is None and match.group("season"): + try: + season_number = int(match.group("season")) + except (TypeError, ValueError): + season_number = None + if match.group("episode_end"): + end_number = _safe_number(match.group("episode_end")) + if end_number: + if end_number not in episode_list: + episode_list.append(end_number) + if episode_number is not None and episode_number not in episode_list: + episode_list.append(episode_number) + + detected_type = ( + MediaItem.TYPE_EPISODE + if season_number is not None and episode_number is not None + else MediaItem.TYPE_SHOW + ) + normalized_title = normalize_title(series_name or base_name) + data = _json_safe(guess_data) + if season_number is not None: + data["season"] = season_number + if episode_number is not None: + data["episode"] = episode_number + if episode_list: + episode_list_sorted = sorted({n for n in episode_list if isinstance(n, int)}) + if episode_list_sorted: + data["episode_list"] = episode_list_sorted + if series_name: + data["series"] = series_name + + return ClassificationResult( + detected_type=detected_type, + title=series_name or base_name, + year=guess_data.get("year"), + season=season_number, + episode=episode_number, + episode_title=episode_title or None, + data=data, + ) + + detected_type = ( + MediaItem.TYPE_MOVIE + if library.library_type == Library.LIBRARY_TYPE_MOVIES + else MediaItem.TYPE_OTHER + ) + data = _json_safe(guess_data) + title_candidate = guessed_title or base_name + title, parsed_year = _sanitize_movie_title( + title_candidate, base_name, _safe_number(guess_data.get("year")) + ) + year = _safe_number(guess_data.get("year")) or parsed_year + if title and title != guessed_title: + data["parsed_title"] = title + if year is not None: + data["parsed_year"] = year + return ClassificationResult( + detected_type=detected_type, + title=title, + year=year, + data=data, + ) diff --git a/apps/media_library/metadata.py b/apps/media_library/metadata.py new file mode 100644 index 00000000..83da08c4 --- /dev/null +++ b/apps/media_library/metadata.py @@ -0,0 +1,903 @@ +import logging +from typing import Any, Dict, Optional, Tuple + +import requests +from dateutil import parser as date_parser +from django.core.cache import cache +from django.utils import timezone + +from apps.media_library.models import ArtworkAsset, MediaItem +from apps.media_library.utils import normalize_title + +logger = logging.getLogger(__name__) + +IMAGE_BASE_URL = "https://image.tmdb.org/t/p/original" +MOVIEDB_SEARCH_URL = "https://movie-db.org/api.php" +MOVIEDB_EPISODE_URL = "https://movie-db.org/episode.php" +MOVIEDB_CREDITS_URL = "https://movie-db.org/credits.php" +METADATA_CACHE_TIMEOUT = 60 * 60 * 6 +MOVIEDB_HEALTH_CACHE_KEY = "movie-db:health" +MOVIEDB_HEALTH_SUCCESS_TTL = 60 * 15 +MOVIEDB_HEALTH_FAILURE_TTL = 60 * 2 + +_REQUESTS_SESSION = requests.Session() +_REQUESTS_SESSION.mount( + "https://", + requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=40, max_retries=3), +) + +_MOVIEDB_SEARCH_CACHE: dict[tuple[str, ...], tuple[dict | None, str | None]] = {} +_MOVIEDB_CREDITS_CACHE: dict[tuple[str, str], tuple[dict | None, str | None]] = {} + +GENRE_ID_MAP: dict[int, str] = { + 12: "Adventure", + 14: "Fantasy", + 16: "Animation", + 18: "Drama", + 27: "Horror", + 28: "Action", + 35: "Comedy", + 36: "History", + 37: "Western", + 53: "Thriller", + 80: "Crime", + 99: "Documentary", + 878: "Science Fiction", + 9648: "Mystery", + 10402: "Music", + 10749: "Romance", + 10751: "Family", + 10752: "War", + 10759: "Action & Adventure", + 10762: "Kids", + 10763: "News", + 10764: "Reality", + 10765: "Sci-Fi & Fantasy", + 10766: "Soap", + 10767: "Talk", + 10768: "War & Politics", + 10770: "TV Movie", +} + + +def _get_library_metadata_prefs(media_item: MediaItem) -> dict[str, Any]: + prefs: dict[str, Any] = {"language": None, "region": None} + library = getattr(media_item, "library", None) + if not library: + return prefs + + prefs["language"] = (library.metadata_language or "").strip() or None + prefs["region"] = (library.metadata_country or "").strip() or None + + options = library.metadata_options or {} + if isinstance(options, dict): + if options.get("language"): + prefs["language"] = str(options["language"]).strip() or prefs["language"] + if options.get("region"): + prefs["region"] = str(options["region"]).strip() or prefs["region"] + + return prefs + + +def _metadata_cache_key(media_item: MediaItem, prefs: dict[str, Any] | None = None) -> str: + normalized = normalize_title(media_item.title) or media_item.normalized_title or "" + key_parts = [ + "movie-db", + str(media_item.item_type or ""), + normalized, + str(media_item.release_year or ""), + ] + if media_item.movie_db_id: + key_parts.append(f"id:{media_item.movie_db_id}") + if media_item.item_type == MediaItem.TYPE_EPISODE: + season = media_item.season_number or "" + episode = media_item.episode_number or "" + key_parts.append(f"s{season}e{episode}") + parent = getattr(media_item, "parent", None) + if parent and parent.movie_db_id: + key_parts.append(f"parent:{parent.movie_db_id}") + + if prefs: + language = (prefs.get("language") or "").strip() + region = (prefs.get("region") or "").strip() + if language or region: + key_parts.append(f"lang:{language}") + key_parts.append(f"region:{region}") + + return "media-metadata:" + ":".join(key_parts) + + +def build_image_url(path: Optional[str]) -> Optional[str]: + if not path: + return None + return f"{IMAGE_BASE_URL}{path}" + + +def _to_serializable(obj): + if isinstance(obj, dict): + return {key: _to_serializable(value) for key, value in obj.items()} + if isinstance(obj, (list, tuple, set)): + return [_to_serializable(item) for item in obj] + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + return str(obj) + + +def check_movie_db_health(*, use_cache: bool = True) -> Tuple[bool, str | None]: + if use_cache: + cached = cache.get(MOVIEDB_HEALTH_CACHE_KEY) + if cached is not None: + return cached + + try: + response = _REQUESTS_SESSION.get( + MOVIEDB_SEARCH_URL, + params={"type": "tv", "query": "The Simpsons"}, + timeout=5, + ) + except requests.RequestException as exc: + message = f"Movie-DB request failed: {exc}" + cache.set(MOVIEDB_HEALTH_CACHE_KEY, (False, message), MOVIEDB_HEALTH_FAILURE_TTL) + return False, message + + if response.status_code != 200: + message = f"Movie-DB returned status {response.status_code}." + cache.set(MOVIEDB_HEALTH_CACHE_KEY, (False, message), MOVIEDB_HEALTH_FAILURE_TTL) + return False, message + + try: + payload = response.json() + except ValueError: + message = "Movie-DB returned an unexpected response format." + cache.set(MOVIEDB_HEALTH_CACHE_KEY, (False, message), MOVIEDB_HEALTH_FAILURE_TTL) + return False, message + + results = [] + if isinstance(payload, dict): + results = payload.get("results") or [] + elif isinstance(payload, list): + results = payload + + if not results: + message = "Movie-DB did not return any results." + cache.set(MOVIEDB_HEALTH_CACHE_KEY, (False, message), MOVIEDB_HEALTH_FAILURE_TTL) + return False, message + + cache.set(MOVIEDB_HEALTH_CACHE_KEY, (True, None), MOVIEDB_HEALTH_SUCCESS_TTL) + return True, None + + +def _parse_release_year(value: Optional[str]) -> Optional[int]: + if not value: + return None + try: + return date_parser.parse(value).year + except (ValueError, TypeError): + return None + + +def _php_value_from_string(value: str): + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + if lowered in {"null", ""}: + return None + try: + if "." in value: + return float(value) + return int(value) + except ValueError: + return value + + +def _convert_numeric_container(value): + if isinstance(value, dict): + converted = {k: _convert_numeric_container(v) for k, v in value.items()} + numeric = True + indices: list[int] = [] + for key in converted.keys(): + try: + indices.append(int(key)) + except (TypeError, ValueError): + numeric = False + break + if numeric: + return [converted[str(idx)] for idx in sorted(indices)] + return converted + if isinstance(value, list): + return [_convert_numeric_container(item) for item in value] + return value + + +def _parse_php_array_lines(lines: list[str], idx: int = 0): + def parse_body(current_idx: int): + result: dict[str, Any] = {} + while current_idx < len(lines): + token = lines[current_idx] + if token == ")": + return result, current_idx + 1 + if token.startswith("[") and "] =>" in token: + key_part, value_part = token.split("] =>", 1) + key = key_part[1:].strip() + value = value_part.strip() + if value == "Array": + sub_value, next_idx = _parse_php_array_lines(lines, current_idx + 1) + result[key] = sub_value + current_idx = next_idx + continue + result[key] = _php_value_from_string(value) + current_idx += 1 + return result, current_idx + + if idx >= len(lines): + return {}, idx + + token = lines[idx] + if token.lower() == "array": + idx += 1 + if idx < len(lines) and lines[idx] == "(": + idx += 1 + parsed, idx = parse_body(idx) + return _convert_numeric_container(parsed), idx + return {}, idx + + +def _parse_movie_db_php_response(payload: str) -> Optional[dict | list]: + if not payload: + return None + stripped = payload.replace("
", "").replace("
", "").strip() + if not stripped.lower().startswith("array"): + return None + lines = [line.strip() for line in stripped.splitlines() if line.strip()] + parsed, _ = _parse_php_array_lines(lines, 0) + return parsed + + +def _extract_imdb_id(payload: dict | None) -> Optional[str]: + if not payload or not isinstance(payload, dict): + return None + for key in ("imdb_id", "imdbId", "imdbID"): + value = payload.get(key) + if value: + return str(value) + external = payload.get("external_ids") or {} + if isinstance(external, dict): + for key in ("imdb_id", "imdbId", "imdbID"): + value = external.get(key) + if value: + return str(value) + return None + + +def _candidate_year(payload: dict) -> Optional[int]: + release_date = ( + payload.get("release_date") + or payload.get("first_air_date") + or payload.get("air_date") + ) + return _parse_release_year(release_date) + + +def _movie_db_title(payload: dict) -> str: + return ( + payload.get("title") + or payload.get("name") + or payload.get("original_title") + or payload.get("original_name") + or "" + ) + + +def _select_movie_db_candidate( + results: list[dict], + title: str, + *, + year: Optional[int] = None, + prefs: dict[str, Any] | None = None, +) -> Optional[dict]: + if not results: + return None + normalized_query = normalize_title(title) + language = (prefs or {}).get("language") + region = (prefs or {}).get("region") + best_score = None + best_result = None + for idx, result in enumerate(results): + if not isinstance(result, dict): + continue + normalized_title = normalize_title(_movie_db_title(result)) + score = 0 + if normalized_query and normalized_title == normalized_query: + score += 6 + elif normalized_query and ( + normalized_query in normalized_title or normalized_title in normalized_query + ): + score += 3 + if year: + candidate_year = _candidate_year(result) + if candidate_year and candidate_year == year: + score += 2 + if language and result.get("original_language") == language: + score += 2 + if region: + origin_country = result.get("origin_country") or [] + if isinstance(origin_country, str): + origin_country = [origin_country] + if region in origin_country: + score += 2 + if result.get("poster_path"): + score += 2 + elif result.get("backdrop_path"): + score += 1 + # Keep earlier results when scores tie. + score = (score, -idx) + if best_score is None or score > best_score: + best_score = score + best_result = result + return best_result or (results[0] if results else None) + + +def _movie_db_search( + media_type: str, + title: str, + *, + year: Optional[int] = None, + prefs: dict[str, Any] | None = None, +) -> tuple[Optional[dict], Optional[str]]: + language = (prefs or {}).get("language") or "" + region = (prefs or {}).get("region") or "" + normalized_key = ( + media_type, + (title or "").strip().lower(), + str(year or ""), + str(language), + str(region), + ) + if normalized_key in _MOVIEDB_SEARCH_CACHE: + return _MOVIEDB_SEARCH_CACHE[normalized_key] + + try: + response = _REQUESTS_SESSION.get( + MOVIEDB_SEARCH_URL, + params={"type": media_type, "query": title}, + timeout=6, + ) + except requests.RequestException as exc: + message = f"Movie-DB search failed: {exc}" + logger.warning(message) + _MOVIEDB_SEARCH_CACHE[normalized_key] = (None, message) + return None, message + + if response.status_code != 200: + message = f"Movie-DB search returned status {response.status_code}." + logger.debug(message) + _MOVIEDB_SEARCH_CACHE[normalized_key] = (None, message) + return None, message + + try: + payload = response.json() + except ValueError: + message = "Movie-DB search returned an unexpected response format." + logger.debug(message) + _MOVIEDB_SEARCH_CACHE[normalized_key] = (None, message) + return None, message + + results = [] + if isinstance(payload, dict): + results = payload.get("results") or [] + elif isinstance(payload, list): + results = payload + + if not results: + message = "Movie-DB search returned no matches." + logger.debug(message) + _MOVIEDB_SEARCH_CACHE[normalized_key] = (None, message) + return None, message + + candidate = _select_movie_db_candidate( + results, title, year=year, prefs=prefs + ) + _MOVIEDB_SEARCH_CACHE[normalized_key] = (candidate, None) + return candidate, None + + +def _map_genre_ids(genre_ids: Any) -> list[str]: + if not genre_ids or not isinstance(genre_ids, (list, tuple, set)): + return [] + genres: list[str] = [] + seen: set[str] = set() + for entry in genre_ids: + try: + genre_id = int(entry) + except (TypeError, ValueError): + continue + name = GENRE_ID_MAP.get(genre_id) + if not name or name in seen: + continue + genres.append(name) + seen.add(name) + return genres + + +def _movie_db_fetch_credits( + media_type: str, content_id: Any +) -> tuple[Optional[dict], Optional[str]]: + if not content_id: + return None, "Movie-DB credits require an id." + normalized_key = (media_type, str(content_id)) + if normalized_key in _MOVIEDB_CREDITS_CACHE: + return _MOVIEDB_CREDITS_CACHE[normalized_key] + + params = {"id": content_id, "media_type": media_type} + try: + response = _REQUESTS_SESSION.get( + MOVIEDB_CREDITS_URL, + params=params, + timeout=6, + ) + except requests.RequestException as exc: + message = f"Movie-DB credits lookup failed: {exc}" + logger.warning(message) + _MOVIEDB_CREDITS_CACHE[normalized_key] = (None, message) + return None, message + + if response.status_code != 200 and media_type == "movie": + try: + response = _REQUESTS_SESSION.get( + MOVIEDB_CREDITS_URL, + params={"movie_id": content_id}, + timeout=6, + ) + except requests.RequestException as exc: + message = f"Movie-DB credits lookup failed: {exc}" + logger.warning(message) + _MOVIEDB_CREDITS_CACHE[normalized_key] = (None, message) + return None, message + + if response.status_code != 200: + message = f"Movie-DB credits returned status {response.status_code}." + logger.debug(message) + _MOVIEDB_CREDITS_CACHE[normalized_key] = (None, message) + return None, message + + try: + payload = response.json() + except ValueError: + message = "Movie-DB credits returned an unexpected response format." + logger.debug(message) + _MOVIEDB_CREDITS_CACHE[normalized_key] = (None, message) + return None, message + + if not isinstance(payload, dict): + message = "Movie-DB credits returned an unexpected payload." + logger.debug(message) + _MOVIEDB_CREDITS_CACHE[normalized_key] = (None, message) + return None, message + + _MOVIEDB_CREDITS_CACHE[normalized_key] = (payload, None) + return payload, None + + +def _normalize_credits(payload: dict | None) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + if not payload: + return [], [] + cast_entries = payload.get("cast") or [] + crew_entries = payload.get("crew") or [] + + cast: list[dict[str, Any]] = [] + for member in cast_entries[:12]: + if not isinstance(member, dict): + continue + name = member.get("name") or member.get("original_name") + if not name: + continue + cast.append( + { + "name": name, + "character": member.get("character"), + "profile_url": build_image_url(member.get("profile_path")), + } + ) + + crew: list[dict[str, Any]] = [] + for member in crew_entries[:15]: + if not isinstance(member, dict): + continue + name = member.get("name") or member.get("original_name") + if not name: + continue + crew.append( + { + "name": name, + "job": member.get("job"), + "department": member.get("department"), + "profile_url": build_image_url(member.get("profile_path")), + } + ) + + return cast, crew + + +def _movie_db_fetch_episode_details( + tv_id: Any, season: int, episode: int +) -> tuple[Optional[dict], Optional[str]]: + try: + response = _REQUESTS_SESSION.get( + MOVIEDB_EPISODE_URL, + params={"tvId": tv_id, "season": season, "episode": episode}, + timeout=6, + ) + except requests.RequestException as exc: + message = f"Movie-DB episode lookup failed: {exc}" + logger.warning(message) + return None, message + + if response.status_code != 200: + message = f"Movie-DB episode endpoint returned status {response.status_code}." + logger.debug(message) + return None, message + + parsed = _parse_movie_db_php_response(response.text) + if not parsed or not isinstance(parsed, dict): + message = "Movie-DB episode endpoint returned an unexpected payload." + logger.debug(message) + return None, message + + return _convert_numeric_container(parsed), None + + +def fetch_movie_db_metadata( + media_item: MediaItem, + *, + use_cache: bool = True, +) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + if media_item.item_type not in { + MediaItem.TYPE_MOVIE, + MediaItem.TYPE_SHOW, + MediaItem.TYPE_EPISODE, + }: + return None, "Movie-DB does not support this media type." + + prefs = _get_library_metadata_prefs(media_item) + cache_key = _metadata_cache_key(media_item, prefs) + if use_cache: + cached = cache.get(cache_key) + if cached: + return cached, None + + if media_item.item_type == MediaItem.TYPE_MOVIE: + candidate, message = _movie_db_search( + "movie", + media_item.title, + year=media_item.release_year, + prefs=prefs, + ) + if not candidate: + return None, message + + movie_id = candidate.get("id") + genres = _map_genre_ids(candidate.get("genre_ids")) + credits_payload, _ = _movie_db_fetch_credits("movie", movie_id) + cast, crew = _normalize_credits(credits_payload) + imdb_id = _extract_imdb_id(candidate) + release_date = candidate.get("release_date") or candidate.get("first_air_date") + release_year = _parse_release_year(release_date) or media_item.release_year + metadata = _to_serializable( + { + "movie_db_id": str(movie_id or media_item.movie_db_id or ""), + "imdb_id": imdb_id, + "title": candidate.get("title") + or candidate.get("name") + or media_item.title, + "synopsis": candidate.get("overview") or media_item.synopsis, + "tagline": candidate.get("tagline") or media_item.tagline, + "release_year": release_year, + "poster": build_image_url(candidate.get("poster_path")), + "backdrop": build_image_url(candidate.get("backdrop_path")), + "runtime_minutes": candidate.get("runtime"), + "genres": genres, + "cast": cast, + "crew": crew, + "source": "movie-db", + "raw": candidate, + } + ) + if use_cache: + cache.set(cache_key, metadata, METADATA_CACHE_TIMEOUT) + return metadata, None + + if media_item.item_type == MediaItem.TYPE_SHOW: + candidate, message = _movie_db_search( + "tv", + media_item.title, + year=media_item.release_year, + prefs=prefs, + ) + if not candidate: + return None, message + tv_id = candidate.get("id") + genres = _map_genre_ids(candidate.get("genre_ids")) + credits_payload, _ = _movie_db_fetch_credits("tv", tv_id) + cast, crew = _normalize_credits(credits_payload) + imdb_id = _extract_imdb_id(candidate) + release_date = candidate.get("first_air_date") or candidate.get("release_date") + release_year = _parse_release_year(release_date) or media_item.release_year + metadata = _to_serializable( + { + "movie_db_id": str(candidate.get("id") or media_item.movie_db_id or ""), + "imdb_id": imdb_id, + "title": candidate.get("name") + or candidate.get("title") + or media_item.title, + "synopsis": candidate.get("overview") or media_item.synopsis, + "tagline": candidate.get("tagline") or media_item.tagline, + "release_year": release_year, + "poster": build_image_url(candidate.get("poster_path")), + "backdrop": build_image_url(candidate.get("backdrop_path")), + "runtime_minutes": candidate.get("episode_run_time"), + "genres": genres, + "cast": cast, + "crew": crew, + "movie_db_tv_id": str(tv_id) if tv_id is not None else None, + "source": "movie-db", + "raw": candidate, + } + ) + if use_cache: + cache.set(cache_key, metadata, METADATA_CACHE_TIMEOUT) + return metadata, None + + parent_series = media_item.parent or MediaItem.objects.filter( + pk=media_item.parent_id + ).first() + if not parent_series: + return None, "Unable to determine parent series for episode." + + tv_id = parent_series.movie_db_id or None + if not tv_id and isinstance(parent_series.metadata, dict): + tv_id = parent_series.metadata.get("movie_db_tv_id") + if tv_id and not parent_series.movie_db_id: + parent_series.movie_db_id = str(tv_id) + parent_series.save(update_fields=["movie_db_id", "updated_at"]) + + if not tv_id: + series_candidate, message = _movie_db_search( + "tv", + parent_series.title, + year=parent_series.release_year or media_item.release_year, + prefs=prefs, + ) + if not series_candidate: + return None, message + tv_id = series_candidate.get("id") + if tv_id is not None: + tv_id_str = str(tv_id) + update_fields = [] + if not parent_series.movie_db_id: + parent_series.movie_db_id = tv_id_str + update_fields.append("movie_db_id") + if isinstance(parent_series.metadata, dict): + updated_metadata = dict(parent_series.metadata) + else: + updated_metadata = {} + updated_metadata["movie_db_tv_id"] = tv_id_str + parent_series.metadata = updated_metadata + update_fields.append("metadata") + update_fields.append("updated_at") + parent_series.save(update_fields=update_fields) + + if tv_id is None: + return None, "Movie-DB did not return a tvId for this series." + + if media_item.season_number is None or media_item.episode_number is None: + return None, "Episode numbers are required for Movie-DB lookup." + + try: + season_number = int(media_item.season_number) + episode_number = int(media_item.episode_number) + except (TypeError, ValueError): + return None, "Episode numbers are required for Movie-DB lookup." + + episode_details, message = _movie_db_fetch_episode_details( + tv_id, season_number, episode_number + ) + if not episode_details: + return None, message + + release_year = ( + _parse_release_year(episode_details.get("air_date")) + or media_item.release_year + ) + guest_stars = episode_details.get("guest_stars") or [] + crew_entries = episode_details.get("crew") or [] + + cast: list[dict[str, Any]] = [] + for star in guest_stars: + if not isinstance(star, dict): + continue + name = star.get("name") or star.get("original_name") + if not name: + continue + cast.append( + { + "name": name, + "character": star.get("character"), + "profile_url": build_image_url(star.get("profile_path")), + } + ) + + crew: list[dict[str, Any]] = [] + for member in crew_entries: + if not isinstance(member, dict): + continue + name = member.get("name") or member.get("original_name") + if not name: + continue + crew.append( + { + "name": name, + "job": member.get("job"), + "department": member.get("department"), + "profile_url": build_image_url(member.get("profile_path")), + } + ) + + metadata = _to_serializable( + { + "movie_db_id": str(episode_details.get("id") or media_item.movie_db_id or ""), + "title": episode_details.get("name") or media_item.title, + "synopsis": episode_details.get("overview") or media_item.synopsis, + "release_year": release_year, + "poster": build_image_url(episode_details.get("still_path")), + "backdrop": build_image_url(episode_details.get("still_path")), + "runtime_minutes": episode_details.get("runtime"), + "genres": [], + "cast": cast, + "crew": crew, + "movie_db_tv_id": str(tv_id) if tv_id is not None else None, + "source": "movie-db", + "raw": episode_details, + } + ) + + if use_cache: + cache.set(cache_key, metadata, METADATA_CACHE_TIMEOUT) + return metadata, None + + +def apply_metadata(media_item: MediaItem, metadata: Dict[str, Any]) -> MediaItem: + changed = False + update_fields: list[str] = [] + + movie_db_id = metadata.get("movie_db_id") + if movie_db_id and movie_db_id != media_item.movie_db_id: + media_item.movie_db_id = movie_db_id + update_fields.append("movie_db_id") + changed = True + + imdb_id = metadata.get("imdb_id") + if imdb_id and imdb_id != media_item.imdb_id: + media_item.imdb_id = imdb_id + update_fields.append("imdb_id") + changed = True + + title = metadata.get("title") + if title and title != media_item.title: + media_item.title = title + update_fields.append("title") + changed = True + + synopsis = metadata.get("synopsis") + if synopsis and synopsis != media_item.synopsis: + media_item.synopsis = synopsis + update_fields.append("synopsis") + changed = True + + tagline = metadata.get("tagline") + if tagline and tagline != media_item.tagline: + media_item.tagline = tagline + update_fields.append("tagline") + changed = True + + release_year = metadata.get("release_year") + if release_year and release_year != media_item.release_year: + try: + media_item.release_year = int(release_year) + except (TypeError, ValueError): + media_item.release_year = media_item.release_year + else: + changed = True + update_fields.append("release_year") + + runtime_minutes = metadata.get("runtime_minutes") + if runtime_minutes: + new_runtime_ms = int(float(runtime_minutes) * 60 * 1000) + if media_item.runtime_ms != new_runtime_ms: + media_item.runtime_ms = new_runtime_ms + update_fields.append("runtime_ms") + changed = True + + genres = metadata.get("genres") + if genres and genres != media_item.genres: + media_item.genres = genres + update_fields.append("genres") + changed = True + + cast = metadata.get("cast") + if cast and cast != media_item.cast: + media_item.cast = cast + update_fields.append("cast") + changed = True + + crew = metadata.get("crew") + if crew and crew != media_item.crew: + media_item.crew = crew + update_fields.append("crew") + changed = True + + poster = metadata.get("poster") + if poster and poster != media_item.poster_url: + media_item.poster_url = poster + update_fields.append("poster_url") + changed = True + + backdrop = metadata.get("backdrop") + if backdrop and backdrop != media_item.backdrop_url: + media_item.backdrop_url = backdrop + update_fields.append("backdrop_url") + changed = True + + raw_payload = metadata.get("raw") + if raw_payload and raw_payload != media_item.metadata: + media_item.metadata = raw_payload + update_fields.append("metadata") + changed = True + + new_source = metadata.get("source", media_item.metadata_source or "unknown") + if new_source and new_source != media_item.metadata_source: + media_item.metadata_source = new_source + update_fields.append("metadata_source") + changed = True + + if changed or not media_item.metadata_last_synced_at: + media_item.metadata_last_synced_at = timezone.now() + if "metadata_last_synced_at" not in update_fields: + update_fields.append("metadata_last_synced_at") + + if update_fields: + update_fields = list(dict.fromkeys(update_fields + ["updated_at"])) + media_item.save(update_fields=update_fields) + + if poster: + ArtworkAsset.objects.update_or_create( + media_item=media_item, + asset_type=ArtworkAsset.TYPE_POSTER, + source=metadata.get("source", "movie-db"), + defaults={"external_url": poster}, + ) + if backdrop: + ArtworkAsset.objects.update_or_create( + media_item=media_item, + asset_type=ArtworkAsset.TYPE_BACKDROP, + source=metadata.get("source", "movie-db"), + defaults={"external_url": backdrop}, + ) + + return media_item + + +def sync_metadata(media_item: MediaItem, *, force: bool = False) -> Optional[MediaItem]: + movie_db_ok, movie_db_message = check_movie_db_health() + if not movie_db_ok: + if movie_db_message: + logger.debug("Metadata sync skipped for %s: %s", media_item, movie_db_message) + return None + + metadata, error = fetch_movie_db_metadata(media_item, use_cache=not force) + if not metadata: + if error: + logger.debug("Metadata sync skipped for %s: %s", media_item, error) + return None + return apply_metadata(media_item, metadata) diff --git a/apps/media_library/migrations/0001_initial.py b/apps/media_library/migrations/0001_initial.py new file mode 100644 index 00000000..cdd20bea --- /dev/null +++ b/apps/media_library/migrations/0001_initial.py @@ -0,0 +1,408 @@ +# Generated by Django 5.2.4 on 2025-12-20 00:00 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("m3u", "0018_add_profile_custom_properties"), + ("vod", "0003_vodlogo_alter_movie_logo_alter_series_logo"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Library", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, default="")), + ( + "library_type", + models.CharField( + choices=[("movies", "Movies"), ("shows", "TV Shows")], + max_length=20, + ), + ), + ("metadata_language", models.CharField(blank=True, default="", max_length=12)), + ("metadata_country", models.CharField(blank=True, default="", max_length=12)), + ("metadata_options", models.JSONField(blank=True, default=dict, null=True)), + ("scan_interval_minutes", models.PositiveIntegerField(default=1440)), + ("auto_scan_enabled", models.BooleanField(default=True)), + ("add_to_vod", models.BooleanField(default=False)), + ("last_scan_at", models.DateTimeField(blank=True, null=True)), + ("last_successful_scan_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "vod_account", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="media_libraries", + to="m3u.m3uaccount", + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + migrations.CreateModel( + name="MediaItem", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "item_type", + models.CharField( + choices=[ + ("movie", "Movie"), + ("show", "Show"), + ("episode", "Episode"), + ("other", "Other"), + ], + max_length=16, + ), + ), + ("title", models.CharField(max_length=512)), + ("sort_title", models.CharField(blank=True, default="", max_length=512)), + ("normalized_title", models.CharField(blank=True, default="", max_length=512)), + ("synopsis", models.TextField(blank=True, default="")), + ("tagline", models.TextField(blank=True, default="")), + ("release_year", models.IntegerField(blank=True, null=True)), + ("rating", models.CharField(blank=True, default="", max_length=10)), + ("runtime_ms", models.PositiveBigIntegerField(blank=True, null=True)), + ("season_number", models.IntegerField(blank=True, null=True)), + ("episode_number", models.IntegerField(blank=True, null=True)), + ("genres", models.JSONField(blank=True, default=list, null=True)), + ("tags", models.JSONField(blank=True, default=list, null=True)), + ("studios", models.JSONField(blank=True, default=list, null=True)), + ("cast", models.JSONField(blank=True, default=list, null=True)), + ("crew", models.JSONField(blank=True, default=list, null=True)), + ("poster_url", models.TextField(blank=True, default="")), + ("backdrop_url", models.TextField(blank=True, default="")), + ("movie_db_id", models.CharField(blank=True, default="", max_length=64)), + ("imdb_id", models.CharField(blank=True, default="", max_length=32)), + ("metadata", models.JSONField(blank=True, null=True)), + ("metadata_source", models.CharField(blank=True, default="", max_length=32)), + ("metadata_last_synced_at", models.DateTimeField(blank=True, null=True)), + ("first_imported_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "library", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="items", + to="media_library.library", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="media_library.mediaitem", + ), + ), + ], + options={ + "ordering": ["sort_title", "title"], + }, + ), + migrations.CreateModel( + name="LibraryLocation", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("path", models.CharField(max_length=1024)), + ("include_subdirectories", models.BooleanField(default=True)), + ("is_primary", models.BooleanField(default=False)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "library", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="locations", + to="media_library.library", + ), + ), + ], + options={ + "unique_together": {("library", "path")}, + }, + ), + migrations.CreateModel( + name="MediaFile", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("path", models.CharField(max_length=1024)), + ("relative_path", models.CharField(blank=True, default="", max_length=1024)), + ("file_name", models.CharField(max_length=512)), + ("size_bytes", models.BigIntegerField(blank=True, null=True)), + ("modified_at", models.DateTimeField(blank=True, null=True)), + ("duration_ms", models.PositiveBigIntegerField(blank=True, null=True)), + ("is_primary", models.BooleanField(default=False)), + ("last_seen_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "library", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="media_library.library", + ), + ), + ( + "media_item", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="media_library.mediaitem", + ), + ), + ], + options={ + "unique_together": {("library", "path")}, + }, + ), + migrations.CreateModel( + name="MediaItemVODLink", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "media_item", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="vod_link", + to="media_library.mediaitem", + ), + ), + ( + "vod_episode", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="media_library_links", + to="vod.episode", + ), + ), + ( + "vod_movie", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="media_library_links", + to="vod.movie", + ), + ), + ( + "vod_series", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="media_library_links", + to="vod.series", + ), + ), + ], + ), + migrations.CreateModel( + name="ArtworkAsset", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("asset_type", models.CharField(choices=[("poster", "Poster"), ("backdrop", "Backdrop")], max_length=16)), + ("source", models.CharField(blank=True, default="", max_length=32)), + ("external_url", models.TextField()), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "media_item", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="artwork_assets", + to="media_library.mediaitem", + ), + ), + ], + ), + migrations.CreateModel( + name="WatchProgress", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("position_ms", models.PositiveBigIntegerField(default=0)), + ("duration_ms", models.PositiveBigIntegerField(blank=True, null=True)), + ("completed", models.BooleanField(default=False)), + ("last_watched_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "file", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="progress_entries", + to="media_library.mediafile", + ), + ), + ( + "media_item", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="progress_entries", + to="media_library.mediaitem", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="media_progress", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "unique_together": {("user", "media_item")}, + }, + ), + migrations.CreateModel( + name="LibraryScan", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "scan_type", + models.CharField( + choices=[("quick", "Quick"), ("full", "Full")], + max_length=16, + ), + ), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("queued", "Queued"), + ("running", "Running"), + ("completed", "Completed"), + ("failed", "Failed"), + ("cancelled", "Cancelled"), + ], + max_length=16, + ), + ), + ("summary", models.CharField(blank=True, default="", max_length=255)), + ("stages", models.JSONField(blank=True, default=dict, null=True)), + ("processed_files", models.PositiveIntegerField(default=0)), + ("total_files", models.PositiveIntegerField(default=0)), + ("new_files", models.PositiveIntegerField(default=0)), + ("updated_files", models.PositiveIntegerField(default=0)), + ("removed_files", models.PositiveIntegerField(default=0)), + ("unmatched_files", models.PositiveIntegerField(default=0)), + ("log", models.TextField(blank=True, default="")), + ("extra", models.JSONField(blank=True, null=True)), + ("task_id", models.CharField(blank=True, default="", max_length=255)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("started_at", models.DateTimeField(blank=True, null=True)), + ("finished_at", models.DateTimeField(blank=True, null=True)), + ( + "library", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="scans", + to="media_library.library", + ), + ), + ], + options={ + "ordering": ["-created_at"], + }, + ), + migrations.AddIndex( + model_name="librarylocation", + index=models.Index(fields=["library", "is_primary"], name="media_libr_library_b13f17_idx"), + ), + migrations.AddIndex( + model_name="mediaitem", + index=models.Index(fields=["library", "item_type"], name="media_libr_library_a7ec3e_idx"), + ), + migrations.AddIndex( + model_name="mediaitem", + index=models.Index(fields=["library", "sort_title"], name="media_libr_library_2dc3a5_idx"), + ), + migrations.AddIndex( + model_name="mediaitem", + index=models.Index(fields=["library", "updated_at"], name="media_libr_library_0f7e9b_idx"), + ), + migrations.AddIndex( + model_name="mediaitem", + index=models.Index(fields=["parent", "season_number", "episode_number"], name="media_libr_parent__f58b2b_idx"), + ), + migrations.AddConstraint( + model_name="mediaitem", + constraint=models.UniqueConstraint( + condition=models.Q(item_type="episode"), + fields=("parent", "season_number", "episode_number"), + name="unique_episode_per_season", + ), + ), + migrations.AddIndex( + model_name="mediafile", + index=models.Index(fields=["media_item"], name="media_libr_media_i_4bf7b4_idx"), + ), + migrations.AddIndex( + model_name="mediafile", + index=models.Index(fields=["library", "is_primary"], name="media_libr_library_0b3ae5_idx"), + ), + migrations.AddIndex( + model_name="mediafile", + index=models.Index(fields=["library", "last_seen_at"], name="media_libr_library_31c4a1_idx"), + ), + migrations.AddIndex( + model_name="mediaitemvodlink", + index=models.Index(fields=["media_item"], name="media_libr_media_i_f9b327_idx"), + ), + migrations.AddIndex( + model_name="mediaitemvodlink", + index=models.Index(fields=["vod_movie"], name="media_libr_vod_mo_6d43ad_idx"), + ), + migrations.AddIndex( + model_name="mediaitemvodlink", + index=models.Index(fields=["vod_series"], name="media_libr_vod_se_05e51a_idx"), + ), + migrations.AddIndex( + model_name="mediaitemvodlink", + index=models.Index(fields=["vod_episode"], name="media_libr_vod_ep_c0b0c7_idx"), + ), + migrations.AddIndex( + model_name="artworkasset", + index=models.Index(fields=["media_item", "asset_type"], name="media_libr_media_i_6fe152_idx"), + ), + migrations.AddIndex( + model_name="watchprogress", + index=models.Index(fields=["user", "media_item"], name="media_libr_user_id_5f9dd5_idx"), + ), + migrations.AddIndex( + model_name="libraryscan", + index=models.Index(fields=["library", "created_at"], name="media_libr_library_1f1b2e_idx"), + ), + migrations.AddIndex( + model_name="libraryscan", + index=models.Index(fields=["library", "status"], name="media_libr_library_0f3a70_idx"), + ), + ] diff --git a/apps/media_library/migrations/__init__.py b/apps/media_library/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/media_library/models.py b/apps/media_library/models.py new file mode 100644 index 00000000..1753b075 --- /dev/null +++ b/apps/media_library/models.py @@ -0,0 +1,327 @@ +from django.conf import settings +from django.db import models + + +class Library(models.Model): + LIBRARY_TYPE_MOVIES = "movies" + LIBRARY_TYPE_SHOWS = "shows" + + LIBRARY_TYPE_CHOICES = [ + (LIBRARY_TYPE_MOVIES, "Movies"), + (LIBRARY_TYPE_SHOWS, "TV Shows"), + ] + + name = models.CharField(max_length=255) + description = models.TextField(blank=True, default="") + library_type = models.CharField(max_length=20, choices=LIBRARY_TYPE_CHOICES) + metadata_language = models.CharField(max_length=12, blank=True, default="") + metadata_country = models.CharField(max_length=12, blank=True, default="") + metadata_options = models.JSONField(default=dict, blank=True, null=True) + scan_interval_minutes = models.PositiveIntegerField(default=1440) + auto_scan_enabled = models.BooleanField(default=True) + add_to_vod = models.BooleanField(default=False) + vod_account = models.ForeignKey( + "m3u.M3UAccount", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="media_libraries", + ) + last_scan_at = models.DateTimeField(null=True, blank=True) + last_successful_scan_at = models.DateTimeField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["name"] + + def __str__(self): + return self.name + + +class LibraryLocation(models.Model): + library = models.ForeignKey( + Library, on_delete=models.CASCADE, related_name="locations" + ) + path = models.CharField(max_length=1024) + include_subdirectories = models.BooleanField(default=True) + is_primary = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = [("library", "path")] + indexes = [ + models.Index(fields=["library", "is_primary"]), + ] + + def __str__(self): + return f"{self.library.name}: {self.path}" + + +class MediaItem(models.Model): + TYPE_MOVIE = "movie" + TYPE_SHOW = "show" + TYPE_EPISODE = "episode" + TYPE_OTHER = "other" + + ITEM_TYPE_CHOICES = [ + (TYPE_MOVIE, "Movie"), + (TYPE_SHOW, "Show"), + (TYPE_EPISODE, "Episode"), + (TYPE_OTHER, "Other"), + ] + + library = models.ForeignKey( + Library, on_delete=models.CASCADE, related_name="items" + ) + parent = models.ForeignKey( + "self", + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="children", + ) + item_type = models.CharField(max_length=16, choices=ITEM_TYPE_CHOICES) + title = models.CharField(max_length=512) + sort_title = models.CharField(max_length=512, blank=True, default="") + normalized_title = models.CharField(max_length=512, blank=True, default="") + synopsis = models.TextField(blank=True, default="") + tagline = models.TextField(blank=True, default="") + release_year = models.IntegerField(null=True, blank=True) + rating = models.CharField(max_length=10, blank=True, default="") + runtime_ms = models.PositiveBigIntegerField(null=True, blank=True) + season_number = models.IntegerField(null=True, blank=True) + episode_number = models.IntegerField(null=True, blank=True) + genres = models.JSONField(default=list, blank=True, null=True) + tags = models.JSONField(default=list, blank=True, null=True) + studios = models.JSONField(default=list, blank=True, null=True) + cast = models.JSONField(default=list, blank=True, null=True) + crew = models.JSONField(default=list, blank=True, null=True) + poster_url = models.TextField(blank=True, default="") + backdrop_url = models.TextField(blank=True, default="") + movie_db_id = models.CharField(max_length=64, blank=True, default="") + imdb_id = models.CharField(max_length=32, blank=True, default="") + metadata = models.JSONField(blank=True, null=True) + metadata_source = models.CharField(max_length=32, blank=True, default="") + metadata_last_synced_at = models.DateTimeField(null=True, blank=True) + first_imported_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["sort_title", "title"] + indexes = [ + models.Index(fields=["library", "item_type"]), + models.Index(fields=["library", "sort_title"]), + models.Index(fields=["library", "updated_at"]), + models.Index(fields=["parent", "season_number", "episode_number"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["parent", "season_number", "episode_number"], + condition=models.Q(item_type="episode"), + name="unique_episode_per_season", + ) + ] + + def __str__(self): + return self.title + + @property + def is_movie(self): + return self.item_type == self.TYPE_MOVIE + + @property + def is_show(self): + return self.item_type == self.TYPE_SHOW + + @property + def is_episode(self): + return self.item_type == self.TYPE_EPISODE + + +class MediaFile(models.Model): + library = models.ForeignKey( + Library, on_delete=models.CASCADE, related_name="files" + ) + media_item = models.ForeignKey( + MediaItem, on_delete=models.CASCADE, related_name="files" + ) + path = models.CharField(max_length=1024) + relative_path = models.CharField(max_length=1024, blank=True, default="") + file_name = models.CharField(max_length=512) + size_bytes = models.BigIntegerField(null=True, blank=True) + modified_at = models.DateTimeField(null=True, blank=True) + duration_ms = models.PositiveBigIntegerField(null=True, blank=True) + is_primary = models.BooleanField(default=False) + last_seen_at = models.DateTimeField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = [("library", "path")] + indexes = [ + models.Index(fields=["media_item"]), + models.Index(fields=["library", "is_primary"]), + models.Index(fields=["library", "last_seen_at"]), + ] + + def __str__(self): + return self.path + + +class MediaItemVODLink(models.Model): + media_item = models.OneToOneField( + MediaItem, on_delete=models.CASCADE, related_name="vod_link" + ) + vod_movie = models.ForeignKey( + "vod.Movie", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="media_library_links", + ) + vod_series = models.ForeignKey( + "vod.Series", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="media_library_links", + ) + vod_episode = models.ForeignKey( + "vod.Episode", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="media_library_links", + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + models.Index(fields=["media_item"]), + models.Index(fields=["vod_movie"]), + models.Index(fields=["vod_series"]), + models.Index(fields=["vod_episode"]), + ] + + def __str__(self): + return f"{self.media_item.title} -> VOD" + + +class ArtworkAsset(models.Model): + TYPE_POSTER = "poster" + TYPE_BACKDROP = "backdrop" + + TYPE_CHOICES = [ + (TYPE_POSTER, "Poster"), + (TYPE_BACKDROP, "Backdrop"), + ] + + media_item = models.ForeignKey( + MediaItem, on_delete=models.CASCADE, related_name="artwork_assets" + ) + asset_type = models.CharField(max_length=16, choices=TYPE_CHOICES) + source = models.CharField(max_length=32, blank=True, default="") + external_url = models.TextField() + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + models.Index(fields=["media_item", "asset_type"]), + ] + + def __str__(self): + return f"{self.media_item.title}: {self.asset_type}" + + +class WatchProgress(models.Model): + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="media_progress", + ) + media_item = models.ForeignKey( + MediaItem, on_delete=models.CASCADE, related_name="progress_entries" + ) + file = models.ForeignKey( + MediaFile, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="progress_entries", + ) + position_ms = models.PositiveBigIntegerField(default=0) + duration_ms = models.PositiveBigIntegerField(null=True, blank=True) + completed = models.BooleanField(default=False) + last_watched_at = models.DateTimeField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = [("user", "media_item")] + indexes = [ + models.Index(fields=["user", "media_item"]), + ] + + def __str__(self): + return f"{self.user} - {self.media_item}" + + +class LibraryScan(models.Model): + SCAN_QUICK = "quick" + SCAN_FULL = "full" + + SCAN_TYPE_CHOICES = [ + (SCAN_QUICK, "Quick"), + (SCAN_FULL, "Full"), + ] + + STATUS_PENDING = "pending" + STATUS_QUEUED = "queued" + STATUS_RUNNING = "running" + STATUS_COMPLETED = "completed" + STATUS_FAILED = "failed" + STATUS_CANCELLED = "cancelled" + + STATUS_CHOICES = [ + (STATUS_PENDING, "Pending"), + (STATUS_QUEUED, "Queued"), + (STATUS_RUNNING, "Running"), + (STATUS_COMPLETED, "Completed"), + (STATUS_FAILED, "Failed"), + (STATUS_CANCELLED, "Cancelled"), + ] + + library = models.ForeignKey( + Library, on_delete=models.CASCADE, related_name="scans" + ) + scan_type = models.CharField(max_length=16, choices=SCAN_TYPE_CHOICES) + status = models.CharField(max_length=16, choices=STATUS_CHOICES) + summary = models.CharField(max_length=255, blank=True, default="") + stages = models.JSONField(default=dict, blank=True, null=True) + processed_files = models.PositiveIntegerField(default=0) + total_files = models.PositiveIntegerField(default=0) + new_files = models.PositiveIntegerField(default=0) + updated_files = models.PositiveIntegerField(default=0) + removed_files = models.PositiveIntegerField(default=0) + unmatched_files = models.PositiveIntegerField(default=0) + log = models.TextField(blank=True, default="") + extra = models.JSONField(blank=True, null=True) + task_id = models.CharField(max_length=255, blank=True, default="") + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + started_at = models.DateTimeField(null=True, blank=True) + finished_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["-created_at"] + indexes = [ + models.Index(fields=["library", "created_at"]), + models.Index(fields=["library", "status"]), + ] + + def __str__(self): + return f"{self.library.name} ({self.scan_type})" diff --git a/apps/media_library/scanner.py b/apps/media_library/scanner.py new file mode 100644 index 00000000..666a8401 --- /dev/null +++ b/apps/media_library/scanner.py @@ -0,0 +1,373 @@ +import os +from datetime import timezone as dt_timezone +from dataclasses import dataclass, field +from typing import Callable, Iterable, Optional + +from django.utils import timezone + +from apps.media_library.classification import classify_media_entry +from apps.media_library.models import Library, MediaFile, MediaItem +from apps.media_library.utils import normalize_title + +VIDEO_EXTENSIONS = { + ".mkv", + ".mp4", + ".m4v", + ".avi", + ".mov", + ".wmv", + ".mpg", + ".mpeg", + ".ts", + ".m2ts", + ".webm", +} + +PROGRESS_UPDATE_INTERVAL = 50 +BULK_UPDATE_SIZE = 200 +BULK_CREATE_SIZE = 200 + + +class ScanCancelled(Exception): + pass + + +@dataclass +class ScanResult: + processed_files: int = 0 + total_files: int = 0 + new_files: int = 0 + updated_files: int = 0 + removed_files: int = 0 + unmatched_files: int = 0 + unmatched_paths: list[str] = field(default_factory=list) + metadata_item_ids: set[int] = field(default_factory=set) + errors: list[dict[str, str]] = field(default_factory=list) + + +def _is_media_file(file_name: str) -> bool: + _, ext = os.path.splitext(file_name) + return ext.lower() in VIDEO_EXTENSIONS + + +def _iter_location_files(base_path: str, include_subdirectories: bool) -> Iterable[str]: + if include_subdirectories: + for root, _dirs, files in os.walk(base_path): + for file_name in files: + yield os.path.join(root, file_name) + else: + for entry in os.scandir(base_path): + if entry.is_file(): + yield entry.path + + +def _resolve_relative_path(full_path: str, base_path: str) -> str: + relative = os.path.relpath(full_path, base_path) + if relative == ".": + return "" + parent = os.path.dirname(relative) + return "" if parent == "." else parent + + +def scan_library_files( + library: Library, + *, + full: bool = False, + progress_callback: Optional[Callable[[int, int], None]] = None, + cancel_check: Optional[Callable[[], bool]] = None, +) -> ScanResult: + result = ScanResult() + scan_start = timezone.now() + + existing_files = { + row["path"]: row + for row in MediaFile.objects.filter(library=library).values( + "id", + "path", + "size_bytes", + "modified_at", + "media_item_id", + "relative_path", + "file_name", + "duration_ms", + "is_primary", + ) + } + + primary_item_ids = set( + MediaFile.objects.filter(library=library, is_primary=True).values_list( + "media_item_id", flat=True + ) + ) + + movie_cache: dict[tuple[str, int | None], MediaItem] = {} + show_cache: dict[str, MediaItem] = {} + episode_cache: dict[tuple[int, int | None, int | None, str], MediaItem] = {} + + file_updates: list[MediaFile] = [] + file_creates: list[MediaFile] = [] + + def flush_updates(): + nonlocal file_updates, file_creates + if file_updates: + MediaFile.objects.bulk_update( + file_updates, + [ + "media_item", + "relative_path", + "file_name", + "size_bytes", + "modified_at", + "duration_ms", + "is_primary", + "last_seen_at", + ], + ) + file_updates = [] + if file_creates: + MediaFile.objects.bulk_create(file_creates) + file_creates = [] + + def get_or_create_movie(title: str, release_year: int | None) -> MediaItem: + normalized = normalize_title(title) + key = (normalized, release_year) + if key in movie_cache: + return movie_cache[key] + query = MediaItem.objects.filter( + library=library, + item_type=MediaItem.TYPE_MOVIE, + normalized_title=normalized, + ) + if release_year: + query = query.filter(release_year=release_year) + item = query.first() + if not item: + item = MediaItem.objects.create( + library=library, + item_type=MediaItem.TYPE_MOVIE, + title=title, + sort_title=title, + normalized_title=normalized, + release_year=release_year, + ) + movie_cache[key] = item + return item + + def get_or_create_show(title: str, release_year: int | None) -> MediaItem: + normalized = normalize_title(title) + if normalized in show_cache: + return show_cache[normalized] + query = MediaItem.objects.filter( + library=library, + item_type=MediaItem.TYPE_SHOW, + normalized_title=normalized, + ) + if release_year: + query = query.filter(release_year=release_year) + item = query.first() + if not item: + item = MediaItem.objects.create( + library=library, + item_type=MediaItem.TYPE_SHOW, + title=title, + sort_title=title, + normalized_title=normalized, + release_year=release_year, + ) + show_cache[normalized] = item + return item + + def get_or_create_episode( + parent: MediaItem, + title: str, + season_number: int | None, + episode_number: int | None, + ) -> MediaItem: + cache_key = (parent.id, season_number, episode_number, title) + if cache_key in episode_cache: + return episode_cache[cache_key] + query = MediaItem.objects.filter( + library=library, + parent=parent, + item_type=MediaItem.TYPE_EPISODE, + ) + if season_number is not None: + query = query.filter(season_number=season_number) + if episode_number is not None: + query = query.filter(episode_number=episode_number) + if season_number is None and episode_number is None: + query = query.filter(title=title) + item = query.first() + if not item: + item = MediaItem.objects.create( + library=library, + parent=parent, + item_type=MediaItem.TYPE_EPISODE, + title=title, + sort_title=title, + normalized_title=normalize_title(title), + season_number=season_number, + episode_number=episode_number, + ) + episode_cache[cache_key] = item + return item + + processed_since_update = 0 + + for location in library.locations.all(): + base_path = os.path.expanduser(location.path) + if not os.path.isdir(base_path): + result.errors.append({"path": base_path, "error": "Path not found"}) + continue + + try: + file_iter = _iter_location_files(base_path, location.include_subdirectories) + for full_path in file_iter: + if cancel_check and cancel_check(): + raise ScanCancelled() + + file_name = os.path.basename(full_path) + if not _is_media_file(file_name): + continue + + result.total_files += 1 + result.processed_files += 1 + processed_since_update += 1 + + stat = os.stat(full_path) + modified_at = timezone.datetime.fromtimestamp( + stat.st_mtime, tz=dt_timezone.utc + ) + size_bytes = stat.st_size + relative_path = _resolve_relative_path(full_path, base_path) + + existing = existing_files.get(full_path) + changed = full or not existing + if existing and not changed: + if existing["size_bytes"] != size_bytes: + changed = True + elif existing["modified_at"] != modified_at: + changed = True + + if changed: + classification = classify_media_entry( + library, relative_path=relative_path, file_name=file_name + ) + item_type = classification.detected_type + item_title = classification.title + media_item: MediaItem + + if library.library_type == Library.LIBRARY_TYPE_SHOWS: + show_item = get_or_create_show( + item_title, classification.year + ) + result.metadata_item_ids.add(show_item.id) + episode_title = classification.episode_title or os.path.splitext(file_name)[0] + media_item = get_or_create_episode( + show_item, + episode_title, + classification.season, + classification.episode, + ) + if media_item.normalized_title != normalize_title(media_item.title): + media_item.normalized_title = normalize_title(media_item.title) + media_item.save(update_fields=["normalized_title", "updated_at"]) + else: + if item_type == MediaItem.TYPE_OTHER: + result.unmatched_files += 1 + result.unmatched_paths.append(full_path) + media_item = get_or_create_movie( + item_title, classification.year + ) + + result.metadata_item_ids.add(media_item.id) + + if existing: + file_updates.append( + MediaFile( + id=existing["id"], + library_id=library.id, + media_item_id=media_item.id, + path=full_path, + relative_path=relative_path, + file_name=file_name, + size_bytes=size_bytes, + modified_at=modified_at, + duration_ms=existing.get("duration_ms"), + is_primary=existing.get("is_primary", False), + last_seen_at=scan_start, + ) + ) + result.updated_files += 1 + else: + is_primary = False + if media_item.id not in primary_item_ids: + is_primary = True + primary_item_ids.add(media_item.id) + file_creates.append( + MediaFile( + library=library, + media_item=media_item, + path=full_path, + relative_path=relative_path, + file_name=file_name, + size_bytes=size_bytes, + modified_at=modified_at, + is_primary=is_primary, + last_seen_at=scan_start, + ) + ) + result.new_files += 1 + else: + file_updates.append( + MediaFile( + id=existing["id"], + library_id=library.id, + media_item_id=existing["media_item_id"], + path=full_path, + relative_path=existing.get("relative_path") or relative_path, + file_name=existing.get("file_name") or file_name, + size_bytes=size_bytes, + modified_at=modified_at, + duration_ms=existing.get("duration_ms"), + is_primary=existing.get("is_primary", False), + last_seen_at=scan_start, + ) + ) + + if processed_since_update >= PROGRESS_UPDATE_INTERVAL: + flush_updates() + if progress_callback: + progress_callback(result.processed_files, result.total_files) + processed_since_update = 0 + + if len(file_updates) >= BULK_UPDATE_SIZE or len(file_creates) >= BULK_CREATE_SIZE: + flush_updates() + + except PermissionError as exc: + result.errors.append({"path": base_path, "error": str(exc)}) + except OSError as exc: + result.errors.append({"path": base_path, "error": str(exc)}) + + flush_updates() + + stale_files = MediaFile.objects.filter( + library=library, last_seen_at__lt=scan_start + ) + result.removed_files = stale_files.count() + stale_item_ids = set(stale_files.values_list("media_item_id", flat=True)) + stale_files.delete() + + if stale_item_ids: + MediaItem.objects.filter( + library=library, + id__in=stale_item_ids, + ).filter( + files__isnull=True, + children__isnull=True, + ).delete() + + if progress_callback: + progress_callback(result.processed_files, result.total_files) + + return result diff --git a/apps/media_library/serializers.py b/apps/media_library/serializers.py new file mode 100644 index 00000000..b61d2cc1 --- /dev/null +++ b/apps/media_library/serializers.py @@ -0,0 +1,305 @@ +from django.db.models import Count +from rest_framework import serializers + +from apps.media_library.models import ( + ArtworkAsset, + Library, + LibraryLocation, + LibraryScan, + MediaFile, + MediaItem, + WatchProgress, +) + + +class LibraryLocationSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False) + + class Meta: + model = LibraryLocation + fields = ["id", "path", "include_subdirectories", "is_primary"] + + +class LibrarySerializer(serializers.ModelSerializer): + locations = LibraryLocationSerializer(many=True) + + class Meta: + model = Library + fields = [ + "id", + "name", + "description", + "library_type", + "metadata_language", + "metadata_country", + "metadata_options", + "scan_interval_minutes", + "auto_scan_enabled", + "add_to_vod", + "last_scan_at", + "last_successful_scan_at", + "locations", + "created_at", + "updated_at", + ] + + def create(self, validated_data): + locations = validated_data.pop("locations", []) + library = Library.objects.create(**validated_data) + for index, location in enumerate(locations): + LibraryLocation.objects.create( + library=library, + path=location["path"], + include_subdirectories=location.get("include_subdirectories", True), + is_primary=location.get("is_primary", index == 0), + ) + return library + + def update(self, instance, validated_data): + locations = validated_data.pop("locations", None) + for attr, value in validated_data.items(): + setattr(instance, attr, value) + instance.save() + + if locations is not None: + existing_ids = {loc.id for loc in instance.locations.all()} + seen_ids: set[int] = set() + for index, location in enumerate(locations): + location_id = location.get("id") + if location_id and location_id in existing_ids: + LibraryLocation.objects.filter(id=location_id).update( + path=location.get("path", ""), + include_subdirectories=location.get("include_subdirectories", True), + is_primary=location.get("is_primary", index == 0), + ) + seen_ids.add(location_id) + else: + created = LibraryLocation.objects.create( + library=instance, + path=location.get("path", ""), + include_subdirectories=location.get("include_subdirectories", True), + is_primary=location.get("is_primary", index == 0), + ) + seen_ids.add(created.id) + + missing_ids = existing_ids - seen_ids + if missing_ids: + LibraryLocation.objects.filter(id__in=missing_ids).delete() + + return instance + + +class MediaFileSerializer(serializers.ModelSerializer): + class Meta: + model = MediaFile + fields = [ + "id", + "path", + "relative_path", + "file_name", + "size_bytes", + "modified_at", + "duration_ms", + ] + + +class MediaItemSerializer(serializers.ModelSerializer): + watch_progress = serializers.SerializerMethodField() + watch_summary = serializers.SerializerMethodField() + + class Meta: + model = MediaItem + fields = [ + "id", + "library", + "parent", + "item_type", + "title", + "sort_title", + "normalized_title", + "synopsis", + "tagline", + "release_year", + "rating", + "runtime_ms", + "season_number", + "episode_number", + "genres", + "tags", + "studios", + "cast", + "crew", + "poster_url", + "backdrop_url", + "movie_db_id", + "imdb_id", + "metadata_source", + "metadata_last_synced_at", + "first_imported_at", + "updated_at", + "watch_progress", + "watch_summary", + ] + + def _progress_payload(self, progress: WatchProgress | None, item: MediaItem): + if not progress: + return None + duration = progress.duration_ms or item.runtime_ms + percentage = None + completed = progress.completed + if duration: + percentage = min(1, max(0, progress.position_ms / duration)) + if percentage >= 0.95: + completed = True + return { + "id": progress.id, + "position_ms": progress.position_ms, + "duration_ms": duration, + "percentage": percentage, + "completed": completed, + "last_watched_at": progress.last_watched_at, + } + + def _summary_for_progress(self, progress: WatchProgress | None, item: MediaItem): + if not progress: + return {"status": "unwatched"} + duration = progress.duration_ms or item.runtime_ms or 0 + completed = progress.completed + if duration and progress.position_ms / max(duration, 1) >= 0.95: + completed = True + if completed: + status = "watched" + elif progress.position_ms > 0: + status = "in_progress" + else: + status = "unwatched" + return { + "status": status, + "position_ms": progress.position_ms, + "duration_ms": duration, + "completed": completed, + } + + def _summary_for_show(self, item: MediaItem, user): + episodes = MediaItem.objects.filter( + parent=item, item_type=MediaItem.TYPE_EPISODE + ).order_by("season_number", "episode_number", "id") + total = episodes.count() + if total == 0: + return { + "status": "unwatched", + "total_episodes": 0, + "completed_episodes": 0, + } + + progress_map = { + entry.media_item_id: entry + for entry in WatchProgress.objects.filter(user=user, media_item__in=episodes) + } + + completed_episodes = 0 + resume_episode_id = None + next_episode_id = None + + for episode in episodes: + progress = progress_map.get(episode.id) + if progress: + duration = progress.duration_ms or episode.runtime_ms or 0 + percent = progress.position_ms / max(duration, 1) if duration else 0 + completed = progress.completed or percent >= 0.95 + if completed: + completed_episodes += 1 + elif progress.position_ms > 0 and resume_episode_id is None: + resume_episode_id = episode.id + if next_episode_id is None: + if not progress: + next_episode_id = episode.id + else: + duration = progress.duration_ms or episode.runtime_ms or 0 + percent = progress.position_ms / max(duration, 1) if duration else 0 + completed = progress.completed or percent >= 0.95 + if not completed: + next_episode_id = episode.id + + if completed_episodes == total: + status = "watched" + elif completed_episodes > 0 or resume_episode_id: + status = "in_progress" + else: + status = "unwatched" + + return { + "status": status, + "total_episodes": total, + "completed_episodes": completed_episodes, + "resume_episode_id": resume_episode_id, + "next_episode_id": next_episode_id, + } + + def get_watch_progress(self, obj: MediaItem): + request = self.context.get("request") + user = getattr(request, "user", None) + if not user or not user.is_authenticated: + return None + progress = WatchProgress.objects.filter(user=user, media_item=obj).first() + return self._progress_payload(progress, obj) + + def get_watch_summary(self, obj: MediaItem): + request = self.context.get("request") + user = getattr(request, "user", None) + if not user or not user.is_authenticated: + return None + if obj.item_type == MediaItem.TYPE_SHOW: + return self._summary_for_show(obj, user) + progress = WatchProgress.objects.filter(user=user, media_item=obj).first() + return self._summary_for_progress(progress, obj) + + +class MediaItemDetailSerializer(MediaItemSerializer): + files = MediaFileSerializer(many=True, read_only=True) + + class Meta(MediaItemSerializer.Meta): + fields = MediaItemSerializer.Meta.fields + ["files", "metadata"] + + +class LibraryScanSerializer(serializers.ModelSerializer): + class Meta: + model = LibraryScan + fields = [ + "id", + "library", + "scan_type", + "status", + "summary", + "stages", + "processed_files", + "total_files", + "new_files", + "updated_files", + "removed_files", + "unmatched_files", + "log", + "extra", + "task_id", + "created_at", + "started_at", + "finished_at", + ] + + +class MediaItemUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = MediaItem + fields = [ + "title", + "synopsis", + "release_year", + "rating", + "genres", + "tags", + "studios", + "movie_db_id", + "imdb_id", + "poster_url", + "backdrop_url", + ] diff --git a/apps/media_library/signals.py b/apps/media_library/signals.py new file mode 100644 index 00000000..ac6236a5 --- /dev/null +++ b/apps/media_library/signals.py @@ -0,0 +1,86 @@ +import json +import logging + +from django.db.models.signals import post_delete, post_save, pre_delete +from django.dispatch import receiver +from django_celery_beat.models import IntervalSchedule, PeriodicTask + +from apps.media_library.models import Library, MediaItem +from apps.media_library.vod import cleanup_library_vod, cleanup_media_item_vod + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=Library) +def create_or_update_scan_task(sender, instance, **kwargs): + interval_minutes = int(instance.scan_interval_minutes or 0) + task_name = f"media-library-scan-{instance.id}" + should_be_enabled = instance.auto_scan_enabled and interval_minutes > 0 + + if interval_minutes <= 0: + if should_be_enabled: + logger.warning("Scan interval invalid for library %s", instance.id) + try: + task = PeriodicTask.objects.get(name=task_name) + if task.enabled: + task.enabled = False + task.save(update_fields=["enabled"]) + except PeriodicTask.DoesNotExist: + pass + return + + interval, _ = IntervalSchedule.objects.get_or_create( + every=interval_minutes, period=IntervalSchedule.MINUTES + ) + + kwargs_payload = json.dumps({"library_id": instance.id, "full": False}) + + try: + task = PeriodicTask.objects.get(name=task_name) + updated_fields = [] + + if task.enabled != should_be_enabled: + task.enabled = should_be_enabled + updated_fields.append("enabled") + if task.interval != interval: + task.interval = interval + updated_fields.append("interval") + if task.kwargs != kwargs_payload: + task.kwargs = kwargs_payload + updated_fields.append("kwargs") + + if updated_fields: + task.save(update_fields=updated_fields) + except PeriodicTask.DoesNotExist: + PeriodicTask.objects.create( + name=task_name, + interval=interval, + task="apps.media_library.tasks.scan_library", + kwargs=kwargs_payload, + enabled=should_be_enabled, + ) + + +@receiver(post_delete, sender=Library) +def delete_scan_task(sender, instance, **kwargs): + task_name = f"media-library-scan-{instance.id}" + try: + PeriodicTask.objects.filter(name=task_name).delete() + except Exception as exc: + logger.warning("Failed to delete scan task for library %s: %s", instance.id, exc) + + +@receiver(pre_delete, sender=Library) +def delete_library_vod(sender, instance, **kwargs): + try: + cleanup_library_vod(instance) + except Exception as exc: + logger.warning("Failed to cleanup VOD for library %s: %s", instance.id, exc) + + +@receiver(pre_delete, sender=MediaItem) +def delete_media_item_vod(sender, instance, **kwargs): + try: + cleanup_media_item_vod(instance) + except Exception as exc: + logger.warning("Failed to cleanup VOD for media item %s: %s", instance.id, exc) diff --git a/apps/media_library/tasks.py b/apps/media_library/tasks.py new file mode 100644 index 00000000..4b5da27b --- /dev/null +++ b/apps/media_library/tasks.py @@ -0,0 +1,255 @@ +import logging +from datetime import timedelta + +from celery import shared_task +from django.db.models import Q +from django.utils import timezone + +from apps.media_library.metadata import METADATA_CACHE_TIMEOUT, sync_metadata +from apps.media_library.models import Library, LibraryScan, MediaItem +from apps.media_library.scanner import ScanCancelled, scan_library_files +from apps.media_library.vod import sync_vod_for_media_item + +logger = logging.getLogger(__name__) + +STAGE_DISCOVERY = "discovery" +STAGE_METADATA = "metadata" +STAGE_ARTWORK = "artwork" + + +def _update_stage(scan: LibraryScan, stage_key: str, *, status=None, processed=None, total=None): + stages = scan.stages or {} + stage = stages.get(stage_key, {"status": "pending", "processed": 0, "total": 0}) + if status is not None: + stage["status"] = status + if processed is not None: + stage["processed"] = processed + if total is not None: + stage["total"] = total + stages[stage_key] = stage + scan.stages = stages + scan.save(update_fields=["stages", "updated_at"]) + + +def _filter_metadata_queryset(queryset, *, force: bool): + if force or not METADATA_CACHE_TIMEOUT: + return queryset + cutoff = timezone.now() - timedelta(seconds=METADATA_CACHE_TIMEOUT) + return queryset.filter( + Q(metadata_last_synced_at__isnull=True) + | Q(metadata_last_synced_at__lt=cutoff) + | Q(poster_url__isnull=True) + | Q(poster_url="") + | Q(backdrop_url__isnull=True) + | Q(backdrop_url="") + ) + + +@shared_task(bind=True) +def scan_library(self, library_id: int, *, full: bool = False, scan_id: int | None = None): + library = Library.objects.get(id=library_id) + scan_type = LibraryScan.SCAN_FULL if full else LibraryScan.SCAN_QUICK + + if scan_id: + scan = LibraryScan.objects.get(id=scan_id, library=library) + else: + scan = LibraryScan.objects.create( + library=library, + scan_type=scan_type, + status=LibraryScan.STATUS_QUEUED, + summary="Full scan" if full else "Quick scan", + stages={}, + ) + + scan.task_id = self.request.id + scan.status = LibraryScan.STATUS_RUNNING + scan.started_at = timezone.now() + scan.save(update_fields=["task_id", "status", "started_at", "updated_at"]) + + library.last_scan_at = scan.started_at + library.save(update_fields=["last_scan_at", "updated_at"]) + + _update_stage(scan, STAGE_DISCOVERY, status="running", processed=0, total=0) + _update_stage(scan, STAGE_METADATA, status="pending", processed=0, total=0) + _update_stage(scan, STAGE_ARTWORK, status="pending", processed=0, total=0) + + def cancel_check(): + scan.refresh_from_db(fields=["status"]) + return scan.status == LibraryScan.STATUS_CANCELLED + + def progress_callback(processed: int, total: int): + scan.processed_files = processed + scan.total_files = total + scan.save(update_fields=["processed_files", "total_files", "updated_at"]) + _update_stage( + scan, + STAGE_DISCOVERY, + status="running", + processed=processed, + total=total, + ) + + try: + result = scan_library_files( + library, + full=full, + progress_callback=progress_callback, + cancel_check=cancel_check, + ) + except ScanCancelled: + scan.status = LibraryScan.STATUS_CANCELLED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + _update_stage(scan, STAGE_DISCOVERY, status="cancelled") + _update_stage(scan, STAGE_METADATA, status="skipped") + _update_stage(scan, STAGE_ARTWORK, status="skipped") + return + except Exception: + logger.exception("Library scan failed for %s", library.name) + scan.status = LibraryScan.STATUS_FAILED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + _update_stage(scan, STAGE_DISCOVERY, status="failed") + _update_stage(scan, STAGE_METADATA, status="skipped") + _update_stage(scan, STAGE_ARTWORK, status="skipped") + return + + scan.processed_files = result.processed_files + scan.total_files = result.total_files + scan.new_files = result.new_files + scan.updated_files = result.updated_files + scan.removed_files = result.removed_files + scan.unmatched_files = result.unmatched_files + scan.extra = { + "errors": result.errors, + "unmatched_paths": result.unmatched_paths, + } + scan.save( + update_fields=[ + "processed_files", + "total_files", + "new_files", + "updated_files", + "removed_files", + "unmatched_files", + "extra", + "updated_at", + ] + ) + + _update_stage( + scan, + STAGE_DISCOVERY, + status="completed", + processed=result.processed_files, + total=result.total_files, + ) + + metadata_ids = list(result.metadata_item_ids) + if not metadata_ids: + _update_stage(scan, STAGE_METADATA, status="completed", processed=0, total=0) + _update_stage(scan, STAGE_ARTWORK, status="completed", processed=0, total=0) + scan.status = LibraryScan.STATUS_COMPLETED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + library.last_successful_scan_at = scan.finished_at + library.save(update_fields=["last_successful_scan_at", "updated_at"]) + return + + metadata_qs = MediaItem.objects.filter(id__in=metadata_ids) + metadata_qs = _filter_metadata_queryset(metadata_qs, force=full) + metadata_total = metadata_qs.count() + if not metadata_total: + _update_stage(scan, STAGE_METADATA, status="completed", processed=0, total=0) + _update_stage(scan, STAGE_ARTWORK, status="completed", processed=0, total=0) + scan.status = LibraryScan.STATUS_COMPLETED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + library.last_successful_scan_at = scan.finished_at + library.save(update_fields=["last_successful_scan_at", "updated_at"]) + return + + _update_stage(scan, STAGE_METADATA, status="running", processed=0, total=metadata_total) + _update_stage(scan, STAGE_ARTWORK, status="running", processed=0, total=metadata_total) + + processed = 0 + artwork_processed = 0 + + for media_item in metadata_qs.iterator(): + if cancel_check(): + scan.status = LibraryScan.STATUS_CANCELLED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + _update_stage(scan, STAGE_METADATA, status="cancelled") + _update_stage(scan, STAGE_ARTWORK, status="cancelled") + return + + updated = sync_metadata(media_item, force=full) + try: + sync_vod_for_media_item(media_item) + except Exception: + logger.exception("Failed to sync VOD for media item %s", media_item.id) + processed += 1 + if updated and (updated.poster_url or updated.backdrop_url): + artwork_processed += 1 + + if processed % 25 == 0 or processed == metadata_total: + _update_stage( + scan, + STAGE_METADATA, + status="running", + processed=processed, + total=metadata_total, + ) + _update_stage( + scan, + STAGE_ARTWORK, + status="running", + processed=artwork_processed, + total=metadata_total, + ) + + _update_stage( + scan, + STAGE_METADATA, + status="completed", + processed=processed, + total=metadata_total, + ) + _update_stage( + scan, + STAGE_ARTWORK, + status="completed", + processed=artwork_processed, + total=metadata_total, + ) + + scan.status = LibraryScan.STATUS_COMPLETED + scan.finished_at = timezone.now() + scan.save(update_fields=["status", "finished_at", "updated_at"]) + library.last_successful_scan_at = scan.finished_at + library.save(update_fields=["last_successful_scan_at", "updated_at"]) + + +@shared_task +def refresh_media_item_metadata(item_id: int): + try: + media_item = MediaItem.objects.get(id=item_id) + except MediaItem.DoesNotExist: + return + sync_metadata(media_item, force=True) + try: + sync_vod_for_media_item(media_item) + except Exception: + logger.exception("Failed to sync VOD for media item %s", media_item.id) + + +@shared_task +def refresh_library_metadata(library_id: int): + items = MediaItem.objects.filter(library_id=library_id).iterator() + for item in items: + sync_metadata(item, force=True) + try: + sync_vod_for_media_item(item) + except Exception: + logger.exception("Failed to sync VOD for media item %s", item.id) diff --git a/apps/media_library/utils.py b/apps/media_library/utils.py new file mode 100644 index 00000000..809e5df6 --- /dev/null +++ b/apps/media_library/utils.py @@ -0,0 +1,32 @@ +import re +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class ClassificationResult: + detected_type: str + title: str + year: Optional[int] = None + season: Optional[int] = None + episode: Optional[int] = None + episode_title: Optional[str] = None + data: dict[str, Any] = field(default_factory=dict) + + +def normalize_title(value: str | None) -> str: + if not value: + return "" + normalized = re.sub(r"[^A-Za-z0-9]+", " ", value) + normalized = re.sub(r"\s+", " ", normalized).strip().lower() + return normalized + + +def _json_safe(value: Any) -> Any: + if isinstance(value, dict): + return {key: _json_safe(val) for key, val in value.items()} + if isinstance(value, (list, tuple, set)): + return [_json_safe(val) for val in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) diff --git a/apps/media_library/vod.py b/apps/media_library/vod.py new file mode 100644 index 00000000..8ade7831 --- /dev/null +++ b/apps/media_library/vod.py @@ -0,0 +1,416 @@ +import mimetypes +import os +from typing import Optional + +from apps.media_library.models import Library, MediaFile, MediaItem, MediaItemVODLink +from apps.m3u.models import M3UAccount +from apps.vod.models import ( + Episode, + M3UEpisodeRelation, + M3UMovieRelation, + M3USeriesRelation, + Movie, + Series, + VODCategory, + VODLogo, +) + +LIBRARY_ACCOUNT_PREFIX = "Media Library" +LOCAL_TMDB_PREFIX = "ml" +UNCATEGORIZED_NAME = "Uncategorized" + + +def _account_name(library: Library) -> str: + return f"{LIBRARY_ACCOUNT_PREFIX} {library.id}: {library.name}" + + +def _local_tmdb_id(media_item: MediaItem) -> str: + return f"{LOCAL_TMDB_PREFIX}:{media_item.id}" + + +def _ensure_uncategorized(category_type: str) -> VODCategory: + category, _ = VODCategory.objects.get_or_create( + name=UNCATEGORIZED_NAME, category_type=category_type + ) + return category + + +def _merge_custom_properties(existing: Optional[dict], updates: dict) -> dict: + merged = dict(existing or {}) + for key, value in updates.items(): + if value in (None, "", [], {}): + continue + merged[key] = value + return merged + + +def _get_primary_file(media_item: MediaItem) -> Optional[MediaFile]: + return ( + media_item.files.filter(is_primary=True).first() + or media_item.files.first() + ) + + +def ensure_library_vod_account(library: Library) -> M3UAccount: + desired_name = _account_name(library) + if library.vod_account_id: + account = library.vod_account + if account and account.name != desired_name: + account.name = desired_name + account.save(update_fields=["name"]) + return account + + account = ( + M3UAccount.objects.filter(custom_properties__media_library_id=library.id).first() + ) + if account: + if account.name != desired_name: + account.name = desired_name + account.save(update_fields=["name"]) + library.vod_account = account + library.save(update_fields=["vod_account"]) + return account + + account = M3UAccount.objects.create( + name=desired_name, + account_type=M3UAccount.Types.STADNARD, + is_active=library.add_to_vod, + locked=True, + priority=0, + custom_properties={ + "media_library_id": library.id, + "source": "media-library", + }, + ) + library.vod_account = account + library.save(update_fields=["vod_account"]) + return account + + +def sync_library_vod_account_state(library: Library) -> M3UAccount: + account = ensure_library_vod_account(library) + if account.is_active != library.add_to_vod: + account.is_active = library.add_to_vod + account.save(update_fields=["is_active"]) + return account + + +def _ensure_logo(media_item: MediaItem) -> Optional[VODLogo]: + if not media_item.poster_url: + return None + logo, _ = VODLogo.objects.get_or_create( + url=media_item.poster_url, defaults={"name": media_item.title} + ) + return logo + + +def _runtime_secs(media_item: MediaItem) -> Optional[int]: + if media_item.runtime_ms: + return int(media_item.runtime_ms / 1000) + return None + + +def _genre_string(media_item: MediaItem) -> str: + if isinstance(media_item.genres, list) and media_item.genres: + return ", ".join([str(entry) for entry in media_item.genres if entry]) + return "" + + +def _file_container_extension(file: Optional[MediaFile]) -> Optional[str]: + if not file or not file.file_name: + return None + _base, ext = os.path.splitext(file.file_name) + return ext[1:].lower() if ext else None + + +def _relation_payload(media_item: MediaItem, file: Optional[MediaFile]) -> dict: + return _merge_custom_properties( + {}, + { + "source": "media-library", + "library_id": media_item.library_id, + "media_item_id": media_item.id, + "file_id": file.id if file else None, + "file_path": file.path if file else None, + "file_name": file.file_name if file else None, + "file_size_bytes": file.size_bytes if file else None, + "file_mime": mimetypes.guess_type(file.path)[0] if file else None, + }, + ) + + +def sync_vod_for_media_item(media_item: MediaItem) -> None: + library = media_item.library + account = sync_library_vod_account_state(library) + link, _ = MediaItemVODLink.objects.get_or_create(media_item=media_item) + + if media_item.item_type == MediaItem.TYPE_MOVIE: + _sync_movie(media_item, account, link) + elif media_item.item_type == MediaItem.TYPE_SHOW: + _sync_series(media_item, account, link) + elif media_item.item_type == MediaItem.TYPE_EPISODE: + _sync_episode(media_item, account, link) + + +def _sync_movie(media_item: MediaItem, account: M3UAccount, link: MediaItemVODLink) -> None: + logo = _ensure_logo(media_item) + custom_properties = _merge_custom_properties( + {}, + { + "source": "media-library", + "movie_db_id": media_item.movie_db_id, + "imdb_id": media_item.imdb_id, + "poster_url": media_item.poster_url, + "backdrop_url": media_item.backdrop_url, + "tags": media_item.tags, + "studios": media_item.studios, + }, + ) + + movie = link.vod_movie + if not movie: + movie = Movie.objects.filter(tmdb_id=_local_tmdb_id(media_item)).first() + if not movie: + movie = Movie.objects.create( + name=media_item.title, + description=media_item.synopsis or "", + year=media_item.release_year, + rating=media_item.rating or "", + genre=_genre_string(media_item), + duration_secs=_runtime_secs(media_item), + tmdb_id=_local_tmdb_id(media_item), + custom_properties=custom_properties, + logo=logo, + ) + else: + movie.name = media_item.title + movie.description = media_item.synopsis or "" + movie.year = media_item.release_year + movie.rating = media_item.rating or "" + movie.genre = _genre_string(media_item) + movie.duration_secs = _runtime_secs(media_item) + movie.custom_properties = _merge_custom_properties(movie.custom_properties, custom_properties) + if logo: + movie.logo = logo + movie.save() + + link.vod_movie = movie + link.vod_series = None + link.vod_episode = None + link.save(update_fields=["vod_movie", "vod_series", "vod_episode", "updated_at"]) + + file = _get_primary_file(media_item) + if not file: + return + + category = _ensure_uncategorized("movie") + M3UMovieRelation.objects.update_or_create( + m3u_account=account, + movie=movie, + defaults={ + "category": category, + "stream_id": str(media_item.id), + "container_extension": _file_container_extension(file), + "custom_properties": _relation_payload(media_item, file), + }, + ) + + +def _sync_series(media_item: MediaItem, account: M3UAccount, link: MediaItemVODLink) -> None: + logo = _ensure_logo(media_item) + custom_properties = _merge_custom_properties( + {}, + { + "source": "media-library", + "movie_db_id": media_item.movie_db_id, + "imdb_id": media_item.imdb_id, + "poster_url": media_item.poster_url, + "backdrop_url": media_item.backdrop_url, + "tags": media_item.tags, + "studios": media_item.studios, + }, + ) + + series = link.vod_series + if not series: + series = Series.objects.filter(tmdb_id=_local_tmdb_id(media_item)).first() + if not series: + series = Series.objects.create( + name=media_item.title, + description=media_item.synopsis or "", + year=media_item.release_year, + rating=media_item.rating or "", + genre=_genre_string(media_item), + tmdb_id=_local_tmdb_id(media_item), + custom_properties=custom_properties, + logo=logo, + ) + else: + series.name = media_item.title + series.description = media_item.synopsis or "" + series.year = media_item.release_year + series.rating = media_item.rating or "" + series.genre = _genre_string(media_item) + series.custom_properties = _merge_custom_properties(series.custom_properties, custom_properties) + if logo: + series.logo = logo + series.save() + + link.vod_series = series + link.vod_movie = None + link.vod_episode = None + link.save(update_fields=["vod_series", "vod_movie", "vod_episode", "updated_at"]) + + category = _ensure_uncategorized("series") + M3USeriesRelation.objects.update_or_create( + m3u_account=account, + series=series, + defaults={ + "category": category, + "external_series_id": str(media_item.id), + "custom_properties": _merge_custom_properties( + {}, + { + "source": "media-library", + "library_id": media_item.library_id, + "media_item_id": media_item.id, + }, + ), + }, + ) + + +def _sync_episode(media_item: MediaItem, account: M3UAccount, link: MediaItemVODLink) -> None: + if not media_item.parent_id: + return + parent = media_item.parent + if not parent: + return + + parent_link, _ = MediaItemVODLink.objects.get_or_create(media_item=parent) + if not parent_link.vod_series: + _sync_series(parent, account, parent_link) + parent_link.refresh_from_db(fields=["vod_series"]) + + if not parent_link.vod_series: + return + + logo = _ensure_logo(media_item) or _ensure_logo(parent) + custom_properties = _merge_custom_properties( + {}, + { + "source": "media-library", + "movie_db_id": media_item.movie_db_id, + "imdb_id": media_item.imdb_id, + "poster_url": media_item.poster_url, + "backdrop_url": media_item.backdrop_url, + "tags": media_item.tags, + "studios": media_item.studios, + }, + ) + + episode = link.vod_episode + if not episode: + episode = Episode.objects.filter(tmdb_id=_local_tmdb_id(media_item)).first() + if not episode: + episode = Episode.objects.create( + name=media_item.title, + description=media_item.synopsis or "", + season_number=media_item.season_number, + episode_number=media_item.episode_number, + series=parent_link.vod_series, + tmdb_id=_local_tmdb_id(media_item), + custom_properties=custom_properties, + duration_secs=_runtime_secs(media_item), + ) + else: + episode.name = media_item.title + episode.description = media_item.synopsis or "" + episode.season_number = media_item.season_number + episode.episode_number = media_item.episode_number + episode.duration_secs = _runtime_secs(media_item) + episode.custom_properties = _merge_custom_properties(episode.custom_properties, custom_properties) + episode.save() + + link.vod_episode = episode + link.vod_movie = None + link.vod_series = None + link.save(update_fields=["vod_episode", "vod_movie", "vod_series", "updated_at"]) + + file = _get_primary_file(media_item) + if not file: + return + + M3UEpisodeRelation.objects.update_or_create( + m3u_account=account, + episode=episode, + defaults={ + "stream_id": str(media_item.id), + "container_extension": _file_container_extension(file), + "custom_properties": _relation_payload(media_item, file), + }, + ) + + +def cleanup_library_vod(library: Library) -> None: + if not library: + return + + link_qs = MediaItemVODLink.objects.filter(media_item__library=library) + movie_ids = set(link_qs.exclude(vod_movie__isnull=True).values_list("vod_movie_id", flat=True)) + series_ids = set(link_qs.exclude(vod_series__isnull=True).values_list("vod_series_id", flat=True)) + episode_ids = set(link_qs.exclude(vod_episode__isnull=True).values_list("vod_episode_id", flat=True)) + + account = library.vod_account + if account: + account.delete() + + if episode_ids: + Episode.objects.filter(id__in=episode_ids, m3u_relations__isnull=True).delete() + + if series_ids: + Series.objects.filter(id__in=series_ids, m3u_relations__isnull=True, episodes__isnull=True).delete() + + if movie_ids: + Movie.objects.filter(id__in=movie_ids, m3u_relations__isnull=True).delete() + + +def cleanup_media_item_vod(media_item: MediaItem) -> None: + if not media_item: + return + link = getattr(media_item, "vod_link", None) + if not link: + return + + account_id = None + try: + account_id = media_item.library.vod_account_id + except Exception: + account_id = None + + movie_id = link.vod_movie_id + series_id = link.vod_series_id + episode_id = link.vod_episode_id + + if account_id: + if movie_id: + M3UMovieRelation.objects.filter( + m3u_account_id=account_id, movie_id=movie_id + ).delete() + if series_id: + M3USeriesRelation.objects.filter( + m3u_account_id=account_id, series_id=series_id + ).delete() + if episode_id: + M3UEpisodeRelation.objects.filter( + m3u_account_id=account_id, episode_id=episode_id + ).delete() + + link.delete() + + if episode_id: + Episode.objects.filter(id=episode_id, m3u_relations__isnull=True).delete() + if series_id: + Series.objects.filter(id=series_id, m3u_relations__isnull=True, episodes__isnull=True).delete() + if movie_id: + Movie.objects.filter(id=movie_id, m3u_relations__isnull=True).delete() diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 2ec95cc3..da17b045 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -6,6 +6,8 @@ Supports M3U profiles for authentication and URL transformation. import time import random import logging +import mimetypes +import os import requests from django.http import StreamingHttpResponse, JsonResponse, Http404, HttpResponse from django.shortcuts import get_object_or_404 @@ -145,6 +147,7 @@ class VODStreamView(View): # Extract preferred M3U account ID and stream ID from query parameters preferred_m3u_account_id = request.GET.get('m3u_account_id') preferred_stream_id = request.GET.get('stream_id') + include_inactive = request.GET.get('include_inactive') in ('1', 'true', 'yes') if preferred_m3u_account_id: try: @@ -157,7 +160,13 @@ class VODStreamView(View): logger.info(f"[VOD-PARAM] Preferred stream ID: {preferred_stream_id}") # Get the content object and its relation - content_obj, relation = self._get_content_and_relation(content_type, content_id, preferred_m3u_account_id, preferred_stream_id) + content_obj, relation = self._get_content_and_relation( + content_type, + content_id, + preferred_m3u_account_id, + preferred_stream_id, + include_inactive=include_inactive, + ) if not content_obj or not relation: logger.error(f"[VOD-ERROR] Content or relation not found: {content_type} {content_id}") raise Http404(f"Content not found: {content_type} {content_id}") @@ -168,6 +177,10 @@ class VODStreamView(View): m3u_account = relation.m3u_account logger.info(f"[VOD-ACCOUNT] Using M3U account: {m3u_account.name}") + local_path = self._get_local_file_path(relation) + if local_path: + return self._stream_local_file(request, local_path, relation) + # Get stream URL from relation stream_url = self._get_stream_url_from_relation(relation) logger.info(f"[VOD-CONTENT] Content URL: {stream_url or 'No URL found'}") @@ -268,11 +281,21 @@ class VODStreamView(View): logger.info(f"[VOD-HEAD] Preferred stream ID: {preferred_stream_id}") # Get content and relation (same as GET) - content_obj, relation = self._get_content_and_relation(content_type, content_id, preferred_m3u_account_id, preferred_stream_id) + content_obj, relation = self._get_content_and_relation( + content_type, + content_id, + preferred_m3u_account_id, + preferred_stream_id, + include_inactive=include_inactive, + ) if not content_obj or not relation: logger.error(f"[VOD-HEAD] Content or relation not found: {content_type} {content_id}") return HttpResponse("Content not found", status=404) + local_path = self._get_local_file_path(relation) + if local_path: + return self._head_local_file(local_path, relation, session_url, session_id) + # Get M3U account and stream URL m3u_account = relation.m3u_account stream_url = self._get_stream_url_from_relation(relation) @@ -385,7 +408,14 @@ class VODStreamView(View): logger.error(f"[VOD-HEAD] Error in HEAD request: {e}", exc_info=True) return HttpResponse(f"HEAD error: {str(e)}", status=500) - def _get_content_and_relation(self, content_type, content_id, preferred_m3u_account_id=None, preferred_stream_id=None): + def _get_content_and_relation( + self, + content_type, + content_id, + preferred_m3u_account_id=None, + preferred_stream_id=None, + include_inactive=False, + ): """Get the content object and its M3U relation""" try: logger.info(f"[CONTENT-LOOKUP] Looking up {content_type} with UUID {content_id}") @@ -399,7 +429,9 @@ class VODStreamView(View): logger.info(f"[CONTENT-FOUND] Movie: {content_obj.name} (ID: {content_obj.id})") # Filter by preferred stream ID first (most specific) - relations_query = content_obj.m3u_relations.filter(m3u_account__is_active=True) + relations_query = content_obj.m3u_relations.all() + if not include_inactive: + relations_query = relations_query.filter(m3u_account__is_active=True) if preferred_stream_id: specific_relation = relations_query.filter(stream_id=preferred_stream_id).first() if specific_relation: @@ -430,7 +462,9 @@ class VODStreamView(View): logger.info(f"[CONTENT-FOUND] Episode: {content_obj.name} (ID: {content_obj.id}, Series: {content_obj.series.name})") # Filter by preferred stream ID first (most specific) - relations_query = content_obj.m3u_relations.filter(m3u_account__is_active=True) + relations_query = content_obj.m3u_relations.all() + if not include_inactive: + relations_query = relations_query.filter(m3u_account__is_active=True) if preferred_stream_id: specific_relation = relations_query.filter(stream_id=preferred_stream_id).first() if specific_relation: @@ -468,7 +502,9 @@ class VODStreamView(View): logger.info(f"[CONTENT-FOUND] First episode: {episode.name} (ID: {episode.id})") # Filter by preferred stream ID first (most specific) - relations_query = episode.m3u_relations.filter(m3u_account__is_active=True) + relations_query = episode.m3u_relations.all() + if not include_inactive: + relations_query = relations_query.filter(m3u_account__is_active=True) if preferred_stream_id: specific_relation = relations_query.filter(stream_id=preferred_stream_id).first() if specific_relation: @@ -524,6 +560,86 @@ class VODStreamView(View): logger.error(f"[VOD-URL] Error getting stream URL from relation: {e}", exc_info=True) return None + def _get_local_file_path(self, relation): + props = getattr(relation, "custom_properties", None) + if not isinstance(props, dict): + return None + path = props.get("file_path") + if not path: + return None + if not os.path.exists(path): + logger.warning("[VOD-LOCAL] File not found at %s", path) + return None + return path + + def _stream_local_file(self, request, file_path, relation): + props = getattr(relation, "custom_properties", {}) if relation else {} + file_name = None + if isinstance(props, dict): + file_name = props.get("file_name") + file_name = file_name or os.path.basename(file_path) + + content_type = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + file_size = os.path.getsize(file_path) + range_header = request.META.get("HTTP_RANGE", "").strip() + + def file_iterator(path, start=0, end=None, chunk_size=8192): + with open(path, "rb") as handle: + handle.seek(start) + remaining = (end - start + 1) if end is not None else None + while True: + if remaining is not None and remaining <= 0: + break + bytes_to_read = min(chunk_size, remaining) if remaining is not None else chunk_size + data = handle.read(bytes_to_read) + if not data: + break + if remaining is not None: + remaining -= len(data) + yield data + + if range_header.startswith("bytes="): + try: + range_spec = range_header.split("=", 1)[1] + start_str, end_str = range_spec.split("-", 1) + start = int(start_str) if start_str else 0 + end = int(end_str) if end_str else file_size - 1 + start = max(0, start) + end = min(file_size - 1, end) + length = end - start + 1 + + resp = StreamingHttpResponse( + file_iterator(file_path, start, end), + status=206, + content_type=content_type, + ) + resp["Content-Range"] = f"bytes {start}-{end}/{file_size}" + resp["Content-Length"] = str(length) + resp["Accept-Ranges"] = "bytes" + resp["Content-Disposition"] = f'inline; filename="{file_name}"' + return resp + except Exception as exc: + logger.warning("[VOD-LOCAL] Range parse failed: %s", exc) + + response = FileResponse(open(file_path, "rb"), content_type=content_type) + response["Content-Length"] = str(file_size) + response["Accept-Ranges"] = "bytes" + response["Content-Disposition"] = f'inline; filename="{file_name}"' + return response + + def _head_local_file(self, file_path, relation, session_url, session_id): + if not os.path.exists(file_path): + return HttpResponse("Content not found", status=404) + content_type = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + file_size = os.path.getsize(file_path) + response = HttpResponse() + response["Content-Length"] = str(file_size) + response["Content-Type"] = content_type + response["Accept-Ranges"] = "bytes" + response["X-Session-URL"] = session_url + response["X-Dispatcharr-Session"] = session_id + return response + def _get_m3u_profile(self, m3u_account, profile_id, session_id=None): """Get appropriate M3U profile for streaming using Redis-based viewer counts diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 1a9a1a44..ce6d9710 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -30,6 +30,7 @@ INSTALLED_APPS = [ "apps.proxy.apps.ProxyConfig", "apps.proxy.ts_proxy", "apps.vod.apps.VODConfig", + "apps.media_library.apps.MediaLibraryConfig", "core", "daphne", "drf_yasg", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 3c7c3877..2e72298f 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -18,6 +18,7 @@ import PluginsPage from './pages/Plugins'; import Users from './pages/Users'; import LogosPage from './pages/Logos'; import VODsPage from './pages/VODs'; +import LibraryPage from './pages/Library'; import useAuthStore from './store/auth'; import useLogosStore from './store/logos'; import FloatingVideo from './components/FloatingVideo'; @@ -153,6 +154,7 @@ const App = () => { } /> } /> } /> + } /> ) : ( } /> diff --git a/frontend/src/api.js b/frontend/src/api.js index 64ce4d77..bf118e46 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -2545,6 +2545,297 @@ export default class API { } } + // Media Library Methods + static async getLibraries() { + try { + const response = await request(`${host}/api/media-library/libraries/`); + return response.results || response; + } catch (e) { + errorNotification('Failed to retrieve libraries', e); + } + } + + static async createLibrary(payload) { + try { + const response = await request(`${host}/api/media-library/libraries/`, { + method: 'POST', + body: payload, + }); + return response; + } catch (e) { + errorNotification('Failed to create library', e); + } + } + + static async updateLibrary(id, payload) { + try { + const response = await request(`${host}/api/media-library/libraries/${id}/`, { + method: 'PATCH', + body: payload, + }); + return response; + } catch (e) { + errorNotification('Failed to update library', e); + } + } + + static async deleteLibrary(id) { + try { + await request(`${host}/api/media-library/libraries/${id}/`, { + method: 'DELETE', + }); + return true; + } catch (e) { + errorNotification('Failed to delete library', e); + } + } + + static async browseLibraryPath(path = '') { + try { + const params = new URLSearchParams(); + if (path) { + params.append('path', path); + } + const response = await request( + `${host}/api/media-library/browse/?${params.toString()}` + ); + return response; + } catch (e) { + errorNotification('Failed to browse library path', e); + } + } + + static async getLibraryScans(libraryId, params = new URLSearchParams()) { + try { + if (libraryId) { + params.append('library', libraryId); + } + const response = await request( + `${host}/api/media-library/scans/?${params.toString()}` + ); + return response.results || response; + } catch (e) { + errorNotification('Failed to retrieve library scans', e); + } + } + + static async triggerLibraryScan(libraryId, { full = false } = {}) { + try { + const response = await request( + `${host}/api/media-library/libraries/${libraryId}/scan/`, + { + method: 'POST', + body: { full }, + } + ); + return response; + } catch (e) { + errorNotification('Failed to start library scan', e); + } + } + + static async cancelLibraryScan(scanId) { + try { + const response = await request( + `${host}/api/media-library/scans/${scanId}/cancel/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to cancel library scan', e); + } + } + + static async deleteLibraryScan(scanId) { + try { + await request(`${host}/api/media-library/scans/${scanId}/`, { + method: 'DELETE', + }); + return true; + } catch (e) { + errorNotification('Failed to delete library scan', e); + } + } + + static async purgeLibraryScans(libraryId = null) { + try { + const params = new URLSearchParams(); + if (libraryId) { + params.append('library', libraryId); + } + const response = await request( + `${host}/api/media-library/scans/purge/?${params.toString()}`, + { + method: 'DELETE', + } + ); + return response; + } catch (e) { + errorNotification('Failed to purge library scans', e); + } + } + + static async getMediaItems(params = new URLSearchParams()) { + try { + const response = await request( + `${host}/api/media-library/items/?${params.toString()}` + ); + return response.results || response; + } catch (e) { + errorNotification('Failed to retrieve media items', e); + } + } + + static async getMediaItem(id, options = {}) { + try { + const response = await request(`${host}/api/media-library/items/${id}/`); + return response; + } catch (e) { + if (!options.suppressErrorNotification) { + errorNotification('Failed to retrieve media item', e); + } + throw e; + } + } + + static async updateMediaItem(id, payload) { + try { + const response = await request(`${host}/api/media-library/items/${id}/`, { + method: 'PATCH', + body: payload, + }); + return response; + } catch (e) { + errorNotification('Failed to update media item', e); + } + } + + static async deleteMediaItem(id) { + try { + await request(`${host}/api/media-library/items/${id}/`, { + method: 'DELETE', + }); + return true; + } catch (e) { + errorNotification('Failed to delete media item', e); + } + } + + static async getMediaItemEpisodes(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/episodes/` + ); + return response; + } catch (e) { + errorNotification('Failed to retrieve episodes', e); + } + } + + static async markMediaItemWatched(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/mark-watched/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to mark item watched', e); + } + } + + static async updateMediaItemProgress(id, payload = {}, options = {}) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/progress/`, + { + method: 'POST', + body: payload, + } + ); + return response; + } catch (e) { + if (!options.suppressErrorNotification) { + errorNotification('Failed to update media progress', e); + } + } + } + + static async clearMediaItemProgress(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/clear-progress/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to clear media progress', e); + } + } + + static async markSeriesWatched(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/series/mark-watched/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to mark series watched', e); + } + } + + static async markSeriesUnwatched(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/series/clear-progress/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to clear series progress', e); + } + } + + static async refreshMediaItemMetadata(id) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/refresh-metadata/`, + { + method: 'POST', + } + ); + return response; + } catch (e) { + errorNotification('Failed to refresh metadata', e); + } + } + + static async streamMediaItem(id, payload = {}) { + try { + const response = await request( + `${host}/api/media-library/items/${id}/stream/`, + { + method: 'POST', + body: payload, + } + ); + return response; + } catch (e) { + errorNotification('Failed to start playback', e); + } + } + // VOD Methods static async getMovies(params = new URLSearchParams()) { try { diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 557767ed..ccaf49c8 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -4,6 +4,8 @@ import Draggable from 'react-draggable'; import useVideoStore from '../store/useVideoStore'; import mpegts from 'mpegts.js'; import { CloseButton, Flex, Loader, Text, Box } from '@mantine/core'; +import API from '../api'; +import useMediaLibraryStore from '../store/mediaLibrary'; export default function FloatingVideo() { const isVisible = useVideoStore((s) => s.isVisible); @@ -14,6 +16,10 @@ export default function FloatingVideo() { const videoRef = useRef(null); const playerRef = useRef(null); const videoContainerRef = useRef(null); + const playbackMetaRef = useRef({ contentType: null, metadata: null }); + const progressStateRef = useRef({ lastSentAt: 0, lastPositionMs: 0 }); + const progressInFlightRef = useRef(false); + const resumeAppliedRef = useRef(false); const [isLoading, setIsLoading] = useState(false); const [loadError, setLoadError] = useState(null); const [showOverlay, setShowOverlay] = useState(true); @@ -105,6 +111,82 @@ export default function FloatingVideo() { }, }, ]; + const isVod = contentType === 'vod' || contentType === 'library'; + + useEffect(() => { + playbackMetaRef.current = { contentType, metadata }; + }, [contentType, metadata]); + + const reportLibraryProgress = useCallback( + async ({ force = false, completed = false } = {}) => { + const metaState = playbackMetaRef.current; + if (!metaState || metaState.contentType !== 'library') { + return; + } + const meta = metaState.metadata || {}; + const mediaItemId = meta.mediaItemId; + if (!mediaItemId || !videoRef.current) { + return; + } + + const video = videoRef.current; + const positionMs = Math.max(0, Math.floor((video.currentTime || 0) * 1000)); + const durationMs = meta.durationMs + ? Math.floor(meta.durationMs) + : video.duration + ? Math.floor(video.duration * 1000) + : null; + + if (!force) { + const now = Date.now(); + const last = progressStateRef.current; + const timeDelta = now - last.lastSentAt; + const positionDelta = Math.abs(positionMs - last.lastPositionMs); + if (timeDelta < 10000 && positionDelta < 5000) { + return; + } + if (positionMs <= 0 && !completed) { + return; + } + progressStateRef.current = { + lastSentAt: now, + lastPositionMs: positionMs, + }; + } else { + progressStateRef.current = { + lastSentAt: Date.now(), + lastPositionMs: positionMs, + }; + } + + if (progressInFlightRef.current && !force) { + return; + } + progressInFlightRef.current = true; + try { + const response = await API.updateMediaItemProgress( + mediaItemId, + { + position_ms: positionMs, + duration_ms: durationMs, + completed, + file_id: meta.fileId || null, + }, + { suppressErrorNotification: true } + ); + if (response?.id) { + const store = useMediaLibraryStore.getState(); + store.upsertItems([response]); + store.setActiveProgress(response.watch_progress || null); + } + } catch (error) { + // Progress updates should not interrupt playback. + } finally { + progressInFlightRef.current = false; + } + }, + [] + ); // Safely destroy the mpegts player to prevent errors const safeDestroyPlayer = () => { @@ -170,16 +252,48 @@ export default function FloatingVideo() { console.log('Initializing VOD player for:', streamUrl); const video = videoRef.current; + resumeAppliedRef.current = false; // Enhanced video element configuration for VOD video.preload = 'metadata'; video.crossOrigin = 'anonymous'; + const applyResumeSeek = () => { + if (resumeAppliedRef.current) return; + const meta = playbackMetaRef.current?.metadata || {}; + const resumePositionMs = Number(meta.resumePositionMs || 0); + const resumeHandledByServer = Boolean(meta.resumeHandledByServer); + if (!resumePositionMs || resumeHandledByServer) { + resumeAppliedRef.current = true; + return; + } + const resumeSeconds = Math.max(0, resumePositionMs / 1000); + if (!Number.isFinite(resumeSeconds) || resumeSeconds <= 0) { + resumeAppliedRef.current = true; + return; + } + const duration = Number.isFinite(video.duration) ? video.duration : null; + const safeSeconds = + duration && duration > 1 + ? Math.min(resumeSeconds, Math.max(0, duration - 1)) + : resumeSeconds; + try { + video.currentTime = safeSeconds; + resumeAppliedRef.current = true; + } catch (error) { + // Keep false so we can retry on the next readiness event. + } + }; + // Set up event listeners const handleLoadStart = () => setIsLoading(true); + const handleLoadedMetadata = () => { + applyResumeSeek(); + }; const handleLoadedData = () => setIsLoading(false); const handleCanPlay = () => { setIsLoading(false); + applyResumeSeek(); // Auto-play for VOD content video.play().catch((e) => { console.log('Auto-play prevented:', e); @@ -188,6 +302,15 @@ export default function FloatingVideo() { // Start overlay timer when video is ready startOverlayTimer(); }; + const handleTimeUpdate = () => { + reportLibraryProgress(); + }; + const handlePause = () => { + reportLibraryProgress({ force: true }); + }; + const handleEnded = () => { + reportLibraryProgress({ force: true, completed: true }); + }; const handleError = (e) => { setIsLoading(false); const error = e.target.error; @@ -229,10 +352,14 @@ export default function FloatingVideo() { // Add event listeners video.addEventListener('loadstart', handleLoadStart); + video.addEventListener('loadedmetadata', handleLoadedMetadata); video.addEventListener('loadeddata', handleLoadedData); video.addEventListener('canplay', handleCanPlay); video.addEventListener('error', handleError); video.addEventListener('progress', handleProgress); + video.addEventListener('timeupdate', handleTimeUpdate); + video.addEventListener('pause', handlePause); + video.addEventListener('ended', handleEnded); // Set the source video.src = streamUrl; @@ -242,10 +369,14 @@ export default function FloatingVideo() { playerRef.current = { destroy: () => { video.removeEventListener('loadstart', handleLoadStart); + video.removeEventListener('loadedmetadata', handleLoadedMetadata); video.removeEventListener('loadeddata', handleLoadedData); video.removeEventListener('canplay', handleCanPlay); video.removeEventListener('error', handleError); video.removeEventListener('progress', handleProgress); + video.removeEventListener('timeupdate', handleTimeUpdate); + video.removeEventListener('pause', handlePause); + video.removeEventListener('ended', handleEnded); video.removeAttribute('src'); video.load(); }, @@ -378,7 +509,7 @@ export default function FloatingVideo() { safeDestroyPlayer(); // Initialize the appropriate player based on content type - if (contentType === 'vod') { + if (isVod) { initializeVODPlayer(); } else { initializeLivePlayer(); @@ -388,7 +519,7 @@ export default function FloatingVideo() { return () => { safeDestroyPlayer(); }; - }, [isVisible, streamUrl, contentType]); + }, [isVisible, streamUrl, contentType, isVod]); // Modified hideVideo handler to clean up player first const handleClose = (e) => { @@ -396,6 +527,7 @@ export default function FloatingVideo() { e.stopPropagation(); e.preventDefault(); } + reportLibraryProgress({ force: true }); safeDestroyPlayer(); setTimeout(() => { hideVideo(); @@ -743,7 +875,7 @@ export default function FloatingVideo() { { - if (contentType === 'vod' && !isLoading) { + if (isVod && !isLoading) { setShowOverlay(true); if (overlayTimeoutRef.current) { clearTimeout(overlayTimeoutRef.current); @@ -751,7 +883,7 @@ export default function FloatingVideo() { } }} onMouseLeave={() => { - if (contentType === 'vod' && !isLoading) { + if (isVod && !isLoading) { startOverlayTimer(); } }} @@ -767,19 +899,19 @@ export default function FloatingVideo() { backgroundColor: '#000', borderRadius: '0 0 8px 8px', // Better controls styling for VOD - ...(contentType === 'vod' && { + ...(isVod && { controlsList: 'nodownload', playsInline: true, }), }} // Add poster for VOD if available - {...(contentType === 'vod' && { + {...(isVod && { poster: metadata?.logo?.url, // Use VOD poster if available })} /> {/* VOD title overlay when not loading - auto-hides after 4 seconds */} - {!isLoading && metadata && contentType === 'vod' && showOverlay && ( + {!isLoading && metadata && isVod && showOverlay && ( { path: '/vods', icon: