diff --git a/.dockerignore b/.dockerignore index e0cc78f0..5073af60 100755 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ **/__pycache__ **/.venv +**/venv **/.classpath **/.dockerignore **/.env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8aa06d0d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,116 @@ +name: CI Pipeline + +on: + push: + branches: [ dev ] + pull_request: + branches: [ dev ] + +# Add explicit permissions for the workflow +permissions: + contents: write # For managing releases and pushing tags + packages: write # For publishing to GitHub Container Registry + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + + - name: Check if commit is from GitHub Actions + id: check_actor + run: | + if [[ "${{ github.actor }}" == "github-actions" ]]; then + echo "is_bot=true" >> $GITHUB_OUTPUT + else + echo "is_bot=false" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Increment Build Number + if: steps.check_actor.outputs.is_bot != 'true' + id: increment_build + run: | + python scripts/increment_build.py + BUILD=$(python -c "import version; print(version.__build__)") + echo "build=${BUILD}" >> $GITHUB_OUTPUT + + - name: Commit Build Number Update + if: steps.check_actor.outputs.is_bot != 'true' + run: | + git add version.py + git commit -m "Increment build number to ${{ steps.increment_build.outputs.build }} [skip ci]" + git push + + - name: Extract version info + id: version + run: | + VERSION=$(python -c "import version; print(version.__version__)") + BUILD=$(python -c "import version; print(version.__build__)") + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "build=${BUILD}" >> $GITHUB_OUTPUT + echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Set repository and image metadata + id: meta + run: | + # Get lowercase repository owner + REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "repo_owner=${REPO_OWNER}" >> $GITHUB_OUTPUT + + # Get repository name + REPO_NAME=$(echo "${{ github.repository }}" | cut -d '/' -f 2 | tr '[:upper:]' '[:lower:]') + echo "repo_name=${REPO_NAME}" >> $GITHUB_OUTPUT + + # Determine branch name + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "branch_tag=latest" >> $GITHUB_OUTPUT + echo "is_main=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then + echo "branch_tag=dev" >> $GITHUB_OUTPUT + echo "is_main=false" >> $GITHUB_OUTPUT + else + # For other branches, use the branch name + BRANCH=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///' | sed 's/[^a-zA-Z0-9]/-/g') + echo "branch_tag=${BRANCH}" >> $GITHUB_OUTPUT + echo "is_main=false" >> $GITHUB_OUTPUT + fi + + # Determine if this is from a fork + if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then + echo "is_fork=true" >> $GITHUB_OUTPUT + else + echo "is_fork=false" >> $GITHUB_OUTPUT + fi + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64 # Fast build - amd64 only + tags: | + ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.meta.outputs.branch_tag }} + ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.version.outputs.version }}-${{ steps.version.outputs.build }} + ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.version.outputs.sha_short }} + build-args: | + BRANCH=${{ github.ref_name }} + REPO_URL=https://github.com/${{ github.repository }} + file: ./docker/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6fb7187b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,93 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + version_type: + description: 'Type of version increment' + required: true + default: 'patch' + type: choice + options: + - major + - minor + - patch + +# Add explicit permissions for the workflow +permissions: + contents: write # For managing releases and pushing tags + packages: write # For publishing to GitHub Container Registry + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + + - name: Update Version + id: update_version + run: | + python scripts/bump_version.py ${{ github.event.inputs.version_type }} + NEW_VERSION=$(python -c "import version; print(f'{version.__version__}')") + echo "new_version=${NEW_VERSION}" >> $GITHUB_OUTPUT + + - name: Set lowercase repo owner + id: repo_owner + run: | + REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "lowercase=${REPO_OWNER}" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Commit and Tag + run: | + git add version.py + git commit -m "Release v${{ steps.update_version.outputs.new_version }}" + git tag -a "v${{ steps.update_version.outputs.new_version }}" -m "Release v${{ steps.update_version.outputs.new_version }}" + git push origin main --tags + + - name: Build and Push Release Image + uses: docker/build-push-action@v4 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64, #linux/arm/v7 # Multi-arch support for releases + tags: | + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }} + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest-amd64 + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest-arm64 + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-amd64 + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-arm64 + ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-armv7 + build-args: | + BRANCH=${{ github.ref_name }} + REPO_URL=https://github.com/${{ github.repository }} + file: ./docker/Dockerfile + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: v${{ steps.update_version.outputs.new_version }} + name: Release v${{ steps.update_version.outputs.new_version }} + draft: false + prerelease: false + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index b6631ac0..a9d76412 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store **/__pycache__/ **/.vscode/ +**/venv *.pyc node_modules/ .history/ @@ -10,4 +11,11 @@ docker/Dockerfile DEV static/ data/ .next -next-env.d.ts \ No newline at end of file +next-env.d.ts +media/ +celerybeat-schedule* +dump.rdb +debugpy* +uwsgi.sock +package-lock.json +models \ No newline at end of file diff --git a/apps/channels/admin.py b/apps/channels/admin.py index 302811af..410f101d 100644 --- a/apps/channels/admin.py +++ b/apps/channels/admin.py @@ -26,10 +26,10 @@ class ChannelAdmin(admin.ModelAdmin): 'uuid', 'name', 'channel_group', - 'tvg_name' + 'epg_data' ) list_filter = ('channel_group',) - search_fields = ('id', 'name', 'channel_group__name', 'tvg_name') # Added 'id' + search_fields = ('id', 'name', 'channel_group__name', 'epg_data') # Added 'id' ordering = ('channel_number',) @admin.register(ChannelGroup) diff --git a/apps/channels/api_urls.py b/apps/channels/api_urls.py index b2cf387e..fc02d7e5 100644 --- a/apps/channels/api_urls.py +++ b/apps/channels/api_urls.py @@ -6,6 +6,11 @@ from .api_views import ( ChannelGroupViewSet, BulkDeleteStreamsAPIView, BulkDeleteChannelsAPIView, + LogoViewSet, + ChannelProfileViewSet, + UpdateChannelMembershipAPIView, + BulkUpdateChannelMembershipAPIView, + RecordingViewSet, ) app_name = 'channels' # for DRF routing @@ -14,11 +19,16 @@ router = DefaultRouter() router.register(r'streams', StreamViewSet, basename='stream') router.register(r'groups', ChannelGroupViewSet, basename='channel-group') router.register(r'channels', ChannelViewSet, basename='channel') +router.register(r'logos', LogoViewSet, basename='logo') +router.register(r'profiles', ChannelProfileViewSet, basename='profile') +router.register(r'recordings', RecordingViewSet, basename='recording') urlpatterns = [ # Bulk delete is a single APIView, not a ViewSet path('streams/bulk-delete/', BulkDeleteStreamsAPIView.as_view(), name='bulk_delete_streams'), path('channels/bulk-delete/', BulkDeleteChannelsAPIView.as_view(), name='bulk_delete_channels'), + path('profiles//channels//', UpdateChannelMembershipAPIView.as_view(), name='update_channel_membership'), + path('profiles//channels/bulk-update/', BulkUpdateChannelMembershipAPIView.as_view(), name='bulk_update_channel_membership'), ] urlpatterns += router.urls diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 579d9dfa..7ea7e3aa 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -1,21 +1,42 @@ from rest_framework import viewsets, status from rest_framework.response import Response from rest_framework.views import APIView -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import IsAuthenticated, AllowAny from rest_framework.decorators import action +from rest_framework.parsers import MultiPartParser, FormParser from drf_yasg.utils import swagger_auto_schema from drf_yasg import openapi from django.shortcuts import get_object_or_404 +from django.db import transaction +import os, json, requests -from .models import Stream, Channel, ChannelGroup -from .serializers import StreamSerializer, ChannelSerializer, ChannelGroupSerializer +from .models import Stream, Channel, ChannelGroup, Logo, ChannelProfile, ChannelProfileMembership, Recording +from .serializers import StreamSerializer, ChannelSerializer, ChannelGroupSerializer, LogoSerializer, ChannelProfileMembershipSerializer, BulkChannelProfileMembershipSerializer, ChannelProfileSerializer, RecordingSerializer from .tasks import match_epg_channels import django_filters from django_filters.rest_framework import DjangoFilterBackend from rest_framework.filters import SearchFilter, OrderingFilter +from apps.epg.models import EPGData +from django.db.models import Q +from django.http import StreamingHttpResponse, FileResponse, Http404 + from rest_framework.pagination import PageNumberPagination + +class OrInFilter(django_filters.Filter): + """ + Custom filter that handles the OR condition instead of AND. + """ + def filter(self, queryset, value): + if value: + # Create a Q object for each value and combine them with OR + query = Q() + for val in value.split(','): + query |= Q(**{self.field_name: val}) + return queryset.filter(query) + return queryset + class StreamPagination(PageNumberPagination): page_size = 25 # Default page size page_size_query_param = 'page_size' # Allow clients to specify page size @@ -23,7 +44,7 @@ class StreamPagination(PageNumberPagination): class StreamFilter(django_filters.FilterSet): name = django_filters.CharFilter(lookup_expr='icontains') - channel_group_name = django_filters.CharFilter(field_name="channel_group__name", lookup_expr="icontains") + channel_group_name = OrInFilter(field_name="channel_group__name", lookup_expr="icontains") m3u_account = django_filters.NumberFilter(field_name="m3u_account__id") m3u_account_name = django_filters.CharFilter(field_name="m3u_account__name", lookup_expr="icontains") m3u_account_is_active = django_filters.BooleanFilter(field_name="m3u_account__is_active") @@ -62,7 +83,8 @@ class StreamViewSet(viewsets.ModelViewSet): channel_group = self.request.query_params.get('channel_group') if channel_group: - qs = qs.filter(channel_group__name=channel_group) + group_names = channel_group.split(',') + qs = qs.filter(channel_group__name__in=group_names) return qs @@ -100,17 +122,54 @@ class ChannelGroupViewSet(viewsets.ModelViewSet): # ───────────────────────────────────────────────────────── # 3) Channel Management (CRUD) # ───────────────────────────────────────────────────────── +class ChannelPagination(PageNumberPagination): + page_size = 25 # Default page size + page_size_query_param = 'page_size' # Allow clients to specify page size + max_page_size = 10000 # Prevent excessive page sizes + +class ChannelFilter(django_filters.FilterSet): + name = django_filters.CharFilter(lookup_expr='icontains') + channel_group_name = OrInFilter(field_name="channel_group__name", lookup_expr="icontains") + + class Meta: + model = Channel + fields = ['name', 'channel_group_name',] + class ChannelViewSet(viewsets.ModelViewSet): queryset = Channel.objects.all() serializer_class = ChannelSerializer permission_classes = [IsAuthenticated] + # pagination_class = ChannelPagination - def get_next_available_channel_number(self, starting_from=1): - used_numbers = set(Channel.objects.all().values_list('channel_number', flat=True)) - n = starting_from - while n in used_numbers: - n += 1 - return n + # filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter] + # filterset_class = ChannelFilter + # search_fields = ['name', 'channel_group__name'] + # ordering_fields = ['channel_number', 'name', 'channel_group__name'] + # ordering = ['-channel_number'] + + def get_queryset(self): + qs = super().get_queryset() + + channel_group = self.request.query_params.get('channel_group') + if channel_group: + group_names = channel_group.split(',') + qs = qs.filter(channel_group__name__in=group_names) + + return qs + + @action(detail=False, methods=['get'], url_path='ids') + def get_ids(self, request, *args, **kwargs): + # Get the filtered queryset + queryset = self.get_queryset() + + # Apply filtering, search, and ordering + queryset = self.filter_queryset(queryset) + + # Return only the IDs from the queryset + channel_ids = queryset.values_list('id', flat=True) + + # Return the response with the list of IDs + return Response(list(channel_ids)) @swagger_auto_schema( method='post', @@ -130,9 +189,11 @@ class ChannelViewSet(viewsets.ModelViewSet): ) @action(detail=False, methods=['post'], url_path='assign') def assign(self, request): - channel_order = request.data.get('channel_order', []) - for order, channel_id in enumerate(channel_order, start=1): - Channel.objects.filter(id=channel_id).update(channel_number=order) + with transaction.atomic(): + channel_order = request.data.get('channel_order', []) + for order, channel_id in enumerate(channel_order, start=1): + Channel.objects.filter(id=channel_id).update(channel_number=order) + return Response({"message": "Channels have been auto-assigned!"}, status=status.HTTP_200_OK) @swagger_auto_schema( @@ -168,34 +229,56 @@ class ChannelViewSet(viewsets.ModelViewSet): stream = get_object_or_404(Stream, pk=stream_id) channel_group = stream.channel_group - # Check if client provided a channel_number; if not, auto-assign one. - provided_number = request.data.get('channel_number') - if provided_number is None: - channel_number = self.get_next_available_channel_number() - else: - try: - channel_number = int(provided_number) - except ValueError: - return Response({"error": "channel_number must be an integer."}, status=status.HTTP_400_BAD_REQUEST) - # If the provided number is already used, return an error. - if Channel.objects.filter(channel_number=channel_number).exists(): - return Response( - {"error": f"Channel number {channel_number} is already in use. Please choose a different number."}, - status=status.HTTP_400_BAD_REQUEST - ) - name = request.data.get('name') if name is None: name = stream.name + # Check if client provided a channel_number; if not, auto-assign one. + stream_custom_props = json.loads(stream.custom_properties) if stream.custom_properties else {} + + channel_number = None + if 'tv-chno' in stream_custom_props: + channel_number = int(stream_custom_props['tv-chno']) + elif 'channel-number' in stream_custom_props: + channel_number = int(stream_custom_props['channel-number']) + + if channel_number is None: + provided_number = request.data.get('channel_number') + if provided_number is None: + channel_number = Channel.get_next_available_channel_number() + else: + try: + channel_number = int(provided_number) + except ValueError: + return Response({"error": "channel_number must be an integer."}, status=status.HTTP_400_BAD_REQUEST) + # If the provided number is already used, return an error. + if Channel.objects.filter(channel_number=channel_number).exists(): + return Response( + {"error": f"Channel number {channel_number} is already in use. Please choose a different number."}, + status=status.HTTP_400_BAD_REQUEST + ) + + + channel_data = { 'channel_number': channel_number, 'name': name, 'tvg_id': stream.tvg_id, 'channel_group_id': channel_group.id, - 'logo_url': stream.logo_url, - 'streams': [stream_id] + 'streams': [stream_id], } + + if stream.logo_url: + logo, _ = Logo.objects.get_or_create(url=stream.logo_url, defaults={ + "name": stream.name or stream.tvg_id + }) + channel_data["logo_id"] = logo.id + + # Attempt to find existing EPGs with the same tvg-id + epgs = EPGData.objects.filter(tvg_id=stream.tvg_id) + if epgs: + channel_data["epg_data_id"] = epgs.first().id + serializer = self.get_serializer(data=channel_data) serializer.is_valid(raise_exception=True) channel = serializer.save() @@ -250,6 +333,10 @@ class ChannelViewSet(viewsets.ModelViewSet): used_numbers.add(next_number) return next_number + logos_to_create = [] + channels_to_create = [] + streams_map = [] + logo_map = [] for item in data_list: stream_id = item.get('stream_id') if not all([stream_id]): @@ -262,44 +349,98 @@ class ChannelViewSet(viewsets.ModelViewSet): errors.append({"item": item, "error": str(e)}) continue - channel_group = stream.channel_group - - # Determine channel number: if provided, use it (if free); else auto assign. - provided_number = item.get('channel_number') - if provided_number is None: - channel_number = get_auto_number() - else: - try: - channel_number = int(provided_number) - except ValueError: - errors.append({"item": item, "error": "channel_number must be an integer."}) - continue - if channel_number in used_numbers or Channel.objects.filter(channel_number=channel_number).exists(): - errors.append({"item": item, "error": f"Channel number {channel_number} is already in use."}) - continue - used_numbers.add(channel_number) - name = item.get('name') if name is None: name = stream.name + channel_group = stream.channel_group + + stream_custom_props = json.loads(stream.custom_properties) if stream.custom_properties else {} + + channel_number = None + if 'tv-chno' in stream_custom_props: + channel_number = int(stream_custom_props['tv-chno']) + elif 'channel-number' in stream_custom_props: + channel_number = int(stream_custom_props['channel-number']) + + # Determine channel number: if provided, use it (if free); else auto assign. + if channel_number is None: + provided_number = item.get('channel_number') + if provided_number is None: + channel_number = get_auto_number() + else: + try: + channel_number = int(provided_number) + except ValueError: + errors.append({"item": item, "error": "channel_number must be an integer."}) + continue + if channel_number in used_numbers or Channel.objects.filter(channel_number=channel_number).exists(): + errors.append({"item": item, "error": f"Channel number {channel_number} is already in use."}) + continue + used_numbers.add(channel_number) + channel_data = { "channel_number": channel_number, "name": name, "tvg_id": stream.tvg_id, "channel_group_id": channel_group.id, - "logo_url": stream.logo_url, - "streams": [stream_id], } + + # Attempt to find existing EPGs with the same tvg-id + epgs = EPGData.objects.filter(tvg_id=stream.tvg_id) + if epgs: + channel_data["epg_data_id"] = epgs.first().id + serializer = self.get_serializer(data=channel_data) if serializer.is_valid(): - channel = serializer.save() - channel.streams.add(stream) - created_channels.append(serializer.data) + validated_data = serializer.validated_data + channel = Channel(**validated_data) + channels_to_create.append(channel) + + streams_map.append([stream_id]) + if stream.logo_url: + logos_to_create.append(Logo( + url=stream.logo_url, + name=stream.name or stream.tvg_id, + )) + logo_map.append(stream.logo_url) + else: + logo_map.append(None) + + # channel = serializer.save() + # channel.streams.add(stream) + # created_channels.append(serializer.data) else: errors.append({"item": item, "error": serializer.errors}) - response_data = {"created": created_channels} + if logos_to_create: + Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True) + + channel_logos = {logo.url: logo for logo in Logo.objects.filter(url__in=[url for url in logo_map if url is not None])} + + profiles = ChannelProfile.objects.all() + channel_profile_memberships = [] + if channels_to_create: + with transaction.atomic(): + created_channels = Channel.objects.bulk_create(channels_to_create) + + update = [] + for channel, stream_ids, logo_url in zip(created_channels, streams_map, logo_map): + if logo_url: + channel.logo = channel_logos[logo_url] + update.append(channel) + channel_profile_memberships = channel_profile_memberships + [ + ChannelProfileMembership(channel_profile=profile, channel=channel) + for profile in profiles + ] + + ChannelProfileMembership.objects.bulk_create(channel_profile_memberships) + Channel.objects.bulk_update(update, ['logo']) + + for channel, stream_ids in zip(created_channels, streams_map): + channel.streams.set(stream_ids) + + response_data = {"created": ChannelSerializer(created_channels, many=True).data} if errors: response_data["errors"] = errors @@ -371,3 +512,105 @@ class BulkDeleteChannelsAPIView(APIView): channel_ids = request.data.get('channel_ids', []) Channel.objects.filter(id__in=channel_ids).delete() return Response({"message": "Channels deleted"}, status=status.HTTP_204_NO_CONTENT) + +class LogoViewSet(viewsets.ModelViewSet): + permission_classes = [IsAuthenticated] + queryset = Logo.objects.all() + serializer_class = LogoSerializer + parser_classes = (MultiPartParser, FormParser) + + @action(detail=False, methods=['post']) + def upload(self, request): + if 'file' not in request.FILES: + return Response({'error': 'No file uploaded'}, status=status.HTTP_400_BAD_REQUEST) + + file = request.FILES['file'] + file_name = file.name + file_path = os.path.join('/data/logos', file_name) + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) + + logo, _ = Logo.objects.get_or_create(url=file_path, defaults={ + "name": file_name, + }) + + return Response({'id': logo.id, 'name': logo.name, 'url': logo.url}, status=status.HTTP_201_CREATED) + + @action(detail=True, methods=['get'], permission_classes=[AllowAny]) + def cache(self, request, pk=None): + """Streams the logo file, whether it's local or remote.""" + logo = self.get_object() + logo_url = logo.url + + if logo_url.startswith("/data"): # Local file + if not os.path.exists(logo_url): + raise Http404("Image not found") + return FileResponse(open(logo_url, "rb"), content_type="image/*") + + else: # Remote image + try: + remote_response = requests.get(logo_url, stream=True) + if remote_response.status_code == 200: + return StreamingHttpResponse(remote_response.iter_content(chunk_size=8192), content_type="image/*") + raise Http404("Remote image not found") + except requests.RequestException: + raise Http404("Error fetching remote image") + +class ChannelProfileViewSet(viewsets.ModelViewSet): + queryset = ChannelProfile.objects.all() + serializer_class = ChannelProfileSerializer + permission_classes = [IsAuthenticated] + +class UpdateChannelMembershipAPIView(APIView): + def patch(self, request, profile_id, channel_id): + """Enable or disable a channel for a specific group""" + channel_profile = get_object_or_404(ChannelProfile, id=profile_id) + channel = get_object_or_404(Channel, id=channel_id) + membership = get_object_or_404(ChannelProfileMembership, channel_profile=channel_profile, channel=channel) + + serializer = ChannelProfileMembershipSerializer(membership, data=request.data, partial=True) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=status.HTTP_200_OK) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + +class BulkUpdateChannelMembershipAPIView(APIView): + def patch(self, request, profile_id): + """Bulk enable or disable channels for a specific profile""" + # Get the channel profile + channel_profile = get_object_or_404(ChannelProfile, id=profile_id) + + # Validate the incoming data using the serializer + serializer = BulkChannelProfileMembershipSerializer(data=request.data) + + if serializer.is_valid(): + updates = serializer.validated_data['channels'] + channel_ids = [entry['channel_id'] for entry in updates] + + memberships = ChannelProfileMembership.objects.filter( + channel_profile=channel_profile, + channel_id__in=channel_ids + ) + + membership_dict = {m.channel.id: m for m in memberships} + + for entry in updates: + channel_id = entry['channel_id'] + enabled_status = entry['enabled'] + if channel_id in membership_dict: + membership_dict[channel_id].enabled = enabled_status + + ChannelProfileMembership.objects.bulk_update(memberships, ['enabled']) + + return Response({"status": "success"}, status=status.HTTP_200_OK) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + +class RecordingViewSet(viewsets.ModelViewSet): + queryset = Recording.objects.all() + serializer_class = RecordingSerializer + permission_classes = [IsAuthenticated] diff --git a/apps/channels/forms.py b/apps/channels/forms.py index bee073b6..342bd0fe 100644 --- a/apps/channels/forms.py +++ b/apps/channels/forms.py @@ -40,7 +40,7 @@ class StreamForm(forms.ModelForm): 'name', 'url', 'logo_url', - 'tvg_id', + 'epg_data', 'local_file', 'channel_group', ] diff --git a/apps/channels/migrations/0009_remove_channel_tvg_name_channel_epg_data.py b/apps/channels/migrations/0009_remove_channel_tvg_name_channel_epg_data.py new file mode 100644 index 00000000..814ea1ff --- /dev/null +++ b/apps/channels/migrations/0009_remove_channel_tvg_name_channel_epg_data.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-03-26 12:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0008_stream_stream_hash'), + ('epg', '0004_epgdata_epg_source_alter_epgdata_tvg_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='channel', + name='tvg_name', + ), + migrations.AddField( + model_name='channel', + name='epg_data', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='channels', to='epg.epgdata'), + ), + ] diff --git a/apps/channels/migrations/0010_stream_custom_properties.py b/apps/channels/migrations/0010_stream_custom_properties.py new file mode 100644 index 00000000..0c21f12f --- /dev/null +++ b/apps/channels/migrations/0010_stream_custom_properties.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-04-01 17:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0009_remove_channel_tvg_name_channel_epg_data'), + ] + + operations = [ + migrations.AddField( + model_name='stream', + name='custom_properties', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/apps/channels/migrations/0011_logo_remove_channel_logo_file_and_more.py b/apps/channels/migrations/0011_logo_remove_channel_logo_file_and_more.py new file mode 100644 index 00000000..0f0db44f --- /dev/null +++ b/apps/channels/migrations/0011_logo_remove_channel_logo_file_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.6 on 2025-04-01 22:14 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0010_stream_custom_properties'), + ] + + operations = [ + migrations.CreateModel( + name='Logo', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('url', models.URLField(unique=True)), + ], + ), + migrations.RemoveField( + model_name='channel', + name='logo_file', + ), + migrations.RemoveField( + model_name='channel', + name='logo_url', + ), + migrations.AddField( + model_name='channel', + name='logo', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='channels', to='dispatcharr_channels.logo'), + ), + ] diff --git a/apps/channels/migrations/0012_channelprofile_channelprofilemembership.py b/apps/channels/migrations/0012_channelprofile_channelprofilemembership.py new file mode 100644 index 00000000..53fbdeff --- /dev/null +++ b/apps/channels/migrations/0012_channelprofile_channelprofilemembership.py @@ -0,0 +1,33 @@ +# Generated by Django 5.1.6 on 2025-04-02 23:27 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0011_logo_remove_channel_logo_file_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ChannelProfile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name='ChannelProfileMembership', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True)), + ('channel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dispatcharr_channels.channel')), + ('channel_profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dispatcharr_channels.channelprofile')), + ], + options={ + 'unique_together': {('channel_profile', 'channel')}, + }, + ), + ] diff --git a/apps/channels/migrations/0013_alter_logo_url.py b/apps/channels/migrations/0013_alter_logo_url.py new file mode 100644 index 00000000..002ec35f --- /dev/null +++ b/apps/channels/migrations/0013_alter_logo_url.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-04-04 15:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0012_channelprofile_channelprofilemembership'), + ] + + operations = [ + migrations.AlterField( + model_name='logo', + name='url', + field=models.TextField(unique=True), + ), + ] diff --git a/apps/channels/migrations/0014_recording.py b/apps/channels/migrations/0014_recording.py new file mode 100644 index 00000000..44fa681c --- /dev/null +++ b/apps/channels/migrations/0014_recording.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-04-05 22:25 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0013_alter_logo_url'), + ] + + operations = [ + migrations.CreateModel( + name='Recording', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('start_time', models.DateTimeField()), + ('end_time', models.DateTimeField()), + ('task_id', models.CharField(blank=True, max_length=255, null=True)), + ('channel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='recordings', to='dispatcharr_channels.channel')), + ], + ), + ] diff --git a/apps/channels/migrations/0015_recording_custom_properties.py b/apps/channels/migrations/0015_recording_custom_properties.py new file mode 100644 index 00000000..3c01fc61 --- /dev/null +++ b/apps/channels/migrations/0015_recording_custom_properties.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-04-07 16:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0014_recording'), + ] + + operations = [ + migrations.AddField( + model_name='recording', + name='custom_properties', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index ec95e309..60c35923 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -3,12 +3,13 @@ from django.core.exceptions import ValidationError from core.models import StreamProfile from django.conf import settings from core.models import StreamProfile, CoreSettings -from core.utils import redis_client, execute_redis_command +from core.utils import RedisClient import logging import uuid from datetime import datetime import hashlib import json +from apps.epg.models import EPGData logger = logging.getLogger(__name__) @@ -18,8 +19,7 @@ from apps.m3u.models import M3UAccount # Add fallback functions if Redis isn't available def get_total_viewers(channel_id): """Get viewer count from Redis or return 0 if Redis isn't available""" - if redis_client is None: - return 0 + redis_client = RedisClient.get_client() try: return int(redis_client.get(f"channel:{channel_id}:viewers") or 0) @@ -90,6 +90,7 @@ class Stream(models.Model): db_index=True, ) last_seen = models.DateTimeField(db_index=True, default=datetime.now) + custom_properties = models.TextField(null=True, blank=True) class Meta: # If you use m3u_account, you might do unique_together = ('name','url','m3u_account') @@ -132,6 +133,77 @@ class Stream(models.Model): stream = cls.objects.create(**fields_to_update) return stream, True # True means it was created + # @TODO: honor stream's stream profile + def get_stream_profile(self): + stream_profile = StreamProfile.objects.get(id=CoreSettings.get_default_stream_profile_id()) + + return stream_profile + + def get_stream(self): + """ + Finds an available stream for the requested channel and returns the selected stream and profile. + """ + redis_client = RedisClient.get_client() + profile_id = redis_client.get(f"stream_profile:{self.id}") + if profile_id: + profile_id = int(profile_id) + return self.id, profile_id + + # Retrieve the M3U account associated with the stream. + m3u_account = self.m3u_account + m3u_profiles = m3u_account.profiles.all() + default_profile = next((obj for obj in m3u_profiles if obj.is_default), None) + profiles = [default_profile] + [obj for obj in m3u_profiles if not obj.is_default] + + for profile in profiles: + logger.info(profile) + # Skip inactive profiles + if profile.is_active == False: + continue + + profile_connections_key = f"profile_connections:{profile.id}" + current_connections = int(redis_client.get(profile_connections_key) or 0) + + # Check if profile has available slots (or unlimited connections) + if profile.max_streams == 0 or current_connections < profile.max_streams: + # Start a new stream + redis_client.set(f"channel_stream:{self.id}", self.id) + redis_client.set(f"stream_profile:{self.id}", profile.id) # Store only the matched profile + + # Increment connection count for profiles with limits + if profile.max_streams > 0: + redis_client.incr(profile_connections_key) + + return self.id, profile.id # Return newly assigned stream and matched profile + + # 4. No available streams + return None, None + + def release_stream(self): + """ + Called when a stream is finished to release the lock. + """ + redis_client = RedisClient.get_client() + + stream_id = self.id + # Get the matched profile for cleanup + profile_id = redis_client.get(f"stream_profile:{stream_id}") + if not profile_id: + logger.debug("Invalid profile ID pulled from stream index") + return + + redis_client.delete(f"stream_profile:{stream_id}") # Remove profile association + + profile_id = int(profile_id) + logger.debug(f"Found profile ID {profile_id} associated with stream {stream_id}") + + profile_connections_key = f"profile_connections:{profile_id}" + + # Only decrement if the profile had a max_connections limit + current_count = int(redis_client.get(profile_connections_key) or 0) + if current_count > 0: + redis_client.decr(profile_connections_key) + class ChannelManager(models.Manager): def active(self): return self.all() @@ -140,11 +212,12 @@ class ChannelManager(models.Manager): class Channel(models.Model): channel_number = models.IntegerField() name = models.CharField(max_length=255) - logo_url = models.URLField(max_length=2000, blank=True, null=True) - logo_file = models.ImageField( - upload_to='logos/', # Will store in MEDIA_ROOT/logos + logo = models.ForeignKey( + 'Logo', + on_delete=models.SET_NULL, + null=True, blank=True, - null=True + related_name='channels', ) # M2M to Stream now in the same file @@ -164,7 +237,13 @@ class Channel(models.Model): help_text="Channel group this channel belongs to." ) tvg_id = models.CharField(max_length=255, blank=True, null=True) - tvg_name = models.CharField(max_length=255, blank=True, null=True) + epg_data = models.ForeignKey( + EPGData, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='channels' + ) stream_profile = models.ForeignKey( StreamProfile, @@ -190,6 +269,15 @@ class Channel(models.Model): def __str__(self): return f"{self.channel_number} - {self.name}" + @classmethod + def get_next_available_channel_number(cls, starting_from=1): + used_numbers = set(cls.objects.all().values_list('channel_number', flat=True)) + n = starting_from + while n in used_numbers: + n += 1 + return n + + # @TODO: honor stream's stream profile def get_stream_profile(self): stream_profile = self.stream_profile if not stream_profile: @@ -200,33 +288,62 @@ class Channel(models.Model): def get_stream(self): """ Finds an available stream for the requested channel and returns the selected stream and profile. + + Returns: + Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason) """ + redis_client = RedisClient.get_client() + error_reason = None - # 2. Check if a stream is already active for this channel - stream_id = redis_client.get(f"channel_stream:{self.id}") - if stream_id: - stream_id = int(stream_id) - profile_id = redis_client.get(f"stream_profile:{stream_id}") - if profile_id: - profile_id = int(profile_id) - return stream_id, profile_id + # Check if this channel has any streams + if not self.streams.exists(): + error_reason = "No streams assigned to channel" + return None, None, error_reason - # 3. Iterate through channel streams and their profiles + # Check if a stream is already active for this channel + stream_id_bytes = redis_client.get(f"channel_stream:{self.id}") + if stream_id_bytes: + try: + stream_id = int(stream_id_bytes) + profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}") + if profile_id_bytes: + try: + profile_id = int(profile_id_bytes) + return stream_id, profile_id, None + except (ValueError, TypeError): + logger.debug(f"Invalid profile ID retrieved from Redis: {profile_id_bytes}") + except (ValueError, TypeError): + logger.debug(f"Invalid stream ID retrieved from Redis: {stream_id_bytes}") + + # No existing active stream, attempt to assign a new one + has_streams_but_maxed_out = False + has_active_profiles = False + + # Iterate through channel streams and their profiles for stream in self.streams.all().order_by('channelstream__order'): # Retrieve the M3U account associated with the stream. m3u_account = stream.m3u_account + if not m3u_account: + logger.debug(f"Stream {stream.id} has no M3U account") + continue + m3u_profiles = m3u_account.profiles.all() default_profile = next((obj for obj in m3u_profiles if obj.is_default), None) + + if not default_profile: + logger.debug(f"M3U account {m3u_account.id} has no default profile") + continue + profiles = [default_profile] + [obj for obj in m3u_profiles if not obj.is_default] - logger.info('profiles') - for profile in profiles: - logger.info(profile) # Skip inactive profiles - if profile.is_active == False: + if not profile.is_active: + logger.debug(f"Skipping inactive profile {profile.id}") continue + has_active_profiles = True + profile_connections_key = f"profile_connections:{profile.id}" current_connections = int(redis_client.get(profile_connections_key) or 0) @@ -234,21 +351,34 @@ class Channel(models.Model): if profile.max_streams == 0 or current_connections < profile.max_streams: # Start a new stream redis_client.set(f"channel_stream:{self.id}", stream.id) - redis_client.set(f"stream_profile:{stream.id}", profile.id) # Store only the matched profile + redis_client.set(f"stream_profile:{stream.id}", profile.id) # Increment connection count for profiles with limits if profile.max_streams > 0: redis_client.incr(profile_connections_key) - return stream.id, profile.id # Return newly assigned stream and matched profile + return stream.id, profile.id, None # Return newly assigned stream and matched profile + else: + # This profile is at max connections + has_streams_but_maxed_out = True + logger.debug(f"Profile {profile.id} at max connections: {current_connections}/{profile.max_streams}") - # 4. No available streams - return None, None + # No available streams - determine specific reason + if has_streams_but_maxed_out: + error_reason = "All M3U profiles have reached maximum connection limits" + elif has_active_profiles: + error_reason = "No compatible profile found for any assigned stream" + else: + error_reason = "No active profiles found for any assigned stream" + + return None, None, error_reason def release_stream(self): """ Called when a stream is finished to release the lock. """ + redis_client = RedisClient.get_client() + stream_id = redis_client.get(f"channel_stream:{self.id}") if not stream_id: logger.debug("Invalid stream ID pulled from channel index") @@ -277,6 +407,18 @@ class Channel(models.Model): if current_count > 0: redis_client.decr(profile_connections_key) + +class ChannelProfile(models.Model): + name = models.CharField(max_length=100, unique=True) + +class ChannelProfileMembership(models.Model): + channel_profile = models.ForeignKey(ChannelProfile, on_delete=models.CASCADE) + channel = models.ForeignKey(Channel, on_delete=models.CASCADE) + enabled = models.BooleanField(default=True) # Track if the channel is enabled for this group + + class Meta: + unique_together = ('channel_profile', 'channel') + class ChannelStream(models.Model): channel = models.ForeignKey(Channel, on_delete=models.CASCADE) stream = models.ForeignKey(Stream, on_delete=models.CASCADE) @@ -303,3 +445,21 @@ class ChannelGroupM3UAccount(models.Model): def __str__(self): return f"{self.channel_group.name} - {self.m3u_account.name} (Enabled: {self.enabled})" + + +class Logo(models.Model): + name = models.CharField(max_length=255) + url = models.TextField(unique=True) + + def __str__(self): + return self.name + +class Recording(models.Model): + channel = models.ForeignKey("Channel", on_delete=models.CASCADE, related_name="recordings") + start_time = models.DateTimeField() + end_time = models.DateTimeField() + task_id = models.CharField(max_length=255, null=True, blank=True) + custom_properties = models.TextField(null=True, blank=True) + + def __str__(self): + return f"{self.channel.name} - {self.start_time} to {self.end_time}" diff --git a/apps/channels/serializers.py b/apps/channels/serializers.py index a075297d..fcab5905 100644 --- a/apps/channels/serializers.py +++ b/apps/channels/serializers.py @@ -1,6 +1,25 @@ from rest_framework import serializers -from .models import Stream, Channel, ChannelGroup, ChannelStream, ChannelGroupM3UAccount +from .models import Stream, Channel, ChannelGroup, ChannelStream, ChannelGroupM3UAccount, Logo, ChannelProfile, ChannelProfileMembership, Recording +from apps.epg.serializers import EPGDataSerializer from core.models import StreamProfile +from apps.epg.models import EPGData +from django.urls import reverse +from rest_framework import serializers +from django.utils import timezone + +class LogoSerializer(serializers.ModelSerializer): + cache_url = serializers.SerializerMethodField() + + class Meta: + model = Logo + fields = ['id', 'name', 'url', 'cache_url'] + + def get_cache_url(self, obj): + # return f"/api/channels/logos/{obj.id}/cache/" + request = self.context.get('request') + if request: + return request.build_absolute_uri(reverse('api:channels:logo-cache', args=[obj.id])) + return reverse('api:channels:logo-cache', args=[obj.id]) # # Stream @@ -12,7 +31,7 @@ class StreamSerializer(serializers.ModelSerializer): allow_null=True, required=False ) - read_only_fields = ['is_custom', 'm3u_account'] + read_only_fields = ['is_custom', 'm3u_account', 'stream_hash'] class Meta: model = Stream @@ -29,6 +48,7 @@ class StreamSerializer(serializers.ModelSerializer): 'stream_profile_id', 'is_custom', 'channel_group', + 'stream_hash', ] def get_fields(self): @@ -55,12 +75,49 @@ class ChannelGroupSerializer(serializers.ModelSerializer): model = ChannelGroup fields = ['id', 'name'] +class ChannelProfileSerializer(serializers.ModelSerializer): + channels = serializers.SerializerMethodField() + + class Meta: + model = ChannelProfile + fields = ['id', 'name', 'channels'] + + def get_channels(self, obj): + memberships = ChannelProfileMembership.objects.filter(channel_profile=obj) + return [ + { + 'id': membership.channel.id, + 'enabled': membership.enabled + } + for membership in memberships + ] + +class ChannelProfileMembershipSerializer(serializers.ModelSerializer): + class Meta: + model = ChannelProfileMembership + fields = ['channel', 'enabled'] + +class ChanneProfilelMembershipUpdateSerializer(serializers.Serializer): + channel_id = serializers.IntegerField() # Ensure channel_id is an integer + enabled = serializers.BooleanField() + +class BulkChannelProfileMembershipSerializer(serializers.Serializer): + channels = serializers.ListField( + child=ChanneProfilelMembershipUpdateSerializer(), # Use the nested serializer + allow_empty=False + ) + + def validate_channels(self, value): + if not value: + raise serializers.ValidationError("At least one channel must be provided.") + return value # # Channel # class ChannelSerializer(serializers.ModelSerializer): # Show nested group data, or ID + channel_number = serializers.IntegerField(allow_null=True, required=False) channel_group = ChannelGroupSerializer(read_only=True) channel_group_id = serializers.PrimaryKeyRelatedField( queryset=ChannelGroup.objects.all(), @@ -68,12 +125,20 @@ class ChannelSerializer(serializers.ModelSerializer): write_only=True, required=False ) + epg_data = EPGDataSerializer(read_only=True) + epg_data_id = serializers.PrimaryKeyRelatedField( + queryset=EPGData.objects.all(), + source="epg_data", + write_only=True, + required=False, + allow_null=True, + ) stream_profile_id = serializers.PrimaryKeyRelatedField( queryset=StreamProfile.objects.all(), source='stream_profile', allow_null=True, - required=False + required=False, ) streams = serializers.SerializerMethodField() @@ -81,22 +146,32 @@ class ChannelSerializer(serializers.ModelSerializer): queryset=Stream.objects.all(), many=True, write_only=True, required=False ) + logo = LogoSerializer(read_only=True) + logo_id = serializers.PrimaryKeyRelatedField( + queryset=Logo.objects.all(), + source='logo', + allow_null=True, + required=False, + write_only=True, + ) + class Meta: model = Channel fields = [ 'id', 'channel_number', 'name', - 'logo_url', - 'logo_file', 'channel_group', 'channel_group_id', 'tvg_id', - 'tvg_name', + 'epg_data', + 'epg_data_id', 'streams', 'stream_ids', 'stream_profile_id', 'uuid', + 'logo', + 'logo_id', ] def get_streams(self, obj): @@ -104,12 +179,17 @@ class ChannelSerializer(serializers.ModelSerializer): ordered_streams = obj.streams.all().order_by('channelstream__order') return StreamSerializer(ordered_streams, many=True).data + def get_logo(self, obj): + return LogoSerializer(obj.logo).data + # def get_stream_ids(self, obj): # """Retrieve ordered stream IDs for GET requests.""" # return list(obj.streams.all().order_by('channelstream__order').values_list('id', flat=True)) def create(self, validated_data): stream_ids = validated_data.pop('streams', []) + channel_number = validated_data.pop('channel_number', Channel.get_next_available_channel_number()) + validated_data["channel_number"] = channel_number channel = Channel.objects.create(**validated_data) # Add streams in the specified order @@ -124,15 +204,16 @@ class ChannelSerializer(serializers.ModelSerializer): # Update the actual Channel fields instance.channel_number = validated_data.get('channel_number', instance.channel_number) instance.name = validated_data.get('name', instance.name) - instance.logo_url = validated_data.get('logo_url', instance.logo_url) instance.tvg_id = validated_data.get('tvg_id', instance.tvg_id) - instance.tvg_name = validated_data.get('tvg_name', instance.tvg_name) + instance.epg_data = validated_data.get('epg_data', None) # If serializer allows changing channel_group or stream_profile: if 'channel_group' in validated_data: instance.channel_group = validated_data['channel_group'] if 'stream_profile' in validated_data: instance.stream_profile = validated_data['stream_profile'] + if 'logo' in validated_data: + instance.logo = validated_data['logo'] instance.save() @@ -156,3 +237,27 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer): # Optionally, if you only need the id of the ChannelGroup, you can customize it like this: # channel_group = serializers.PrimaryKeyRelatedField(queryset=ChannelGroup.objects.all()) + + +class RecordingSerializer(serializers.ModelSerializer): + class Meta: + model = Recording + fields = '__all__' + read_only_fields = ['task_id'] + + def validate(self, data): + start_time = data.get('start_time') + end_time = data.get('end_time') + + now = timezone.now() # timezone-aware current time + + if end_time < now: + raise serializers.ValidationError("End time must be in the future.") + + if start_time < now: + # Optional: Adjust start_time if it's in the past but end_time is in the future + data['start_time'] = now # or: timezone.now() + timedelta(seconds=1) + if end_time <= data['start_time']: + raise serializers.ValidationError("End time must be after start time.") + + return data diff --git a/apps/channels/signals.py b/apps/channels/signals.py index 4a76ed05..660de04c 100644 --- a/apps/channels/signals.py +++ b/apps/channels/signals.py @@ -1,17 +1,24 @@ # apps/channels/signals.py -from django.db.models.signals import m2m_changed, pre_save +from django.db.models.signals import m2m_changed, pre_save, post_save, post_delete from django.dispatch import receiver -from .models import Channel, Stream +from django.utils.timezone import now +from celery.result import AsyncResult +from .models import Channel, Stream, ChannelProfile, ChannelProfileMembership, Recording from apps.m3u.models import M3UAccount +from apps.epg.tasks import parse_programs_for_tvg_id +import logging, requests, time +from .tasks import run_recording +from django.utils.timezone import now, is_aware, make_aware +from datetime import timedelta + +logger = logging.getLogger(__name__) @receiver(m2m_changed, sender=Channel.streams.through) def update_channel_tvg_id_and_logo(sender, instance, action, reverse, model, pk_set, **kwargs): """ Whenever streams are added to a channel: 1) If the channel doesn't have a tvg_id, fill it from the first newly-added stream that has one. - 2) If the channel doesn't have a logo_url, fill it from the first newly-added stream that has one. - This way if an M3U or EPG entry carried a logo, newly created channels automatically get that logo. """ # We only care about post_add, i.e. once the new streams are fully associated if action == "post_add": @@ -23,14 +30,6 @@ def update_channel_tvg_id_and_logo(sender, instance, action, reverse, model, pk_ instance.tvg_id = streams_with_tvg.first().tvg_id instance.save(update_fields=['tvg_id']) - # --- 2) Populate channel.logo_url if empty --- - if not instance.logo_url: - # Look for newly added streams that have a nonempty logo_url - streams_with_logo = model.objects.filter(pk__in=pk_set).exclude(logo_url__exact='') - if streams_with_logo.exists(): - instance.logo_url = streams_with_logo.first().logo_url - instance.save(update_fields=['logo_url']) - @receiver(pre_save, sender=Stream) def set_default_m3u_account(sender, instance, **kwargs): """ @@ -45,3 +44,87 @@ def set_default_m3u_account(sender, instance, **kwargs): instance.m3u_account = default_account else: raise ValueError("No default M3UAccount found.") + +@receiver(post_save, sender=Channel) +def refresh_epg_programs(sender, instance, created, **kwargs): + if instance.epg_data: + parse_programs_for_tvg_id.delay(instance.epg_data.id) + +@receiver(post_save, sender=Channel) +def add_new_channel_to_groups(sender, instance, created, **kwargs): + if created: + profiles = ChannelProfile.objects.all() + ChannelProfileMembership.objects.bulk_create([ + ChannelProfileMembership(channel_profile=profile, channel=instance) + for profile in profiles + ]) + +@receiver(post_save, sender=ChannelProfile) +def create_profile_memberships(sender, instance, created, **kwargs): + if created: + channels = Channel.objects.all() + ChannelProfileMembership.objects.bulk_create([ + ChannelProfileMembership(channel_profile=instance, channel=channel) + for channel in channels + ]) + +def schedule_recording_task(instance): + eta = instance.start_time + task = run_recording.apply_async( + args=[instance.channel_id, str(instance.start_time), str(instance.end_time)], + eta=eta + ) + return task.id + +def revoke_task(task_id): + if task_id: + AsyncResult(task_id).revoke() + +@receiver(pre_save, sender=Recording) +def revoke_old_task_on_update(sender, instance, **kwargs): + if not instance.pk: + return # New instance + try: + old = Recording.objects.get(pk=instance.pk) + if old.task_id and ( + old.start_time != instance.start_time or + old.end_time != instance.end_time or + old.channel_id != instance.channel_id + ): + revoke_task(old.task_id) + instance.task_id = None + except Recording.DoesNotExist: + pass + +@receiver(post_save, sender=Recording) +def schedule_task_on_save(sender, instance, created, **kwargs): + try: + if not instance.task_id: + start_time = instance.start_time + + # Make both datetimes aware (in UTC) + if not is_aware(start_time): + print("Start time was not aware, making aware") + start_time = make_aware(start_time) + + current_time = now() + + # Debug log + print(f"Start time: {start_time}, Now: {current_time}") + + # Optionally allow slight fudge factor (1 second) to ensure scheduling happens + if start_time > current_time - timedelta(seconds=1): + print("Scheduling recording task!") + task_id = schedule_recording_task(instance) + instance.task_id = task_id + instance.save(update_fields=['task_id']) + else: + print("Start time is in the past. Not scheduling.") + except Exception as e: + import traceback + print("Error in post_save signal:", e) + traceback.print_exc() + +@receiver(post_delete, sender=Recording) +def revoke_task_on_delete(sender, instance, **kwargs): + revoke_task(instance.task_id) diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py old mode 100644 new mode 100755 index f9b4992b..94bb8ca9 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -2,38 +2,28 @@ import logging import os import re +import requests +import time +import json +import subprocess +from datetime import datetime from celery import shared_task -from rapidfuzz import fuzz -from sentence_transformers import SentenceTransformer, util -from django.conf import settings -from django.db import transaction +from django.utils.text import slugify from apps.channels.models import Channel -from apps.epg.models import EPGData, EPGSource +from apps.epg.models import EPGData from core.models import CoreSettings -from apps.epg.tasks import parse_programs_for_tvg_id # <-- we import our new helper + +from channels.layers import get_channel_layer +from asgiref.sync import async_to_sync + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +import tempfile logger = logging.getLogger(__name__) -# Load the sentence-transformers model once at the module level -SENTENCE_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2" -MODEL_PATH = os.path.join(settings.MEDIA_ROOT, "models", "all-MiniLM-L6-v2") -os.makedirs(MODEL_PATH, exist_ok=True) - -# If not present locally, download: -if not os.path.exists(os.path.join(MODEL_PATH, "config.json")): - logger.info(f"Local model not found in {MODEL_PATH}; downloading from {SENTENCE_MODEL_NAME}...") - st_model = SentenceTransformer(SENTENCE_MODEL_NAME, cache_folder=MODEL_PATH) -else: - logger.info(f"Loading local model from {MODEL_PATH}") - st_model = SentenceTransformer(MODEL_PATH) - -# Thresholds -BEST_FUZZY_THRESHOLD = 85 -LOWER_FUZZY_THRESHOLD = 40 -EMBED_SIM_THRESHOLD = 0.65 - # Words we remove to help with fuzzy + embedding matching COMMON_EXTRANEOUS_WORDS = [ "tv", "channel", "network", "television", @@ -70,8 +60,7 @@ def match_epg_channels(): 1) If channel.tvg_id is valid in EPGData, skip. 2) If channel has a tvg_id but not found in EPGData, attempt direct EPGData lookup. 3) Otherwise, perform name-based fuzzy matching with optional region-based bonus. - 4) If a match is found, we set channel.tvg_id and also parse its programs - from the cached EPG file (parse_programs_for_tvg_id). + 4) If a match is found, we set channel.tvg_id 5) Summarize and log results. """ logger.info("Starting EPG matching logic...") @@ -83,132 +72,82 @@ def match_epg_channels(): except CoreSettings.DoesNotExist: region_code = None - # Gather EPGData rows so we can do fuzzy matching in memory - all_epg = list(EPGData.objects.all()) - epg_rows = [] - for e in all_epg: - epg_rows.append({ - "epg_id": e.id, - "tvg_id": e.tvg_id or "", - "raw_name": e.name, - "norm_name": normalize_name(e.name), - }) - - epg_embeddings = None - if any(row["norm_name"] for row in epg_rows): - epg_embeddings = st_model.encode( - [row["norm_name"] for row in epg_rows], - convert_to_tensor=True - ) - matched_channels = [] + channels_to_update = [] - source = EPGSource.objects.filter(is_active=True).first() - epg_file_path = getattr(source, 'file_path', None) if source else None + channels_json = [{ + "id": channel.id, + "name": channel.name, + "tvg_id": channel.tvg_id, + "fallback_name": channel.tvg_id.strip() if channel.tvg_id else channel.name, + "norm_chan": normalize_name(channel.tvg_id.strip() if channel.tvg_id else channel.name) + } for channel in Channel.objects.all() if not channel.epg_data] - with transaction.atomic(): - for chan in Channel.objects.all(): + epg_json = [{ + 'id': epg.id, + 'tvg_id': epg.tvg_id, + 'name': epg.name, + 'norm_name': normalize_name(epg.name), + 'epg_source_id': epg.epg_source.id, + } for epg in EPGData.objects.all()] - # A) Skip if channel.tvg_id is already valid - if chan.tvg_id and EPGData.objects.filter(tvg_id=chan.tvg_id).exists(): - continue + payload = { + "channels": channels_json, + "epg_data": epg_json, + "region_code": region_code, + } - # B) If channel has a tvg_id that doesn't exist in EPGData, do direct check - if chan.tvg_id: - epg_match = EPGData.objects.filter(tvg_id=chan.tvg_id).first() - if epg_match: - logger.info(f"Channel {chan.id} '{chan.name}' => EPG found by tvg_id={chan.tvg_id}") - continue + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_file.write(json.dumps(payload).encode('utf-8')) + temp_file_path = temp_file.name - # C) Perform name-based fuzzy matching - fallback_name = chan.tvg_name.strip() if chan.tvg_name else chan.name - norm_chan = normalize_name(fallback_name) - if not norm_chan: - logger.info(f"Channel {chan.id} '{chan.name}' => empty after normalization, skipping") - continue + process = subprocess.Popen( + ['python', '/app/scripts/epg_match.py', temp_file_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) - best_score = 0 - best_epg = None - for row in epg_rows: - if not row["norm_name"]: - continue - base_score = fuzz.ratio(norm_chan, row["norm_name"]) - bonus = 0 - # Region-based bonus/penalty - combined_text = row["tvg_id"].lower() + " " + row["raw_name"].lower() - dot_regions = re.findall(r'\.([a-z]{2})', combined_text) - if region_code: - if dot_regions: - if region_code in dot_regions: - bonus = 30 # bigger bonus if .us or .ca matches - else: - bonus = -15 - elif region_code in combined_text: - bonus = 15 - score = base_score + bonus + # Log stderr in real-time + for line in iter(process.stderr.readline, ''): + if line: + logger.info(line.strip()) - logger.debug( - f"Channel {chan.id} '{fallback_name}' => EPG row {row['epg_id']}: " - f"raw_name='{row['raw_name']}', norm_name='{row['norm_name']}', " - f"combined_text='{combined_text}', dot_regions={dot_regions}, " - f"base_score={base_score}, bonus={bonus}, total_score={score}" - ) + process.stderr.close() + stdout, stderr = process.communicate() - if score > best_score: - best_score = score - best_epg = row + os.remove(temp_file_path) - # If no best match was found, skip - if not best_epg: - logger.info(f"Channel {chan.id} '{fallback_name}' => no EPG match at all.") - continue + if process.returncode != 0: + return f"Failed to process EPG matching: {stderr}" - # If best_score is above BEST_FUZZY_THRESHOLD => direct accept - if best_score >= BEST_FUZZY_THRESHOLD: - chan.tvg_id = best_epg["tvg_id"] - chan.save() + result = json.loads(stdout) + # This returns lists of dicts, not model objects + channels_to_update_dicts = result["channels_to_update"] + matched_channels = result["matched_channels"] - # Attempt to parse program data for this channel - if epg_file_path: - parse_programs_for_tvg_id(epg_file_path, best_epg["tvg_id"]) - logger.info(f"Loaded program data for tvg_id={best_epg['tvg_id']}") + # Convert your dict-based 'channels_to_update' into real Channel objects + if channels_to_update_dicts: + # Extract IDs of the channels that need updates + channel_ids = [d["id"] for d in channels_to_update_dicts] - matched_channels.append((chan.id, fallback_name, best_epg["tvg_id"])) - logger.info( - f"Channel {chan.id} '{fallback_name}' => matched tvg_id={best_epg['tvg_id']} " - f"(score={best_score})" - ) + # Fetch them from DB + channels_qs = Channel.objects.filter(id__in=channel_ids) + channels_list = list(channels_qs) - # If best_score is in the “middle range,” do embedding check - elif best_score >= LOWER_FUZZY_THRESHOLD and epg_embeddings is not None: - chan_embedding = st_model.encode(norm_chan, convert_to_tensor=True) - sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0] - top_index = int(sim_scores.argmax()) - top_value = float(sim_scores[top_index]) - if top_value >= EMBED_SIM_THRESHOLD: - matched_epg = epg_rows[top_index] - chan.tvg_id = matched_epg["tvg_id"] - chan.save() + # Build a map from channel_id -> epg_data_id (or whatever fields you need) + epg_mapping = { + d["id"]: d["epg_data_id"] for d in channels_to_update_dicts + } - if epg_file_path: - parse_programs_for_tvg_id(epg_file_path, matched_epg["tvg_id"]) - logger.info(f"Loaded program data for tvg_id={matched_epg['tvg_id']}") + # Populate each Channel object with the updated epg_data_id + for channel_obj in channels_list: + # The script sets 'epg_data_id' in the returned dict + # We either assign directly, or fetch the EPGData instance if needed. + channel_obj.epg_data_id = epg_mapping.get(channel_obj.id) - matched_channels.append((chan.id, fallback_name, matched_epg["tvg_id"])) - logger.info( - f"Channel {chan.id} '{fallback_name}' => matched EPG tvg_id={matched_epg['tvg_id']} " - f"(fuzzy={best_score}, cos-sim={top_value:.2f})" - ) - else: - logger.info( - f"Channel {chan.id} '{fallback_name}' => fuzzy={best_score}, " - f"cos-sim={top_value:.2f} < {EMBED_SIM_THRESHOLD}, skipping" - ) - else: - logger.info( - f"Channel {chan.id} '{fallback_name}' => fuzzy={best_score} < " - f"{LOWER_FUZZY_THRESHOLD}, skipping" - ) + # Now we have real model objects, so bulk_update will work + Channel.objects.bulk_update(channels_list, ["epg_data"]) total_matched = len(matched_channels) if total_matched: @@ -219,4 +158,63 @@ def match_epg_channels(): logger.info("No new channels were matched.") logger.info("Finished EPG matching logic.") + + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": {"success": True, "type": "epg_match"} + } + ) + return f"Done. Matched {total_matched} channel(s)." + + +@shared_task +def run_recording(channel_id, start_time_str, end_time_str): + channel = Channel.objects.get(id=channel_id) + + start_time = datetime.fromisoformat(start_time_str) + end_time = datetime.fromisoformat(end_time_str) + + duration_seconds = int((end_time - start_time).total_seconds()) + filename = f'{slugify(channel.name)}-{start_time.strftime("%Y-%m-%d_%H-%M-%S")}.mp4' + + channel_layer = get_channel_layer() + + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": {"success": True, "type": "recording_started", "channel": channel.name} + }, + ) + + logger.info(f"Starting recording for channel {channel.name}") + with requests.get(f"http://localhost:5656/proxy/ts/stream/{channel.uuid}", headers={ + 'User-Agent': 'Dispatcharr-DVR', + }, stream=True) as response: + # Raise an exception for bad responses (4xx, 5xx) + response.raise_for_status() + + # Open the file in write-binary mode + with open(f"/data/recordings/{filename}", 'wb') as file: + start_time = time.time() # Start the timer + for chunk in response.iter_content(chunk_size=8192): # 8KB chunks + if time.time() - start_time > duration_seconds: + print(f"Timeout reached: {duration_seconds} seconds") + break + # Write the chunk to the file + file.write(chunk) + + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": {"success": True, "type": "recording_ended", "channel": channel.name} + }, + ) + + # After the loop, the file and response are closed automatically. + logger.info(f"Finished recording for channel {channel.name}") diff --git a/apps/epg/api_views.py b/apps/epg/api_views.py index 571a7165..ffa5260d 100644 --- a/apps/epg/api_views.py +++ b/apps/epg/api_views.py @@ -1,8 +1,9 @@ -import logging +import logging, os from rest_framework import viewsets, status from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.permissions import IsAuthenticated +from rest_framework.decorators import action from drf_yasg.utils import swagger_auto_schema from drf_yasg import openapi from django.utils import timezone @@ -26,6 +27,29 @@ class EPGSourceViewSet(viewsets.ModelViewSet): logger.debug("Listing all EPG sources.") return super().list(request, *args, **kwargs) + @action(detail=False, methods=['post']) + def upload(self, request): + if 'file' not in request.FILES: + return Response({'error': 'No file uploaded'}, status=status.HTTP_400_BAD_REQUEST) + + file = request.FILES['file'] + file_name = file.name + file_path = os.path.join('/data/uploads/epgs', file_name) + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) + + new_obj_data = request.data.copy() + new_obj_data['file_path'] = file_path + + serializer = self.get_serializer(data=new_obj_data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + # ───────────────────────────── # 2) Program API (CRUD) # ───────────────────────────── @@ -43,23 +67,28 @@ class ProgramViewSet(viewsets.ModelViewSet): # 3) EPG Grid View # ───────────────────────────── class EPGGridAPIView(APIView): - """Returns all programs airing in the next 12 hours""" + """Returns all programs airing in the next 24 hours including currently running ones and recent ones""" @swagger_auto_schema( - operation_description="Retrieve upcoming EPG programs within the next 12 hours", + operation_description="Retrieve programs from the previous hour, currently running and upcoming for the next 24 hours", responses={200: ProgramDataSerializer(many=True)} ) def get(self, request, format=None): - # Get current date and reset time to midnight (00:00) - now = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) - twelve_hours_later = now + timedelta(hours=24) - logger.debug(f"EPGGridAPIView: Querying programs between {now} and {twelve_hours_later}.") - # Use select_related to prefetch EPGData (no channel relation now) + # Use current time instead of midnight + now = timezone.now() + one_hour_ago = now - timedelta(hours=1) + twenty_four_hours_later = now + timedelta(hours=24) + logger.debug(f"EPGGridAPIView: Querying programs between {one_hour_ago} and {twenty_four_hours_later}.") + + # Use select_related to prefetch EPGData and include programs from the last hour programs = ProgramData.objects.select_related('epg').filter( - start_time__gte=now, start_time__lte=twelve_hours_later + # Programs that end after one hour ago (includes recently ended programs) + end_time__gt=one_hour_ago, + # AND start before the end time window + start_time__lt=twenty_four_hours_later ) count = programs.count() - logger.debug(f"EPG`Grid`APIView: Found {count} program(s).") + logger.debug(f"EPGGridAPIView: Found {count} program(s), including recently ended, currently running, and upcoming shows.") serializer = ProgramDataSerializer(programs, many=True) return Response({'data': serializer.data}, status=status.HTTP_200_OK) @@ -75,7 +104,7 @@ class EPGImportAPIView(APIView): ) def post(self, request, format=None): logger.info("EPGImportAPIView: Received request to import EPG data.") - refresh_epg_data.delay() # Trigger Celery task + refresh_epg_data.delay(request.data.get('id', None)) # Trigger Celery task logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.") return Response({'success': True, 'message': 'EPG data import initiated.'}, status=status.HTTP_202_ACCEPTED) @@ -90,4 +119,3 @@ class EPGDataViewSet(viewsets.ReadOnlyModelViewSet): queryset = EPGData.objects.all() serializer_class = EPGDataSerializer permission_classes = [IsAuthenticated] - diff --git a/apps/epg/migrations/0003_alter_epgdata_tvg_id.py b/apps/epg/migrations/0003_alter_epgdata_tvg_id.py new file mode 100644 index 00000000..f339b981 --- /dev/null +++ b/apps/epg/migrations/0003_alter_epgdata_tvg_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-03-25 19:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0002_epgsource_file_path'), + ] + + operations = [ + migrations.AlterField( + model_name='epgdata', + name='tvg_id', + field=models.CharField(blank=True, max_length=255, null=True, unique=True), + ), + ] diff --git a/apps/epg/migrations/0004_epgdata_epg_source_alter_epgdata_tvg_id.py b/apps/epg/migrations/0004_epgdata_epg_source_alter_epgdata_tvg_id.py new file mode 100644 index 00000000..ff8f7a11 --- /dev/null +++ b/apps/epg/migrations/0004_epgdata_epg_source_alter_epgdata_tvg_id.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-03-26 12:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0003_alter_epgdata_tvg_id'), + ] + + operations = [ + migrations.AddField( + model_name='epgdata', + name='epg_source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='epgs', to='epg.epgsource'), + ), + migrations.AlterField( + model_name='epgdata', + name='tvg_id', + field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + ), + ] diff --git a/apps/epg/migrations/0005_programdata_custom_properties_and_more.py b/apps/epg/migrations/0005_programdata_custom_properties_and_more.py new file mode 100644 index 00000000..35d8d1c3 --- /dev/null +++ b/apps/epg/migrations/0005_programdata_custom_properties_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.6 on 2025-03-27 17:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0004_epgdata_epg_source_alter_epgdata_tvg_id'), + ] + + operations = [ + migrations.AddField( + model_name='programdata', + name='custom_properties', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterUniqueTogether( + name='epgdata', + unique_together={('tvg_id', 'epg_source')}, + ), + ] diff --git a/apps/epg/migrations/0006_epgsource_refresh_interval_epgsource_refresh_task.py b/apps/epg/migrations/0006_epgsource_refresh_interval_epgsource_refresh_task.py new file mode 100644 index 00000000..10cf17ba --- /dev/null +++ b/apps/epg/migrations/0006_epgsource_refresh_interval_epgsource_refresh_task.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.6 on 2025-03-29 17:31 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_celery_beat', '0019_alter_periodictasks_options'), + ('epg', '0005_programdata_custom_properties_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='epgsource', + name='refresh_interval', + field=models.IntegerField(default=24), + ), + migrations.AddField( + model_name='epgsource', + name='refresh_task', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_celery_beat.periodictask'), + ), + ] diff --git a/apps/epg/migrations/0007_populate_periodic_tasks.py b/apps/epg/migrations/0007_populate_periodic_tasks.py new file mode 100644 index 00000000..edca8db9 --- /dev/null +++ b/apps/epg/migrations/0007_populate_periodic_tasks.py @@ -0,0 +1,52 @@ +from django.db import migrations +import json + +def create_default_refresh_tasks(apps, schema_editor): + """ + Creates a PeriodicTask for each existing EPGSource that doesn't have one. + """ + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + EPGSource = apps.get_model("epg", "EPGSource") + + default_interval, _ = IntervalSchedule.objects.get_or_create( + every=24, + period="hours", + ) + + for account in EPGSource.objects.all(): + if account.refresh_task: + continue + + task_name = f"epg_source-refresh-{account.id}" + + refresh_task = PeriodicTask.objects.create( + name=task_name, + interval=default_interval, + task="apps.epg.tasks.refresh_epg_data", + kwargs=json.dumps({"account_id": account.id}), + ) + + account.refresh_task = refresh_task + account.save(update_fields=["refresh_task"]) + +def reverse_migration(apps, schema_editor): + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + EPGSource = apps.get_model("epg", "EPGSource") + + for account in EPGSource.objects.all(): + IntervalSchedule.objects.all().delete() + PeriodicTask.objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("epg", "0006_epgsource_refresh_interval_epgsource_refresh_task"), + ("django_celery_beat", "0019_alter_periodictasks_options"), + ] + + operations = [ + migrations.RunPython(create_default_refresh_tasks, reverse_migration), + ] diff --git a/apps/epg/migrations/0008_epgsource_created_at_epgsource_updated_at.py b/apps/epg/migrations/0008_epgsource_created_at_epgsource_updated_at.py new file mode 100644 index 00000000..1dcfeed0 --- /dev/null +++ b/apps/epg/migrations/0008_epgsource_created_at_epgsource_updated_at.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-04-07 16:29 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0007_populate_periodic_tasks'), + ] + + operations = [ + migrations.AddField( + model_name='epgsource', + name='created_at', + field=models.DateTimeField(default=django.utils.timezone.now, help_text='Time when this source was created'), + ), + migrations.AddField( + model_name='epgsource', + name='updated_at', + field=models.DateTimeField(default=django.utils.timezone.now, help_text='Time when this source was last updated'), + ), + ] diff --git a/apps/epg/migrations/0009_alter_epgsource_created_at_and_more.py b/apps/epg/migrations/0009_alter_epgsource_created_at_and_more.py new file mode 100644 index 00000000..cb8088eb --- /dev/null +++ b/apps/epg/migrations/0009_alter_epgsource_created_at_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.6 on 2025-04-07 16:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0008_epgsource_created_at_epgsource_updated_at'), + ] + + operations = [ + migrations.AlterField( + model_name='epgsource', + name='created_at', + field=models.DateTimeField(auto_now_add=True, help_text='Time when this source was created'), + ), + migrations.AlterField( + model_name='epgsource', + name='updated_at', + field=models.DateTimeField(auto_now=True, help_text='Time when this source was last updated'), + ), + ] diff --git a/apps/epg/models.py b/apps/epg/models.py index 305d30ed..09986bfe 100644 --- a/apps/epg/models.py +++ b/apps/epg/models.py @@ -1,5 +1,6 @@ from django.db import models from django.utils import timezone +from django_celery_beat.models import PeriodicTask class EPGSource(models.Model): SOURCE_TYPE_CHOICES = [ @@ -12,6 +13,18 @@ class EPGSource(models.Model): api_key = models.CharField(max_length=255, blank=True, null=True) # For Schedules Direct is_active = models.BooleanField(default=True) file_path = models.CharField(max_length=1024, blank=True, null=True) + refresh_interval = models.IntegerField(default=24) + refresh_task = models.ForeignKey( + PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True + ) + created_at = models.DateTimeField( + auto_now_add=True, + help_text="Time when this source was created" + ) + updated_at = models.DateTimeField( + auto_now=True, + help_text="Time when this source was last updated" + ) def __str__(self): return self.name @@ -19,8 +32,18 @@ class EPGSource(models.Model): class EPGData(models.Model): # Removed the Channel foreign key. We now just store the original tvg_id # and a name (which might simply be the tvg_id if no real channel exists). - tvg_id = models.CharField(max_length=255, null=True, blank=True, unique=True) + tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) name = models.CharField(max_length=255) + epg_source = models.ForeignKey( + EPGSource, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="epgs", + ) + + class Meta: + unique_together = ('tvg_id', 'epg_source') def __str__(self): return f"EPG Data for {self.name}" @@ -34,6 +57,7 @@ class ProgramData(models.Model): sub_title = models.CharField(max_length=255, blank=True, null=True) description = models.TextField(blank=True, null=True) tvg_id = models.CharField(max_length=255, null=True, blank=True) + custom_properties = models.TextField(null=True, blank=True) def __str__(self): return f"{self.title} ({self.start_time} - {self.end_time})" diff --git a/apps/epg/serializers.py b/apps/epg/serializers.py index b10e7371..e4ff932e 100644 --- a/apps/epg/serializers.py +++ b/apps/epg/serializers.py @@ -3,9 +3,15 @@ from .models import EPGSource, EPGData, ProgramData from apps.channels.models import Channel class EPGSourceSerializer(serializers.ModelSerializer): + epg_data_ids = serializers.SerializerMethodField() + read_only_fields = ['created_at', 'updated_at'] + class Meta: model = EPGSource - fields = ['id', 'name', 'source_type', 'url', 'api_key', 'is_active'] + fields = ['id', 'name', 'source_type', 'url', 'api_key', 'is_active', 'epg_data_ids', 'refresh_interval', 'created_at', 'updated_at'] + + def get_epg_data_ids(self, obj): + return list(obj.epgs.values_list('id', flat=True)) class ProgramDataSerializer(serializers.ModelSerializer): class Meta: @@ -17,10 +23,13 @@ class EPGDataSerializer(serializers.ModelSerializer): Only returns the tvg_id and the 'name' field from EPGData. We assume 'name' is effectively the channel name. """ + read_only_fields = ['epg_source'] + class Meta: model = EPGData fields = [ 'id', 'tvg_id', 'name', - ] \ No newline at end of file + 'epg_source', + ] diff --git a/apps/epg/signals.py b/apps/epg/signals.py index c8f4a62c..82db7fad 100644 --- a/apps/epg/signals.py +++ b/apps/epg/signals.py @@ -1,10 +1,56 @@ -from django.db.models.signals import post_save +from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from .models import EPGSource from .tasks import refresh_epg_data +from django_celery_beat.models import PeriodicTask, IntervalSchedule +import json @receiver(post_save, sender=EPGSource) def trigger_refresh_on_new_epg_source(sender, instance, created, **kwargs): # Trigger refresh only if the source is newly created and active if created and instance.is_active: - refresh_epg_data.delay() + refresh_epg_data.delay(instance.id) + +@receiver(post_save, sender=EPGSource) +def create_or_update_refresh_task(sender, instance, **kwargs): + """ + Create or update a Celery Beat periodic task when an EPGSource is created/updated. + """ + task_name = f"epg_source-refresh-{instance.id}" + interval, _ = IntervalSchedule.objects.get_or_create( + every=int(instance.refresh_interval), + period=IntervalSchedule.HOURS + ) + + task, created = PeriodicTask.objects.get_or_create(name=task_name, defaults={ + "interval": interval, + "task": "apps.epg.tasks.refresh_epg_data", + "kwargs": json.dumps({"source_id": instance.id}), + "enabled": instance.refresh_interval != 0, + }) + + update_fields = [] + if created: + task.interval = interval + + if task.interval != interval: + task.interval = interval + update_fields.append("interval") + if task.enabled != (instance.refresh_interval != 0): + task.enabled = instance.refresh_interval != 0 + update_fields.append("enabled") + + if update_fields: + task.save(update_fields=update_fields) + + if instance.refresh_task != task: + instance.refresh_task = task + instance.save(update_fields=update_fields) + +@receiver(post_delete, sender=EPGSource) +def delete_refresh_task(sender, instance, **kwargs): + """ + Delete the associated Celery Beat periodic task when a Channel is deleted. + """ + if instance.refresh_task: + instance.refresh_task.delete() diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 9b9fa49f..9bfce9fe 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -14,30 +14,54 @@ from django.db import transaction from django.utils import timezone from apps.channels.models import Channel +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer from .models import EPGSource, EPGData, ProgramData +from core.utils import acquire_task_lock, release_task_lock logger = logging.getLogger(__name__) @shared_task -def refresh_epg_data(): +def refresh_all_epg_data(): logger.info("Starting refresh_epg_data task.") active_sources = EPGSource.objects.filter(is_active=True) logger.debug(f"Found {active_sources.count()} active EPGSource(s).") for source in active_sources: - logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") - if source.source_type == 'xmltv': - fetch_xmltv(source) - elif source.source_type == 'schedules_direct': - fetch_schedules_direct(source) + refresh_epg_data(source.id) logger.info("Finished refresh_epg_data task.") return "EPG data refreshed." +@shared_task +def refresh_epg_data(source_id): + if not acquire_task_lock('refresh_epg_data', source_id): + logger.debug(f"EPG refresh for {source_id} already running") + return + + source = EPGSource.objects.get(id=source_id) + if not source.is_active: + logger.info(f"EPG source {source_id} is not active. Skipping.") + return + + logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})") + if source.source_type == 'xmltv': + fetch_xmltv(source) + parse_channels_only(source) + parse_programs_for_source(source) + elif source.source_type == 'schedules_direct': + fetch_schedules_direct(source) + + source.save(update_fields=['updated_at']) + + release_task_lock('refresh_epg_data', source_id) def fetch_xmltv(source): + if not source.url: + return + logger.info(f"Fetching XMLTV data from source: {source.name}") try: response = requests.get(source.url, timeout=30) @@ -62,21 +86,14 @@ def fetch_xmltv(source): source.file_path = file_path source.save(update_fields=['file_path']) - epg_entries = EPGData.objects.exclude(tvg_id__isnull=True).exclude(tvg_id__exact='') - for epg in epg_entries: - if Channel.objects.filter(tvg_id=epg.tvg_id).exists(): - logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") - parse_programs_for_tvg_id(file_path, epg.tvg_id) - - # Now parse blocks only - parse_channels_only(file_path) - except Exception as e: logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True) -def parse_channels_only(file_path): +def parse_channels_only(source): + file_path = source.file_path logger.info(f"Parsing channels from EPG file: {file_path}") + existing_epgs = {e.tvg_id: e for e in EPGData.objects.filter(epg_source=source)} # Read entire file (decompress if .gz) if file_path.endswith('.gz'): @@ -90,77 +107,190 @@ def parse_channels_only(file_path): root = ET.fromstring(xml_data) channels = root.findall('channel') + epgs_to_create = [] + epgs_to_update = [] + logger.info(f"Found {len(channels)} entries in {file_path}") - with transaction.atomic(): - for channel_elem in channels: - tvg_id = channel_elem.get('id', '').strip() - if not tvg_id: - continue # skip blank/invalid IDs + for channel_elem in channels: + tvg_id = channel_elem.get('id', '').strip() + if not tvg_id: + continue # skip blank/invalid IDs - display_name = channel_elem.findtext('display-name', default=tvg_id).strip() + display_name = channel_elem.findtext('display-name', default=tvg_id).strip() - epg_obj, created = EPGData.objects.get_or_create( + if tvg_id in existing_epgs: + epg_obj = existing_epgs[tvg_id] + if epg_obj.name != display_name: + epg_obj.name = display_name + epgs_to_update.append(epg_obj) + else: + epgs_to_create.append(EPGData( tvg_id=tvg_id, - defaults={'name': display_name} - ) - if not created: - # Optionally update if new name is different - if epg_obj.name != display_name: - epg_obj.name = display_name - epg_obj.save() - logger.debug(f"Channel <{tvg_id}> => EPGData.id={epg_obj.id}, created={created}") + name=display_name, + epg_source=source, + )) - parse_programs_for_tvg_id(file_path, tvg_id) + if epgs_to_create: + EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True) + if epgs_to_update: + EPGData.objects.bulk_update(epgs_to_update, ["name"]) + + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": {"success": True, "type": "epg_channels"} + } + ) logger.info("Finished parsing channel info.") +@shared_task +def parse_programs_for_tvg_id(epg_id): + if not acquire_task_lock('parse_epg_programs', epg_id): + logger.debug(f"Program parse for {epg_id} already in progress") + return -def parse_programs_for_tvg_id(file_path, tvg_id): - logger.info(f"Parsing for tvg_id={tvg_id} from {file_path}") + epg = EPGData.objects.get(id=epg_id) + epg_source = epg.epg_source + + if not Channel.objects.filter(epg_data=epg).exists(): + logger.info(f"No channels matched to EPG {epg.tvg_id}") + release_task_lock('parse_epg_programs', epg_id) + return + + logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}") + + # First, remove all existing programs + ProgramData.objects.filter(epg=epg).delete() # Read entire file (decompress if .gz) - if file_path.endswith('.gz'): - with open(file_path, 'rb') as gz_file: + if epg_source.file_path.endswith('.gz'): + with open(epg_source.file_path, 'rb') as gz_file: decompressed = gzip.decompress(gz_file.read()) xml_data = decompressed.decode('utf-8') else: - with open(file_path, 'r', encoding='utf-8') as xml_file: + with open(epg_source.file_path, 'r', encoding='utf-8') as xml_file: xml_data = xml_file.read() root = ET.fromstring(xml_data) - # Retrieve the EPGData record - try: - epg_obj = EPGData.objects.get(tvg_id=tvg_id) - except EPGData.DoesNotExist: - logger.warning(f"No EPGData record found for tvg_id={tvg_id}") - return # Find only elements for this tvg_id - matched_programmes = [p for p in root.findall('programme') if p.get('channel') == tvg_id] - logger.debug(f"Found {len(matched_programmes)} programmes for tvg_id={tvg_id}") + matched_programmes = [p for p in root.findall('programme') if p.get('channel') == epg.tvg_id] + logger.debug(f"Found {len(matched_programmes)} programmes for tvg_id={epg.tvg_id}") - with transaction.atomic(): - for prog in matched_programmes: - start_time = parse_xmltv_time(prog.get('start')) - end_time = parse_xmltv_time(prog.get('stop')) - title = prog.findtext('title', default='No Title') - desc = prog.findtext('desc', default='') + programs_to_create = [] + for prog in matched_programmes: + start_time = parse_xmltv_time(prog.get('start')) + end_time = parse_xmltv_time(prog.get('stop')) + title = prog.findtext('title', default='No Title') + desc = prog.findtext('desc', default='') + sub_title = prog.findtext('sub-title', default='') - obj, created = ProgramData.objects.update_or_create( - epg=epg_obj, - start_time=start_time, - title=title, - defaults={ - 'end_time': end_time, - 'description': desc, - 'sub_title': '', - 'tvg_id': tvg_id, - } - ) - if created: - logger.debug(f"Created ProgramData: {title} [{start_time} - {end_time}]") - logger.info(f"Completed program parsing for tvg_id={tvg_id}.") + # Extract custom properties + custom_props = {} + # Extract categories + categories = [] + for cat_elem in prog.findall('category'): + if cat_elem.text and cat_elem.text.strip(): + categories.append(cat_elem.text.strip()) + if categories: + custom_props['categories'] = categories + + # Extract episode numbers + for ep_num in prog.findall('episode-num'): + system = ep_num.get('system', '') + if system == 'xmltv_ns' and ep_num.text: + # Parse XMLTV episode-num format (season.episode.part) + parts = ep_num.text.split('.') + if len(parts) >= 2: + if parts[0].strip() != '': + try: + season = int(parts[0]) + 1 # XMLTV format is zero-based + custom_props['season'] = season + except ValueError: + pass + if parts[1].strip() != '': + try: + episode = int(parts[1]) + 1 # XMLTV format is zero-based + custom_props['episode'] = episode + except ValueError: + pass + elif system == 'onscreen' and ep_num.text: + # Just store the raw onscreen format + custom_props['onscreen_episode'] = ep_num.text.strip() + + # Extract ratings + for rating_elem in prog.findall('rating'): + if rating_elem.findtext('value'): + custom_props['rating'] = rating_elem.findtext('value').strip() + if rating_elem.get('system'): + custom_props['rating_system'] = rating_elem.get('system') + break # Just use the first rating + + # Extract credits (actors, directors, etc.) + credits_elem = prog.find('credits') + if credits_elem is not None: + credits = {} + for credit_type in ['director', 'actor', 'writer', 'presenter', 'producer']: + elements = credits_elem.findall(credit_type) + if elements: + names = [e.text.strip() for e in elements if e.text and e.text.strip()] + if names: + credits[credit_type] = names + if credits: + custom_props['credits'] = credits + + # Extract other common program metadata + if prog.findtext('date'): + custom_props['year'] = prog.findtext('date').strip()[:4] # Just the year part + + if prog.findtext('country'): + custom_props['country'] = prog.findtext('country').strip() + + for icon_elem in prog.findall('icon'): + if icon_elem.get('src'): + custom_props['icon'] = icon_elem.get('src') + break # Just use the first icon + + for kw in ['previously-shown', 'premiere', 'new']: + if prog.find(kw) is not None: + custom_props[kw.replace('-', '_')] = True + + # Convert custom_props to JSON string if not empty + custom_properties_json = None + if custom_props: + import json + try: + custom_properties_json = json.dumps(custom_props) + except Exception as e: + logger.error(f"Error serializing custom properties to JSON: {e}", exc_info=True) + + programs_to_create.append(ProgramData( + epg=epg, + start_time=start_time, + end_time=end_time, + title=title, + description=desc, + sub_title=sub_title, + tvg_id=epg.tvg_id, + custom_properties=custom_properties_json + )) + + ProgramData.objects.bulk_create(programs_to_create) + + release_task_lock('parse_epg_programs', epg_id) + + logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.") + +def parse_programs_for_source(epg_source, tvg_id=None): + file_path = epg_source.file_path + epg_entries = EPGData.objects.filter(epg_source=epg_source) + for epg in epg_entries: + if epg.tvg_id: + parse_programs_for_tvg_id(epg.id) def fetch_schedules_direct(source): logger.info(f"Fetching Schedules Direct data from source: {source.name}") diff --git a/apps/hdhr/api_views.py b/apps/hdhr/api_views.py index 4dd9c07d..4aefcc9a 100644 --- a/apps/hdhr/api_views.py +++ b/apps/hdhr/api_views.py @@ -6,7 +6,7 @@ from django.http import JsonResponse, HttpResponseForbidden, HttpResponse from drf_yasg.utils import swagger_auto_schema from drf_yasg import openapi from django.shortcuts import get_object_or_404 -from apps.channels.models import Channel +from apps.channels.models import Channel, ChannelProfile from .models import HDHRDevice from .serializers import HDHRDeviceSerializer from django.contrib.auth.decorators import login_required @@ -38,8 +38,12 @@ class DiscoverAPIView(APIView): operation_description="Retrieve HDHomeRun device discovery information", responses={200: openapi.Response("HDHR Discovery JSON")} ) - def get(self, request): - base_url = request.build_absolute_uri('/hdhr/').rstrip('/') + def get(self, request, profile=None): + uri_parts = ["hdhr"] + if profile is not None: + uri_parts.append(profile) + + base_url = request.build_absolute_uri(f'/{"/".join(uri_parts)}/').rstrip('/') device = HDHRDevice.objects.first() if not device: @@ -52,6 +56,7 @@ class DiscoverAPIView(APIView): "DeviceAuth": "test_auth_token", "BaseURL": base_url, "LineupURL": f"{base_url}/lineup.json", + "TunerCount": 10, } else: data = { @@ -63,6 +68,7 @@ class DiscoverAPIView(APIView): "DeviceAuth": "test_auth_token", "BaseURL": base_url, "LineupURL": f"{base_url}/lineup.json", + "TunerCount": 10, } return JsonResponse(data) @@ -75,13 +81,23 @@ class LineupAPIView(APIView): operation_description="Retrieve the available channel lineup", responses={200: openapi.Response("Channel Lineup JSON")} ) - def get(self, request): - channels = Channel.objects.all().order_by('channel_number') + def get(self, request, profile=None): + if profile is not None: + channel_profile = ChannelProfile.objects.get(name=profile) + channels = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True + ).order_by('channel_number') + else: + channels = Channel.objects.all().order_by('channel_number') + lineup = [ { "GuideNumber": str(ch.channel_number), "GuideName": ch.name, - "URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}") + "URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}"), + "Guide_ID": str(ch.channel_number), + "Station": str(ch.channel_number), } for ch in channels ] @@ -96,7 +112,7 @@ class LineupStatusAPIView(APIView): operation_description="Retrieve the HDHomeRun lineup status", responses={200: openapi.Response("Lineup Status JSON")} ) - def get(self, request): + def get(self, request, profile=None): data = { "ScanInProgress": 0, "ScanPossible": 0, diff --git a/apps/hdhr/urls.py b/apps/hdhr/urls.py index 2a1a06e0..2659cd7b 100644 --- a/apps/hdhr/urls.py +++ b/apps/hdhr/urls.py @@ -10,9 +10,12 @@ router.register(r'devices', HDHRDeviceViewSet, basename='hdhr-device') urlpatterns = [ path('dashboard/', hdhr_dashboard_view, name='hdhr_dashboard'), path('', hdhr_dashboard_view, name='hdhr_dashboard'), - path('discover.json', DiscoverAPIView.as_view(), name='discover'), - path('lineup.json', LineupAPIView.as_view(), name='lineup'), - path('lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status'), + path('/discover.json', DiscoverAPIView.as_view(), name='discover_with_profile'), + path('discover.json', DiscoverAPIView.as_view(), name='discover_no_profile'), + path('/lineup.json', LineupAPIView.as_view(), name='lineup_with_profile'), + path('lineup.json', LineupAPIView.as_view(), name='lineup_no_profile'), + path('/lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status_with_profile'), + path('lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status_no_profile'), path('device.xml', HDHRDeviceXMLAPIView.as_view(), name='device_xml'), ] diff --git a/apps/hdhr/views.py b/apps/hdhr/views.py index 4dd9c07d..48c48f80 100644 --- a/apps/hdhr/views.py +++ b/apps/hdhr/views.py @@ -52,6 +52,7 @@ class DiscoverAPIView(APIView): "DeviceAuth": "test_auth_token", "BaseURL": base_url, "LineupURL": f"{base_url}/lineup.json", + "TunerCount": "10", } else: data = { @@ -63,6 +64,7 @@ class DiscoverAPIView(APIView): "DeviceAuth": "test_auth_token", "BaseURL": base_url, "LineupURL": f"{base_url}/lineup.json", + "TunerCount": "10", } return JsonResponse(data) diff --git a/apps/m3u/api_views.py b/apps/m3u/api_views.py index 508b6a89..054bdaa9 100644 --- a/apps/m3u/api_views.py +++ b/apps/m3u/api_views.py @@ -7,6 +7,9 @@ from drf_yasg import openapi from django.shortcuts import get_object_or_404 from django.http import JsonResponse from django.core.cache import cache +import os +from rest_framework.decorators import action +from django.conf import settings # Import all models, including UserAgent. from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile @@ -22,6 +25,8 @@ from .serializers import ( ) from .tasks import refresh_single_m3u_account, refresh_m3u_accounts +from django.core.files.storage import default_storage +from django.core.files.base import ContentFile class M3UAccountViewSet(viewsets.ModelViewSet): """Handles CRUD operations for M3U accounts""" @@ -29,38 +34,60 @@ class M3UAccountViewSet(viewsets.ModelViewSet): serializer_class = M3UAccountSerializer permission_classes = [IsAuthenticated] + def create(self, request, *args, **kwargs): + # Handle file upload first, if any + file_path = None + if 'file' in request.FILES: + file = request.FILES['file'] + file_name = file.name + file_path = os.path.join('/data/uploads/m3us', file_name) + + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) + + # Add file_path to the request data so it's available during creation + request.data._mutable = True # Allow modification of the request data + request.data['file_path'] = file_path # Include the file path if a file was uploaded + request.data.pop('server_url') + request.data._mutable = False # Make the request data immutable again + + # Now call super().create() to create the instance + response = super().create(request, *args, **kwargs) + + # After the instance is created, return the response + return response + def update(self, request, *args, **kwargs): - # Get the M3UAccount instance we're updating instance = self.get_object() - # Handle updates to the 'enabled' flag of the related ChannelGroupM3UAccount instances - updates = request.data.get('channel_groups', []) + # Handle file upload first, if any + file_path = None + if 'file' in request.FILES: + file = request.FILES['file'] + file_name = file.name + file_path = os.path.join('/data/uploads/m3us', file_name) - for update_data in updates: - channel_group_id = update_data.get('channel_group') - enabled = update_data.get('enabled') + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) - try: - # Get the specific relationship to update - relationship = ChannelGroupM3UAccount.objects.get( - m3u_account=instance, channel_group_id=channel_group_id - ) - relationship.enabled = enabled - relationship.save() - except ChannelGroupM3UAccount.DoesNotExist: - return Response( - {"error": "ChannelGroupM3UAccount not found for the given M3UAccount and ChannelGroup."}, - status=status.HTTP_400_BAD_REQUEST - ) + # Add file_path to the request data so it's available during creation + request.data._mutable = True # Allow modification of the request data + request.data['file_path'] = file_path # Include the file path if a file was uploaded + request.data.pop('server_url') + request.data._mutable = False # Make the request data immutable again - # After updating the ChannelGroupM3UAccount relationships, reload the M3UAccount instance - instance.refresh_from_db() + if instance.file_path and os.path.exists(instance.file_path): + os.remove(instance.file_path) - refresh_single_m3u_account.delay(instance.id) + # Now call super().create() to create the instance + response = super().update(request, *args, **kwargs) - # Serialize and return the updated M3UAccount data - serializer = self.get_serializer(instance) - return Response(serializer.data) + # After the instance is created, return the response + return response class M3UFilterViewSet(viewsets.ModelViewSet): """Handles CRUD operations for M3U filters""" diff --git a/apps/m3u/migrations/0005_m3uaccount_custom_properties_and_more.py b/apps/m3u/migrations/0005_m3uaccount_custom_properties_and_more.py new file mode 100644 index 00000000..3728bf7f --- /dev/null +++ b/apps/m3u/migrations/0005_m3uaccount_custom_properties_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.1.6 on 2025-03-29 13:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_celery_beat', '0019_alter_periodictasks_options'), + ('m3u', '0004_m3uaccount_stream_profile'), + ] + + operations = [ + migrations.AddField( + model_name='m3uaccount', + name='custom_properties', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='m3uaccount', + name='refresh_interval', + field=models.IntegerField(default=24), + ), + migrations.AddField( + model_name='m3uaccount', + name='refresh_task', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_celery_beat.periodictask'), + ), + ] diff --git a/apps/m3u/migrations/0006_populate_periodic_tasks.py b/apps/m3u/migrations/0006_populate_periodic_tasks.py new file mode 100644 index 00000000..bc387979 --- /dev/null +++ b/apps/m3u/migrations/0006_populate_periodic_tasks.py @@ -0,0 +1,52 @@ +from django.db import migrations +import json + +def create_default_refresh_tasks(apps, schema_editor): + """ + Creates a PeriodicTask for each existing M3UAccount that doesn't have one. + """ + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + M3UAccount = apps.get_model("m3u", "M3UAccount") + + default_interval, _ = IntervalSchedule.objects.get_or_create( + every=24, + period="hours", + ) + + for account in M3UAccount.objects.all(): + if account.refresh_task: + continue + + task_name = f"m3u_account-refresh-{account.id}" + + refresh_task = PeriodicTask.objects.create( + name=task_name, + interval=default_interval, + task="apps.m3u.tasks.refresh_single_m3u_account", + kwargs=json.dumps({"account_id": account.id}), + ) + + account.refresh_task = refresh_task + account.save(update_fields=["refresh_task"]) + +def reverse_migration(apps, schema_editor): + IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + M3UAccount = apps.get_model("m3u", "M3UAccount") + + for account in M3UAccount.objects.all(): + IntervalSchedule.objects.filter(name=f"m3u_account-refresh-interval-{account.id}").delete() + PeriodicTask.objects.filter(name=f"m3u_account-refresh-{account.id}").delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("m3u", "0005_m3uaccount_custom_properties_and_more"), + ("django_celery_beat", "0019_alter_periodictasks_options"), + ] + + operations = [ + migrations.RunPython(create_default_refresh_tasks, reverse_migration), + ] diff --git a/apps/m3u/migrations/0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path.py b/apps/m3u/migrations/0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path.py new file mode 100644 index 00000000..086eff29 --- /dev/null +++ b/apps/m3u/migrations/0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.6 on 2025-04-06 19:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('m3u', '0006_populate_periodic_tasks'), + ] + + operations = [ + migrations.RemoveField( + model_name='m3uaccount', + name='uploaded_file', + ), + migrations.AddField( + model_name='m3uaccount', + name='file_path', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 773261df..25a332c6 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -4,6 +4,8 @@ from core.models import UserAgent import re from django.dispatch import receiver from apps.channels.models import StreamProfile +from django_celery_beat.models import PeriodicTask +from core.models import CoreSettings, UserAgent CUSTOM_M3U_ACCOUNT_NAME="custom" @@ -19,8 +21,8 @@ class M3UAccount(models.Model): null=True, help_text="The base URL of the M3U server (optional if a file is uploaded)" ) - uploaded_file = models.FileField( - upload_to='m3u_uploads/', + file_path = models.CharField( + max_length=255, blank=True, null=True ) @@ -67,6 +69,11 @@ class M3UAccount(models.Model): blank=True, related_name='m3u_accounts' ) + custom_properties = models.TextField(null=True, blank=True) + refresh_interval = models.IntegerField(default=24) + refresh_task = models.ForeignKey( + PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True + ) def __str__(self): return self.name @@ -94,6 +101,13 @@ class M3UAccount(models.Model): def get_custom_account(cls): return cls.objects.get(name=CUSTOM_M3U_ACCOUNT_NAME, locked=True) + def get_user_agent(self): + user_agent = self.user_agent + if not user_agent: + user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) + + return user_agent + # def get_channel_groups(self): # return ChannelGroup.objects.filter(m3u_account__m3u_account=self) diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index 3946aac5..d79b0117 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -1,4 +1,5 @@ from rest_framework import serializers +from rest_framework.response import Response from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile from core.models import UserAgent from apps.channels.models import ChannelGroup, ChannelGroupM3UAccount @@ -32,6 +33,19 @@ class M3UAccountProfileSerializer(serializers.ModelSerializer): return super().create(validated_data) + def update(self, instance, validated_data): + if instance.is_default: + raise serializers.ValidationError("Default profiles cannot be modified.") + return super().update(instance, validated_data) + + def destroy(self, request, *args, **kwargs): + instance = self.get_object() + if instance.is_default: + return Response( + {"error": "Default profiles cannot be deleted."}, + status=status.HTTP_400_BAD_REQUEST + ) + return super().destroy(request, *args, **kwargs) class M3UAccountSerializer(serializers.ModelSerializer): """Serializer for M3U Account""" @@ -39,77 +53,52 @@ class M3UAccountSerializer(serializers.ModelSerializer): # Include user_agent as a mandatory field using its primary key. user_agent = serializers.PrimaryKeyRelatedField( queryset=UserAgent.objects.all(), - required=True + required=False, + allow_null=True, ) profiles = M3UAccountProfileSerializer(many=True, read_only=True) - read_only_fields = ['locked'] + read_only_fields = ['locked', 'created_at', 'updated_at'] # channel_groups = serializers.SerializerMethodField() - channel_groups = ChannelGroupM3UAccountSerializer(source='channel_group.all', many=True, required=False) - + channel_groups = ChannelGroupM3UAccountSerializer(source='channel_group', many=True, required=False) class Meta: model = M3UAccount fields = [ - 'id', 'name', 'server_url', 'uploaded_file', 'server_group', + 'id', 'name', 'server_url', 'file_path', 'server_group', 'max_streams', 'is_active', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', - 'channel_groups', + 'channel_groups', 'refresh_interval' ] - # def get_channel_groups(self, obj): - # # Retrieve related ChannelGroupM3UAccount records for this M3UAccount - # relations = ChannelGroupM3UAccount.objects.filter(m3u_account=obj).select_related('channel_group') + def update(self, instance, validated_data): + # Pop out channel group memberships so we can handle them manually + channel_group_data = validated_data.pop('channel_group', []) - # # Serialize the channel groups with their enabled status - # return [ - # { - # 'channel_group_name': relation.channel_group.name, - # 'channel_group_id': relation.channel_group.id, - # 'enabled': relation.enabled, - # } - # for relation in relations - # ] + # First, update the M3UAccount itself + for attr, value in validated_data.items(): + setattr(instance, attr, value) + instance.save() - # def to_representation(self, instance): - # """Override the default to_representation method to include channel_groups""" - # representation = super().to_representation(instance) + # Prepare a list of memberships to update + memberships_to_update = [] + for group_data in channel_group_data: + group = group_data.get('channel_group') + enabled = group_data.get('enabled') - # # Manually add the channel_groups to the representation - # channel_groups = ChannelGroupM3UAccount.objects.filter(m3u_account=instance).select_related('channel_group') - # representation['channel_groups'] = [ - # { - # 'id': relation.id, - # 'channel_group_name': relation.channel_group.name, - # 'channel_group_id': relation.channel_group.id, - # 'enabled': relation.enabled, - # } - # for relation in channel_groups - # ] + try: + membership = ChannelGroupM3UAccount.objects.get( + m3u_account=instance, + channel_group=group + ) + membership.enabled = enabled + memberships_to_update.append(membership) + except ChannelGroupM3UAccount.DoesNotExist: + continue - # return representation + # Perform the bulk update + if memberships_to_update: + ChannelGroupM3UAccount.objects.bulk_update(memberships_to_update, ['enabled']) - # def update(self, instance, validated_data): - # logger.info(validated_data) - # channel_groups_data = validated_data.pop('channel_groups', None) - # instance = super().update(instance, validated_data) - - # if channel_groups_data is not None: - # logger.info(json.dumps(channel_groups_data)) - # # Remove existing relationships not included in the request - # existing_groups = {cg.channel_group_id: cg for cg in instance.channel_group.all()} - - # # for group_id in set(existing_groups.keys()) - sent_group_ids: - # # existing_groups[group_id].delete() - - # # Create or update relationships - # for cg_data in channel_groups_data: - # logger.info(json.dumps(cg_data)) - # ChannelGroupM3UAccount.objects.update_or_create( - # channel_group=existing_groups[cg_data['channel_group_id']], - # m3u_account=instance, - # defaults={'enabled': cg_data.get('enabled', True)} - # ) - - # return instance + return instance class ServerGroupSerializer(serializers.ModelSerializer): """Serializer for Server Group""" diff --git a/apps/m3u/signals.py b/apps/m3u/signals.py index c07c1c4c..6e46a0ff 100644 --- a/apps/m3u/signals.py +++ b/apps/m3u/signals.py @@ -1,8 +1,10 @@ # apps/m3u/signals.py -from django.db.models.signals import post_save +from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from .models import M3UAccount from .tasks import refresh_single_m3u_account, refresh_m3u_groups +from django_celery_beat.models import PeriodicTask, IntervalSchedule +import json @receiver(post_save, sender=M3UAccount) def refresh_account_on_save(sender, instance, created, **kwargs): @@ -12,4 +14,49 @@ def refresh_account_on_save(sender, instance, created, **kwargs): if it is active or newly created. """ if created: - refresh_single_m3u_account.delay(instance.id) + refresh_m3u_groups.delay(instance.id) + +@receiver(post_save, sender=M3UAccount) +def create_or_update_refresh_task(sender, instance, **kwargs): + """ + Create or update a Celery Beat periodic task when an M3UAccount is created/updated. + """ + task_name = f"m3u_account-refresh-{instance.id}" + + interval, _ = IntervalSchedule.objects.get_or_create( + every=int(instance.refresh_interval), + period=IntervalSchedule.HOURS + ) + + if not instance.refresh_task: + refresh_task = PeriodicTask.objects.create( + name=task_name, + interval=interval, + task="apps.m3u.tasks.refresh_single_m3u_account", + kwargs=json.dumps({"account_id": instance.id}), + enabled=instance.refresh_interval != 0, + ) + M3UAccount.objects.filter(id=instance.id).update(refresh_task=refresh_task) + else: + task = instance.refresh_task + updated_fields = [] + + if task.enabled != (instance.refresh_interval != 0): + task.enabled = instance.refresh_interval != 0 + updated_fields.append("enabled") + + if task.interval != interval: + task.interval = interval + updated_fields.append("interval") + + if updated_fields: + task.save(update_fields=updated_fields) + +@receiver(post_delete, sender=M3UAccount) +def delete_refresh_task(sender, instance, **kwargs): + """ + Delete the associated Celery Beat periodic task when a Channel is deleted. + """ + if instance.refresh_task: + instance.refresh_task.interval.delete() + instance.refresh_task.delete() diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 82f2310b..36de852b 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -3,6 +3,8 @@ import logging import re import requests import os +import gc +import gzip, zipfile from celery.app.control import Inspect from celery.result import AsyncResult from celery import shared_task, current_app, group @@ -15,15 +17,13 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer from django.utils import timezone import time -from channels.layers import get_channel_layer import json -from core.utils import redis_client +from core.utils import RedisClient, acquire_task_lock, release_task_lock from core.models import CoreSettings from asgiref.sync import async_to_sync logger = logging.getLogger(__name__) -LOCK_EXPIRE = 300 BATCH_SIZE = 1000 SKIP_EXTS = {} m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u") @@ -35,30 +35,74 @@ def fetch_m3u_lines(account, use_cache=False): """Fetch M3U file lines efficiently.""" if account.server_url: if not use_cache or not os.path.exists(file_path): - headers = {"User-Agent": account.user_agent.user_agent} + user_agent = account.get_user_agent() + headers = {"User-Agent": user_agent.user_agent} logger.info(f"Fetching from URL {account.server_url}") try: response = requests.get(account.server_url, headers=headers, stream=True) - response.raise_for_status() # This will raise an HTTPError if the status is not 200 + response.raise_for_status() + + total_size = int(response.headers.get('Content-Length', 0)) + downloaded = 0 + start_time = time.time() + last_update_time = start_time + progress = 0 + with open(file_path, 'wb') as file: - # Stream the content in chunks and write to the file - for chunk in response.iter_content(chunk_size=8192): # You can adjust the chunk size - if chunk: # Ensure chunk is not empty + send_m3u_update(account.id, "downloading", 0) + for chunk in response.iter_content(chunk_size=8192): + if chunk: file.write(chunk) + + downloaded += len(chunk) + elapsed_time = time.time() - start_time + + # Calculate download speed in KB/s + speed = downloaded / elapsed_time / 1024 # in KB/s + + # Calculate progress percentage + if total_size and total_size > 0: + progress = (downloaded / total_size) * 100 + + # Time remaining (in seconds) + time_remaining = (total_size - downloaded) / (speed * 1024) + + current_time = time.time() + if current_time - last_update_time >= 0.5: + last_update_time = current_time + if progress > 0: + send_m3u_update(account.id, "downloading", progress, speed=speed, elapsed_time=elapsed_time, time_remaining=time_remaining) + + send_m3u_update(account.id, "downloading", 100) except requests.exceptions.RequestException as e: logger.error(f"Error fetching M3U from URL {account.server_url}: {e}") - return [] # Return an empty list in case of error + return [] with open(file_path, 'r', encoding='utf-8') as f: return f.readlines() - elif account.uploaded_file: + elif account.file_path: try: - # Open the file and return the lines as a list or iterator - with open(account.uploaded_file.path, 'r', encoding='utf-8') as f: - return f.readlines() # Ensure you return lines from the file, not the file object - except IOError as e: - logger.error(f"Error opening file {account.uploaded_file.path}: {e}") - return [] # Return an empty list in case of error + if account.file_path.endswith('.gz'): + with gzip.open(account.file_path, 'rt', encoding='utf-8') as f: + return f.readlines() + + elif account.file_path.endswith('.zip'): + with zipfile.ZipFile(account.file_path, 'r') as zip_file: + for name in zip_file.namelist(): + if name.endswith('.m3u'): + with zip_file.open(name) as f: + return [line.decode('utf-8') for line in f.readlines()] + logger.warning(f"No .m3u file found in ZIP archive: {account.file_path}") + return [] + + else: + with open(account.file_path, 'r', encoding='utf-8') as f: + return f.readlines() + + except (IOError, OSError, zipfile.BadZipFile, gzip.BadGzipFile) as e: + logger.error(f"Error opening file {account.file_path}: {e}") + return [] + # Return an empty list if neither server_url nor uploaded_file is available return [] @@ -105,19 +149,6 @@ def _matches_filters(stream_name: str, group_name: str, filters): return exclude return False -def acquire_lock(task_name, account_id): - """Acquire a lock to prevent concurrent task execution.""" - lock_id = f"task_lock_{task_name}_{account_id}" - lock_acquired = cache.add(lock_id, "locked", timeout=LOCK_EXPIRE) - if not lock_acquired: - logger.warning(f"Lock for {task_name} and account_id={account_id} already acquired. Task will not proceed.") - return lock_acquired - -def release_lock(task_name, account_id): - """Release the lock after task execution.""" - lock_id = f"task_lock_{task_name}_{account_id}" - cache.delete(lock_id) - @shared_task def refresh_m3u_accounts(): """Queue background parse for all active M3UAccounts.""" @@ -175,20 +206,15 @@ def process_groups(account, group_names): ) @shared_task -def process_m3u_batch(account_id, batch, group_names, hash_keys): +def process_m3u_batch(account_id, batch, groups, hash_keys): """Processes a batch of M3U streams using bulk operations.""" account = M3UAccount.objects.get(id=account_id) - existing_groups = {group.name: group for group in ChannelGroup.objects.filter( - m3u_account__m3u_account=account, # Filter by the M3UAccount - m3u_account__enabled=True # Filter by the enabled flag in the join table - )} streams_to_create = [] streams_to_update = [] stream_hashes = {} # compiled_filters = [(f.filter_type, re.compile(f.regex_pattern, re.IGNORECASE)) for f in filters] - logger.debug(f"Processing batch of {len(batch)}") for stream_info in batch: name, url = stream_info["name"], stream_info["url"] @@ -196,7 +222,7 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys): group_title = stream_info["attributes"].get("group-title", "Default Group") # Filter out disabled groups for this account - if group_title not in existing_groups: + if group_title not in groups: logger.debug(f"Skipping stream in disabled group: {group_title}") continue @@ -213,19 +239,20 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys): try: stream_hash = Stream.generate_hash_key(name, url, tvg_id, hash_keys) - if redis_client.exists(f"m3u_refresh:{stream_hash}"): - # duplicate already processed by another batch - continue + # if redis_client.exists(f"m3u_refresh:{stream_hash}"): + # # duplicate already processed by another batch + # continue - redis_client.set(f"m3u_refresh:{stream_hash}", "true") + # redis_client.set(f"m3u_refresh:{stream_hash}", "true") stream_props = { "name": name, "url": url, "logo_url": tvg_logo, "tvg_id": tvg_id, "m3u_account": account, - "channel_group": existing_groups[group_title], + "channel_group_id": int(groups.get(group_title)), "stream_hash": stream_hash, + "custom_properties": json.dumps(stream_info["attributes"]), } if stream_hash not in stream_hashes: @@ -235,20 +262,17 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys): logger.error(json.dumps(stream_info)) existing_streams = {s.stream_hash: s for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys())} - logger.info(f"Hashed {len(stream_hashes.keys())} unique streams") for stream_hash, stream_props in stream_hashes.items(): if stream_hash in existing_streams: obj = existing_streams[stream_hash] - changed = False - for key, value in stream_props.items(): - if getattr(obj, key) == value: - continue - changed = True - setattr(obj, key, value) + existing_attr = {field.name: getattr(obj, field.name) for field in Stream._meta.fields if field != 'channel_group_id'} + changed = any(existing_attr[key] != value for key, value in stream_props.items() if key != 'channel_group_id') - obj.last_seen = timezone.now() if changed: + for key, value in stream_props.items(): + setattr(obj, key, value) + obj.last_seen = timezone.now() streams_to_update.append(obj) del existing_streams[stream_hash] else: @@ -258,17 +282,23 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys): streams_to_create.append(Stream(**stream_props)) try: - if streams_to_create: - Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) - if streams_to_update: - Stream.objects.bulk_update(streams_to_update, stream_props.keys()) - if len(existing_streams.keys()) > 0: - Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) + with transaction.atomic(): + if streams_to_create: + Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) + if streams_to_update: + Stream.objects.bulk_update(streams_to_update, { key for key in stream_props.keys() if key not in ["m3u_account", "stream_hash"] and key not in hash_keys}) + # if len(existing_streams.keys()) > 0: + # Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) except Exception as e: logger.error(f"Bulk create failed: {str(e)}") + retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." - return f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + # Aggressive garbage collection + del streams_to_create, streams_to_update, stream_hashes, existing_streams + gc.collect() + + return retval def cleanup_streams(account_id): account = M3UAccount.objects.get(id=account_id, is_active=True) @@ -290,25 +320,21 @@ def cleanup_streams(account_id): logger.info(f"Cleanup complete") -def refresh_m3u_groups(account_id): - if not acquire_lock('refresh_m3u_account_groups', account_id): +@shared_task +def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): + if not acquire_task_lock('refresh_m3u_account_groups', account_id): return f"Task already running for account_id={account_id}.", None - # Record start time - start_time = time.time() - send_progress_update(0, account_id) - try: account = M3UAccount.objects.get(id=account_id, is_active=True) except M3UAccount.DoesNotExist: - release_lock('refresh_m3u_account_groups', account_id) - return f"M3UAccount with ID={account_id} not found or inactive." + release_task_lock('refresh_m3u_account_groups', account_id) + return f"M3UAccount with ID={account_id} not found or inactive.", None - lines = fetch_m3u_lines(account) extinf_data = [] groups = set(["Default Group"]) - for line in lines: + for line in fetch_m3u_lines(account, use_cache): line = line.strip() if line.startswith("#EXTINF"): parsed = parse_extinf_line(line) @@ -321,6 +347,8 @@ def refresh_m3u_groups(account_id): # Associate URL with the last EXTINF line extinf_data[-1]["url"] = line + send_m3u_update(account_id, "processing_groups", 0) + groups = list(groups) cache_path = os.path.join(m3u_dir, f"{account_id}.json") with open(cache_path, 'w', encoding='utf-8') as f: @@ -331,25 +359,41 @@ def refresh_m3u_groups(account_id): process_groups(account, groups) - release_lock('refresh_m3u_account_groups`', account_id) + release_task_lock('refresh_m3u_account_groups', account_id) + + send_m3u_update(account_id, "processing_groups", 100) + + if not full_refresh: + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": {"success": True, "type": "m3u_group_refresh", "account": account_id} + } + ) return extinf_data, groups @shared_task -def refresh_single_m3u_account(account_id, use_cache=False): +def refresh_single_m3u_account(account_id): """Splits M3U processing into chunks and dispatches them as parallel tasks.""" - if not acquire_lock('refresh_single_m3u_account', account_id): + if not acquire_task_lock('refresh_single_m3u_account', account_id): return f"Task already running for account_id={account_id}." + # redis_client = RedisClient.get_client() # Record start time start_time = time.time() - send_progress_update(0, account_id) try: account = M3UAccount.objects.get(id=account_id, is_active=True) + if not account.is_active: + logger.info(f"Account {account_id} is not active, skipping.") + return + filters = list(account.filters.all()) except M3UAccount.DoesNotExist: - release_lock('refresh_single_m3u_account', account_id) + release_task_lock('refresh_single_m3u_account', account_id) return f"M3UAccount with ID={account_id} not found or inactive." # Fetch M3U lines and handle potential issues @@ -358,7 +402,7 @@ def refresh_single_m3u_account(account_id, use_cache=False): groups = None cache_path = os.path.join(m3u_dir, f"{account_id}.json") - if use_cache and os.path.exists(cache_path): + if os.path.exists(cache_path): with open(cache_path, 'r') as file: data = json.load(file) @@ -367,15 +411,24 @@ def refresh_single_m3u_account(account_id, use_cache=False): if not extinf_data: try: - extinf_data, groups = refresh_m3u_groups(account_id) + extinf_data, groups = refresh_m3u_groups(account_id, full_refresh=True) + if not extinf_data or not groups: + release_task_lock('refresh_single_m3u_account', account_id) + return "Failed to update m3u account, task may already be running" except: + release_task_lock('refresh_single_m3u_account', account_id) return "Failed to update m3u account" hash_keys = CoreSettings.get_m3u_hash_key().split(",") + existing_groups = {group.name: group.id for group in ChannelGroup.objects.filter( + m3u_account__m3u_account=account, # Filter by the M3UAccount + m3u_account__enabled=True # Filter by the enabled flag in the join table + )} + # Break into batches and process in parallel batches = [extinf_data[i:i + BATCH_SIZE] for i in range(0, len(extinf_data), BATCH_SIZE)] - task_group = group(process_m3u_batch.s(account_id, batch, groups, hash_keys) for batch in batches) + task_group = group(process_m3u_batch.s(account_id, batch, existing_groups, hash_keys) for batch in batches) total_batches = len(batches) completed_batches = 0 @@ -399,7 +452,7 @@ def refresh_single_m3u_account(account_id, use_cache=False): if progress == 100: progress = 99 - send_progress_update(progress, account_id) + send_m3u_update(account_id, "parsing", progress) # Optionally remove completed task from the group to prevent processing it again result.remove(async_result) @@ -408,33 +461,54 @@ def refresh_single_m3u_account(account_id, use_cache=False): # Run cleanup cleanup_streams(account_id) - send_progress_update(100, account_id) + send_m3u_update(account_id, "parsing", 100) end_time = time.time() # Calculate elapsed time elapsed_time = end_time - start_time + account.save(update_fields=['updated_at']) print(f"Function took {elapsed_time} seconds to execute.") - release_lock('refresh_single_m3u_account', account_id) + # Aggressive garbage collection + del existing_groups, extinf_data, groups, batches + gc.collect() - cursor = 0 - while True: - cursor, keys = redis_client.scan(cursor, match=f"m3u_refresh:*", count=BATCH_SIZE) - if keys: - redis_client.delete(*keys) # Delete the matching keys - if cursor == 0: - break + # Clean up cache file since we've fully processed it + if os.path.exists(cache_path): + os.remove(cache_path) + + release_task_lock('refresh_single_m3u_account', account_id) + + # cursor = 0 + # while True: + # cursor, keys = redis_client.scan(cursor, match=f"m3u_refresh:*", count=BATCH_SIZE) + # if keys: + # redis_client.delete(*keys) # Delete the matching keys + # if cursor == 0: + # break return f"Dispatched jobs complete." -def send_progress_update(progress, account_id): +def send_m3u_update(account_id, action, progress, **kwargs): + # Start with the base data dictionary + data = { + "progress": progress, + "type": "m3u_refresh", + "account": account_id, + "action": action, + } + + # Add the additional key-value pairs from kwargs + data.update(kwargs) + + # Now, send the updated data dictionary channel_layer = get_channel_layer() async_to_sync(channel_layer.group_send)( 'updates', { 'type': 'update', - "data": {"progress": progress, "type": "m3u_refresh", "account": account_id} + 'data': data } ) diff --git a/apps/output/urls.py b/apps/output/urls.py index 02e43c83..92774adb 100644 --- a/apps/output/urls.py +++ b/apps/output/urls.py @@ -5,12 +5,12 @@ from core.views import stream_view app_name = 'output' urlpatterns = [ - # Allow both `/m3u` and `/m3u/` - re_path(r'^m3u/?$', generate_m3u, name='generate_m3u'), - - # Allow both `/epg` and `/epg/` - re_path(r'^epg/?$', generate_epg, name='generate_epg'), - + # Allow `/m3u`, `/m3u/`, `/m3u/profile_name`, and `/m3u/profile_name/` + re_path(r'^m3u(?:/(?P[^/]+))?/?$', generate_m3u, name='generate_m3u'), + + # Allow `/epg`, `/epg/`, `/epg/profile_name`, and `/epg/profile_name/` + re_path(r'^epg(?:/(?P[^/]+))?/?$', generate_epg, name='generate_epg'), + # Allow both `/stream/` and `/stream//` re_path(r'^stream/(?P[0-9a-fA-F\-]+)/?$', stream_view, name='stream'), ] diff --git a/apps/output/views.py b/apps/output/views.py index cffc20e1..3d268e10 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -1,21 +1,32 @@ from django.http import HttpResponse from django.urls import reverse -from apps.channels.models import Channel +from apps.channels.models import Channel, ChannelProfile from apps.epg.models import ProgramData from django.utils import timezone +from datetime import datetime, timedelta +import re +import html # Add this import for XML escaping -def generate_m3u(request): +def generate_m3u(request, profile_name=None): """ Dynamically generate an M3U file from channels. The stream URL now points to the new stream_view that uses StreamProfile. """ + if profile_name is not None: + channel_profile = ChannelProfile.objects.get(name=profile_name) + channels = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True + ).order_by('channel_number') + else: + channels = Channel.objects.order_by('channel_number') + m3u_content = "#EXTM3U\n" - channels = Channel.objects.order_by('channel_number') for channel in channels: group_title = channel.channel_group.name if channel.channel_group else "Default" - tvg_id = channel.tvg_id or "" - tvg_name = channel.tvg_name or channel.name - tvg_logo = channel.logo_url or "" + tvg_id = channel.channel_number or channel.id + tvg_name = channel.name + tvg_logo = channel.logo.url if channel.logo else "" channel_number = channel.channel_number extinf_line = ( @@ -33,44 +44,162 @@ def generate_m3u(request): response['Content-Disposition'] = 'attachment; filename="channels.m3u"' return response -def generate_epg(request): +def generate_dummy_epg(name, channel_id, num_days=7, interval_hours=4): + xml_lines = [] + + # Loop through the number of days + for day_offset in range(num_days): + current_day = datetime.now() + timedelta(days=day_offset) + + # Loop through each 4-hour interval in the day + for hour in range(0, 24, interval_hours): + start_time = current_day.replace(hour=hour, minute=0, second=0, microsecond=0) + stop_time = start_time + timedelta(hours=interval_hours) + + # Format the times as per the requested format + start_str = start_time.strftime("%Y%m%d%H%M%S") + " 0000" + stop_str = stop_time.strftime("%Y%m%d%H%M%S") + " 0000" + + # Create the XML-like programme entry with escaped name + xml_lines.append(f'') + xml_lines.append(f' {html.escape(name)}') + xml_lines.append(f'') + + return xml_lines + +def generate_epg(request, profile_name=None): """ Dynamically generate an XMLTV (EPG) file using the new EPGData/ProgramData models. Since the EPG data is stored independently of Channels, we group programmes by their associated EPGData record. This version does not filter by time, so it includes the entire EPG saved in the DB. """ - # Retrieve all ProgramData records and join the related EPGData record. - programs = ProgramData.objects.select_related('epg').all().order_by('start_time') - - # Group programmes by their EPGData record. - epg_programs = {} - for prog in programs: - epg = prog.epg - epg_programs.setdefault(epg, []).append(prog) - xml_lines = [] xml_lines.append('') - xml_lines.append('') + xml_lines.append('') - # Output channel definitions based on EPGData. - # Use the EPGData's tvg_id (or a fallback) as the channel identifier. - for epg in epg_programs.keys(): - channel_id = epg.tvg_id if epg.tvg_id else f"default-{epg.id}" + if profile_name is not None: + channel_profile = ChannelProfile.objects.get(name=profile_name) + channels = Channel.objects.filter( + channelprofilemembership__channel_profile=channel_profile, + channelprofilemembership__enabled=True + ) + else: + channels = Channel.objects.all() + + # Retrieve all active channels + for channel in channels: + channel_id = channel.channel_number or channel.id + display_name = channel.epg_data.name if channel.epg_data else channel.name xml_lines.append(f' ') - xml_lines.append(f' {epg.name}') + xml_lines.append(f' {html.escape(display_name)}') + + # Add channel logo if available + if channel.logo: + logo_url = channel.logo.url + + # Convert to absolute URL if it's relative + if logo_url.startswith('/data'): + # Use the full URL for the logo + logo_uri = re.sub(r"^\/data", '', logo_url) + base_url = request.build_absolute_uri('/')[:-1] + logo_url = f"{base_url}{logo_uri}" + + xml_lines.append(f' ') + xml_lines.append(' ') - # Output programme entries referencing the channel id from EPGData. - for epg, progs in epg_programs.items(): - channel_id = epg.tvg_id if epg.tvg_id else f"default-{epg.id}" - for prog in progs: - start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") - stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") - xml_lines.append(f' ') - xml_lines.append(f' {prog.title}') - xml_lines.append(f' {prog.description}') - xml_lines.append(' ') + for channel in channels: + channel_id = channel.channel_number or channel.id + display_name = channel.epg_data.name if channel.epg_data else channel.name + if not channel.epg_data: + xml_lines = xml_lines + generate_dummy_epg(display_name, channel_id) + else: + programs = channel.epg_data.programs.all() + for prog in programs: + start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z") + stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z") + xml_lines.append(f' ') + xml_lines.append(f' {html.escape(prog.title)}') + + # Add subtitle if available + if prog.sub_title: + xml_lines.append(f' {html.escape(prog.sub_title)}') + + # Add description if available + if prog.description: + xml_lines.append(f' {html.escape(prog.description)}') + + # Process custom properties if available + if prog.custom_properties: + try: + import json + custom_data = json.loads(prog.custom_properties) + + # Add categories if available + if 'categories' in custom_data and custom_data['categories']: + for category in custom_data['categories']: + xml_lines.append(f' {html.escape(category)}') + + # Handle episode numbering - multiple formats supported + # Standard episode number if available + if 'episode' in custom_data: + xml_lines.append(f' E{custom_data["episode"]}') + + # Handle onscreen episode format (like S06E128) + if 'onscreen_episode' in custom_data: + xml_lines.append(f' {html.escape(custom_data["onscreen_episode"])}') + + # Add season and episode numbers in xmltv_ns format if available + if 'season' in custom_data and 'episode' in custom_data: + season = int(custom_data['season']) - 1 if str(custom_data['season']).isdigit() else 0 + episode = int(custom_data['episode']) - 1 if str(custom_data['episode']).isdigit() else 0 + xml_lines.append(f' {season}.{episode}.') + + # Add rating if available + if 'rating' in custom_data: + rating_system = custom_data.get('rating_system', 'TV Parental Guidelines') + xml_lines.append(f' ') + xml_lines.append(f' {html.escape(custom_data["rating"])}') + xml_lines.append(f' ') + + # Add actors/directors/writers if available + if 'credits' in custom_data: + xml_lines.append(f' ') + for role, people in custom_data['credits'].items(): + if isinstance(people, list): + for person in people: + xml_lines.append(f' <{role}>{html.escape(person)}') + else: + xml_lines.append(f' <{role}>{html.escape(people)}') + xml_lines.append(f' ') + + # Add program date/year if available + if 'year' in custom_data: + xml_lines.append(f' {html.escape(custom_data["year"])}') + + # Add country if available + if 'country' in custom_data: + xml_lines.append(f' {html.escape(custom_data["country"])}') + + # Add icon if available + if 'icon' in custom_data: + xml_lines.append(f' ') + + # Add special flags as proper tags + if custom_data.get('previously_shown', False): + xml_lines.append(f' ') + + if custom_data.get('premiere', False): + xml_lines.append(f' ') + + if custom_data.get('new', False): + xml_lines.append(f' ') + + except Exception as e: + xml_lines.append(f' ') + + xml_lines.append(' ') xml_lines.append('') xml_content = "\n".join(xml_lines) diff --git a/apps/proxy/apps.py b/apps/proxy/apps.py index c8c42088..d1c8b966 100644 --- a/apps/proxy/apps.py +++ b/apps/proxy/apps.py @@ -1,3 +1,4 @@ +import sys from django.apps import AppConfig class ProxyConfig(AppConfig): @@ -7,9 +8,10 @@ class ProxyConfig(AppConfig): def ready(self): """Initialize proxy servers when Django starts""" - from .hls_proxy.server import ProxyServer as HLSProxyServer - from .ts_proxy.server import ProxyServer as TSProxyServer - - # Initialize proxy servers - self.hls_proxy = HLSProxyServer() - self.ts_proxy = TSProxyServer() \ No newline at end of file + if 'manage.py' not in sys.argv: + from .hls_proxy.server import ProxyServer as HLSProxyServer + from .ts_proxy.server import ProxyServer as TSProxyServer + + # Initialize proxy servers + self.hls_proxy = HLSProxyServer() + self.ts_proxy = TSProxyServer() diff --git a/apps/proxy/config.py b/apps/proxy/config.py index 28d3b872..a465b5d7 100644 --- a/apps/proxy/config.py +++ b/apps/proxy/config.py @@ -5,6 +5,10 @@ class BaseConfig: CHUNK_SIZE = 8192 CLIENT_POLL_INTERVAL = 0.1 MAX_RETRIES = 3 + RETRY_WAIT_INTERVAL = 0.5 # seconds to wait between retries + CONNECTION_TIMEOUT = 10 # seconds to wait for initial connection + MAX_STREAM_SWITCHES = 10 # Maximum number of stream switch attempts before giving up + BUFFER_CHUNK_SIZE = 188 * 1361 # ~256KB # Redis settings REDIS_CHUNK_TTL = 60 # Number in seconds - Chunks expire after 1 minute @@ -24,10 +28,6 @@ class HLSConfig(BaseConfig): class TSConfig(BaseConfig): """Configuration settings for TS proxy""" - # Connection settings - CONNECTION_TIMEOUT = 10 # seconds to wait for initial connection - MAX_RETRIES = 3 # maximum connection retry attempts - # Buffer settings INITIAL_BEHIND_CHUNKS = 4 # How many chunks behind to start a client (4 chunks = ~1MB) CHUNK_BATCH_SIZE = 5 # How many chunks to fetch in one batch @@ -52,8 +52,13 @@ class TSConfig(BaseConfig): # TS packets are 188 bytes # Make chunk size a multiple of TS packet size for perfect alignment # ~1MB is ideal for streaming (matches typical media buffer sizes) - BUFFER_CHUNK_SIZE = 188 * 1361 # ~256KB - # Maximum number of stream switch attempts before giving up - MAX_STREAM_SWITCHES = 10 + # Stream health and recovery settings + MAX_HEALTH_RECOVERY_ATTEMPTS = 2 # Maximum times to attempt recovery for a single stream + MAX_RECONNECT_ATTEMPTS = 3 # Maximum reconnects to try before switching streams + MIN_STABLE_TIME_BEFORE_RECONNECT = 30 # Minimum seconds a stream must be stable to try reconnect + FAILOVER_GRACE_PERIOD = 20 # Extra time (seconds) to allow for stream switching before disconnecting clients + + + diff --git a/apps/proxy/tasks.py b/apps/proxy/tasks.py index 37a1f8f9..a4aaf8e5 100644 --- a/apps/proxy/tasks.py +++ b/apps/proxy/tasks.py @@ -6,7 +6,7 @@ import redis import json import logging import re -from core.utils import redis_client +from core.utils import RedisClient from apps.proxy.ts_proxy.channel_status import ChannelStatus logger = logging.getLogger(__name__) @@ -16,6 +16,8 @@ last_known_data = {} @shared_task def fetch_channel_stats(): + redis_client = RedisClient.get_client() + try: # Basic info for all channels channel_pattern = "ts_proxy:channel:*:metadata" diff --git a/apps/proxy/ts_proxy/__init__.py b/apps/proxy/ts_proxy/__init__.py index 2f48eb4b..e69de29b 100644 --- a/apps/proxy/ts_proxy/__init__.py +++ b/apps/proxy/ts_proxy/__init__.py @@ -1,9 +0,0 @@ -"""Transport Stream proxy module""" - -# Only class imports, no instance creation -from .server import ProxyServer -from .stream_manager import StreamManager -from .stream_buffer import StreamBuffer -from .client_manager import ClientManager - -proxy_server = ProxyServer() diff --git a/apps/proxy/ts_proxy/apps.py b/apps/proxy/ts_proxy/apps.py new file mode 100644 index 00000000..116720df --- /dev/null +++ b/apps/proxy/ts_proxy/apps.py @@ -0,0 +1,13 @@ +import sys +from django.apps import AppConfig + +class TSProxyConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.proxy.ts_proxy' + verbose_name = "TS Stream Proxies" + + def ready(self): + """Initialize proxy servers when Django starts""" + if 'manage.py' not in sys.argv: + from .server import ProxyServer + ProxyServer.get_instance() diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 9338fb7f..d4e33f02 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -1,9 +1,9 @@ import logging import time import re -from . import proxy_server +from .server import ProxyServer from .redis_keys import RedisKeys -from .constants import TS_PACKET_SIZE +from .constants import TS_PACKET_SIZE, ChannelMetadataField from redis.exceptions import ConnectionError, TimeoutError from .utils import get_logger @@ -22,6 +22,8 @@ class ChannelStatus: return (total_bytes * 8) / duration / 1000 def get_detailed_channel_info(channel_id): + proxy_server = ProxyServer.get_instance() + # Get channel metadata metadata_key = RedisKeys.channel_metadata(channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) @@ -35,28 +37,31 @@ class ChannelStatus: info = { 'channel_id': channel_id, - 'state': metadata.get(b'state', b'unknown').decode('utf-8'), - 'url': metadata.get(b'url', b'').decode('utf-8'), - 'profile': metadata.get(b'profile', b'unknown').decode('utf-8'), - 'started_at': metadata.get(b'init_time', b'0').decode('utf-8'), - 'owner': metadata.get(b'owner', b'unknown').decode('utf-8'), + 'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'), + 'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'), + 'started_at': metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'), + 'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'), 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, } # Add timing information - if b'state_changed_at' in metadata: - state_changed_at = float(metadata[b'state_changed_at'].decode('utf-8')) + state_changed_field = ChannelMetadataField.STATE_CHANGED_AT.encode('utf-8') + if state_changed_field in metadata: + state_changed_at = float(metadata[state_changed_field].decode('utf-8')) info['state_changed_at'] = state_changed_at info['state_duration'] = time.time() - state_changed_at - if b'init_time' in metadata: - created_at = float(metadata[b'init_time'].decode('utf-8')) + init_time_field = ChannelMetadataField.INIT_TIME.encode('utf-8') + if init_time_field in metadata: + created_at = float(metadata[init_time_field].decode('utf-8')) info['started_at'] = created_at info['uptime'] = time.time() - created_at # Add data throughput information - if b'total_bytes' in metadata: - total_bytes = int(metadata[b'total_bytes'].decode('utf-8')) + total_bytes_field = ChannelMetadataField.TOTAL_BYTES.encode('utf-8') + if total_bytes_field in metadata: + total_bytes = int(metadata[total_bytes_field].decode('utf-8')) info['total_bytes'] = total_bytes # Format total bytes in human-readable form @@ -87,7 +92,7 @@ class ChannelStatus: for client_id in client_ids: client_id_str = client_id.decode('utf-8') - client_key = f"ts_proxy:channel:{channel_id}:clients:{client_id_str}" + client_key = RedisKeys.client_metadata(channel_id, client_id_str) client_data = proxy_server.redis_client.hgetall(client_key) if client_data: @@ -227,6 +232,8 @@ class ChannelStatus: @staticmethod def _execute_redis_command(command_func): """Execute Redis command with error handling""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return None @@ -242,6 +249,8 @@ class ChannelStatus: @staticmethod def get_basic_channel_info(channel_id): """Get basic channel information with Redis error handling""" + proxy_server = ProxyServer.get_instance() + try: # Use _execute_redis_command for Redis operations metadata_key = RedisKeys.channel_metadata(channel_id) @@ -261,23 +270,23 @@ class ChannelStatus: client_count = proxy_server.redis_client.scard(client_set_key) or 0 # Calculate uptime - created_at = float(metadata.get(b'init_time', b'0').decode('utf-8')) + created_at = float(metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8')) uptime = time.time() - created_at if created_at > 0 else 0 # Simplified info info = { 'channel_id': channel_id, - 'state': metadata.get(b'state', b'unknown').decode('utf-8'), - 'url': metadata.get(b'url', b'').decode('utf-8'), - 'profile': metadata.get(b'profile', b'unknown').decode('utf-8'), - 'owner': metadata.get(b'owner', b'unknown').decode('utf-8'), + 'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'), + 'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'), + 'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'), 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, 'client_count': client_count, 'uptime': uptime } # Add data throughput information to basic info - total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, 'total_bytes') + total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES.encode('utf-8')) if total_bytes_bytes: total_bytes = int(total_bytes_bytes.decode('utf-8')) info['total_bytes'] = total_bytes @@ -307,7 +316,7 @@ class ChannelStatus: # Get up to 10 clients for the basic view for client_id in list(client_ids)[:10]: client_id_str = client_id.decode('utf-8') - client_key = f"ts_proxy:channel:{channel_id}:clients:{client_id_str}" + client_key = RedisKeys.client_metadata(channel_id, client_id_str) # Efficient way - just retrieve the essentials client_info = { diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index 42d7e04d..98dbf072 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -38,6 +38,11 @@ class ClientManager: def _start_heartbeat_thread(self): """Start thread to regularly refresh client presence in Redis""" def heartbeat_task(): + no_clients_count = 0 # Track consecutive empty cycles + max_empty_cycles = 3 # Exit after this many consecutive empty checks + + logger.debug(f"Started heartbeat thread for channel {self.channel_id} (interval: {self.heartbeat_interval}s)") + while True: try: # Wait for the interval @@ -46,7 +51,19 @@ class ClientManager: # Send heartbeat for all local clients with self.lock: if not self.clients or not self.redis_client: + # No clients left, increment our counter + no_clients_count += 1 + + # If we've seen no clients for several consecutive checks, exit the thread + if no_clients_count >= max_empty_cycles: + logger.info(f"No clients for channel {self.channel_id} after {no_clients_count} consecutive checks, exiting heartbeat thread") + return # This exits the thread + + # Skip this cycle if we have no clients continue + else: + # Reset counter when we see clients + no_clients_count = 0 # IMPROVED GHOST DETECTION: Check for stale clients before sending heartbeats current_time = time.time() diff --git a/apps/proxy/ts_proxy/config_helper.py b/apps/proxy/ts_proxy/config_helper.py index c0576cb7..f78ba0b6 100644 --- a/apps/proxy/ts_proxy/config_helper.py +++ b/apps/proxy/ts_proxy/config_helper.py @@ -70,3 +70,8 @@ class ConfigHelper: def max_stream_switches(): """Get maximum number of stream switch attempts""" return ConfigHelper.get('MAX_STREAM_SWITCHES', 10) + + @staticmethod + def retry_wait_interval(): + """Get wait interval between connection retries in seconds""" + return ConfigHelper.get('RETRY_WAIT_INTERVAL', 0.5) # Default to 0.5 second diff --git a/apps/proxy/ts_proxy/constants.py b/apps/proxy/ts_proxy/constants.py index 56caacbe..4827b24b 100644 --- a/apps/proxy/ts_proxy/constants.py +++ b/apps/proxy/ts_proxy/constants.py @@ -35,6 +35,45 @@ class StreamType: TS = "ts" UNKNOWN = "unknown" +# Channel metadata field names stored in Redis +class ChannelMetadataField: + # Basic fields + URL = "url" + USER_AGENT = "user_agent" + STATE = "state" + OWNER = "owner" + STREAM_ID = "stream_id" + + # Profile fields + STREAM_PROFILE = "stream_profile" + M3U_PROFILE = "m3u_profile" + + # Status and error fields + ERROR_MESSAGE = "error_message" + ERROR_TIME = "error_time" + STATE_CHANGED_AT = "state_changed_at" + INIT_TIME = "init_time" + CONNECTION_READY_TIME = "connection_ready_time" + + # Buffer and data tracking + BUFFER_CHUNKS = "buffer_chunks" + TOTAL_BYTES = "total_bytes" + + # Stream switching + STREAM_SWITCH_TIME = "stream_switch_time" + STREAM_SWITCH_REASON = "stream_switch_reason" + + # Client metadata fields + CONNECTED_AT = "connected_at" + LAST_ACTIVE = "last_active" + BYTES_SENT = "bytes_sent" + AVG_RATE_KBPS = "avg_rate_KBps" + CURRENT_RATE_KBPS = "current_rate_KBps" + IP_ADDRESS = "ip_address" + WORKER_ID = "worker_id" + CHUNKS_SENT = "chunks_sent" + STATS_UPDATED_AT = "stats_updated_at" + # TS packet constants TS_PACKET_SIZE = 188 TS_SYNC_BYTE = 0x47 diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index 2dd923fd..1a04ffb5 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -17,8 +17,8 @@ import os import json from typing import Dict, Optional, Set from apps.proxy.config import TSConfig as Config -from apps.channels.models import Channel -from core.utils import redis_client as global_redis_client, redis_pubsub_client as global_redis_pubsub_client # Import both global Redis clients +from apps.channels.models import Channel, Stream +from core.utils import RedisClient from redis.exceptions import ConnectionError, TimeoutError from .stream_manager import StreamManager from .stream_buffer import StreamBuffer @@ -32,6 +32,19 @@ logger = get_logger() class ProxyServer: """Manages TS proxy server instance with worker coordination""" + _instance = None + + @classmethod + def get_instance(cls): + if cls._instance is None: + from .server import ProxyServer + from .stream_manager import StreamManager + from .stream_buffer import StreamBuffer + from .client_manager import ClientManager + + cls._instance = ProxyServer() + + return cls._instance def __init__(self): """Initialize proxy server with worker identification""" @@ -46,17 +59,17 @@ class ProxyServer: hostname = socket.gethostname() self.worker_id = f"{hostname}:{pid}" - # Connect to Redis - try using global client first + # Connect to Redis - use dedicated client for proxy self.redis_client = None self.redis_connection_attempts = 0 self.redis_max_retries = 3 self.redis_retry_interval = 5 # seconds try: - # First try to use the global client from core.utils - if global_redis_client is not None: - self.redis_client = global_redis_client - logger.info(f"Using global Redis client") + # Use dedicated Redis client for proxy + self.redis_client = RedisClient.get_client() + if self.redis_client is not None: + logger.info(f"Using dedicated Redis client for proxy server") logger.info(f"Worker ID: {self.worker_id}") else: # Fall back to direct connection with retry @@ -75,50 +88,14 @@ class ProxyServer: def _setup_redis_connection(self): """Setup Redis connection with retry logic""" - import redis - from django.conf import settings - - while self.redis_connection_attempts < self.redis_max_retries: - try: - logger.info(f"Attempting to connect to Redis ({self.redis_connection_attempts+1}/{self.redis_max_retries})") - - # Get connection parameters from settings or environment - redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) - redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) - redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) - - # Create Redis client with reasonable timeouts - self.redis_client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - socket_timeout=5, - socket_connect_timeout=5, - retry_on_timeout=True, - health_check_interval=30 - ) - - # Test connection - self.redis_client.ping() - logger.info(f"Successfully connected to Redis at {redis_host}:{redis_port}/{redis_db}") - logger.info(f"Worker ID: {self.worker_id}") - break - - except (ConnectionError, TimeoutError) as e: - self.redis_connection_attempts += 1 - if self.redis_connection_attempts >= self.redis_max_retries: - logger.error(f"Failed to connect to Redis after {self.redis_max_retries} attempts: {e}") - self.redis_client = None - else: - # Exponential backoff with a maximum of 30 seconds - retry_delay = min(self.redis_retry_interval * (2 ** (self.redis_connection_attempts - 1)), 30) - logger.warning(f"Redis connection failed. Retrying in {retry_delay}s... ({self.redis_connection_attempts}/{self.redis_max_retries})") - time.sleep(retry_delay) - - except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}", exc_info=True) - self.redis_client = None - break + # Try to use get_redis_client utility instead of direct connection + self.redis_client = RedisClient.get_client(max_retries=self.redis_max_retries, + retry_interval=self.redis_retry_interval) + if self.redis_client: + logger.info(f"Successfully connected to Redis using utility function") + logger.info(f"Worker ID: {self.worker_id}") + else: + logger.error(f"Failed to connect to Redis after {self.redis_max_retries} attempts") def _execute_redis_command(self, command_func, *args, **kwargs): """Execute Redis command with error handling and reconnection logic""" @@ -156,12 +133,13 @@ class ProxyServer: while True: try: - # Use the global PubSub client if available - if global_redis_pubsub_client: - pubsub_client = global_redis_pubsub_client - logger.info("Using global Redis PubSub client for event listener") + # Use dedicated PubSub client for event listener + pubsub_client = RedisClient.get_pubsub_client() + if pubsub_client: + logger.info("Using dedicated Redis PubSub client for event listener") else: - # Fall back to creating a dedicated client if global one is unavailable + # Fall back to creating a dedicated client if utility fails + logger.warning("Utility function for PubSub client failed, creating direct connection") from django.conf import settings import redis @@ -178,7 +156,7 @@ class ProxyServer: socket_keepalive=True, health_check_interval=30 ) - logger.info("Created dedicated Redis PubSub client for event listener") + logger.info("Created fallback Redis PubSub client for event listener") # Test connection before subscribing pubsub_client.ping() @@ -740,12 +718,16 @@ class ProxyServer: # Force release resources in the Channel model try: - from apps.channels.models import Channel channel = Channel.objects.get(uuid=channel_id) channel.release_stream() logger.info(f"Released stream allocation for zombie channel {channel_id}") except Exception as e: - logger.error(f"Error releasing stream for zombie channel {channel_id}: {e}") + try: + stream = Stream.objects.get(stream_hash=channel_id) + stream.release_stream() + logger.info(f"Released stream allocation for zombie channel {channel_id}") + except Exception as e: + logger.error(f"Error releasing stream for zombie channel {channel_id}: {e}") return True except Exception as e: @@ -1067,8 +1049,12 @@ class ProxyServer: def _clean_redis_keys(self, channel_id): """Clean up all Redis keys for a channel more efficiently""" # Release the channel, stream, and profile keys from the channel - channel = Channel.objects.get(uuid=channel_id) - channel.release_stream() + try: + channel = Channel.objects.get(uuid=channel_id) + channel.release_stream() + except: + stream = Stream.objects.get(stream_hash=channel_id) + stream.release_stream() if not self.redis_client: return 0 @@ -1149,7 +1135,7 @@ class ProxyServer: self.redis_client.hset(metadata_key, mapping=update_data) # Log the transition - logger.info(f"Channel {channel_id} state transition: {current_state or 'None'} → {new_state}") + logger.info(f"Channel {channel_id} state transition: {current_state or 'None'} -> {new_state}") return True except Exception as e: logger.error(f"Error updating channel state: {e}") diff --git a/apps/proxy/ts_proxy/services/channel_service.py b/apps/proxy/ts_proxy/services/channel_service.py index 210e4b0f..3ac62af4 100644 --- a/apps/proxy/ts_proxy/services/channel_service.py +++ b/apps/proxy/ts_proxy/services/channel_service.py @@ -9,9 +9,9 @@ import json from django.shortcuts import get_object_or_404 from apps.channels.models import Channel from apps.proxy.config import TSConfig as Config -from .. import proxy_server +from ..server import ProxyServer from ..redis_keys import RedisKeys -from ..constants import EventType, ChannelState +from ..constants import EventType, ChannelState, ChannelMetadataField from ..url_utils import get_stream_info_for_switch logger = logging.getLogger("ts_proxy") @@ -20,7 +20,7 @@ class ChannelService: """Service class for channel operations""" @staticmethod - def initialize_channel(channel_id, stream_url, user_agent, transcode=False, profile_value=None, stream_id=None): + def initialize_channel(channel_id, stream_url, user_agent, transcode=False, stream_profile_value=None, stream_id=None, m3u_profile_id=None): """ Initialize a channel with the given parameters. @@ -29,12 +29,14 @@ class ChannelService: stream_url: URL of the stream user_agent: User agent for the stream connection transcode: Whether to transcode the stream - profile_value: Stream profile value to store in metadata + stream_profile_value: Stream profile value to store in metadata stream_id: ID of the stream being used + m3u_profile_id: ID of the M3U profile being used Returns: bool: Success status """ + proxy_server = ProxyServer.get_instance() # FIXED: First, ensure that Redis metadata including stream_id is set BEFORE channel initialization # This ensures the stream ID is available when the StreamManager looks it up if stream_id and proxy_server.redis_client: @@ -42,19 +44,19 @@ class ChannelService: # Check if metadata already exists if proxy_server.redis_client.exists(metadata_key): # Just update the existing metadata with stream_id - proxy_server.redis_client.hset(metadata_key, "stream_id", str(stream_id)) + proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STREAM_ID, str(stream_id)) logger.info(f"Pre-set stream ID {stream_id} in Redis for channel {channel_id}") else: # Create initial metadata with essential values initial_metadata = { - "stream_id": str(stream_id), + ChannelMetadataField.STREAM_ID: str(stream_id), "temp_init": str(time.time()) } proxy_server.redis_client.hset(metadata_key, mapping=initial_metadata) logger.info(f"Created initial metadata with stream_id {stream_id} for channel {channel_id}") # Verify the stream_id was set - stream_id_value = proxy_server.redis_client.hget(metadata_key, "stream_id") + stream_id_value = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) if stream_id_value: logger.info(f"Verified stream_id {stream_id_value.decode('utf-8')} is now set in Redis") else: @@ -67,10 +69,12 @@ class ChannelService: if success and proxy_server.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) update_data = {} - if profile_value: - update_data["profile"] = profile_value + if stream_profile_value: + update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value if stream_id: - update_data["stream_id"] = str(stream_id) + update_data[ChannelMetadataField.STREAM_ID] = str(stream_id) + if m3u_profile_id: + update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id) if update_data: proxy_server.redis_client.hset(metadata_key, mapping=update_data) @@ -91,7 +95,10 @@ class ChannelService: Returns: dict: Result information including success status and diagnostics """ + proxy_server = ProxyServer.get_instance() + # If no direct URL is provided but a target stream is, get URL from target stream + stream_id = None if not new_url and target_stream_id: stream_info = get_stream_info_for_switch(channel_id, target_stream_id) if 'error' in stream_info: @@ -101,6 +108,10 @@ class ChannelService: } new_url = stream_info['url'] user_agent = stream_info['user_agent'] + stream_id = target_stream_id + elif target_stream_id: + # If we have both URL and target_stream_id, use the target_stream_id + stream_id = target_stream_id # Check if channel exists in_local_managers = channel_id in proxy_server.stream_managers @@ -152,7 +163,7 @@ class ChannelService: # Update metadata in Redis regardless of ownership if proxy_server.redis_client: try: - ChannelService._update_channel_metadata(channel_id, new_url, user_agent) + ChannelService._update_channel_metadata(channel_id, new_url, user_agent, stream_id) result['metadata_updated'] = True except Exception as e: logger.error(f"Error updating Redis metadata: {e}", exc_info=True) @@ -177,7 +188,7 @@ class ChannelService: # If we're not the owner, publish an event for the owner to pick up logger.info(f"Not the owner, requesting URL change via Redis PubSub") if proxy_server.redis_client: - ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent) + ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent, stream_id) result.update({ 'direct_update': False, 'event_published': True, @@ -203,6 +214,8 @@ class ChannelService: Returns: dict: Result information including previous state if available """ + proxy_server = ProxyServer.get_instance() + # Check if channel exists channel_exists = proxy_server.check_if_channel_exists(channel_id) if not channel_exists: @@ -220,8 +233,8 @@ class ChannelService: channel_info = {"state": state} # Immediately mark as stopping in metadata so clients detect it faster - proxy_server.redis_client.hset(metadata_key, "state", ChannelState.STOPPING) - proxy_server.redis_client.hset(metadata_key, "state_changed_at", str(time.time())) + proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE, ChannelState.STOPPING) + proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE_CHANGED_AT, str(time.time())) except Exception as e: logger.error(f"Error fetching channel state: {e}") @@ -248,8 +261,11 @@ class ChannelService: logger.info(f"Released channel {channel_id} stream allocation") model_released = True except Channel.DoesNotExist: - logger.warning(f"Could not find Channel model for UUID {channel_id}") - model_released = False + logger.warning(f"Could not find Channel model for UUID {channel_id}, attempting stream hash") + stream = Stream.objects.get(stream_hash=channel_id) + stream.release_stream() + logger.info(f"Released stream {channel_id} stream allocation") + model_released = True except Exception as e: logger.error(f"Error releasing channel stream: {e}") model_released = False @@ -276,6 +292,7 @@ class ChannelService: dict: Result information """ logger.info(f"Request to stop client {client_id} on channel {channel_id}") + proxy_server = ProxyServer.get_instance() # Set a Redis key for immediate detection key_set = False @@ -339,6 +356,8 @@ class ChannelService: Returns: tuple: (valid, state, owner, details) - validity status, current state, owner, and diagnostic info """ + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return False, None, None, {"error": "Redis not available"} @@ -350,8 +369,8 @@ class ChannelService: metadata = proxy_server.redis_client.hgetall(metadata_key) # Extract state and owner - state = metadata.get(b'state', b'unknown').decode('utf-8') - owner = metadata.get(b'owner', b'unknown').decode('utf-8') + state = metadata.get(ChannelMetadataField.STATE.encode(), b'unknown').decode('utf-8') + owner = metadata.get(ChannelMetadataField.OWNER.encode(), b'unknown').decode('utf-8') # Valid states indicate channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING] @@ -360,7 +379,7 @@ class ChannelService: return False, state, owner, {"error": f"Invalid state: {state}"} # Check if owner is still active - owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat" + owner_heartbeat_key = RedisKeys.worker_heartbeat(owner) owner_alive = proxy_server.redis_client.exists(owner_heartbeat_key) if not owner_alive: @@ -394,8 +413,10 @@ class ChannelService: # Helper methods for Redis operations @staticmethod - def _update_channel_metadata(channel_id, url, user_agent=None): + def _update_channel_metadata(channel_id, url, user_agent=None, stream_id=None): """Update channel metadata in Redis""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return False @@ -405,23 +426,22 @@ class ChannelService: key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8') logger.debug(f"Redis key {metadata_key} is of type: {key_type}") + # Build metadata update dict + metadata = {ChannelMetadataField.URL: url} + if user_agent: + metadata[ChannelMetadataField.USER_AGENT] = user_agent + if stream_id: + metadata[ChannelMetadataField.STREAM_ID] = str(stream_id) + logger.info(f"Updating stream ID to {stream_id} in Redis for channel {channel_id}") + # Use the appropriate method based on the key type if key_type == 'hash': - proxy_server.redis_client.hset(metadata_key, "url", url) - if user_agent: - proxy_server.redis_client.hset(metadata_key, "user_agent", user_agent) + proxy_server.redis_client.hset(metadata_key, mapping=metadata) elif key_type == 'none': # Key doesn't exist yet - # Create new hash with all required fields - metadata = {"url": url} - if user_agent: - metadata["user_agent"] = user_agent proxy_server.redis_client.hset(metadata_key, mapping=metadata) else: # If key exists with wrong type, delete it and recreate proxy_server.redis_client.delete(metadata_key) - metadata = {"url": url} - if user_agent: - metadata["user_agent"] = user_agent proxy_server.redis_client.hset(metadata_key, mapping=metadata) # Set switch request flag to ensure all workers see it @@ -432,16 +452,19 @@ class ChannelService: return True @staticmethod - def _publish_stream_switch_event(channel_id, new_url, user_agent=None): + def _publish_stream_switch_event(channel_id, new_url, user_agent=None, stream_id=None): """Publish a stream switch event to Redis pubsub""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return False switch_request = { - "event": EventType.STREAM_SWITCH, # Use constant instead of string + "event": EventType.STREAM_SWITCH, "channel_id": channel_id, "url": new_url, "user_agent": user_agent, + "stream_id": stream_id, "requester": proxy_server.worker_id, "timestamp": time.time() } @@ -455,11 +478,13 @@ class ChannelService: @staticmethod def _publish_channel_stop_event(channel_id): """Publish a channel stop event to Redis pubsub""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return False stop_request = { - "event": EventType.CHANNEL_STOP, # Use constant instead of string + "event": EventType.CHANNEL_STOP, "channel_id": channel_id, "requester_worker_id": proxy_server.worker_id, "timestamp": time.time() @@ -476,11 +501,13 @@ class ChannelService: @staticmethod def _publish_client_stop_event(channel_id, client_id): """Publish a client stop event to Redis pubsub""" + proxy_server = ProxyServer.get_instance() + if not proxy_server.redis_client: return False stop_request = { - "event": EventType.CLIENT_STOP, # Use constant instead of string + "event": EventType.CLIENT_STOP, "channel_id": channel_id, "client_id": client_id, "requester_worker_id": proxy_server.worker_id, diff --git a/apps/proxy/ts_proxy/stream_buffer.py b/apps/proxy/ts_proxy/stream_buffer.py index a94204ab..4d73bdc2 100644 --- a/apps/proxy/ts_proxy/stream_buffer.py +++ b/apps/proxy/ts_proxy/stream_buffer.py @@ -94,7 +94,7 @@ class StreamBuffer: writes_done += 1 if writes_done > 0: - logger.debug(f"Added {writes_done} optimized chunks ({self.target_chunk_size} bytes each) to Redis") + logger.debug(f"Added {writes_done} chunks ({self.target_chunk_size} bytes each) to Redis for channel {self.channel_id} at index {self.index}") return True diff --git a/apps/proxy/ts_proxy/stream_generator.py b/apps/proxy/ts_proxy/stream_generator.py index dc6c2fc2..9377a079 100644 --- a/apps/proxy/ts_proxy/stream_generator.py +++ b/apps/proxy/ts_proxy/stream_generator.py @@ -7,10 +7,11 @@ import time import logging import threading from apps.proxy.config import TSConfig as Config -from . import proxy_server +from .server import ProxyServer from .utils import create_ts_packet, get_logger from .redis_keys import RedisKeys from .utils import get_logger +from .constants import ChannelMetadataField logger = get_logger() @@ -96,6 +97,7 @@ class StreamGenerator: max_init_wait = getattr(Config, 'CLIENT_WAIT_TIMEOUT', 30) keepalive_interval = 0.5 last_keepalive = 0 + proxy_server = ProxyServer.get_instance() # While init is happening, send keepalive packets while time.time() - initialization_start < max_init_wait: @@ -142,6 +144,8 @@ class StreamGenerator: def _setup_streaming(self): """Setup streaming parameters and check resources.""" + proxy_server = ProxyServer.get_instance() + # Get buffer - stream manager may not exist in this worker buffer = proxy_server.stream_buffers.get(self.channel_id) stream_manager = proxy_server.stream_managers.get(self.channel_id) @@ -217,6 +221,8 @@ class StreamGenerator: def _check_resources(self): """Check if required resources still exist.""" + proxy_server = ProxyServer.get_instance() + # Enhanced resource checks if self.channel_id not in proxy_server.stream_buffers: logger.info(f"[{self.client_id}] Channel buffer no longer exists, terminating stream") @@ -263,6 +269,7 @@ class StreamGenerator: # Process and send chunks total_size = sum(len(c) for c in chunks) logger.debug(f"[{self.client_id}] Retrieved {len(chunks)} chunks ({total_size} bytes) from index {self.local_index+1} to {next_index}") + proxy_server = ProxyServer.get_instance() # Send the chunks to the client for chunk in chunks: @@ -298,11 +305,11 @@ class StreamGenerator: try: client_key = RedisKeys.client_metadata(self.channel_id, self.client_id) stats = { - "chunks_sent": str(self.chunks_sent), - "bytes_sent": str(self.bytes_sent), - "avg_rate_KBps": str(round(avg_rate, 1)), - "current_rate_KBps": str(round(self.current_rate, 1)), - "stats_updated_at": str(current_time) + ChannelMetadataField.CHUNKS_SENT: str(self.chunks_sent), + ChannelMetadataField.BYTES_SENT: str(self.bytes_sent), + ChannelMetadataField.AVG_RATE_KBPS: str(round(avg_rate, 1)), + ChannelMetadataField.CURRENT_RATE_KBPS: str(round(self.current_rate, 1)), + ChannelMetadataField.STATS_UPDATED_AT: str(current_time) } proxy_server.redis_client.hset(client_key, mapping=stats) # No need to set expiration as client heartbeat will refresh this key @@ -328,14 +335,24 @@ class StreamGenerator: def _is_timeout(self): """Check if the stream has timed out.""" + # Get a more generous timeout for stream switching + stream_timeout = getattr(Config, 'STREAM_TIMEOUT', 10) + failover_grace_period = getattr(Config, 'FAILOVER_GRACE_PERIOD', 20) + total_timeout = stream_timeout + failover_grace_period + # Disconnect after long inactivity - if time.time() - self.last_yield_time > Config.STREAM_TIMEOUT: + if time.time() - self.last_yield_time > total_timeout: if self.stream_manager and not self.stream_manager.healthy: - logger.warning(f"[{self.client_id}] No data for {Config.STREAM_TIMEOUT}s and stream unhealthy, disconnecting") + # Check if stream manager is actively switching or reconnecting + if (hasattr(self.stream_manager, 'url_switching') and self.stream_manager.url_switching): + logger.info(f"[{self.client_id}] Stream switching in progress, giving more time") + return False + + logger.warning(f"[{self.client_id}] No data for {total_timeout}s and stream unhealthy, disconnecting") return True elif not self.is_owner_worker and self.consecutive_empty > 100: # Non-owner worker without data for too long - logger.warning(f"[{self.client_id}] Non-owner worker with no data for {Config.STREAM_TIMEOUT}s, disconnecting") + logger.warning(f"[{self.client_id}] Non-owner worker with no data for {total_timeout}s, disconnecting") return True return False @@ -345,6 +362,34 @@ class StreamGenerator: elapsed = time.time() - self.stream_start_time local_clients = 0 total_clients = 0 + proxy_server = ProxyServer.get_instance() + + # Release M3U profile stream allocation if this is the last client + stream_released = False + if proxy_server.redis_client: + try: + metadata_key = RedisKeys.channel_metadata(self.channel_id) + metadata = proxy_server.redis_client.hgetall(metadata_key) + if metadata: + stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) + if stream_id_bytes: + stream_id = int(stream_id_bytes.decode('utf-8')) + + # Check if we're the last client + if self.channel_id in proxy_server.client_managers: + client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() + # Only the last client or owner should release the stream + if client_count <= 1 and proxy_server.am_i_owner(self.channel_id): + from apps.channels.models import Stream + try: + stream = Stream.objects.get(pk=stream_id) + stream.release_stream() + stream_released = True + logger.debug(f"[{self.client_id}] Released stream {stream_id} for channel {self.channel_id}") + except Exception as e: + logger.error(f"[{self.client_id}] Error releasing stream {stream_id}: {e}") + except Exception as e: + logger.error(f"[{self.client_id}] Error checking stream data for release: {e}") if self.channel_id in proxy_server.client_managers: client_manager = proxy_server.client_managers[self.channel_id] @@ -353,12 +398,15 @@ class StreamGenerator: logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})") # Schedule channel shutdown if no clients left - self._schedule_channel_shutdown_if_needed(local_clients) + if not stream_released: # Only if we haven't already released the stream + self._schedule_channel_shutdown_if_needed(local_clients) def _schedule_channel_shutdown_if_needed(self, local_clients): """ Schedule channel shutdown if there are no clients left and we're the owner. """ + proxy_server = ProxyServer.get_instance() + # If no clients left and we're the owner, schedule shutdown using the config value if local_clients == 0 and proxy_server.am_i_owner(self.channel_id): logger.info(f"No local clients left for channel {self.channel_id}, scheduling shutdown") diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index db0d6bb9..ac5f801b 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -15,9 +15,9 @@ from core.models import UserAgent, CoreSettings from .stream_buffer import StreamBuffer from .utils import detect_stream_type, get_logger from .redis_keys import RedisKeys -from .constants import ChannelState, EventType, StreamType, TS_PACKET_SIZE +from .constants import ChannelState, EventType, StreamType, ChannelMetadataField, TS_PACKET_SIZE from .config_helper import ConfigHelper -from .url_utils import get_alternate_streams, get_stream_info_for_switch +from .url_utils import get_alternate_streams, get_stream_info_for_switch, get_stream_object logger = get_logger() @@ -284,10 +284,10 @@ class StreamManager: # Update metadata to indicate error state update_data = { - "state": ChannelState.ERROR, - "state_changed_at": str(time.time()), - "error_message": error_message, - "error_time": str(time.time()) + ChannelMetadataField.STATE: ChannelState.ERROR, + ChannelMetadataField.STATE_CHANGED_AT: str(time.time()), + ChannelMetadataField.ERROR_MESSAGE: error_message, + ChannelMetadataField.ERROR_TIME: str(time.time()) } self.buffer.redis_client.hset(metadata_key, mapping=update_data) logger.info(f"Updated channel {self.channel_id} state to ERROR in Redis after stream failure") @@ -304,7 +304,7 @@ class StreamManager: """Establish a connection using transcoding""" try: logger.debug(f"Building transcode command for channel {self.channel_id}") - channel = get_object_or_404(Channel, uuid=self.channel_id) + channel = get_stream_object(self.channel_id) # Use FFmpeg specifically for HLS streams if hasattr(self, 'force_ffmpeg') and self.force_ffmpeg: @@ -336,6 +336,9 @@ class StreamManager: self.socket = self.transcode_process.stdout # Read from std output self.connected = True + # Set connection start time for stability tracking + self.connection_start_time = time.time() + # Set channel state to waiting for clients self._set_waiting_for_clients() @@ -367,6 +370,9 @@ class StreamManager: self.healthy = True logger.info(f"Successfully connected to stream source") + # Store connection start time for stability tracking + self.connection_start_time = time.time() + # Set channel state to waiting for clients self._set_waiting_for_clients() @@ -398,13 +404,13 @@ class StreamManager: metadata_key = RedisKeys.channel_metadata(self.channel_id) # Use hincrby to atomically increment the total_bytes field - self.buffer.redis_client.hincrby(metadata_key, "total_bytes", self.bytes_processed) + self.buffer.redis_client.hincrby(metadata_key, ChannelMetadataField.TOTAL_BYTES, self.bytes_processed) # Reset local counter after updating Redis self.bytes_processed = 0 self.last_bytes_update = now - logger.debug(f"Updated total_bytes in Redis for channel {self.channel_id}") + logger.debug(f"Updated {ChannelMetadataField.TOTAL_BYTES} in Redis for channel {self.channel_id}") except Exception as e: logger.error(f"Error updating bytes processed: {e}") @@ -490,6 +496,21 @@ class StreamManager: # Add at the beginning of your stop method self.stopping = True + # Release stream resources if we're the owner + if self.current_stream_id and hasattr(self, 'worker_id') and self.worker_id: + if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: + owner_key = RedisKeys.channel_owner(self.channel_id) + current_owner = self.buffer.redis_client.get(owner_key) + + if current_owner and current_owner.decode('utf-8') == self.worker_id: + try: + from apps.channels.models import Stream + stream = Stream.objects.get(pk=self.current_stream_id) + stream.release_stream() + logger.info(f"Released stream {self.current_stream_id} for channel {self.channel_id}") + except Exception as e: + logger.error(f"Error releasing stream {self.current_stream_id}: {e}") + # Cancel all buffer check timers for timer in list(self._buffer_check_timers): try: @@ -573,24 +594,112 @@ class StreamManager: def _monitor_health(self): """Monitor stream health and attempt recovery if needed""" + consecutive_unhealthy_checks = 0 + health_recovery_attempts = 0 + reconnect_attempts = 0 + max_health_recovery_attempts = ConfigHelper.get('MAX_HEALTH_RECOVERY_ATTEMPTS', 2) + max_reconnect_attempts = ConfigHelper.get('MAX_RECONNECT_ATTEMPTS', 3) + min_stable_time = ConfigHelper.get('MIN_STABLE_TIME_BEFORE_RECONNECT', 30) # seconds + while self.running: try: now = time.time() - if now - self.last_data_time > getattr(Config, 'CONNECTION_TIMEOUT', 10) and self.connected: + inactivity_duration = now - self.last_data_time + timeout_threshold = getattr(Config, 'CONNECTION_TIMEOUT', 10) + + if inactivity_duration > timeout_threshold and self.connected: # Mark unhealthy if no data for too long if self.healthy: - logger.warning(f"Stream unhealthy - no data for {now - self.last_data_time:.1f}s") + logger.warning(f"Stream unhealthy - no data for {inactivity_duration:.1f}s") self.healthy = False + + # Track consecutive unhealthy checks + consecutive_unhealthy_checks += 1 + + # After several unhealthy checks in a row, try recovery + if consecutive_unhealthy_checks >= 3 and health_recovery_attempts < max_health_recovery_attempts: + # Calculate how long the stream was stable before failing + connection_start_time = getattr(self, 'connection_start_time', 0) + stable_time = self.last_data_time - connection_start_time if connection_start_time > 0 else 0 + + if stable_time >= min_stable_time and reconnect_attempts < max_reconnect_attempts: + # Stream was stable for a while, try reconnecting first + logger.warning(f"Stream was stable for {stable_time:.1f}s before failing. " + f"Attempting reconnect {reconnect_attempts + 1}/{max_reconnect_attempts}") + reconnect_attempts += 1 + threading.Thread(target=self._attempt_reconnect, daemon=True).start() + else: + # Stream was not stable long enough, or reconnects failed too many times + # Try switching to another stream + if reconnect_attempts > 0: + logger.warning(f"Reconnect attempts exhausted ({reconnect_attempts}/{max_reconnect_attempts}). " + f"Attempting stream switch recovery") + else: + logger.warning(f"Stream was only stable for {stable_time:.1f}s (<{min_stable_time}s). " + f"Skipping reconnect, attempting stream switch") + + health_recovery_attempts += 1 + reconnect_attempts = 0 # Reset for next time + threading.Thread(target=self._attempt_health_recovery, daemon=True).start() elif self.connected and not self.healthy: # Auto-recover health when data resumes logger.info(f"Stream health restored") self.healthy = True + consecutive_unhealthy_checks = 0 + health_recovery_attempts = 0 + reconnect_attempts = 0 + + # If healthy, reset unhealthy counter (but keep other state) + if self.healthy: + consecutive_unhealthy_checks = 0 except Exception as e: logger.error(f"Error in health monitor: {e}") time.sleep(self.health_check_interval) + def _attempt_reconnect(self): + """Attempt to reconnect to the current stream""" + try: + logger.info(f"Attempting reconnect to current stream for channel {self.channel_id}") + + # Don't try to reconnect if we're already switching URLs + if self.url_switching: + logger.info("URL switching already in progress, skipping reconnect") + return + + # Close existing connection + if self.transcode or self.socket: + self._close_socket() + else: + self._close_connection() + + self.connected = False + + # Attempt to establish a new connection using the same URL + connection_result = False + try: + if self.transcode: + connection_result = self._establish_transcode_connection() + else: + connection_result = self._establish_http_connection() + + if connection_result: + # Store connection start time to measure stability + self.connection_start_time = time.time() + logger.info(f"Reconnect successful for channel {self.channel_id}") + return True + else: + logger.warning(f"Reconnect failed for channel {self.channel_id}") + return False + except Exception as e: + logger.error(f"Error during reconnect: {e}", exc_info=True) + return False + + except Exception as e: + logger.error(f"Error in reconnect attempt: {e}", exc_info=True) + return False + def _close_connection(self): """Close HTTP connection resources""" # Close response if it exists @@ -743,8 +852,9 @@ class StreamManager: current_state = None try: metadata = redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - current_state = metadata[b'state'].decode('utf-8') + state_field = ChannelMetadataField.STATE.encode('utf-8') + if metadata and state_field in metadata: + current_state = metadata[state_field].decode('utf-8') except Exception as e: logger.error(f"Error checking current state: {e}") @@ -758,8 +868,8 @@ class StreamManager: # Not enough buffer yet - set to connecting state if not already if current_state != ChannelState.CONNECTING: update_data = { - "state": ChannelState.CONNECTING, - "state_changed_at": current_time + ChannelMetadataField.STATE: ChannelState.CONNECTING, + ChannelMetadataField.STATE_CHANGED_AT: current_time } redis_client.hset(metadata_key, mapping=update_data) logger.info(f"Channel {channel_id} connected but waiting for buffer to fill: {current_buffer_index}/{initial_chunks_needed} chunks") @@ -772,16 +882,16 @@ class StreamManager: # We have enough buffer, proceed with state change update_data = { - "state": ChannelState.WAITING_FOR_CLIENTS, - "connection_ready_time": current_time, - "state_changed_at": current_time, - "buffer_chunks": str(current_buffer_index) + ChannelMetadataField.STATE: ChannelState.WAITING_FOR_CLIENTS, + ChannelMetadataField.CONNECTION_READY_TIME: current_time, + ChannelMetadataField.STATE_CHANGED_AT: current_time, + ChannelMetadataField.BUFFER_CHUNKS: str(current_buffer_index) } redis_client.hset(metadata_key, mapping=update_data) # Get configured grace period or default grace_period = ConfigHelper.get('CHANNEL_INIT_GRACE_PERIOD', 20) - logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'} → {ChannelState.WAITING_FOR_CLIENTS} with {current_buffer_index} buffer chunks") + logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'} -> {ChannelState.WAITING_FOR_CLIENTS} with {current_buffer_index} buffer chunks") logger.info(f"Started initial connection grace period ({grace_period}s) for channel {channel_id}") else: logger.debug(f"Not changing state: channel {channel_id} already in {current_state} state") @@ -885,12 +995,13 @@ class StreamManager: if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client: metadata_key = RedisKeys.channel_metadata(self.channel_id) self.buffer.redis_client.hset(metadata_key, mapping={ - "url": new_url, - "user_agent": new_user_agent, - "profile": stream_info['profile'], - "stream_id": str(stream_id), - "stream_switch_time": str(time.time()), - "stream_switch_reason": "max_retries_exceeded" + ChannelMetadataField.URL: new_url, + ChannelMetadataField.USER_AGENT: new_user_agent, + ChannelMetadataField.STREAM_PROFILE: stream_info['stream_profile'], + ChannelMetadataField.M3U_PROFILE: stream_info['m3u_profile_id'], + ChannelMetadataField.STREAM_ID: str(stream_id), + ChannelMetadataField.STREAM_SWITCH_TIME: str(time.time()), + ChannelMetadataField.STREAM_SWITCH_REASON: "max_retries_exceeded" }) # Log the switch @@ -908,5 +1019,3 @@ class StreamManager: except Exception as e: logger.error(f"Error trying next stream for channel {self.channel_id}: {e}", exc_info=True) return False - - diff --git a/apps/proxy/ts_proxy/url_utils.py b/apps/proxy/ts_proxy/url_utils.py index a0ed4476..f80aae75 100644 --- a/apps/proxy/ts_proxy/url_utils.py +++ b/apps/proxy/ts_proxy/url_utils.py @@ -10,10 +10,21 @@ from apps.channels.models import Channel, Stream from apps.m3u.models import M3UAccount, M3UAccountProfile from core.models import UserAgent, CoreSettings from .utils import get_logger +from uuid import UUID logger = get_logger() -def generate_stream_url(channel_id: str) -> Tuple[str, str, bool]: +def get_stream_object(id: str): + try: + uuid_obj = UUID(id, version=4) + logger.info(f"Fetching channel ID {id}") + return get_object_or_404(Channel, uuid=id) + except: + # UUID check failed, assume stream hash + logger.info(f"Fetching stream hash {id}") + return get_object_or_404(Stream, stream_hash=id) + +def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]]: """ Generate the appropriate stream URL for a channel based on its profile settings. @@ -21,43 +32,55 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool]: channel_id: The UUID of the channel Returns: - Tuple[str, str, bool]: (stream_url, user_agent, transcode_flag) + Tuple[str, str, bool, Optional[int]]: (stream_url, user_agent, transcode_flag, profile_id) """ - # Get channel and related objects - channel = get_object_or_404(Channel, uuid=channel_id) - stream_id, profile_id = channel.get_stream() + try: + channel = get_stream_object(channel_id) - if stream_id is None or profile_id is None: - logger.error(f"No stream assigned to channel {channel_id}") - return None, None, False + # Get stream and profile for this channel + # Note: get_stream now returns 3 values (stream_id, profile_id, error_reason) + stream_id, profile_id, error_reason = channel.get_stream() - # Get the M3U account profile for URL pattern - stream = get_object_or_404(Stream, pk=stream_id) - profile = get_object_or_404(M3UAccountProfile, pk=profile_id) + if not stream_id or not profile_id: + logger.error(f"No stream available for channel {channel_id}: {error_reason}") + return None, None, False, None - # Get the appropriate user agent - m3u_account = M3UAccount.objects.get(id=profile.m3u_account.id) - stream_user_agent = UserAgent.objects.get(id=m3u_account.user_agent.id).user_agent + # Look up the Stream and Profile objects + try: + stream = Stream.objects.get(id=stream_id) + profile = M3UAccountProfile.objects.get(id=profile_id) + except (Stream.DoesNotExist, M3UAccountProfile.DoesNotExist) as e: + logger.error(f"Error getting stream or profile: {e}") + return None, None, False, None - if stream_user_agent is None: - stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) - logger.debug(f"No user agent found for account, using default: {stream_user_agent}") + # Get the M3U account profile for URL pattern + m3u_profile = profile - # Generate stream URL based on the selected profile - input_url = stream.url - stream_url = transform_url(input_url, profile.search_pattern, profile.replace_pattern) + # Get the appropriate user agent + m3u_account = M3UAccount.objects.get(id=m3u_profile.m3u_account.id) + stream_user_agent = UserAgent.objects.get(id=m3u_account.user_agent.id).user_agent - # Check if transcoding is needed - stream_profile = channel.get_stream_profile() - if stream_profile.is_proxy() or stream_profile is None: - transcode = False - else: - transcode = True + if stream_user_agent is None: + stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) + logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - # Get profile name as string - profile_value = stream_profile.id + # Generate stream URL based on the selected profile + input_url = stream.url + stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern) - return stream_url, stream_user_agent, transcode, profile_value + # Check if transcoding is needed + stream_profile = channel.get_stream_profile() + if stream_profile.is_proxy() or stream_profile is None: + transcode = False + else: + transcode = True + + stream_profile_id = stream_profile.id + + return stream_url, stream_user_agent, transcode, stream_profile_id + except Exception as e: + logger.error(f"Error generating stream URL: {e}") + return None, None, False, None def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: """ @@ -72,18 +95,18 @@ def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str: The transformed URL """ try: - logger.debug("Executing URL pattern replacement:") - logger.debug(f" base URL: {input_url}") - logger.debug(f" search: {search_pattern}") + logger.info("Executing URL pattern replacement:") + logger.info(f" base URL: {input_url}") + logger.info(f" search: {search_pattern}") # Handle backreferences in the replacement pattern safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace_pattern) - logger.debug(f" replace: {replace_pattern}") - logger.debug(f" safe replace: {safe_replace_pattern}") + logger.info(f" replace: {replace_pattern}") + logger.info(f" safe replace: {safe_replace_pattern}") # Apply the transformation stream_url = re.sub(search_pattern, safe_replace_pattern, input_url) - logger.debug(f"Generated stream url: {stream_url}") + logger.info(f"Generated stream url: {stream_url}") return stream_url except Exception as e: @@ -122,21 +145,21 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] ).first() if default_profile: - profile_id = default_profile.id + m3u_profile_id = default_profile.id else: logger.error(f"No profile found for stream {stream_id}") return {'error': 'No profile found for stream'} else: # Use first available profile - profile_id = profiles.first().id + m3u_profile_id = profiles.first().id else: - stream_id, profile_id = channel.get_stream() - if stream_id is None or profile_id is None: - return {'error': 'No stream assigned to channel'} + stream_id, m3u_profile_id, error_reason = channel.get_stream() + if stream_id is None or m3u_profile_id is None: + return {'error': error_reason or 'No stream assigned to channel'} # Get the stream and profile objects directly stream = get_object_or_404(Stream, pk=stream_id) - profile = get_object_or_404(M3UAccountProfile, pk=profile_id) + profile = get_object_or_404(M3UAccountProfile, pk=m3u_profile_id) # Get the user agent from the M3U account m3u_account = M3UAccount.objects.get(id=profile.m3u_account.id) @@ -156,9 +179,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] 'url': stream_url, 'user_agent': user_agent, 'transcode': transcode, - 'profile': profile_value, + 'stream_profile': profile_value, 'stream_id': stream_id, - 'profile_id': profile_id + 'm3u_profile_id': m3u_profile_id } except Exception as e: logger.error(f"Error getting stream info for switch: {e}", exc_info=True) @@ -177,11 +200,15 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No """ try: # Get channel object - channel = get_object_or_404(Channel, uuid=channel_id) + channel = get_stream_object(channel_id) + if isinstance(channel, Stream): + logger.error(f"Stream is not a channel") + return [] + logger.debug(f"Looking for alternate streams for channel {channel_id}, current stream ID: {current_stream_id}") - # Get all assigned streams for this channel - streams = channel.streams.all() + # Get all assigned streams for this channel using the correct ordering from the channelstream table + streams = channel.streams.all().order_by('channelstream__order') logger.debug(f"Channel {channel_id} has {streams.count()} total assigned streams") if not streams.exists(): @@ -190,7 +217,7 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No alternate_streams = [] - # Process each stream + # Process each stream in the user-defined order for stream in streams: # Log each stream we're checking logger.debug(f"Checking stream ID {stream.id} ({stream.name}) for channel {channel_id}") @@ -201,8 +228,6 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No continue # Find compatible profiles for this stream - # FIX: Looking at the error message, M3UAccountProfile doesn't have a 'stream' field - # We need to find which field relates M3UAccountProfile to Stream try: # Check if we can find profiles via m3u_account profiles = M3UAccountProfile.objects.filter(m3u_account=stream.m3u_account) diff --git a/apps/proxy/ts_proxy/urls.py b/apps/proxy/ts_proxy/urls.py index cba06b6f..cb236aa2 100644 --- a/apps/proxy/ts_proxy/urls.py +++ b/apps/proxy/ts_proxy/urls.py @@ -10,4 +10,5 @@ urlpatterns = [ path('status/', views.channel_status, name='channel_status_detail'), path('stop/', views.stop_channel, name='stop_channel'), path('stop_client/', views.stop_client, name='stop_client'), + path('next_stream/', views.next_stream, name='next_stream'), ] diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index fe87e677..32d552da 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -7,7 +7,7 @@ from django.http import StreamingHttpResponse, JsonResponse, HttpResponseRedirec from django.views.decorators.csrf import csrf_exempt from django.shortcuts import get_object_or_404 from apps.proxy.config import TSConfig as Config -from . import proxy_server +from .server import ProxyServer from .channel_status import ChannelStatus from .stream_generator import create_stream_generator from .utils import get_client_ip @@ -18,11 +18,12 @@ from apps.m3u.models import M3UAccount, M3UAccountProfile from core.models import UserAgent, CoreSettings, PROXY_PROFILE_NAME from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated -from .constants import ChannelState, EventType, StreamType +from .constants import ChannelState, EventType, StreamType, ChannelMetadataField from .config_helper import ConfigHelper from .services.channel_service import ChannelService -from .url_utils import generate_stream_url, transform_url, get_stream_info_for_switch +from .url_utils import generate_stream_url, transform_url, get_stream_info_for_switch, get_stream_object, get_alternate_streams from .utils import get_logger +from uuid import UUID logger = get_logger() @@ -30,9 +31,10 @@ logger = get_logger() @api_view(['GET']) def stream_ts(request, channel_id): """Stream TS data to client with immediate response and keep-alive packets during initialization""" + channel = get_stream_object(channel_id) + client_user_agent = None - logger.info(f"Fetching channel ID {channel_id}") - channel = get_object_or_404(Channel, uuid=channel_id) + proxy_server = ProxyServer.get_instance() try: # Generate a unique client ID @@ -56,15 +58,17 @@ def stream_ts(request, channel_id): metadata_key = RedisKeys.channel_metadata(channel_id) if proxy_server.redis_client.exists(metadata_key): metadata = proxy_server.redis_client.hgetall(metadata_key) - if b'state' in metadata: - channel_state = metadata[b'state'].decode('utf-8') + state_field = ChannelMetadataField.STATE.encode('utf-8') + if state_field in metadata: + channel_state = metadata[state_field].decode('utf-8') # Only skip initialization if channel is in a healthy state valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS] if channel_state in valid_states: # Verify the owner is still active - if b'owner' in metadata: - owner = metadata[b'owner'].decode('utf-8') + owner_field = ChannelMetadataField.OWNER.encode('utf-8') + if owner_field in metadata: + owner = metadata[owner_field].decode('utf-8') owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat" if proxy_server.redis_client.exists(owner_heartbeat_key): # Owner is active and channel is in good state @@ -82,14 +86,62 @@ def stream_ts(request, channel_id): # Initialize the channel (but don't wait for completion) logger.info(f"[{client_id}] Starting channel {channel_id} initialization") - # Use the utility function to get stream URL and settings - stream_url, stream_user_agent, transcode, profile_value = generate_stream_url(channel_id) + # Use max retry attempts and connection timeout from config + max_retries = ConfigHelper.max_retries() + retry_timeout = ConfigHelper.connection_timeout() + wait_start_time = time.time() + + stream_url = None + stream_user_agent = None + transcode = False + profile_value = None + error_reason = None + + # Try to get a stream with configured retries + for attempt in range(max_retries): + stream_url, stream_user_agent, transcode, profile_value = generate_stream_url(channel_id) + + if stream_url is not None: + logger.info(f"[{client_id}] Successfully obtained stream for channel {channel_id}") + break + + # If we failed because there are no streams assigned, don't retry + _, _, error_reason = channel.get_stream() + if error_reason and 'maximum connection limits' not in error_reason: + logger.warning(f"[{client_id}] Can't retry - error not related to connection limits: {error_reason}") + break + + # Don't exceed the overall connection timeout + if time.time() - wait_start_time > retry_timeout: + logger.warning(f"[{client_id}] Connection wait timeout exceeded ({retry_timeout}s)") + break + + # Wait before retrying (using exponential backoff with a cap) + wait_time = min(0.5 * (2 ** attempt), 2.0) # Caps at 2 seconds + logger.info(f"[{client_id}] Waiting {wait_time:.1f}s for a connection to become available (attempt {attempt+1}/{max_retries})") + time.sleep(wait_time) + if stream_url is None: - return JsonResponse({'error': 'Channel not available'}, status=404) + # Make sure to release any stream locks that might have been acquired + if hasattr(channel, 'streams') and channel.streams.exists(): + for stream in channel.streams.all(): + try: + stream.release_stream() + logger.info(f"[{client_id}] Released stream {stream.id} for channel {channel_id}") + except Exception as e: + logger.error(f"[{client_id}] Error releasing stream: {e}") + + # Get the specific error message if available + wait_duration = f"{int(time.time() - wait_start_time)}s" + error_msg = error_reason if error_reason else 'No available streams for this channel' + return JsonResponse({ + 'error': error_msg, + 'waited': wait_duration + }, status=503) # 503 Service Unavailable is appropriate here # Get the stream ID from the channel - stream_id, profile_id = channel.get_stream() - logger.info(f"Channel {channel_id} using stream ID {stream_id}, profile ID {profile_id}") + stream_id, m3u_profile_id, _ = channel.get_stream() + logger.info(f"Channel {channel_id} using stream ID {stream_id}, m3u account profile ID {m3u_profile_id}") # Generate transcode command if needed stream_profile = channel.get_stream_profile() @@ -98,7 +150,7 @@ def stream_ts(request, channel_id): # Initialize channel with the stream's user agent (not the client's) success = ChannelService.initialize_channel( - channel_id, stream_url, stream_user_agent, transcode, profile_value, stream_id + channel_id, stream_url, stream_user_agent, transcode, profile_value, stream_id, m3u_profile_id ) if not success: @@ -134,9 +186,9 @@ def stream_ts(request, channel_id): if proxy_server.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) - url_bytes = proxy_server.redis_client.hget(metadata_key, "url") - ua_bytes = proxy_server.redis_client.hget(metadata_key, "user_agent") - profile_bytes = proxy_server.redis_client.hget(metadata_key, "profile") + url_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.URL) + ua_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.USER_AGENT) + profile_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_PROFILE) if url_bytes: url = url_bytes.decode('utf-8') @@ -189,6 +241,8 @@ def stream_ts(request, channel_id): @permission_classes([IsAuthenticated]) def change_stream(request, channel_id): """Change stream URL for existing channel with enhanced diagnostics""" + proxy_server = ProxyServer.get_instance() + try: data = json.loads(request.body) new_url = data.get('url') @@ -240,6 +294,8 @@ def channel_status(request, channel_id=None): - /status/ returns basic summary of all channels - /status/{channel_id} returns detailed info about specific channel """ + proxy_server = ProxyServer.get_instance() + try: # Check if Redis is available if not proxy_server.redis_client: @@ -334,3 +390,115 @@ def stop_client(request, channel_id): except Exception as e: logger.error(f"Failed to stop client: {e}", exc_info=True) return JsonResponse({'error': str(e)}, status=500) + +@csrf_exempt +@api_view(['POST']) +@permission_classes([IsAuthenticated]) +def next_stream(request, channel_id): + """Switch to the next available stream for a channel""" + proxy_server = ProxyServer.get_instance() + + try: + logger.info(f"Request to switch to next stream for channel {channel_id} received") + + # Check if the channel exists + channel = get_stream_object(channel_id) + + # First check if channel is active in Redis + current_stream_id = None + profile_id = None + + if proxy_server.redis_client: + metadata_key = RedisKeys.channel_metadata(channel_id) + if proxy_server.redis_client.exists(metadata_key): + # Get current stream ID from Redis + stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) + if stream_id_bytes: + current_stream_id = int(stream_id_bytes.decode('utf-8')) + logger.info(f"Found current stream ID {current_stream_id} in Redis for channel {channel_id}") + + # Get M3U profile from Redis if available + profile_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.M3U_PROFILE) + if profile_id_bytes: + profile_id = int(profile_id_bytes.decode('utf-8')) + logger.info(f"Found M3U profile ID {profile_id} in Redis for channel {channel_id}") + + if not current_stream_id: + # Channel is not running + return JsonResponse({'error': 'No current stream found for channel'}, status=404) + + # Get all streams for this channel in their defined order + streams = list(channel.streams.all().order_by('channelstream__order')) + + if len(streams) <= 1: + return JsonResponse({ + 'error': 'No alternate streams available for this channel', + 'current_stream_id': current_stream_id + }, status=404) + + # Find the current stream's position in the list + current_index = None + for i, stream in enumerate(streams): + if stream.id == current_stream_id: + current_index = i + break + + if current_index is None: + logger.warning(f"Current stream ID {current_stream_id} not found in channel's streams list") + # Fall back to the first stream that's not the current one + next_stream = next((s for s in streams if s.id != current_stream_id), None) + if not next_stream: + return JsonResponse({ + 'error': 'Could not find current stream in channel list', + 'current_stream_id': current_stream_id + }, status=404) + else: + # Get the next stream in the rotation (with wrap-around) + next_index = (current_index + 1) % len(streams) + next_stream = streams[next_index] + + next_stream_id = next_stream.id + logger.info(f"Rotating to next stream ID {next_stream_id} for channel {channel_id}") + + # Get full stream info including URL for the next stream + stream_info = get_stream_info_for_switch(channel_id, next_stream_id) + + if 'error' in stream_info: + return JsonResponse({ + 'error': stream_info['error'], + 'current_stream_id': current_stream_id, + 'next_stream_id': next_stream_id + }, status=404) + + # Now use the ChannelService to change the stream URL + result = ChannelService.change_stream_url( + channel_id, + stream_info['url'], + stream_info['user_agent'], + next_stream_id # Pass the stream_id to be stored in Redis + ) + + if result.get('status') == 'error': + return JsonResponse({ + 'error': result.get('message', 'Unknown error'), + 'diagnostics': result.get('diagnostics', {}), + 'current_stream_id': current_stream_id, + 'next_stream_id': next_stream_id + }, status=404) + + # Format success response + response_data = { + 'message': 'Stream switched to next available', + 'channel': channel_id, + 'previous_stream_id': current_stream_id, + 'new_stream_id': next_stream_id, + 'new_url': stream_info['url'], + 'owner': result.get('direct_update', False), + 'worker_id': proxy_server.worker_id + } + + return JsonResponse(response_data) + + except Exception as e: + logger.error(f"Failed to switch to next stream: {e}", exc_info=True) + return JsonResponse({'error': str(e)}, status=500) diff --git a/core/api_urls.py b/core/api_urls.py index 724a3311..e30eb698 100644 --- a/core/api_urls.py +++ b/core/api_urls.py @@ -2,14 +2,15 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .api_views import UserAgentViewSet, StreamProfileViewSet, CoreSettingsViewSet, environment +from .api_views import UserAgentViewSet, StreamProfileViewSet, CoreSettingsViewSet, environment, version router = DefaultRouter() router.register(r'useragents', UserAgentViewSet, basename='useragent') router.register(r'streamprofiles', StreamProfileViewSet, basename='streamprofile') router.register(r'settings', CoreSettingsViewSet, basename='coresettings') - +router.register(r'settings', CoreSettingsViewSet, basename='settings') urlpatterns = [ path('settings/env/', environment, name='token_refresh'), + path('version/', version, name='version'), path('', include(router.urls)), ] diff --git a/core/api_views.py b/core/api_views.py index eab5f44e..d9c0aba4 100644 --- a/core/api_views.py +++ b/core/api_views.py @@ -42,6 +42,8 @@ class CoreSettingsViewSet(viewsets.ModelViewSet): @api_view(['GET']) @permission_classes([IsAuthenticated]) def environment(request): + + public_ip = None local_ip = None country_code = None @@ -84,3 +86,17 @@ def environment(request): 'country_name': country_name, 'env_mode': "dev" if os.getenv('DISPATCHARR_ENV') == "dev" else "prod", }) + +@swagger_auto_schema( + method='get', + operation_description="Get application version information", + responses={200: "Version information"} +) +@api_view(['GET']) +def version(request): + # Import version information + from version import __version__, __build__ + return Response({ + 'version': __version__, + 'build': __build__, + }) diff --git a/core/apps.py b/core/apps.py index 8115ae60..3a01f0bd 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig - +from django.conf import settings +import os, logging class CoreConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' diff --git a/core/migrations/0010_reload_additional_settings.py b/core/migrations/0010_reload_additional_settings.py new file mode 100644 index 00000000..5395b89a --- /dev/null +++ b/core/migrations/0010_reload_additional_settings.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.6 on 2025-03-01 14:01 + +from django.db import migrations +from django.utils.text import slugify + +def preload_core_settings(apps, schema_editor): + CoreSettings = apps.get_model("core", "CoreSettings") + CoreSettings.objects.create( + key=slugify("Preferred Region"), + name="Preferred Region", + value="us", + ) + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0009_m3u_hash_settings'), + ] + + operations = [ + migrations.RunPython(preload_core_settings), + ] diff --git a/core/migrations/0011_fix_stream_profiles_and_user_agents.py b/core/migrations/0011_fix_stream_profiles_and_user_agents.py new file mode 100644 index 00000000..c582f965 --- /dev/null +++ b/core/migrations/0011_fix_stream_profiles_and_user_agents.py @@ -0,0 +1,27 @@ +# Generated by Django 5.1.6 on 2025-04-04 + +from django.db import migrations + +def fix_stream_profiles_and_user_agents(apps, schema_editor): + # Get the model + StreamProfile = apps.get_model("core", "StreamProfile") + + streamlink_profile = StreamProfile.objects.get(name="streamlink", locked=True) + streamlink_profile.parameters = "{streamUrl} --http-header User-Agent={userAgent} best --stdout" + streamlink_profile.save() + + UserAgent = apps.get_model("core", "UserAgent") + tivimate = UserAgent.objects.get(name="TiviMate", user_agent="TiviMate/5.16 (Android 12)") + if tivimate: + tivimate.user_agent = "TiviMate/5.1.6 (Android 12)" + tivimate.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0010_reload_additional_settings'), + ] + + operations = [ + migrations.RunPython(fix_stream_profiles_and_user_agents), + ] diff --git a/core/migrations/0012_default_active_m3u_accounts.py b/core/migrations/0012_default_active_m3u_accounts.py new file mode 100644 index 00000000..11888772 --- /dev/null +++ b/core/migrations/0012_default_active_m3u_accounts.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.6 on 2025-03-01 14:01 + +from django.db import migrations +from django.utils.text import slugify + +def preload_core_settings(apps, schema_editor): + CoreSettings = apps.get_model("core", "CoreSettings") + CoreSettings.objects.create( + key=slugify("Auto-Import Mapped Files"), + name="Auto-Import Mapped Files", + value=True, + ) + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_fix_stream_profiles_and_user_agents'), + ] + + operations = [ + migrations.RunPython(preload_core_settings), + ] diff --git a/core/models.py b/core/models.py index a4fa92d4..a8571b5c 100644 --- a/core/models.py +++ b/core/models.py @@ -144,6 +144,7 @@ DEFAULT_USER_AGENT_KEY= slugify("Default User-Agent") DEFAULT_STREAM_PROFILE_KEY = slugify("Default Stream Profile") STREAM_HASH_KEY = slugify("M3U Hash Key") PREFERRED_REGION_KEY = slugify("Preferred Region") +AUTO_IMPORT_MAPPED_FILES = slugify("Auto-Import Mapped Files") class CoreSettings(models.Model): key = models.CharField( @@ -173,9 +174,18 @@ class CoreSettings(models.Model): def get_m3u_hash_key(cls): return cls.objects.get(key=STREAM_HASH_KEY).value + @classmethod def get_preferred_region(cls): """Retrieve the preferred region setting (or return None if not found).""" try: return cls.objects.get(key=PREFERRED_REGION_KEY).value except cls.DoesNotExist: return None + + @classmethod + def get_auto_import_mapped_files(cls): + """Retrieve the preferred region setting (or return None if not found).""" + try: + return cls.objects.get(key=AUTO_IMPORT_MAPPED_FILES).value + except cls.DoesNotExist: + return None diff --git a/core/tasks.py b/core/tasks.py new file mode 100644 index 00000000..7e808310 --- /dev/null +++ b/core/tasks.py @@ -0,0 +1,178 @@ +# yourapp/tasks.py +from celery import shared_task +from channels.layers import get_channel_layer +from asgiref.sync import async_to_sync +import redis +import json +import logging +import re +import time +import os +from core.utils import RedisClient +from apps.proxy.ts_proxy.channel_status import ChannelStatus +from apps.m3u.models import M3UAccount +from apps.epg.models import EPGSource +from apps.m3u.tasks import refresh_single_m3u_account +from apps.epg.tasks import refresh_epg_data +from .models import CoreSettings + +logger = logging.getLogger(__name__) + +EPG_WATCH_DIR = '/data/epgs' +M3U_WATCH_DIR = '/data/m3us' +MIN_AGE_SECONDS = 6 +STARTUP_SKIP_AGE = 30 +REDIS_PREFIX = "processed_file:" +REDIS_TTL = 60 * 60 * 24 * 3 # expire keys after 3 days (optional) + +# Store the last known value to compare with new data +last_known_data = {} + +@shared_task +def beat_periodic_task(): + fetch_channel_stats() + scan_and_process_files() + +@shared_task +def scan_and_process_files(): + redis_client = RedisClient.get_client() + now = time.time() + + for filename in os.listdir(M3U_WATCH_DIR): + filepath = os.path.join(M3U_WATCH_DIR, filename) + + if not os.path.isfile(filepath): + continue + + if not filename.endswith('.m3u') and not filename.endswith('.m3u8'): + continue + + mtime = os.path.getmtime(filepath) + age = now - mtime + redis_key = REDIS_PREFIX + filepath + stored_mtime = redis_client.get(redis_key) + + # Startup safety: skip old untracked files + if not stored_mtime and age > STARTUP_SKIP_AGE: + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + continue # Assume already processed before startup + + # File too new — probably still being written + if age < MIN_AGE_SECONDS: + continue + + # Skip if we've already processed this mtime + if stored_mtime and float(stored_mtime) >= mtime: + continue + + + m3u_account, _ = M3UAccount.objects.get_or_create(file_path=filepath, defaults={ + "name": filename, + "is_active": True if CoreSettings.get_auto_import_mapped_files() == "true" else False, + }) + + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + + if not m3u_account.is_active: + logger.info("M3U account is inactive, skipping.") + continue + + refresh_single_m3u_account.delay(m3u_account.id) + + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": {"success": True, "type": "m3u_file", "filename": filename} + }, + ) + + for filename in os.listdir(EPG_WATCH_DIR): + filepath = os.path.join(EPG_WATCH_DIR, filename) + + if not os.path.isfile(filepath): + continue + + if not filename.endswith('.xml') and not filename.endswith('.gz'): + continue + + mtime = os.path.getmtime(filepath) + age = now - mtime + redis_key = REDIS_PREFIX + filepath + stored_mtime = redis_client.get(redis_key) + + # Startup safety: skip old untracked files + if not stored_mtime and age > STARTUP_SKIP_AGE: + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + continue # Assume already processed before startup + + # File too new — probably still being written + if age < MIN_AGE_SECONDS: + continue + + # Skip if we've already processed this mtime + if stored_mtime and float(stored_mtime) >= mtime: + continue + + epg_source, _ = EPGSource.objects.get_or_create(file_path=filepath, defaults={ + "name": filename, + "source_type": "xmltv", + "is_active": True if CoreSettings.get_auto_import_mapped_files() == "true" else False, + }) + + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + redis_client.set(redis_key, mtime, ex=REDIS_TTL) + + if not epg_source.is_active: + logger.info("EPG source is inactive, skipping.") + continue + + refresh_epg_data.delay(epg_source.id) # Trigger Celery task + + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": {"success": True, "type": "epg_file", "filename": filename} + }, + ) + +def fetch_channel_stats(): + redis_client = RedisClient.get_client() + + try: + # Basic info for all channels + channel_pattern = "ts_proxy:channel:*:metadata" + all_channels = [] + + # Extract channel IDs from keys + cursor = 0 + while True: + cursor, keys = redis_client.scan(cursor, match=channel_pattern) + for key in keys: + channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8')) + if channel_id_match: + ch_id = channel_id_match.group(1) + channel_info = ChannelStatus.get_basic_channel_info(ch_id) + if channel_info: + all_channels.append(channel_info) + + if cursor == 0: + break + + except Exception as e: + logger.error(f"Error in channel_status: {e}", exc_info=True) + return + # return JsonResponse({'error': str(e)}, status=500) + + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + "updates", + { + "type": "update", + "data": {"success": True, "type": "channel_stats", "stats": json.dumps({'channels': all_channels, 'count': len(all_channels)})} + }, + ) diff --git a/core/utils.py b/core/utils.py index 073c2169..3a5d84f4 100644 --- a/core/utils.py +++ b/core/utils.py @@ -5,161 +5,158 @@ import os import threading from django.conf import settings from redis.exceptions import ConnectionError, TimeoutError +from django.core.cache import cache +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +import gc logger = logging.getLogger(__name__) # Import the command detector from .command_utils import is_management_command -def get_redis_client(max_retries=5, retry_interval=1): - """Get Redis client with connection validation and retry logic""" - # Skip Redis connection for management commands like collectstatic - if is_management_command(): - logger.info("Running as management command - skipping Redis initialization") - return None +class RedisClient: + _client = None + _pubsub_client = None - retry_count = 0 - while retry_count < max_retries: - try: - # Get connection parameters from settings or environment - redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) - redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) - redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) + @classmethod + def get_client(cls, max_retries=5, retry_interval=1): + if cls._client is None: + retry_count = 0 + while retry_count < max_retries: + try: + # Get connection parameters from settings or environment + redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) + redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) + redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) - # Use standardized settings - socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) - socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) - health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) - socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) - retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + # Use standardized settings + socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) + socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) + health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) + socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) + retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) - # Create Redis client with better defaults - client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - socket_timeout=socket_timeout, - socket_connect_timeout=socket_connect_timeout, - socket_keepalive=socket_keepalive, - health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout - ) + # Create Redis client with better defaults + client = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + socket_timeout=socket_timeout, + socket_connect_timeout=socket_connect_timeout, + socket_keepalive=socket_keepalive, + health_check_interval=health_check_interval, + retry_on_timeout=retry_on_timeout + ) - # Validate connection with ping - client.ping() - logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") - return client + # Validate connection with ping + client.ping() + client.flushdb() + logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") - except (ConnectionError, TimeoutError) as e: - retry_count += 1 - if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") - return None - else: - # Use exponential backoff for retries - wait_time = retry_interval * (2 ** (retry_count - 1)) - logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") - time.sleep(wait_time) + cls._client = client + break - except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}") - return None + except (ConnectionError, TimeoutError) as e: + retry_count += 1 + if retry_count >= max_retries: + logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") + return None + else: + # Use exponential backoff for retries + wait_time = retry_interval * (2 ** (retry_count - 1)) + logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") + time.sleep(wait_time) -def get_redis_pubsub_client(max_retries=5, retry_interval=1): - """Get Redis client optimized for PubSub operations""" - # Skip Redis connection for management commands like collectstatic - if is_management_command(): - logger.info("Running as management command - skipping Redis PubSub initialization") - return None + except Exception as e: + logger.error(f"Unexpected error connecting to Redis: {e}") + return None - retry_count = 0 - while retry_count < max_retries: - try: - # Get connection parameters from settings or environment - redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) - redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) - redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) + return cls._client - # Use standardized settings but without socket timeouts for PubSub - # Important: socket_timeout is None for PubSub operations - socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) - socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) - health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) - retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + @classmethod + def get_pubsub_client(cls, max_retries=5, retry_interval=1): + """Get Redis client optimized for PubSub operations""" + if cls._pubsub_client is None: + retry_count = 0 + while retry_count < max_retries: + try: + # Get connection parameters from settings or environment + redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) + redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) + redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) - # Create Redis client with PubSub-optimized settings - no timeout - client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - socket_timeout=None, # Critical: No timeout for PubSub operations - socket_connect_timeout=socket_connect_timeout, - socket_keepalive=socket_keepalive, - health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout - ) + # Use standardized settings but without socket timeouts for PubSub + # Important: socket_timeout is None for PubSub operations + socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) + socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) + health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) + retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) - # Validate connection with ping - client.ping() - logger.info(f"Connected to Redis for PubSub at {redis_host}:{redis_port}/{redis_db}") + # Create Redis client with PubSub-optimized settings - no timeout + client = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + socket_timeout=None, # Critical: No timeout for PubSub operations + socket_connect_timeout=socket_connect_timeout, + socket_keepalive=socket_keepalive, + health_check_interval=health_check_interval, + retry_on_timeout=retry_on_timeout + ) - # We don't need the keepalive thread anymore since we're using proper PubSub handling - return client + # Validate connection with ping + client.ping() + logger.info(f"Connected to Redis for PubSub at {redis_host}:{redis_port}/{redis_db}") - except (ConnectionError, TimeoutError) as e: - retry_count += 1 - if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}") - return None - else: - # Use exponential backoff for retries - wait_time = retry_interval * (2 ** (retry_count - 1)) - logger.warning(f"Redis PubSub connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") - time.sleep(wait_time) + # We don't need the keepalive thread anymore since we're using proper PubSub handling + cls._pubsub_client = client + break - except Exception as e: - logger.error(f"Unexpected error connecting to Redis for PubSub: {e}") - return None + except (ConnectionError, TimeoutError) as e: + retry_count += 1 + if retry_count >= max_retries: + logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}") + return None + else: + # Use exponential backoff for retries + wait_time = retry_interval * (2 ** (retry_count - 1)) + logger.warning(f"Redis PubSub connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") + time.sleep(wait_time) -def execute_redis_command(redis_client, command_func, default_return=None): - """ - Execute a Redis command with proper error handling + except Exception as e: + logger.error(f"Unexpected error connecting to Redis for PubSub: {e}") + return None - Args: - redis_client: The Redis client instance - command_func: Lambda function containing the Redis command to execute - default_return: Value to return if command fails + return cls._pubsub_client - Returns: - Command result or default_return on failure - """ - if redis_client is None: - return default_return +def acquire_task_lock(task_name, id): + """Acquire a lock to prevent concurrent task execution.""" + redis_client = RedisClient.get_client() + lock_id = f"task_lock_{task_name}_{id}" - try: - return command_func() - except (ConnectionError, TimeoutError) as e: - logger.warning(f"Redis connection error: {e}") - return default_return - except Exception as e: - logger.error(f"Redis command error: {e}") - return default_return + # Use the Redis SET command with NX (only set if not exists) and EX (set expiration) + lock_acquired = redis_client.set(lock_id, "locked", ex=300, nx=True) -# Initialize the global clients with retry logic -# Skip Redis initialization if running as a management command -if is_management_command(): - redis_client = None - redis_pubsub_client = None - logger.info("Running as management command - Redis clients set to None") -else: - redis_client = get_redis_client() - redis_pubsub_client = get_redis_pubsub_client() + if not lock_acquired: + logger.warning(f"Lock for {task_name} and id={id} already acquired. Task will not proceed.") -# Import and initialize the PubSub manager -# Skip if running as management command or if Redis client is None -if not is_management_command() and redis_client is not None: - from .redis_pubsub import get_pubsub_manager - pubsub_manager = get_pubsub_manager(redis_client) -else: - logger.info("PubSub manager not initialized (running as management command or Redis not available)") - pubsub_manager = None \ No newline at end of file + return lock_acquired + +def release_task_lock(task_name, id): + """Release the lock after task execution.""" + redis_client = RedisClient.get_client() + lock_id = f"task_lock_{task_name}_{id}" + + # Remove the lock + redis_client.delete(lock_id) + +def send_websocket_event(event, success, data): + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'updates', + { + 'type': 'update', + "data": {"success": True, "type": "epg_channels"} + } + ) diff --git a/dispatcharr/consumers.py b/dispatcharr/consumers.py index 356422d7..8d92c4fa 100644 --- a/dispatcharr/consumers.py +++ b/dispatcharr/consumers.py @@ -1,5 +1,8 @@ import json from channels.generic.websocket import AsyncWebsocketConsumer +import re, logging + +logger = logging.getLogger(__name__) class MyWebSocketConsumer(AsyncWebsocketConsumer): async def connect(self): @@ -12,7 +15,29 @@ class MyWebSocketConsumer(AsyncWebsocketConsumer): async def receive(self, text_data): data = json.loads(text_data) - print("Received:", data) + + if data["type"] == "m3u_profile_test": + from apps.proxy.ts_proxy.url_utils import transform_url + + def replace_with_mark(match): + # Wrap the match in tags + return f"{match.group(0)}" + + # Apply the transformation using the replace_with_mark function + try: + search_preview = re.sub(data["search"], replace_with_mark, data["url"]) + except Exception as e: + search_preview = data["search"] + logger.error(f"Failed to generate replace preview: {e}") + + result = transform_url(data["url"], data["search"], data["replace"]) + await self.send(text_data=json.dumps({ + "data": { + 'type': 'm3u_profile_test', + 'search_preview': search_preview, + 'result': result, + } + })) async def update(self, event): await self.send(text_data=json.dumps(event)) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 9b381bfd..96bda89b 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -1,7 +1,6 @@ import os from pathlib import Path from datetime import timedelta -from celery.schedules import crontab BASE_DIR = Path(__file__).resolve().parent.parent @@ -9,7 +8,12 @@ SECRET_KEY = 'REPLACE_ME_WITH_A_REAL_SECRET' REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") REDIS_DB = os.environ.get("REDIS_DB", "0") -DEBUG = True +# Set DEBUG to True for development, False for production +if os.environ.get('DISPATCHARR_DEBUG', 'False').lower() == 'true': + DEBUG = True +else: + DEBUG = False + ALLOWED_HOSTS = ["*"] INSTALLED_APPS = [ @@ -22,9 +26,10 @@ INSTALLED_APPS = [ 'apps.m3u', 'apps.output', 'apps.proxy.apps.ProxyConfig', + 'apps.proxy.ts_proxy', 'core', - 'drf_yasg', 'daphne', + 'drf_yasg', 'channels', 'django.contrib.admin', 'django.contrib.auth', @@ -35,10 +40,9 @@ INSTALLED_APPS = [ 'rest_framework', 'corsheaders', 'django_filters', + 'django_celery_beat', ] - - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -153,7 +157,7 @@ CELERY_RESULT_BACKEND = CELERY_BROKER_URL # Configure Redis key prefix CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = { - 'prefix': 'celery-task:', # Set the Redis key prefix for Celery + 'global_keyprefix': 'celery-tasks:', # Set the Redis key prefix for Celery } # Set TTL (Time-to-Live) for task results (in seconds) @@ -164,9 +168,13 @@ CELERY_BROKER_TRANSPORT_OPTIONS = { 'visibility_timeout': 3600, # Time in seconds that a task remains invisible during retries } +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' + +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler" CELERY_BEAT_SCHEDULE = { 'fetch-channel-statuses': { - 'task': 'apps.proxy.tasks.fetch_channel_stats', + 'task': 'core.tasks.beat_periodic_task', 'schedule': 2.0, }, } diff --git a/docker/Dockerfile b/docker/Dockerfile index fd910759..e3f8a165 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,30 +14,27 @@ ENV PATH="/dispatcharrpy/bin:$PATH" \ RUN apt-get update && \ apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - gcc \ - git \ - libpcre3 \ - libpcre3-dev \ - python3-dev \ - wget && \ - echo "=== setting up nodejs ===" && \ - curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - nodejs && \ + build-essential \ + curl \ + gcc \ + git \ + libpcre3 \ + libpcre3-dev \ + python3-dev \ + wget && \ python -m pip install virtualenv && \ virtualenv /dispatcharrpy && \ git clone -b ${BRANCH} ${REPO_URL} /app && \ cd /app && \ rm -rf .git && \ cd /app && \ - pip install --no-cache-dir -r requirements.txt && \ - python manage.py collectstatic --noinput && \ - cd /app/frontend && \ - npm install --legacy-peer-deps && \ + pip install --no-cache-dir -r requirements.txt + +# Use a dedicated Node.js stage for frontend building +FROM node:20-slim AS frontend-builder +WORKDIR /app/frontend +COPY --from=builder /app /app +RUN npm install --legacy-peer-deps && \ npm run build && \ find . -maxdepth 1 ! -name '.' ! -name 'dist' -exec rm -rf '{}' \; @@ -51,20 +48,24 @@ ENV PATH="/dispatcharrpy/bin:$PATH" \ # Copy the virtual environment and application from the builder stage COPY --from=builder /dispatcharrpy /dispatcharrpy COPY --from=builder /app /app +COPY --from=frontend-builder /app/frontend/dist /app/frontend/dist + +# Run collectstatic after frontend assets are copied +RUN cd /app && python manage.py collectstatic --noinput # Install base dependencies with memory optimization RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - curl \ - ffmpeg \ - libpcre3 \ - libpq-dev \ - nginx \ - procps \ - streamlink \ - wget \ - gnupg2 \ - lsb-release && \ + curl \ + ffmpeg \ + libpcre3 \ + libpq-dev \ + nginx \ + procps \ + streamlink \ + wget \ + gnupg2 \ + lsb-release && \ cp /app/docker/nginx.conf /etc/nginx/sites-enabled/default && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/build-dev.sh b/docker/build-dev.sh index c5c79474..65d643a7 100755 --- a/docker/build-dev.sh +++ b/docker/build-dev.sh @@ -1,3 +1,13 @@ #!/bin/bash - docker build --build-arg BRANCH=dev -t dispatcharr/dispatcharr:dev -f Dockerfile .. + +# Get version information +VERSION=$(python -c "import sys; sys.path.append('..'); import version; print(version.__version__)") +BUILD=$(python -c "import sys; sys.path.append('..'); import version; print(version.__build__)") + +# Build with version tags +docker build --build-arg BRANCH=dev \ + -t dispatcharr/dispatcharr:dev \ + -t dispatcharr/dispatcharr:${VERSION}-${BUILD} \ + -f Dockerfile .. +. diff --git a/docker/docker-compose.aio.yml b/docker/docker-compose.aio.yml index f689bb3f..77b9bec1 100644 --- a/docker/docker-compose.aio.yml +++ b/docker/docker-compose.aio.yml @@ -3,16 +3,16 @@ services: # build: # context: . # dockerfile: Dockerfile - image: dispatcharr/dispatcharr:latest + image: ghcr.io/dispatcharr/dispatcharr:latest container_name: dispatcharr ports: - 9191:9191 volumes: - - dispatcharr_db:/data + - dispatcharr_data:/data environment: - DISPATCHARR_ENV=aio - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 volumes: - dispatcharr_db: + dispatcharr_data: diff --git a/docker/docker-compose.debug.yml b/docker/docker-compose.debug.yml new file mode 100644 index 00000000..40a87bfe --- /dev/null +++ b/docker/docker-compose.debug.yml @@ -0,0 +1,19 @@ +services: + dispatcharr: + # build: + # context: .. + # dockerfile: docker/Dockerfile.dev + image: dispatcharr/dispatcharr + container_name: dispatcharr_debug + ports: + - 5656:5656 # API port + - 9193:9191 # Web UI port + - 8001:8001 # Socket port + - 5678:5678 # Debugging port + volumes: + - ../:/app + environment: + - DISPATCHARR_ENV=dev + - DISPATCHARR_DEBUG=true + - REDIS_HOST=localhost + - CELERY_BROKER_URL=redis://localhost:6379/0 diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 35c5087b..3b6f53df 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -3,7 +3,7 @@ services: # build: # context: .. # dockerfile: docker/Dockerfile.dev - image: dispatcharr/dispatcharr + image: ghcr.io/dispatcharr/dispatcharr:dev container_name: dispatcharr_dev ports: - 5656:5656 @@ -11,7 +11,30 @@ services: - 8001:8001 volumes: - ../:/app + # - ./data/db:/data environment: - DISPATCHARR_ENV=dev - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 + + pgadmin: + image: dpage/pgadmin4 + environment: + PGADMIN_DEFAULT_EMAIL: admin@admin.com + PGADMIN_DEFAULT_PASSWORD: admin + volumes: + - dispatcharr_dev_pgadmin:/var/lib/pgadmin + ports: + - 8082:80 + + redis-commander: + image: rediscommander/redis-commander:latest + environment: + - REDIS_HOSTS=dispatcharr:dispatcharr:6379:0 + - TRUST_PROXY=true + - ADDRESS=0.0.0.0 + ports: + - 8081:8081 + +volumes: + dispatcharr_dev_pgadmin: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d04edcb0..478d94d0 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -36,6 +36,15 @@ export POSTGRES_PORT=${POSTGRES_PORT:-5432} export REDIS_HOST=${REDIS_HOST:-localhost} export REDIS_DB=${REDIS_DB:-0} +export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191} + +# Extract version information from version.py +export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)") +export DISPATCHARR_BUILD=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__build__)") +echo "📦 Dispatcharr version: ${DISPATCHARR_VERSION}-${DISPATCHARR_BUILD}" + +# READ-ONLY - don't let users change these +export POSTGRES_DIR=/data/db # Global variables, stored so other users inherit them if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then @@ -49,8 +58,13 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export POSTGRES_HOST=$POSTGRES_HOST" >> /etc/profile.d/dispatcharr.sh echo "export POSTGRES_PORT=$POSTGRES_PORT" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_ENV=$DISPATCHARR_ENV" >> /etc/profile.d/dispatcharr.sh + echo "export DISPATCHARR_DEBUG=$DISPATCHARR_DEBUG" >> /etc/profile.d/dispatcharr.sh echo "export REDIS_HOST=$REDIS_HOST" >> /etc/profile.d/dispatcharr.sh echo "export REDIS_DB=$REDIS_DB" >> /etc/profile.d/dispatcharr.sh + echo "export POSTGRES_DIR=$POSTGRES_DIR" >> /etc/profile.d/dispatcharr.sh + echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh + echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh + echo "export DISPATCHARR_BUILD=$DISPATCHARR_BUILD" >> /etc/profile.d/dispatcharr.sh fi chmod +x /etc/profile.d/dispatcharr.sh @@ -65,18 +79,23 @@ echo "Starting init process..." # Start PostgreSQL echo "Starting Postgres..." -su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data start -w -t 300 -o '-c port=${POSTGRES_PORT}'" +su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} start -w -t 300 -o '-c port=${POSTGRES_PORT}'" # Wait for PostgreSQL to be ready until su - postgres -c "/usr/lib/postgresql/14/bin/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do echo_with_timestamp "Waiting for PostgreSQL to be ready..." sleep 1 done -postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p') +postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p') echo "✅ Postgres started with PID $postgres_pid" pids+=("$postgres_pid") -if [ "$DISPATCHARR_ENV" = "dev" ]; then +if [[ "$DISPATCHARR_ENV" = "dev" ]]; then . /app/docker/init/99-init-dev.sh + echo "Starting frontend dev environment" + su - $POSTGRES_USER -c "cd /app/frontend && npm run dev &" + npm_pid=$(pgrep vite | sort | head -n1) + echo "✅ vite started with PID $npm_pid" + pids+=("$npm_pid") else echo "🚀 Starting nginx..." nginx @@ -85,22 +104,57 @@ else pids+=("$nginx_pid") fi -uwsgi_file="/app/docker/uwsgi.ini" -if [ "$DISPATCHARR_ENV" = "dev" ]; then +cd /app +python manage.py migrate --noinput +python manage.py collectstatic --noinput + +# Select proper uwsgi config based on environment +if [ "$DISPATCHARR_ENV" = "dev" ] && [ "$DISPATCHARR_DEBUG" != "true" ]; then + echo "🚀 Starting uwsgi in dev mode..." uwsgi_file="/app/docker/uwsgi.dev.ini" +elif [ "$DISPATCHARR_DEBUG" = "true" ]; then + echo "🚀 Starting uwsgi in debug mode..." + uwsgi_file="/app/docker/uwsgi.debug.ini" +else + echo "🚀 Starting uwsgi in production mode..." + uwsgi_file="/app/docker/uwsgi.ini" fi -echo "🚀 Starting uwsgi..." su - $POSTGRES_USER -c "cd /app && uwsgi --ini $uwsgi_file &" uwsgi_pid=$(pgrep uwsgi | sort | head -n1) echo "✅ uwsgi started with PID $uwsgi_pid" pids+=("$uwsgi_pid") +# sed -i 's/protected-mode yes/protected-mode no/g' /etc/redis/redis.conf +# su - $POSTGRES_USER -c "redis-server --protected-mode no &" +# redis_pid=$(pgrep redis) +# echo "✅ redis started with PID $redis_pid" +# pids+=("$redis_pid") +# echo "🚀 Starting gunicorn..." +# su - $POSTGRES_USER -c "cd /app && gunicorn dispatcharr.asgi:application \ +# --bind 0.0.0.0:5656 \ +# --worker-class uvicorn.workers.UvicornWorker \ +# --workers 2 \ +# --threads 1 \ +# --timeout 0 \ +# --keep-alive 30 \ +# --access-logfile - \ +# --error-logfile - &" +# gunicorn_pid=$(pgrep gunicorn | sort | head -n1) +# echo "✅ gunicorn started with PID $gunicorn_pid" +# pids+=("$gunicorn_pid") -cd /app -python manage.py migrate --noinput -python manage.py collectstatic --noinput +# echo "Starting celery and beat..." +# su - $POSTGRES_USER -c "cd /app && celery -A dispatcharr worker -l info --autoscale=8,2 &" +# celery_pid=$(pgrep celery | sort | head -n1) +# echo "✅ celery started with PID $celery_pid" +# pids+=("$celery_pid") + +# su - $POSTGRES_USER -c "cd /app && celery -A dispatcharr beat -l info &" +# beat_pid=$(pgrep beat | sort | head -n1) +# echo "✅ celery beat started with PID $beat_pid" +# pids+=("$beat_pid") # Wait for at least one process to exit and log the process that exited first if [ ${#pids[@]} -gt 0 ]; then diff --git a/docker/init/02-postgres.sh b/docker/init/02-postgres.sh index ddaa15b7..69a81dd4 100644 --- a/docker/init/02-postgres.sh +++ b/docker/init/02-postgres.sh @@ -1,37 +1,64 @@ #!/bin/bash -# Inwitialize PostgreSQL database -if [ -z "$(ls -A "/data")" ]; then - echo_with_timestamp "Initializing PostgreSQL database..." - mkdir -p "/data" - chown -R postgres:postgres "/data" - chmod 700 "/data" +# Temporary migration from postgres in /data to $POSTGRES_DIR. Can likely remove +# some time in the future. +if [ -e "/data/postgresql.conf" ]; then + echo "Migrating PostgreSQL data from /data to $POSTGRES_DIR..." + + # Create a temporary directory outside of /data + mkdir -p /tmp/postgres_migration + + # Move the PostgreSQL files to the temporary directory + mv /data/* /tmp/postgres_migration/ + + # Create the target directory + mkdir -p $POSTGRES_DIR + + # Move the files from temporary directory to the final location + mv /tmp/postgres_migration/* $POSTGRES_DIR/ + + # Clean up the temporary directory + rmdir /tmp/postgres_migration + + # Set proper ownership and permissions for PostgreSQL data directory + chown -R postgres:postgres $POSTGRES_DIR + chmod 700 $POSTGRES_DIR + + echo "Migration completed successfully." +fi + +# Initialize PostgreSQL database +if [ -z "$(ls -A $POSTGRES_DIR)" ]; then + echo "Initializing PostgreSQL database..." + mkdir -p $POSTGRES_DIR + chown -R postgres:postgres $POSTGRES_DIR + chmod 700 $POSTGRES_DIR # Initialize PostgreSQL - su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D /data" + su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D ${POSTGRES_DIR}" # Configure PostgreSQL - echo "host all all 0.0.0.0/0 md5" >> "/data/pg_hba.conf" - echo "listen_addresses='*'" >> "/data/postgresql.conf" + echo "host all all 0.0.0.0/0 md5" >> "${POSTGRES_DIR}/pg_hba.conf" + echo "listen_addresses='*'" >> "${POSTGRES_DIR}/postgresql.conf" # Start PostgreSQL echo "Starting Postgres..." - su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data start -w -t 300 -o '-c port=${POSTGRES_PORT}'" + su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} start -w -t 300 -o '-c port=${POSTGRES_PORT}'" # Wait for PostgreSQL to be ready until su - postgres -c "/usr/lib/postgresql/14/bin/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do - echo_with_timestamp "Waiting for PostgreSQL to be ready..." + echo "Waiting for PostgreSQL to be ready..." sleep 1 done - postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p') + postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p') # Setup database if needed if ! su - postgres -c "psql -p ${POSTGRES_PORT} -tAc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB';\"" | grep -q 1; then # Create PostgreSQL database - echo_with_timestamp "Creating PostgreSQL database..." + echo "Creating PostgreSQL database..." su - postgres -c "createdb -p ${POSTGRES_PORT} ${POSTGRES_DB}" # Create user, set ownership, and grant privileges - echo_with_timestamp "Creating PostgreSQL user..." + echo "Creating PostgreSQL user..." su - postgres -c "psql -p ${POSTGRES_PORT} -d ${POSTGRES_DB}" <\d+)/cache/ { + proxy_pass http://127.0.0.1:5656; + proxy_cache logo_cache; + proxy_cache_key "$scheme$request_uri"; # Cache per logo URL + proxy_cache_valid 200 24h; # Cache for 24 hours + proxy_cache_use_stale error timeout updating; # Serve stale if Django is slow + } + + location ~ ^/api/channels/logos/(?\d+)/cache/ { + proxy_pass http://127.0.0.1:5656; + proxy_cache logo_cache; + proxy_cache_key "$scheme$request_uri"; # Cache per logo URL + proxy_cache_valid 200 24h; # Cache for 24 hours + proxy_cache_use_stale error timeout updating; # Serve stale if Django is slow + } + # admin disabled when not in dev mode location /admin { return 301 /login; @@ -24,7 +52,13 @@ server { # Route HDHR request to Django location /hdhr { - proxy_pass http://127.0.0.1:5656; + include uwsgi_params; + uwsgi_pass unix:/app/uwsgi.sock; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; } # Serve FFmpeg streams efficiently @@ -32,8 +66,9 @@ server { proxy_pass http://127.0.0.1:5656; proxy_buffering off; proxy_set_header Connection keep-alive; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; } # WebSockets for real-time communication @@ -42,6 +77,8 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; } @@ -55,7 +92,8 @@ server { proxy_read_timeout 3600s; proxy_send_timeout 3600s; client_max_body_size 0; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; } } diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini new file mode 100644 index 00000000..957f4f4c --- /dev/null +++ b/docker/uwsgi.debug.ini @@ -0,0 +1,81 @@ +[uwsgi] +; exec-before = python manage.py collectstatic --noinput +; exec-before = python manage.py migrate --noinput + +; First run Redis availability check script once +exec-before = python /app/scripts/wait_for_redis.py + +; Start Redis first +attach-daemon = redis-server +; Then start other services +attach-daemon = celery -A dispatcharr worker -l info +attach-daemon = celery -A dispatcharr beat -l info +attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application +attach-daemon = cd /app/frontend && npm run dev + +# Core settings +chdir = /app +module = scripts.debug_wrapper:application +virtualenv = /dispatcharrpy +master = true +env = DJANGO_SETTINGS_MODULE=dispatcharr.settings +socket = /app/uwsgi.sock +chmod-socket = 777 +vacuum = true +die-on-term = true +static-map = /static=/app/static + +# Worker configuration +workers = 1 +threads = 8 +enable-threads = true +lazy-apps = true + +# HTTP server +http = 0.0.0.0:5656 +http-keepalive = 1 +buffer-size = 65536 +http-timeout = 600 + +# Async mode (use gevent for high concurrency) +gevent = 100 +async = 100 + +# Performance tuning +thunder-lock = true +log-4xx = true +log-5xx = true +disable-logging = false + +; Longer timeouts for debugging sessions +harakiri = 3600 +socket-timeout = 3600 +http-timeout = 3600 + + +# Ignore unknown options +ignore-sigpipe = true +ignore-write-errors = true +disable-write-exception = true + +# Explicitly disable for-server option that confuses debugpy +for-server = false + +# Debugging settings +py-autoreload = 1 +honour-stdin = true + +# Environment variables +env = PYTHONPATH=/app +env = PYTHONUNBUFFERED=1 +env = PYDEVD_DISABLE_FILE_VALIDATION=1 +env = PYTHONUTF8=1 +env = PYTHONXOPT=-Xfrozen_modules=off +env = PYDEVD_DEBUG=1 +env = DEBUGPY_LOG_DIR=/app/debugpy_logs + +# Debugging control variables +env = WAIT_FOR_DEBUGGER=false +env = DEBUG_TIMEOUT=30 + + diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 5b23b183..191423bf 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker -l info +attach-daemon = celery -A dispatcharr worker -l info --concurrency=4 attach-daemon = celery -A dispatcharr beat -l info attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 1b1de50a..e014e030 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker -l error +attach-daemon = celery -A dispatcharr worker -l error --concurrency=4 attach-daemon = celery -A dispatcharr beat -l error attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application @@ -25,9 +25,8 @@ die-on-term = true static-map = /static=/app/static # Worker management (Optimize for I/O bound tasks) -workers = 4 -threads = 2 -enable-threads = true +workers = 2 +enable-threads = false # Optimize for streaming http = 0.0.0.0:5656 diff --git a/frontend/index.html b/frontend/index.html index de12df40..86207098 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -2,7 +2,13 @@ - + + + + + + + Dispatcharr diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1b91bb9b..4f1856f1 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,9 +8,11 @@ "name": "vite", "version": "0.0.0", "dependencies": { + "@mantine/charts": "^7.17.2", "@mantine/core": "^7.17.2", "@mantine/dates": "^7.17.2", "@mantine/dropzone": "^7.17.2", + "@mantine/form": "^7.17.3", "@mantine/hooks": "^7.17.2", "@mantine/notifications": "^7.17.2", "@tabler/icons-react": "^3.31.0", @@ -29,6 +31,8 @@ "react-draggable": "^4.4.6", "react-pro-sidebar": "^1.1.0", "react-router-dom": "^7.3.0", + "react-window": "^1.8.11", + "recharts": "^2.15.1", "video.js": "^8.21.0", "yup": "^1.6.1", "zustand": "^5.0.3" @@ -1082,6 +1086,19 @@ "integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==", "license": "Apache-2.0" }, + "node_modules/@mantine/charts": { + "version": "7.17.2", + "resolved": "https://registry.npmjs.org/@mantine/charts/-/charts-7.17.2.tgz", + "integrity": "sha512-ckB23pIqRjzysUz2EiWZD9AVyf7t0r7o7zfJbl01nzOezFgYq5RGeRoxvpcsfBC+YoSbB/43rjNcXtYhtA7QzA==", + "license": "MIT", + "peerDependencies": { + "@mantine/core": "7.17.2", + "@mantine/hooks": "7.17.2", + "react": "^18.x || ^19.x", + "react-dom": "^18.x || ^19.x", + "recharts": "^2.13.3" + } + }, "node_modules/@mantine/core": { "version": "7.17.2", "resolved": "https://registry.npmjs.org/@mantine/core/-/core-7.17.2.tgz", @@ -1132,6 +1149,19 @@ "react-dom": "^18.x || ^19.x" } }, + "node_modules/@mantine/form": { + "version": "7.17.3", + "resolved": "https://registry.npmjs.org/@mantine/form/-/form-7.17.3.tgz", + "integrity": "sha512-ktERldD8f9lrjjz6wIbwMnNbAZq8XEWPx4K5WuFyjXaK0PI8D+gsXIGKMtA5rVrAUFHCWCdbK3yLgtjJNki8ew==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "klona": "^2.0.6" + }, + "peerDependencies": { + "react": "^18.x || ^19.x" + } + }, "node_modules/@mantine/hooks": { "version": "7.17.2", "resolved": "https://registry.npmjs.org/@mantine/hooks/-/hooks-7.17.2.tgz", @@ -1776,6 +1806,69 @@ "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", "license": "MIT" }, + "node_modules/@types/d3-array": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", + "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz", + "integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -2205,6 +2298,127 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "license": "MIT" }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/dayjs": { "version": "1.11.13", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", @@ -2228,6 +2442,12 @@ } } }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", + "license": "MIT" + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2589,9 +2809,17 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, + "node_modules/fast-equals": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz", + "integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -2947,6 +3175,15 @@ "node": ">=0.8.19" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -3072,6 +3309,15 @@ "json-buffer": "3.0.1" } }, + "node_modules/klona": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", + "integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -3215,6 +3461,12 @@ "node": ">= 0.4" } }, + "node_modules/memoize-one": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", + "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==", + "license": "MIT" + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -3640,6 +3892,12 @@ "integrity": "sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw==", "license": "MIT" }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, "node_modules/react-number-format": { "version": "5.4.3", "resolved": "https://registry.npmjs.org/react-number-format/-/react-number-format-5.4.3.tgz", @@ -3753,6 +4011,21 @@ "react-dom": ">=18" } }, + "node_modules/react-smooth": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz", + "integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==", + "license": "MIT", + "dependencies": { + "fast-equals": "^5.0.1", + "prop-types": "^15.8.1", + "react-transition-group": "^4.4.5" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/react-style-singleton": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", @@ -3808,6 +4081,61 @@ "react-dom": ">=16.6.0" } }, + "node_modules/react-window": { + "version": "1.8.11", + "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.11.tgz", + "integrity": "sha512-+SRbUVT2scadgFSWx+R1P754xHPEqvcfSfVX10QYg6POOz+WNgkN48pS+BtZNIMGiL1HYrSEiCkwsMS15QogEQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.0.0", + "memoize-one": ">=3.1.1 <6" + }, + "engines": { + "node": ">8.0.0" + }, + "peerDependencies": { + "react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/recharts": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.1.tgz", + "integrity": "sha512-v8PUTUlyiDe56qUj82w/EDVuzEFXwEHp9/xOowGAZwfLjB9uAy3GllQVIYMWF6nU+qibx85WF75zD7AjqoT54Q==", + "license": "MIT", + "dependencies": { + "clsx": "^2.0.0", + "eventemitter3": "^4.0.1", + "lodash": "^4.17.21", + "react-is": "^18.3.1", + "react-smooth": "^4.0.4", + "recharts-scale": "^0.4.4", + "tiny-invariant": "^1.3.1", + "victory-vendor": "^36.6.8" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/recharts-scale": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz", + "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==", + "license": "MIT", + "dependencies": { + "decimal.js-light": "^2.4.1" + } + }, + "node_modules/recharts/node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -3998,6 +4326,12 @@ "integrity": "sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==", "license": "MIT" }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, "node_modules/tiny-warning": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", @@ -4145,6 +4479,28 @@ } } }, + "node_modules/victory-vendor": { + "version": "36.9.2", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", + "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/video.js": { "version": "8.22.0", "resolved": "https://registry.npmjs.org/video.js/-/video.js-8.22.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index de53fcc4..5677558b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,9 +10,11 @@ "preview": "vite preview" }, "dependencies": { + "@mantine/charts": "^7.17.2", "@mantine/core": "^7.17.2", "@mantine/dates": "^7.17.2", "@mantine/dropzone": "^7.17.2", + "@mantine/form": "^7.17.3", "@mantine/hooks": "^7.17.2", "@mantine/notifications": "^7.17.2", "@tabler/icons-react": "^3.31.0", @@ -31,6 +33,8 @@ "react-draggable": "^4.4.6", "react-pro-sidebar": "^1.1.0", "react-router-dom": "^7.3.0", + "react-window": "^1.8.11", + "recharts": "^2.15.1", "video.js": "^8.21.0", "yup": "^1.6.1", "zustand": "^5.0.3" diff --git a/frontend/public/android-chrome-192x192.png b/frontend/public/android-chrome-192x192.png new file mode 100644 index 00000000..59248862 Binary files /dev/null and b/frontend/public/android-chrome-192x192.png differ diff --git a/frontend/public/android-chrome-512x512.png b/frontend/public/android-chrome-512x512.png new file mode 100644 index 00000000..cd17f001 Binary files /dev/null and b/frontend/public/android-chrome-512x512.png differ diff --git a/frontend/public/apple-touch-icon.png b/frontend/public/apple-touch-icon.png new file mode 100644 index 00000000..7185f2ee Binary files /dev/null and b/frontend/public/apple-touch-icon.png differ diff --git a/frontend/public/favicon-16x16.png b/frontend/public/favicon-16x16.png new file mode 100644 index 00000000..27c13e08 Binary files /dev/null and b/frontend/public/favicon-16x16.png differ diff --git a/frontend/public/favicon-32x32.png b/frontend/public/favicon-32x32.png new file mode 100644 index 00000000..187a9e71 Binary files /dev/null and b/frontend/public/favicon-32x32.png differ diff --git a/frontend/public/favicon.ico b/frontend/public/favicon.ico new file mode 100644 index 00000000..9a9a94da Binary files /dev/null and b/frontend/public/favicon.ico differ diff --git a/frontend/public/site.webmanifest b/frontend/public/site.webmanifest new file mode 100644 index 00000000..45dc8a20 --- /dev/null +++ b/frontend/public/site.webmanifest @@ -0,0 +1 @@ +{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} \ No newline at end of file diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index c769dcc6..a57cd1b0 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -9,12 +9,11 @@ import { import Sidebar from './components/Sidebar'; import Login from './pages/Login'; import Channels from './pages/Channels'; -import M3U from './pages/M3U'; -import EPG from './pages/EPG'; +import ContentSources from './pages/ContentSources'; import Guide from './pages/Guide'; import Stats from './pages/Stats'; +import DVR from './pages/DVR'; import Settings from './pages/Settings'; -import StreamProfiles from './pages/StreamProfiles'; import useAuthStore from './store/auth'; import FloatingVideo from './components/FloatingVideo'; import { WebsocketProvider } from './WebSocket'; @@ -23,11 +22,13 @@ import '@mantine/core/styles.css'; // Ensure Mantine global styles load import '@mantine/notifications/styles.css'; import 'mantine-react-table/styles.css'; import '@mantine/dropzone/styles.css'; +import '@mantine/dates/styles.css'; import './index.css'; import mantineTheme from './mantineTheme'; import API from './api'; import { Notifications } from '@mantine/notifications'; import M3URefreshNotification from './components/M3URefreshNotification'; +import 'allotment/dist/style.css'; const drawerWidth = 240; const miniDrawerWidth = 60; @@ -84,72 +85,67 @@ const App = () => { withGlobalStyles withNormalizeCSS > - - - - - + + + - - - - - {isAuthenticated ? ( - <> - } /> - } /> - } /> - } - /> - } /> - } /> - } /> - - ) : ( - } /> - )} - - } - /> - - + + + + + {isAuthenticated ? ( + <> + } /> + } /> + } /> + } /> + } /> + } /> + + ) : ( + } /> + )} + + } + /> + - - - - + + + + + + + - - + ); }; diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index bd9f0e5d..72533910 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -4,12 +4,15 @@ import React, { useRef, createContext, useContext, + useMemo, } from 'react'; import useStreamsStore from './store/streams'; import { notifications } from '@mantine/notifications'; import useChannelsStore from './store/channels'; import usePlaylistsStore from './store/playlists'; import useEPGsStore from './store/epgs'; +import { Box, Button, Stack } from '@mantine/core'; +import API from './api'; export const WebsocketContext = createContext(false, null, () => {}); @@ -18,9 +21,12 @@ export const WebsocketProvider = ({ children }) => { const [val, setVal] = useState(null); const { fetchStreams } = useStreamsStore(); - const { setChannelStats, fetchChannelGroups } = useChannelsStore(); - const { fetchPlaylists, setRefreshProgress } = usePlaylistsStore(); - const { fetchEPGData } = useEPGsStore(); + const { fetchChannels, setChannelStats, fetchChannelGroups } = + useChannelsStore(); + const { fetchPlaylists, setRefreshProgress, setProfilePreview } = + usePlaylistsStore(); + const { fetchEPGData, fetchEPGs } = useEPGsStore(); + const { playlists } = usePlaylistsStore(); const ws = useRef(null); @@ -55,29 +61,91 @@ export const WebsocketProvider = ({ children }) => { socket.onmessage = async (event) => { event = JSON.parse(event.data); switch (event.data.type) { + case 'epg_file': + fetchEPGs(); + notifications.show({ + title: 'EPG File Detected', + message: `Processing ${event.data.filename}`, + }); + break; + + case 'm3u_file': + fetchPlaylists(); + notifications.show({ + title: 'M3U File Detected', + message: `Processing ${event.data.filename}`, + }); + break; + + case 'm3u_group_refresh': + fetchChannelGroups(); + fetchPlaylists(); + + notifications.show({ + title: 'Group processing finished!', + autoClose: 5000, + message: ( + + Refresh M3U or filter out groups to pull in streams. + + + ), + color: 'green.5', + }); + break; + case 'm3u_refresh': - console.log('inside m3u_refresh event'); - if (event.data.success) { - fetchStreams(); - notifications.show({ - message: event.data.message, - color: 'green.5', - }); - } else if (event.data.progress) { - if (event.data.progress == 100) { - fetchStreams(); - fetchChannelGroups(); - fetchEPGData(); - fetchPlaylists(); - } - setRefreshProgress(event.data.account, event.data.progress); - } + setRefreshProgress(event.data); break; case 'channel_stats': setChannelStats(JSON.parse(event.data.stats)); break; + case 'epg_channels': + notifications.show({ + message: 'EPG channels updated!', + color: 'green.5', + }); + fetchEPGData(); + break; + + case 'epg_match': + notifications.show({ + message: 'EPG match is complete!', + color: 'green.5', + }); + fetchChannels(); + fetchEPGData(); + break; + + case 'm3u_profile_test': + setProfilePreview(event.data.search_preview, event.data.result); + break; + + case 'recording_started': + notifications.show({ + title: 'Recording started!', + message: `Started recording channel ${event.data.channel}`, + }); + break; + + case 'recording_ended': + notifications.show({ + title: 'Recording finished!', + message: `Stopped recording channel ${event.data.channel}`, + }); + break; + default: console.error(`Unknown websocket event type: ${event.type}`); break; @@ -91,7 +159,9 @@ export const WebsocketProvider = ({ children }) => { }; }, []); - const ret = [isReady, val, ws.current?.send.bind(ws.current)]; + const ret = useMemo(() => { + return [isReady, ws.current?.send.bind(ws.current), val]; + }, [isReady, val]); return ( diff --git a/frontend/src/api.js b/frontend/src/api.js index 369c49c2..92dc84e8 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -7,12 +7,76 @@ import useEPGsStore from './store/epgs'; import useStreamsStore from './store/streams'; import useStreamProfilesStore from './store/streamProfiles'; import useSettingsStore from './store/settings'; +import { notifications } from '@mantine/notifications'; // If needed, you can set a base host or keep it empty if relative requests const host = import.meta.env.DEV ? `http://${window.location.hostname}:5656` : ''; +const errorNotification = (message, error) => { + message = + `${message}: ` + + (error.status ? `${error.status} - ${error.body}` : error.message); + + notifications.show({ + title: 'Error', + message, + autoClose: false, + color: 'red', + }); + + throw error; +}; + +const request = async (url, options = {}) => { + if ( + options.body && + !(options.body instanceof FormData) && + typeof options.body === 'object' + ) { + options.body = JSON.stringify(options.body); + options.headers = { + ...options.headers, + 'Content-Type': 'application/json', + }; + } + + if (options.auth !== false) { + options.headers = { + ...options.headers, + Authorization: `Bearer ${await API.getAuthToken()}`, + }; + } + + const response = await fetch(url, options); + + if (!response.ok) { + const error = new Error(`HTTP error! Status: ${response.status}`); + + let errorBody = await response.text(); + + try { + errorBody = JSON.parse(errorBody); + } catch (e) { + // If parsing fails, leave errorBody as the raw text + } + + error.status = response.status; + error.response = response; + error.body = errorBody; + + throw error; + } + + try { + const retval = await response.json(); + return retval; + } catch (e) { + return ''; + } +}; + export default class API { /** * A static method so we can do: await API.getAuthToken() @@ -22,883 +86,1052 @@ export default class API { } static async fetchSuperUser() { - const response = await fetch(`${host}/api/accounts/initialize-superuser/`); - return await response.json(); + try { + const response = await request( + `${host}/api/accounts/initialize-superuser/`, + { auth: false } + ); + + return response; + } catch (e) { + errorNotification('Failed to fetch superuser', e); + } } static async createSuperUser({ username, email, password }) { - const response = await fetch(`${host}/api/accounts/initialize-superuser/`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - username, - password, - email, - }), - }); + try { + const response = await request( + `${host}/api/accounts/initialize-superuser/`, + { + auth: false, + method: 'POST', + body: { + username, + password, + email, + }, + } + ); - return await response.json(); + return response; + } catch (e) { + errorNotification('Failed to create superuser', e); + } } static async login(username, password) { - const response = await fetch(`${host}/api/accounts/token/`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ username, password }), - }); + try { + const response = await request(`${host}/api/accounts/token/`, { + auth: false, + method: 'POST', + body: { username, password }, + }); - return await response.json(); + return response; + } catch (e) { + errorNotification('Login failed', e); + } } static async refreshToken(refresh) { - const response = await fetch(`${host}/api/accounts/token/refresh/`, { + return await request(`${host}/api/accounts/token/refresh/`, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ refresh }), + body: { auth: false, refresh }, }); - - const retval = await response.json(); - return retval; } static async logout() { - const response = await fetch(`${host}/api/accounts/auth/logout/`, { + return await request(`${host}/api/accounts/auth/logout/`, { + auth: false, method: 'POST', }); - - return response.data.data; } static async getChannels() { - const response = await fetch(`${host}/api/channels/channels/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/channels/channels/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve channels', e); + } + } + + static async queryChannels(params) { + try { + const response = await request( + `${host}/api/channels/channels/?${params.toString()}` + ); + + return response; + } catch (e) { + errorNotification('Failed to fetch channels', e); + } + } + + static async getAllChannelIds(params) { + try { + const response = await request( + `${host}/api/channels/channels/ids/?${params.toString()}` + ); + + return response; + } catch (e) { + errorNotification('Failed to fetch channel IDs', e); + } } static async getChannelGroups() { - const response = await fetch(`${host}/api/channels/groups/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/channels/groups/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve channel groups', e); + } } static async addChannelGroup(values) { - const response = await fetch(`${host}/api/channels/groups/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - }); + try { + const response = await request(`${host}/api/channels/groups/`, { + method: 'POST', + body: values, + }); - const retval = await response.json(); - if (retval.id) { - useChannelsStore.getState().addChannelGroup(retval); + if (response.id) { + useChannelsStore.getState().addChannelGroup(response); + } + + return response; + } catch (e) { + errorNotification('Failed to create channel group', e); } - - return retval; } static async updateChannelGroup(values) { - const { id, ...payload } = values; - const response = await fetch(`${host}/api/channels/groups/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); + try { + const { id, ...payload } = values; + const response = await request(`${host}/api/channels/groups/${id}/`, { + method: 'PUT', + body: payload, + }); - const retval = await response.json(); - if (retval.id) { - useChannelsStore.getState().updateChannelGroup(retval); + if (response.id) { + useChannelsStore.getState().updateChannelGroup(response); + } + + return response; + } catch (e) { + errorNotification('Failed to update channel group', e); } - - return retval; } static async addChannel(channel) { - let body = null; - if (channel.logo_file) { - // Must send FormData for file upload - body = new FormData(); - for (const prop in channel) { - body.append(prop, channel[prop]); + try { + let body = null; + if (channel.logo_file) { + // Must send FormData for file upload + body = new FormData(); + for (const prop in channel) { + body.append(prop, channel[prop]); + } + } else { + body = { ...channel }; + delete body.logo_file; } - } else { - body = { ...channel }; - delete body.logo_file; - body = JSON.stringify(body); + + const response = await request(`${host}/api/channels/channels/`, { + method: 'POST', + body: body, + }); + + if (response.id) { + useChannelsStore.getState().addChannel(response); + } + + return response; + } catch (e) { + errorNotification('Failed to create channel', e); } - - const response = await fetch(`${host}/api/channels/channels/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - ...(channel.logo_file - ? {} - : { - 'Content-Type': 'application/json', - }), - }, - body: body, - }); - - const retval = await response.json(); - if (retval.id) { - useChannelsStore.getState().addChannel(retval); - } - - return retval; } static async deleteChannel(id) { - const response = await fetch(`${host}/api/channels/channels/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/channels/channels/${id}/`, { + method: 'DELETE', + }); - useChannelsStore.getState().removeChannels([id]); + useChannelsStore.getState().removeChannels([id]); + } catch (e) { + errorNotification('Failed to delete channel', e); + } } // @TODO: the bulk delete endpoint is currently broken static async deleteChannels(channel_ids) { - const response = await fetch(`${host}/api/channels/channels/bulk-delete/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ channel_ids }), - }); + try { + await request(`${host}/api/channels/channels/bulk-delete/`, { + method: 'DELETE', + body: { channel_ids }, + }); - useChannelsStore.getState().removeChannels(channel_ids); + useChannelsStore.getState().removeChannels(channel_ids); + } catch (e) { + errorNotification('Failed to delete channels', e); + } } static async updateChannel(values) { - const { id, ...payload } = values; + try { + const { id, ...payload } = values; - let body = null; - if (values.logo_file) { - // Must send FormData for file upload - body = new FormData(); - for (const prop in values) { - body.append(prop, values[prop]); + let body = null; + if (payload.logo_file) { + // Must send FormData for file upload + body = new FormData(); + for (const prop in payload) { + body.append(prop, payload[prop]); + } + } else { + body = { ...payload }; + delete body.logo_file; } - } else { - body = { ...values }; - delete body.logo_file; - body = JSON.stringify(body); + + const response = await request(`${host}/api/channels/channels/${id}/`, { + method: 'PUT', + body, + }); + + if (response.id) { + useChannelsStore.getState().updateChannel(response); + } + + return response; + } catch (e) { + errorNotification('Failed to update channel', e); } - - console.log(body); - - const response = await fetch(`${host}/api/channels/channels/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - ...(values.logo_file - ? {} - : { - 'Content-Type': 'application/json', - }), - }, - body: body, - }); - - const retval = await response.json(); - if (retval.id) { - useChannelsStore.getState().updateChannel(retval); - } - - return retval; } static async assignChannelNumbers(channelIds) { - // Make the request - const response = await fetch(`${host}/api/channels/channels/assign/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ channel_order: channelIds }), - }); + try { + const response = await request(`${host}/api/channels/channels/assign/`, { + method: 'POST', + body: { channel_order: channelIds }, + }); - if (!response.ok) { - const text = await response.text(); - throw new Error(`Assign channels failed: ${response.status} => ${text}`); + // Optionally refesh the channel list in Zustand + await useChannelsStore.getState().fetchChannels(); + + return response; + } catch (e) { + errorNotification('Failed to assign channel #s', e); } - - const retval = await response.json(); - - // Optionally refresh the channel list in Zustand - await useChannelsStore.getState().fetchChannels(); - - return retval; } static async createChannelFromStream(values) { - const response = await fetch(`${host}/api/channels/channels/from-stream/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - }); + try { + const response = await request( + `${host}/api/channels/channels/from-stream/`, + { + method: 'POST', + body: values, + } + ); - const retval = await response.json(); - if (retval.id) { - useChannelsStore.getState().addChannel(retval); + if (response.id) { + useChannelsStore.getState().addChannel(response); + } + + return response; + } catch (e) { + errorNotification('Failed to create channel', e); } - - return retval; } static async createChannelsFromStreams(values) { - const response = await fetch( - `${host}/api/channels/channels/from-stream/bulk/`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), + try { + const response = await request( + `${host}/api/channels/channels/from-stream/bulk/`, + { + method: 'POST', + body: values, + } + ); + + if (response.created.length > 0) { + useChannelsStore.getState().addChannels(response.created); } - ); - const retval = await response.json(); - if (retval.created.length > 0) { - useChannelsStore.getState().addChannels(retval.created); + return response; + } catch (e) { + errorNotification('Failed to create channels', e); } - - return retval; } static async getStreams() { - const response = await fetch(`${host}/api/channels/streams/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/channels/streams/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve streams', e); + } } static async queryStreams(params) { - const response = await fetch( - `${host}/api/channels/streams/?${params.toString()}`, - { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - } - ); + try { + const response = await request( + `${host}/api/channels/streams/?${params.toString()}` + ); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to fetch streams', e); + } } static async getAllStreamIds(params) { - const response = await fetch( - `${host}/api/channels/streams/ids/?${params.toString()}`, - { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - } - ); + try { + const response = await request( + `${host}/api/channels/streams/ids/?${params.toString()}` + ); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to fetch stream IDs', e); + } } static async getStreamGroups() { - const response = await fetch(`${host}/api/channels/streams/groups/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/channels/streams/groups/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve stream groups', e); + } } static async addStream(values) { - const response = await fetch(`${host}/api/channels/streams/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - }); + try { + const response = await request(`${host}/api/channels/streams/`, { + method: 'POST', + body: values, + }); - const retval = await response.json(); - if (retval.id) { - useStreamsStore.getState().addStream(retval); + if (response.id) { + useStreamsStore.getState().addStream(response); + } + + return response; + } catch (e) { + errorNotification('Failed to add stream', e); } - - return retval; } static async updateStream(values) { - const { id, ...payload } = values; - const response = await fetch(`${host}/api/channels/streams/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); + try { + const { id, ...payload } = values; + const response = await request(`${host}/api/channels/streams/${id}/`, { + method: 'PUT', + body: payload, + }); - const retval = await response.json(); - if (retval.id) { - useStreamsStore.getState().updateStream(retval); + if (response.id) { + useStreamsStore.getState().updateStream(response); + } + + return response; + } catch (e) { + errorNotification('Failed to update stream', e); } - - return retval; } static async deleteStream(id) { - const response = await fetch(`${host}/api/channels/streams/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/channels/streams/${id}/`, { + method: 'DELETE', + }); - useStreamsStore.getState().removeStreams([id]); + useStreamsStore.getState().removeStreams([id]); + } catch (e) { + errorNotification('Failed to delete stream', e); + } } static async deleteStreams(ids) { - const response = await fetch(`${host}/api/channels/streams/bulk-delete/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ stream_ids: ids }), - }); + try { + await request(`${host}/api/channels/streams/bulk-delete/`, { + method: 'DELETE', + body: { stream_ids: ids }, + }); - useStreamsStore.getState().removeStreams(ids); + useStreamsStore.getState().removeStreams(ids); + } catch (e) { + errorNotification('Failed to delete streams', e); + } } static async getUserAgents() { - const response = await fetch(`${host}/api/core/useragents/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/core/useragents/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve user-agents', e); + } } static async addUserAgent(values) { - const response = await fetch(`${host}/api/core/useragents/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - }); + try { + const response = await request(`${host}/api/core/useragents/`, { + method: 'POST', + body: values, + }); - const retval = await response.json(); - if (retval.id) { - useUserAgentsStore.getState().addUserAgent(retval); + useUserAgentsStore.getState().addUserAgent(response); + + return response; + } catch (e) { + errorNotification('Failed to create user-agent', e); } - - return retval; } static async updateUserAgent(values) { - const { id, ...payload } = values; - const response = await fetch(`${host}/api/core/useragents/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); + try { + const { id, ...payload } = values; + const response = await request(`${host}/api/core/useragents/${id}/`, { + method: 'PUT', + body: payload, + }); - const retval = await response.json(); - if (retval.id) { - useUserAgentsStore.getState().updateUserAgent(retval); + useUserAgentsStore.getState().updateUserAgent(response); + + return response; + } catch (e) { + errorNotification('Failed to update user-agent', e); } - - return retval; } static async deleteUserAgent(id) { - const response = await fetch(`${host}/api/core/useragents/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/core/useragents/${id}/`, { + method: 'DELETE', + }); - useUserAgentsStore.getState().removeUserAgents([id]); + useUserAgentsStore.getState().removeUserAgents([id]); + } catch (e) { + errorNotification('Failed to delete user-agent', e); + } } static async getPlaylist(id) { - const response = await fetch(`${host}/api/m3u/accounts/${id}/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/m3u/accounts/${id}/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification(`Failed to retrieve M3U account ${id}`, e); + } } static async getPlaylists() { - const response = await fetch(`${host}/api/m3u/accounts/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/m3u/accounts/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve M3U accounts', e); + } } static async addPlaylist(values) { - let body = null; - if (values.uploaded_file) { - body = new FormData(); - for (const prop in values) { - body.append(prop, values[prop]); + try { + let body = null; + if (values.file) { + body = new FormData(); + for (const prop in values) { + body.append(prop, values[prop]); + } + } else { + body = { ...values }; + delete body.file; } - } else { - body = { ...values }; - delete body.uploaded_file; - body = JSON.stringify(body); + + const response = await request(`${host}/api/m3u/accounts/`, { + method: 'POST', + body, + }); + + usePlaylistsStore.getState().addPlaylist(response); + + return response; + } catch (e) { + errorNotification('Failed to create M3U account', e); } - - const response = await fetch(`${host}/api/m3u/accounts/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - ...(values.uploaded_file - ? {} - : { - 'Content-Type': 'application/json', - }), - }, - body, - }); - - const retval = await response.json(); - if (retval.id) { - usePlaylistsStore.getState().addPlaylist(retval); - } - - return retval; } static async refreshPlaylist(id) { - const response = await fetch(`${host}/api/m3u/refresh/${id}/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/m3u/refresh/${id}/`, { + method: 'POST', + }); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to refresh M3U account', e); + } } static async refreshAllPlaylist() { - const response = await fetch(`${host}/api/m3u/refresh/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/m3u/refresh/`, { + method: 'POST', + }); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to refresh all M3U accounts', e); + } } static async deletePlaylist(id) { - const response = await fetch(`${host}/api/m3u/accounts/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/m3u/accounts/${id}/`, { + method: 'DELETE', + }); - usePlaylistsStore.getState().removePlaylists([id]); + usePlaylistsStore.getState().removePlaylists([id]); + // @TODO: MIGHT need to optimize this later if someone has thousands of channels + // but I'm feeling laze right now + useChannelsStore.getState().fetchChannels(); + } catch (e) { + errorNotification(`Failed to delete playlist ${id}`, e); + } } static async updatePlaylist(values) { const { id, ...payload } = values; - const response = await fetch(`${host}/api/m3u/accounts/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); - const retval = await response.json(); - if (retval.id) { - usePlaylistsStore.getState().updatePlaylist(retval); + try { + let body = null; + if (payload.file) { + delete payload.server_url; + + body = new FormData(); + for (const prop in values) { + body.append(prop, values[prop]); + } + } else { + delete payload.file; + if (!payload.server_url) { + delete payload.sever_url; + } + + body = { ...payload }; + delete body.file; + } + + const response = await request(`${host}/api/m3u/accounts/${id}/`, { + method: 'PATCH', + body, + }); + + usePlaylistsStore.getState().updatePlaylist(response); + + return response; + } catch (e) { + errorNotification(`Failed to update M3U account ${id}`, e); } - - return retval; } static async getEPGs() { - const response = await fetch(`${host}/api/epg/sources/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/epg/sources/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve EPGs', e); + } } static async getEPGData() { - const response = await fetch(`${host}/api/epg/epgdata/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/epg/epgdata/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve EPG data', e); + } } // Notice there's a duplicated "refreshPlaylist" method above; // you might want to rename or remove one if it's not needed. static async addEPG(values) { - let body = null; - if (values.epg_file) { - body = new FormData(); - for (const prop in values) { - body.append(prop, values[prop]); + try { + let body = null; + if (values.files) { + body = new FormData(); + for (const prop in values) { + body.append(prop, values[prop]); + } + } else { + body = { ...values }; + delete body.file; } - } else { - body = { ...values }; - delete body.epg_file; - body = JSON.stringify(body); + + const response = await request(`${host}/api/epg/sources/`, { + method: 'POST', + body, + }); + + useEPGsStore.getState().addEPG(response); + + return response; + } catch (e) { + errorNotification('Failed to create EPG', e); } + } - const response = await fetch(`${host}/api/epg/sources/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - ...(values.epg_file - ? {} - : { - 'Content-Type': 'application/json', - }), - }, - body, - }); + static async updateEPG(values) { + const { id, ...payload } = values; - const retval = await response.json(); - if (retval.id) { - useEPGsStore.getState().addEPG(retval); + try { + let body = null; + if (payload.files) { + body = new FormData(); + for (const prop in payload) { + if (prop == 'url') { + continue; + } + body.append(prop, payload[prop]); + } + } else { + delete payload.file; + if (!payload.url) { + delete payload.url; + } + body = payload; + } + + const response = await request(`${host}/api/epg/sources/${id}/`, { + method: 'PATCH', + body, + }); + + useEPGsStore.getState().updateEPG(response); + + return response; + } catch (e) { + errorNotification(`Failed to update EPG ${id}`, e); } - - return retval; } static async deleteEPG(id) { - const response = await fetch(`${host}/api/epg/sources/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/epg/sources/${id}/`, { + method: 'DELETE', + }); - useEPGsStore.getState().removeEPGs([id]); + useEPGsStore.getState().removeEPGs([id]); + } catch (e) { + errorNotification(`Failed to delete EPG ${id}`, e); + } } static async refreshEPG(id) { - const response = await fetch(`${host}/api/epg/import/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ id }), - }); + try { + const response = await request(`${host}/api/epg/import/`, { + method: 'POST', + body: { id }, + }); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification(`Failed to refresh EPG ${id}`, e); + } } static async getStreamProfiles() { - const response = await fetch(`${host}/api/core/streamprofiles/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/core/streamprofiles/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve sream profiles', e); + } } static async addStreamProfile(values) { - const response = await fetch(`${host}/api/core/streamprofiles/`, { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - }); + try { + const response = await request(`${host}/api/core/streamprofiles/`, { + method: 'POST', + body: values, + }); - const retval = await response.json(); - if (retval.id) { - useStreamProfilesStore.getState().addStreamProfile(retval); + useStreamProfilesStore.getState().addStreamProfile(response); + + return response; + } catch (e) { + errorNotification('Failed to create stream profile', e); } - return retval; } static async updateStreamProfile(values) { const { id, ...payload } = values; - const response = await fetch(`${host}/api/core/streamprofiles/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); - const retval = await response.json(); - if (retval.id) { - useStreamProfilesStore.getState().updateStreamProfile(retval); + try { + const response = await request(`${host}/api/core/streamprofiles/${id}/`, { + method: 'PUT', + body: payload, + }); + + useStreamProfilesStore.getState().updateStreamProfile(response); + + return response; + } catch (e) { + errorNotification(`Failed to update stream profile ${id}`, e); } - - return retval; } static async deleteStreamProfile(id) { - const response = await fetch(`${host}/api/core/streamprofiles/${id}/`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + await request(`${host}/api/core/streamprofiles/${id}/`, { + method: 'DELETE', + }); - useStreamProfilesStore.getState().removeStreamProfiles([id]); + useStreamProfilesStore.getState().removeStreamProfiles([id]); + } catch (e) { + errorNotification(`Failed to delete stream propfile ${id}`, e); + } } static async getGrid() { - const response = await fetch(`${host}/api/epg/grid/`, { - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - }); + try { + const response = await request(`${host}/api/epg/grid/`); - const retval = await response.json(); - return retval.data; + return response.data; + } catch (e) { + errorNotification('Failed to retrieve program grid', e); + } } static async addM3UProfile(accountId, values) { - const response = await fetch( - `${host}/api/m3u/accounts/${accountId}/profiles/`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(values), - } - ); + try { + const response = await request( + `${host}/api/m3u/accounts/${accountId}/profiles/`, + { + method: 'POST', + body: values, + } + ); - const retval = await response.json(); - if (retval.id) { // Refresh the playlist const playlist = await API.getPlaylist(accountId); usePlaylistsStore .getState() .updateProfiles(playlist.id, playlist.profiles); - } - return retval; + return response; + } catch (e) { + errorNotification(`Failed to add profile to account ${accountId}`, e); + } } static async deleteM3UProfile(accountId, id) { - const response = await fetch( - `${host}/api/m3u/accounts/${accountId}/profiles/${id}/`, - { + try { + await request(`${host}/api/m3u/accounts/${accountId}/profiles/${id}/`, { method: 'DELETE', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - } - ); + }); - const playlist = await API.getPlaylist(accountId); - usePlaylistsStore.getState().updatePlaylist(playlist); + const playlist = await API.getPlaylist(accountId); + usePlaylistsStore.getState().updatePlaylist(playlist); + } catch (e) { + errorNotification(`Failed to delete profile for account ${accountId}`, e); + } } static async updateM3UProfile(accountId, values) { const { id, ...payload } = values; - const response = await fetch( - `${host}/api/m3u/accounts/${accountId}/profiles/${id}/`, - { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - } - ); - const playlist = await API.getPlaylist(accountId); - usePlaylistsStore.getState().updateProfiles(playlist.id, playlist.profiles); + try { + await request(`${host}/api/m3u/accounts/${accountId}/profiles/${id}/`, { + method: 'PUT', + body: payload, + }); + + const playlist = await API.getPlaylist(accountId); + usePlaylistsStore + .getState() + .updateProfiles(playlist.id, playlist.profiles); + } catch (e) { + errorNotification(`Failed to update profile for account ${accountId}`, e); + } } static async getSettings() { - const response = await fetch(`${host}/api/core/settings/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/core/settings/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve settings', e); + } } static async getEnvironmentSettings() { - const response = await fetch(`${host}/api/core/settings/env/`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/api/core/settings/env/`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve environment settings', e); + } + } + + static async getVersion() { + try { + const response = await request(`${host}/api/core/version/`, { + auth: false, + }); + + return response; + } catch (e) { + errorNotification('Failed to retrieve version', e); + } } static async updateSetting(values) { const { id, ...payload } = values; - const response = await fetch(`${host}/api/core/settings/${id}/`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${await API.getAuthToken()}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); - const retval = await response.json(); - if (retval.id) { - useSettingsStore.getState().updateSetting(retval); + try { + const response = await request(`${host}/api/core/settings/${id}/`, { + method: 'PUT', + body: payload, + }); + + useSettingsStore.getState().updateSetting(response); + + return response; + } catch (e) { + errorNotification('Failed to update settings', e); } - - return retval; } static async getChannelStats(uuid = null) { - const response = await fetch(`${host}/proxy/ts/status`, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/proxy/ts/status`); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to retrieve channel stats', e); + } } static async stopChannel(id) { - const response = await fetch(`${host}/proxy/ts/stop/${id}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request(`${host}/proxy/ts/stop/${id}`, { + method: 'POST', + }); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to stop channel', e); + } } static async stopClient(channelId, clientId) { - const response = await fetch(`${host}/proxy/ts/stop_client/${channelId}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - body: JSON.stringify({ client_id: clientId }), - }); + try { + const response = await request( + `${host}/proxy/ts/stop_client/${channelId}`, + { + method: 'POST', + body: { client_id: clientId }, + } + ); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to stop client', e); + } } static async matchEpg() { - const response = await fetch(`${host}/api/channels/channels/match-epg/`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${await API.getAuthToken()}`, - }, - }); + try { + const response = await request( + `${host}/api/channels/channels/match-epg/`, + { + method: 'POST', + } + ); - const retval = await response.json(); - return retval; + return response; + } catch (e) { + errorNotification('Failed to run EPG auto-match', e); + } + } + + static async getLogos() { + try { + const response = await request(`${host}/api/channels/logos/`); + + return response; + } catch (e) { + errorNotification('Failed to retrieve logos', e); + } + } + + static async uploadLogo(file) { + try { + const formData = new FormData(); + formData.append('file', file); + + const response = await request(`${host}/api/channels/logos/upload/`, { + method: 'POST', + body: formData, + }); + + useChannelsStore.getState().addLogo(response); + + return response; + } catch (e) { + errorNotification('Failed to upload logo', e); + } + } + + static async getChannelProfiles() { + try { + const response = await request(`${host}/api/channels/profiles/`); + + return response; + } catch (e) { + errorNotification('Failed to get channel profiles', e); + } + } + + static async addChannelProfile(values) { + try { + const response = await request(`${host}/api/channels/profiles/`, { + method: 'POST', + body: values, + }); + + useChannelsStore.getState().addProfile(response); + + return response; + } catch (e) { + errorNotification('Failed to create channle profile', e); + } + } + + static async updateChannelProfile(values) { + const { id, ...payload } = values; + + try { + const response = await request(`${host}/api/channels/profiles/${id}/`, { + method: 'PUT', + body: payload, + }); + + useChannelsStore.getState().updateProfile(response); + + return response; + } catch (e) { + errorNotification('Failed to update channel profile', e); + } + } + + static async deleteChannelProfile(id) { + try { + await request(`${host}/api/channels/profiles/${id}/`, { + method: 'DELETE', + }); + + useChannelsStore.getState().removeProfiles([id]); + } catch (e) { + errorNotification(`Failed to delete channel profile ${id}`, e); + } + } + + static async updateProfileChannel(channelId, profileId, enabled) { + try { + await request( + `${host}/api/channels/profiles/${profileId}/channels/${channelId}/`, + { + method: 'PATCH', + body: { enabled }, + } + ); + + useChannelsStore + .getState() + .updateProfileChannels([channelId], profileId, enabled); + } catch (e) { + errorNotification(`Failed to update channel for profile ${profileId}`, e); + } + } + + static async updateProfileChannels(channelIds, profileId, enabled) { + try { + await request( + `${host}/api/channels/profiles/${profileId}/channels/bulk-update/`, + { + method: 'PATCH', + body: { + channels: channelIds.map((id) => ({ + channel_id: id, + enabled, + })), + }, + } + ); + + useChannelsStore + .getState() + .updateProfileChannels(channelIds, profileId, enabled); + } catch (e) { + errorNotification( + `Failed to bulk update channels for profile ${profileId}`, + e + ); + } + } + + static async getRecordings() { + try { + const response = await request(`${host}/api/channels/recordings/`); + + return response; + } catch (e) { + errorNotification('Failed to retrieve recordings', e); + } + } + + static async createRecording(values) { + try { + const response = await request(`${host}/api/channels/recordings/`, { + method: 'POST', + body: values, + }); + + useChannelsStore.getState().fetchRecordings(); + + return response; + } catch (e) { + errorNotification('Failed to create recording', e); + } + } + + static async deleteRecording(id) { + try { + await request(`${host}/api/channels/recordings/${id}/`, { + method: 'DELETE', + }); + + useChannelsStore.getState().fetchRecordings(); + } catch (e) { + errorNotification(`Failed to delete recording ${id}`, e); + } } } diff --git a/frontend/src/assets/android-chrome-192x192.png b/frontend/src/assets/android-chrome-192x192.png new file mode 100644 index 00000000..59248862 Binary files /dev/null and b/frontend/src/assets/android-chrome-192x192.png differ diff --git a/frontend/src/assets/android-chrome-512x512.png b/frontend/src/assets/android-chrome-512x512.png new file mode 100644 index 00000000..cd17f001 Binary files /dev/null and b/frontend/src/assets/android-chrome-512x512.png differ diff --git a/frontend/src/assets/apple-touch-icon.png b/frontend/src/assets/apple-touch-icon.png new file mode 100644 index 00000000..7185f2ee Binary files /dev/null and b/frontend/src/assets/apple-touch-icon.png differ diff --git a/frontend/src/assets/favicon-16x16.png b/frontend/src/assets/favicon-16x16.png new file mode 100644 index 00000000..27c13e08 Binary files /dev/null and b/frontend/src/assets/favicon-16x16.png differ diff --git a/frontend/src/assets/favicon-32x32.png b/frontend/src/assets/favicon-32x32.png new file mode 100644 index 00000000..187a9e71 Binary files /dev/null and b/frontend/src/assets/favicon-32x32.png differ diff --git a/frontend/src/assets/favicon.ico b/frontend/src/assets/favicon.ico new file mode 100644 index 00000000..9a9a94da Binary files /dev/null and b/frontend/src/assets/favicon.ico differ diff --git a/frontend/src/assets/logo.png b/frontend/src/assets/logo.png new file mode 100644 index 00000000..99c3c19f Binary files /dev/null and b/frontend/src/assets/logo.png differ diff --git a/frontend/src/assets/site.webmanifest b/frontend/src/assets/site.webmanifest new file mode 100644 index 00000000..45dc8a20 --- /dev/null +++ b/frontend/src/assets/site.webmanifest @@ -0,0 +1 @@ +{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} \ No newline at end of file diff --git a/frontend/src/assets/vite.svg b/frontend/src/assets/vite.svg new file mode 100644 index 00000000..e7b8dfb1 --- /dev/null +++ b/frontend/src/assets/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/components/M3URefreshNotification.jsx b/frontend/src/components/M3URefreshNotification.jsx index b023b866..90123dc4 100644 --- a/frontend/src/components/M3URefreshNotification.jsx +++ b/frontend/src/components/M3URefreshNotification.jsx @@ -1,79 +1,85 @@ // frontend/src/components/FloatingVideo.js -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import usePlaylistsStore from '../store/playlists'; import { notifications } from '@mantine/notifications'; import { IconCheck } from '@tabler/icons-react'; +import useStreamsStore from '../store/streams'; +import useChannelsStore from '../store/channels'; +import useEPGsStore from '../store/epgs'; export default function M3URefreshNotification() { - const { playlists, refreshProgress, removeRefreshProgress } = - usePlaylistsStore(); - const [progress, setProgress] = useState({}); + const { playlists, refreshProgress } = usePlaylistsStore(); + const { fetchStreams } = useStreamsStore(); + const { fetchChannelGroups } = useChannelsStore(); + const { fetchPlaylists } = usePlaylistsStore(); + const { fetchEPGData } = useEPGsStore(); - const clearAccountNotification = (id) => { - removeRefreshProgress(id); - setProgress({ - ...progress, - [id]: null, + const [notificationStatus, setNotificationStatus] = useState({}); + + const handleM3UUpdate = (data) => { + if ( + JSON.stringify(notificationStatus[data.account]) == JSON.stringify(data) + ) { + return; + } + + console.log(data); + const playlist = playlists.find((pl) => pl.id == data.account); + + setNotificationStatus({ + ...notificationStatus, + [data.account]: data, + }); + + const taskProgress = data.progress; + + if (data.progress != 0 && data.progress != 100) { + console.log('not 0 or 100'); + return; + } + + let message = ''; + switch (data.action) { + case 'downloading': + message = 'Downloading'; + break; + + case 'parsing': + message = 'Stream parsing'; + break; + + case 'processing_groups': + message = 'Group parsing'; + break; + } + + if (taskProgress == 0) { + message = `${message} starting...`; + } else if (taskProgress == 100) { + message = `${message} complete!`; + + if (data.action == 'parsing') { + fetchStreams(); + } else if (data.action == 'processing_groups') { + fetchStreams(); + fetchChannelGroups(); + fetchEPGData(); + fetchPlaylists(); + } + } + + notifications.show({ + title: `M3U Processing: ${playlist.name}`, + message, + loading: taskProgress == 0, + autoClose: 2000, + icon: taskProgress == 100 ? : null, }); }; - for (const id in refreshProgress) { - const playlist = playlists.find((pl) => pl.id == id); - if (!progress[id]) { - if (refreshProgress[id] == 100) { - // This situation is if it refreshes so fast we only get the 100% complete notification - const notificationId = notifications.show({ - loading: false, - title: `M3U Refresh: ${playlist.name}`, - message: `Refresh complete!`, - icon: , - }); - setProgress({ - ...progress, - [id]: notificationId, - }); - setTimeout(() => clearAccountNotification(id), 2000); - - return; - } - - console.log('starting progress bar'); - const notificationId = notifications.show({ - loading: true, - title: `M3U Refresh: ${playlist.name}`, - message: `Starting...`, - autoClose: false, - withCloseButton: false, - }); - - setProgress({ - ...progress, - [id]: notificationId, - }); - } else { - if (refreshProgress[id] == 0) { - notifications.update({ - id: progress[id], - message: `Starting...`, - }); - } else if (refreshProgress[id] == 100) { - notifications.update({ - id: progress[id], - message: `Refresh complete!`, - loading: false, - autoClose: 2000, - icon: , - }); - - setTimeout(() => clearAccountNotification(id), 2000); - } else { - notifications.update({ - id: progress[id], - message: `Updating M3U: ${refreshProgress[id]}%`, - }); - } - } - } + useEffect(() => { + Object.values(refreshProgress).map((data) => handleM3UUpdate(data)); + }, [playlists, refreshProgress]); return <>; } diff --git a/frontend/src/components/Sidebar.jsx b/frontend/src/components/Sidebar.jsx index 157381f0..1b1d348d 100644 --- a/frontend/src/components/Sidebar.jsx +++ b/frontend/src/components/Sidebar.jsx @@ -1,4 +1,4 @@ -import React, { useRef } from 'react'; +import React, { useRef, useEffect, useState } from 'react'; import { Link, useLocation } from 'react-router-dom'; import { ListOrdered, @@ -9,6 +9,7 @@ import { Settings as LucideSettings, Copy, ChartLine, + Video, } from 'lucide-react'; import { Avatar, @@ -25,6 +26,8 @@ import logo from '../images/logo.png'; import useChannelsStore from '../store/channels'; import './sidebar.css'; import useSettingsStore from '../store/settings'; +import useAuthStore from '../store/auth'; // Add this import +import API from '../api'; const NavLink = ({ item, isActive, collapsed }) => { return ( @@ -62,8 +65,40 @@ const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { const location = useLocation(); const { channels } = useChannelsStore(); const { environment } = useSettingsStore(); + const { isAuthenticated } = useAuthStore(); // Add this line to get authentication state const publicIPRef = useRef(null); + const [appVersion, setAppVersion] = useState({ version: '', build: '' }); + // Fetch environment settings including version on component mount + useEffect(() => { + if (!isAuthenticated) { + return; + } + + const fetchEnvironment = async () => { + API.getEnvironmentSettings(); + }; + + fetchEnvironment(); + }, [isAuthenticated]); + + // Fetch version information on component mount (regardless of authentication) + useEffect(() => { + const fetchVersion = async () => { + try { + const versionData = await API.getVersion(); + setAppVersion({ + version: versionData.version || '', + build: versionData.build || '', + }); + } catch (error) { + console.error('Failed to fetch version information:', error); + // Keep using default values from useState initialization + } + }; + + fetchVersion(); + }, []); // Navigation Items const navItems = [ { @@ -72,14 +107,9 @@ const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { path: '/channels', badge: `(${Object.keys(channels).length})`, }, - { label: 'M3U', icon: , path: '/m3u' }, - { label: 'EPG', icon: , path: '/epg' }, - { - label: 'Stream Profiles', - icon: , - path: '/stream-profiles', - }, + { label: 'M3U & EPG Manager', icon: , path: '/sources' }, { label: 'TV Guide', icon: , path: '/guide' }, + { label: 'DVR', icon: