Mirrors/Dispatcharr
1
0
Fork 1
mirror of https://github.com/Dispatcharr/Dispatcharr.git synced 2026-07-25 19:14:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #49 from Dispatcharr/dev

Browse source 
Beta
This commit is contained in:
dekzter 2025-04-12 17:37:43 -04:00 • committed by GitHub
commit ebb016858e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
152 changed files with 9626 additions and 3416 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -1,5 +1,6 @@
**/__pycache__
**/.venv
**/venv
**/.classpath
**/.dockerignore
**/.env

116
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,116 @@
name: CI Pipeline
on:
push:
branches: [ dev ]
pull_request:
branches: [ dev ]
# Add explicit permissions for the workflow
permissions:
contents: write # For managing releases and pushing tags
packages: write # For publishing to GitHub Container Registry
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Configure Git
run: |
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
- name: Check if commit is from GitHub Actions
id: check_actor
run: |
if [[ "${{ github.actor }}" == "github-actions" ]]; then
echo "is_bot=true" >> $GITHUB_OUTPUT
else
echo "is_bot=false" >> $GITHUB_OUTPUT
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Increment Build Number
if: steps.check_actor.outputs.is_bot != 'true'
id: increment_build
run: |
python scripts/increment_build.py
BUILD=$(python -c "import version; print(version.__build__)")
echo "build=${BUILD}" >> $GITHUB_OUTPUT
- name: Commit Build Number Update
if: steps.check_actor.outputs.is_bot != 'true'
run: |
git add version.py
git commit -m "Increment build number to ${{ steps.increment_build.outputs.build }} [skip ci]"
git push
- name: Extract version info
id: version
run: |
VERSION=$(python -c "import version; print(version.__version__)")
BUILD=$(python -c "import version; print(version.__build__)")
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "build=${BUILD}" >> $GITHUB_OUTPUT
echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
- name: Set repository and image metadata
id: meta
run: |
# Get lowercase repository owner
REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
echo "repo_owner=${REPO_OWNER}" >> $GITHUB_OUTPUT
# Get repository name
REPO_NAME=$(echo "${{ github.repository }}" | cut -d '/' -f 2 | tr '[:upper:]' '[:lower:]')
echo "repo_name=${REPO_NAME}" >> $GITHUB_OUTPUT
# Determine branch name
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
echo "branch_tag=latest" >> $GITHUB_OUTPUT
echo "is_main=true" >> $GITHUB_OUTPUT
elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
echo "branch_tag=dev" >> $GITHUB_OUTPUT
echo "is_main=false" >> $GITHUB_OUTPUT
else
# For other branches, use the branch name
BRANCH=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///' | sed 's/[^a-zA-Z0-9]/-/g')
echo "branch_tag=${BRANCH}" >> $GITHUB_OUTPUT
echo "is_main=false" >> $GITHUB_OUTPUT
fi
# Determine if this is from a fork
if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then
echo "is_fork=true" >> $GITHUB_OUTPUT
else
echo "is_fork=false" >> $GITHUB_OUTPUT
fi
- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
platforms: linux/amd64 # Fast build - amd64 only
tags: |
ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.meta.outputs.branch_tag }}
ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.version.outputs.version }}-${{ steps.version.outputs.build }}
ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.version.outputs.sha_short }}
build-args: |
BRANCH=${{ github.ref_name }}
REPO_URL=https://github.com/${{ github.repository }}
file: ./docker/Dockerfile

93
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,93 @@
name: Create Release
on:
workflow_dispatch:
inputs:
version_type:
description: 'Type of version increment'
required: true
default: 'patch'
type: choice
options:
- major
- minor
- patch
# Add explicit permissions for the workflow
permissions:
contents: write # For managing releases and pushing tags
packages: write # For publishing to GitHub Container Registry
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
- name: Update Version
id: update_version
run: |
python scripts/bump_version.py ${{ github.event.inputs.version_type }}
NEW_VERSION=$(python -c "import version; print(f'{version.__version__}')")
echo "new_version=${NEW_VERSION}" >> $GITHUB_OUTPUT
- name: Set lowercase repo owner
id: repo_owner
run: |
REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
echo "lowercase=${REPO_OWNER}" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Commit and Tag
run: |
git add version.py
git commit -m "Release v${{ steps.update_version.outputs.new_version }}"
git tag -a "v${{ steps.update_version.outputs.new_version }}" -m "Release v${{ steps.update_version.outputs.new_version }}"
git push origin main --tags
- name: Build and Push Release Image
uses: docker/build-push-action@v4
with:
context: .
push: true
platforms: linux/amd64,linux/arm64, #linux/arm/v7 # Multi-arch support for releases
tags: |
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest-amd64
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:latest-arm64
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-amd64
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-arm64
ghcr.io/${{ steps.repo_owner.outputs.lowercase }}/dispatcharr:${{ steps.update_version.outputs.new_version }}-armv7
build-args: |
BRANCH=${{ github.ref_name }}
REPO_URL=https://github.com/${{ github.repository }}
file: ./docker/Dockerfile
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
tag_name: v${{ steps.update_version.outputs.new_version }}
name: Release v${{ steps.update_version.outputs.new_version }}
draft: false
prerelease: false
token: ${{ secrets.GITHUB_TOKEN }}

10
.gitignore vendored
View file

@ -1,6 +1,7 @@
.DS_Store
**/__pycache__/
**/.vscode/
**/venv
*.pyc
node_modules/
.history/
@ -10,4 +11,11 @@ docker/Dockerfile DEV
static/
data/
.next
next-env.d.ts
next-env.d.ts
media/
celerybeat-schedule*
dump.rdb
debugpy*
uwsgi.sock
package-lock.json
models

4
apps/channels/admin.py
View file

@ -26,10 +26,10 @@ class ChannelAdmin(admin.ModelAdmin):
'uuid',
'name',
'channel_group',
'tvg_name'
'epg_data'
)
list_filter = ('channel_group',)
search_fields = ('id', 'name', 'channel_group__name', 'tvg_name') # Added 'id'
search_fields = ('id', 'name', 'channel_group__name', 'epg_data') # Added 'id'
ordering = ('channel_number',)
@admin.register(ChannelGroup)

10
apps/channels/api_urls.py
View file

@ -6,6 +6,11 @@ from .api_views import (
ChannelGroupViewSet,
BulkDeleteStreamsAPIView,
BulkDeleteChannelsAPIView,
LogoViewSet,
ChannelProfileViewSet,
UpdateChannelMembershipAPIView,
BulkUpdateChannelMembershipAPIView,
RecordingViewSet,
)
app_name = 'channels' # for DRF routing
@ -14,11 +19,16 @@ router = DefaultRouter()
router.register(r'streams', StreamViewSet, basename='stream')
router.register(r'groups', ChannelGroupViewSet, basename='channel-group')
router.register(r'channels', ChannelViewSet, basename='channel')
router.register(r'logos', LogoViewSet, basename='logo')
router.register(r'profiles', ChannelProfileViewSet, basename='profile')
router.register(r'recordings', RecordingViewSet, basename='recording')
urlpatterns = [
# Bulk delete is a single APIView, not a ViewSet
path('streams/bulk-delete/', BulkDeleteStreamsAPIView.as_view(), name='bulk_delete_streams'),
path('channels/bulk-delete/', BulkDeleteChannelsAPIView.as_view(), name='bulk_delete_channels'),
path('profiles/<int:profile_id>/channels/<int:channel_id>/', UpdateChannelMembershipAPIView.as_view(), name='update_channel_membership'),
path('profiles/<int:profile_id>/channels/bulk-update/', BulkUpdateChannelMembershipAPIView.as_view(), name='bulk_update_channel_membership'),
]
urlpatterns += router.urls

353
apps/channels/api_views.py
View file

@ -1,21 +1,42 @@
from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.decorators import action
from rest_framework.parsers import MultiPartParser, FormParser
from drf_yasg.utils import swagger_auto_schema
from drf_yasg import openapi
from django.shortcuts import get_object_or_404
from django.db import transaction
import os, json, requests
from .models import Stream, Channel, ChannelGroup
from .serializers import StreamSerializer, ChannelSerializer, ChannelGroupSerializer
from .models import Stream, Channel, ChannelGroup, Logo, ChannelProfile, ChannelProfileMembership, Recording
from .serializers import StreamSerializer, ChannelSerializer, ChannelGroupSerializer, LogoSerializer, ChannelProfileMembershipSerializer, BulkChannelProfileMembershipSerializer, ChannelProfileSerializer, RecordingSerializer
from .tasks import match_epg_channels
import django_filters
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.filters import SearchFilter, OrderingFilter
from apps.epg.models import EPGData
from django.db.models import Q
from django.http import StreamingHttpResponse, FileResponse, Http404
from rest_framework.pagination import PageNumberPagination
class OrInFilter(django_filters.Filter):
"""
Custom filter that handles the OR condition instead of AND.
"""
def filter(self, queryset, value):
if value:
# Create a Q object for each value and combine them with OR
query = Q()
for val in value.split(','):
query |= Q(**{self.field_name: val})
return queryset.filter(query)
return queryset
class StreamPagination(PageNumberPagination):
page_size = 25 # Default page size
page_size_query_param = 'page_size' # Allow clients to specify page size
@ -23,7 +44,7 @@ class StreamPagination(PageNumberPagination):
class StreamFilter(django_filters.FilterSet):
name = django_filters.CharFilter(lookup_expr='icontains')
channel_group_name = django_filters.CharFilter(field_name="channel_group__name", lookup_expr="icontains")
channel_group_name = OrInFilter(field_name="channel_group__name", lookup_expr="icontains")
m3u_account = django_filters.NumberFilter(field_name="m3u_account__id")
m3u_account_name = django_filters.CharFilter(field_name="m3u_account__name", lookup_expr="icontains")
m3u_account_is_active = django_filters.BooleanFilter(field_name="m3u_account__is_active")
@ -62,7 +83,8 @@ class StreamViewSet(viewsets.ModelViewSet):
channel_group = self.request.query_params.get('channel_group')
if channel_group:
qs = qs.filter(channel_group__name=channel_group)
group_names = channel_group.split(',')
qs = qs.filter(channel_group__name__in=group_names)
return qs
@ -100,17 +122,54 @@ class ChannelGroupViewSet(viewsets.ModelViewSet):
# ─────────────────────────────────────────────────────────
# 3) Channel Management (CRUD)
# ─────────────────────────────────────────────────────────
class ChannelPagination(PageNumberPagination):
page_size = 25 # Default page size
page_size_query_param = 'page_size' # Allow clients to specify page size
max_page_size = 10000 # Prevent excessive page sizes
class ChannelFilter(django_filters.FilterSet):
name = django_filters.CharFilter(lookup_expr='icontains')
channel_group_name = OrInFilter(field_name="channel_group__name", lookup_expr="icontains")
class Meta:
model = Channel
fields = ['name', 'channel_group_name',]
class ChannelViewSet(viewsets.ModelViewSet):
queryset = Channel.objects.all()
serializer_class = ChannelSerializer
permission_classes = [IsAuthenticated]
# pagination_class = ChannelPagination
def get_next_available_channel_number(self, starting_from=1):
used_numbers = set(Channel.objects.all().values_list('channel_number', flat=True))
n = starting_from
while n in used_numbers:
n += 1
return n
# filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter]
# filterset_class = ChannelFilter
# search_fields = ['name', 'channel_group__name']
# ordering_fields = ['channel_number', 'name', 'channel_group__name']
# ordering = ['-channel_number']
def get_queryset(self):
qs = super().get_queryset()
channel_group = self.request.query_params.get('channel_group')
if channel_group:
group_names = channel_group.split(',')
qs = qs.filter(channel_group__name__in=group_names)
return qs
@action(detail=False, methods=['get'], url_path='ids')
def get_ids(self, request, *args, **kwargs):
# Get the filtered queryset
queryset = self.get_queryset()
# Apply filtering, search, and ordering
queryset = self.filter_queryset(queryset)
# Return only the IDs from the queryset
channel_ids = queryset.values_list('id', flat=True)
# Return the response with the list of IDs
return Response(list(channel_ids))
@swagger_auto_schema(
method='post',
@ -130,9 +189,11 @@ class ChannelViewSet(viewsets.ModelViewSet):
)
@action(detail=False, methods=['post'], url_path='assign')
def assign(self, request):
channel_order = request.data.get('channel_order', [])
for order, channel_id in enumerate(channel_order, start=1):
Channel.objects.filter(id=channel_id).update(channel_number=order)
with transaction.atomic():
channel_order = request.data.get('channel_order', [])
for order, channel_id in enumerate(channel_order, start=1):
Channel.objects.filter(id=channel_id).update(channel_number=order)
return Response({"message": "Channels have been auto-assigned!"}, status=status.HTTP_200_OK)
@swagger_auto_schema(
@ -168,34 +229,56 @@ class ChannelViewSet(viewsets.ModelViewSet):
stream = get_object_or_404(Stream, pk=stream_id)
channel_group = stream.channel_group
# Check if client provided a channel_number; if not, auto-assign one.
provided_number = request.data.get('channel_number')
if provided_number is None:
channel_number = self.get_next_available_channel_number()
else:
try:
channel_number = int(provided_number)
except ValueError:
return Response({"error": "channel_number must be an integer."}, status=status.HTTP_400_BAD_REQUEST)
# If the provided number is already used, return an error.
if Channel.objects.filter(channel_number=channel_number).exists():
return Response(
{"error": f"Channel number {channel_number} is already in use. Please choose a different number."},
status=status.HTTP_400_BAD_REQUEST
)
name = request.data.get('name')
if name is None:
name = stream.name
# Check if client provided a channel_number; if not, auto-assign one.
stream_custom_props = json.loads(stream.custom_properties) if stream.custom_properties else {}
channel_number = None
if 'tv-chno' in stream_custom_props:
channel_number = int(stream_custom_props['tv-chno'])
elif 'channel-number' in stream_custom_props:
channel_number = int(stream_custom_props['channel-number'])
if channel_number is None:
provided_number = request.data.get('channel_number')
if provided_number is None:
channel_number = Channel.get_next_available_channel_number()
else:
try:
channel_number = int(provided_number)
except ValueError:
return Response({"error": "channel_number must be an integer."}, status=status.HTTP_400_BAD_REQUEST)
# If the provided number is already used, return an error.
if Channel.objects.filter(channel_number=channel_number).exists():
return Response(
{"error": f"Channel number {channel_number} is already in use. Please choose a different number."},
status=status.HTTP_400_BAD_REQUEST
)
channel_data = {
'channel_number': channel_number,
'name': name,
'tvg_id': stream.tvg_id,
'channel_group_id': channel_group.id,
'logo_url': stream.logo_url,
'streams': [stream_id]
'streams': [stream_id],
}
if stream.logo_url:
logo, _ = Logo.objects.get_or_create(url=stream.logo_url, defaults={
"name": stream.name or stream.tvg_id
})
channel_data["logo_id"] = logo.id
# Attempt to find existing EPGs with the same tvg-id
epgs = EPGData.objects.filter(tvg_id=stream.tvg_id)
if epgs:
channel_data["epg_data_id"] = epgs.first().id
serializer = self.get_serializer(data=channel_data)
serializer.is_valid(raise_exception=True)
channel = serializer.save()
@ -250,6 +333,10 @@ class ChannelViewSet(viewsets.ModelViewSet):
used_numbers.add(next_number)
return next_number
logos_to_create = []
channels_to_create = []
streams_map = []
logo_map = []
for item in data_list:
stream_id = item.get('stream_id')
if not all([stream_id]):
@ -262,44 +349,98 @@ class ChannelViewSet(viewsets.ModelViewSet):
errors.append({"item": item, "error": str(e)})
continue
channel_group = stream.channel_group
# Determine channel number: if provided, use it (if free); else auto assign.
provided_number = item.get('channel_number')
if provided_number is None:
channel_number = get_auto_number()
else:
try:
channel_number = int(provided_number)
except ValueError:
errors.append({"item": item, "error": "channel_number must be an integer."})
continue
if channel_number in used_numbers or Channel.objects.filter(channel_number=channel_number).exists():
errors.append({"item": item, "error": f"Channel number {channel_number} is already in use."})
continue
used_numbers.add(channel_number)
name = item.get('name')
if name is None:
name = stream.name
channel_group = stream.channel_group
stream_custom_props = json.loads(stream.custom_properties) if stream.custom_properties else {}
channel_number = None
if 'tv-chno' in stream_custom_props:
channel_number = int(stream_custom_props['tv-chno'])
elif 'channel-number' in stream_custom_props:
channel_number = int(stream_custom_props['channel-number'])
# Determine channel number: if provided, use it (if free); else auto assign.
if channel_number is None:
provided_number = item.get('channel_number')
if provided_number is None:
channel_number = get_auto_number()
else:
try:
channel_number = int(provided_number)
except ValueError:
errors.append({"item": item, "error": "channel_number must be an integer."})
continue
if channel_number in used_numbers or Channel.objects.filter(channel_number=channel_number).exists():
errors.append({"item": item, "error": f"Channel number {channel_number} is already in use."})
continue
used_numbers.add(channel_number)
channel_data = {
"channel_number": channel_number,
"name": name,
"tvg_id": stream.tvg_id,
"channel_group_id": channel_group.id,
"logo_url": stream.logo_url,
"streams": [stream_id],
}
# Attempt to find existing EPGs with the same tvg-id
epgs = EPGData.objects.filter(tvg_id=stream.tvg_id)
if epgs:
channel_data["epg_data_id"] = epgs.first().id
serializer = self.get_serializer(data=channel_data)
if serializer.is_valid():
channel = serializer.save()
channel.streams.add(stream)
created_channels.append(serializer.data)
validated_data = serializer.validated_data
channel = Channel(**validated_data)
channels_to_create.append(channel)
streams_map.append([stream_id])
if stream.logo_url:
logos_to_create.append(Logo(
url=stream.logo_url,
name=stream.name or stream.tvg_id,
))
logo_map.append(stream.logo_url)
else:
logo_map.append(None)
# channel = serializer.save()
# channel.streams.add(stream)
# created_channels.append(serializer.data)
else:
errors.append({"item": item, "error": serializer.errors})
response_data = {"created": created_channels}
if logos_to_create:
Logo.objects.bulk_create(logos_to_create, ignore_conflicts=True)
channel_logos = {logo.url: logo for logo in Logo.objects.filter(url__in=[url for url in logo_map if url is not None])}
profiles = ChannelProfile.objects.all()
channel_profile_memberships = []
if channels_to_create:
with transaction.atomic():
created_channels = Channel.objects.bulk_create(channels_to_create)
update = []
for channel, stream_ids, logo_url in zip(created_channels, streams_map, logo_map):
if logo_url:
channel.logo = channel_logos[logo_url]
update.append(channel)
channel_profile_memberships = channel_profile_memberships + [
ChannelProfileMembership(channel_profile=profile, channel=channel)
for profile in profiles
]
ChannelProfileMembership.objects.bulk_create(channel_profile_memberships)
Channel.objects.bulk_update(update, ['logo'])
for channel, stream_ids in zip(created_channels, streams_map):
channel.streams.set(stream_ids)
response_data = {"created": ChannelSerializer(created_channels, many=True).data}
if errors:
response_data["errors"] = errors
@ -371,3 +512,105 @@ class BulkDeleteChannelsAPIView(APIView):
channel_ids = request.data.get('channel_ids', [])
Channel.objects.filter(id__in=channel_ids).delete()
return Response({"message": "Channels deleted"}, status=status.HTTP_204_NO_CONTENT)
class LogoViewSet(viewsets.ModelViewSet):
permission_classes = [IsAuthenticated]
queryset = Logo.objects.all()
serializer_class = LogoSerializer
parser_classes = (MultiPartParser, FormParser)
@action(detail=False, methods=['post'])
def upload(self, request):
if 'file' not in request.FILES:
return Response({'error': 'No file uploaded'}, status=status.HTTP_400_BAD_REQUEST)
file = request.FILES['file']
file_name = file.name
file_path = os.path.join('/data/logos', file_name)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
logo, _ = Logo.objects.get_or_create(url=file_path, defaults={
"name": file_name,
})
return Response({'id': logo.id, 'name': logo.name, 'url': logo.url}, status=status.HTTP_201_CREATED)
@action(detail=True, methods=['get'], permission_classes=[AllowAny])
def cache(self, request, pk=None):
"""Streams the logo file, whether it's local or remote."""
logo = self.get_object()
logo_url = logo.url
if logo_url.startswith("/data"): # Local file
if not os.path.exists(logo_url):
raise Http404("Image not found")
return FileResponse(open(logo_url, "rb"), content_type="image/*")
else: # Remote image
try:
remote_response = requests.get(logo_url, stream=True)
if remote_response.status_code == 200:
return StreamingHttpResponse(remote_response.iter_content(chunk_size=8192), content_type="image/*")
raise Http404("Remote image not found")
except requests.RequestException:
raise Http404("Error fetching remote image")
class ChannelProfileViewSet(viewsets.ModelViewSet):
queryset = ChannelProfile.objects.all()
serializer_class = ChannelProfileSerializer
permission_classes = [IsAuthenticated]
class UpdateChannelMembershipAPIView(APIView):
def patch(self, request, profile_id, channel_id):
"""Enable or disable a channel for a specific group"""
channel_profile = get_object_or_404(ChannelProfile, id=profile_id)
channel = get_object_or_404(Channel, id=channel_id)
membership = get_object_or_404(ChannelProfileMembership, channel_profile=channel_profile, channel=channel)
serializer = ChannelProfileMembershipSerializer(membership, data=request.data, partial=True)
if serializer.is_valid():
serializer.save()
return Response(serializer.data, status=status.HTTP_200_OK)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
class BulkUpdateChannelMembershipAPIView(APIView):
def patch(self, request, profile_id):
"""Bulk enable or disable channels for a specific profile"""
# Get the channel profile
channel_profile = get_object_or_404(ChannelProfile, id=profile_id)
# Validate the incoming data using the serializer
serializer = BulkChannelProfileMembershipSerializer(data=request.data)
if serializer.is_valid():
updates = serializer.validated_data['channels']
channel_ids = [entry['channel_id'] for entry in updates]
memberships = ChannelProfileMembership.objects.filter(
channel_profile=channel_profile,
channel_id__in=channel_ids
)
membership_dict = {m.channel.id: m for m in memberships}
for entry in updates:
channel_id = entry['channel_id']
enabled_status = entry['enabled']
if channel_id in membership_dict:
membership_dict[channel_id].enabled = enabled_status
ChannelProfileMembership.objects.bulk_update(memberships, ['enabled'])
return Response({"status": "success"}, status=status.HTTP_200_OK)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
class RecordingViewSet(viewsets.ModelViewSet):
queryset = Recording.objects.all()
serializer_class = RecordingSerializer
permission_classes = [IsAuthenticated]

2
apps/channels/forms.py
View file

@ -40,7 +40,7 @@ class StreamForm(forms.ModelForm):
'name',
'url',
'logo_url',
'tvg_id',
'epg_data',
'local_file',
'channel_group',
]

24
apps/channels/migrations/0009_remove_channel_tvg_name_channel_epg_data.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-03-26 12:59
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0008_stream_stream_hash'),
('epg', '0004_epgdata_epg_source_alter_epgdata_tvg_id'),
]
operations = [
migrations.RemoveField(
model_name='channel',
name='tvg_name',
),
migrations.AddField(
model_name='channel',
name='epg_data',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='channels', to='epg.epgdata'),
),
]

18
apps/channels/migrations/0010_stream_custom_properties.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.1.6 on 2025-04-01 17:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0009_remove_channel_tvg_name_channel_epg_data'),
]
operations = [
migrations.AddField(
model_name='stream',
name='custom_properties',
field=models.TextField(blank=True, null=True),
),
]

35
apps/channels/migrations/0011_logo_remove_channel_logo_file_and_more.py Normal file
View file

@ -0,0 +1,35 @@
# Generated by Django 5.1.6 on 2025-04-01 22:14
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0010_stream_custom_properties'),
]
operations = [
migrations.CreateModel(
name='Logo',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=255)),
('url', models.URLField(unique=True)),
],
),
migrations.RemoveField(
model_name='channel',
name='logo_file',
),
migrations.RemoveField(
model_name='channel',
name='logo_url',
),
migrations.AddField(
model_name='channel',
name='logo',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='channels', to='dispatcharr_channels.logo'),
),
]

33
apps/channels/migrations/0012_channelprofile_channelprofilemembership.py Normal file
View file

@ -0,0 +1,33 @@
# Generated by Django 5.1.6 on 2025-04-02 23:27
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0011_logo_remove_channel_logo_file_and_more'),
]
operations = [
migrations.CreateModel(
name='ChannelProfile',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100, unique=True)),
],
),
migrations.CreateModel(
name='ChannelProfileMembership',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('enabled', models.BooleanField(default=True)),
('channel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dispatcharr_channels.channel')),
('channel_profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dispatcharr_channels.channelprofile')),
],
options={
'unique_together': {('channel_profile', 'channel')},
},
),
]

18
apps/channels/migrations/0013_alter_logo_url.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.1.6 on 2025-04-04 15:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0012_channelprofile_channelprofilemembership'),
]
operations = [
migrations.AlterField(
model_name='logo',
name='url',
field=models.TextField(unique=True),
),
]

24
apps/channels/migrations/0014_recording.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-04-05 22:25
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0013_alter_logo_url'),
]
operations = [
migrations.CreateModel(
name='Recording',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('start_time', models.DateTimeField()),
('end_time', models.DateTimeField()),
('task_id', models.CharField(blank=True, max_length=255, null=True)),
('channel', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='recordings', to='dispatcharr_channels.channel')),
],
),
]

18
apps/channels/migrations/0015_recording_custom_properties.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.1.6 on 2025-04-07 16:47
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dispatcharr_channels', '0014_recording'),
]
operations = [
migrations.AddField(
model_name='recording',
name='custom_properties',
field=models.TextField(blank=True, null=True),
),
]

210
apps/channels/models.py
View file

@ -3,12 +3,13 @@ from django.core.exceptions import ValidationError
from core.models import StreamProfile
from django.conf import settings
from core.models import StreamProfile, CoreSettings
from core.utils import redis_client, execute_redis_command
from core.utils import RedisClient
import logging
import uuid
from datetime import datetime
import hashlib
import json
from apps.epg.models import EPGData
logger = logging.getLogger(__name__)
@ -18,8 +19,7 @@ from apps.m3u.models import M3UAccount
# Add fallback functions if Redis isn't available
def get_total_viewers(channel_id):
"""Get viewer count from Redis or return 0 if Redis isn't available"""
if redis_client is None:
return 0
redis_client = RedisClient.get_client()
try:
return int(redis_client.get(f"channel:{channel_id}:viewers") or 0)
@ -90,6 +90,7 @@ class Stream(models.Model):
db_index=True,
)
last_seen = models.DateTimeField(db_index=True, default=datetime.now)
custom_properties = models.TextField(null=True, blank=True)
class Meta:
# If you use m3u_account, you might do unique_together = ('name','url','m3u_account')
@ -132,6 +133,77 @@ class Stream(models.Model):
stream = cls.objects.create(**fields_to_update)
return stream, True # True means it was created
# @TODO: honor stream's stream profile
def get_stream_profile(self):
stream_profile = StreamProfile.objects.get(id=CoreSettings.get_default_stream_profile_id())
return stream_profile
def get_stream(self):
"""
Finds an available stream for the requested channel and returns the selected stream and profile.
"""
redis_client = RedisClient.get_client()
profile_id = redis_client.get(f"stream_profile:{self.id}")
if profile_id:
profile_id = int(profile_id)
return self.id, profile_id
# Retrieve the M3U account associated with the stream.
m3u_account = self.m3u_account
m3u_profiles = m3u_account.profiles.all()
default_profile = next((obj for obj in m3u_profiles if obj.is_default), None)
profiles = [default_profile] + [obj for obj in m3u_profiles if not obj.is_default]
for profile in profiles:
logger.info(profile)
# Skip inactive profiles
if profile.is_active == False:
continue
profile_connections_key = f"profile_connections:{profile.id}"
current_connections = int(redis_client.get(profile_connections_key) or 0)
# Check if profile has available slots (or unlimited connections)
if profile.max_streams == 0 or current_connections < profile.max_streams:
# Start a new stream
redis_client.set(f"channel_stream:{self.id}", self.id)
redis_client.set(f"stream_profile:{self.id}", profile.id) # Store only the matched profile
# Increment connection count for profiles with limits
if profile.max_streams > 0:
redis_client.incr(profile_connections_key)
return self.id, profile.id # Return newly assigned stream and matched profile
# 4. No available streams
return None, None
def release_stream(self):
"""
Called when a stream is finished to release the lock.
"""
redis_client = RedisClient.get_client()
stream_id = self.id
# Get the matched profile for cleanup
profile_id = redis_client.get(f"stream_profile:{stream_id}")
if not profile_id:
logger.debug("Invalid profile ID pulled from stream index")
return
redis_client.delete(f"stream_profile:{stream_id}") # Remove profile association
profile_id = int(profile_id)
logger.debug(f"Found profile ID {profile_id} associated with stream {stream_id}")
profile_connections_key = f"profile_connections:{profile_id}"
# Only decrement if the profile had a max_connections limit
current_count = int(redis_client.get(profile_connections_key) or 0)
if current_count > 0:
redis_client.decr(profile_connections_key)
class ChannelManager(models.Manager):
def active(self):
return self.all()
@ -140,11 +212,12 @@ class ChannelManager(models.Manager):
class Channel(models.Model):
channel_number = models.IntegerField()
name = models.CharField(max_length=255)
logo_url = models.URLField(max_length=2000, blank=True, null=True)
logo_file = models.ImageField(
upload_to='logos/', # Will store in MEDIA_ROOT/logos
logo = models.ForeignKey(
'Logo',
on_delete=models.SET_NULL,
null=True,
blank=True,
null=True
related_name='channels',
)
# M2M to Stream now in the same file
@ -164,7 +237,13 @@ class Channel(models.Model):
help_text="Channel group this channel belongs to."
)
tvg_id = models.CharField(max_length=255, blank=True, null=True)
tvg_name = models.CharField(max_length=255, blank=True, null=True)
epg_data = models.ForeignKey(
EPGData,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name='channels'
)
stream_profile = models.ForeignKey(
StreamProfile,
@ -190,6 +269,15 @@ class Channel(models.Model):
def __str__(self):
return f"{self.channel_number} - {self.name}"
@classmethod
def get_next_available_channel_number(cls, starting_from=1):
used_numbers = set(cls.objects.all().values_list('channel_number', flat=True))
n = starting_from
while n in used_numbers:
n += 1
return n
# @TODO: honor stream's stream profile
def get_stream_profile(self):
stream_profile = self.stream_profile
if not stream_profile:
@ -200,33 +288,62 @@ class Channel(models.Model):
def get_stream(self):
"""
Finds an available stream for the requested channel and returns the selected stream and profile.
Returns:
Tuple[Optional[int], Optional[int], Optional[str]]: (stream_id, profile_id, error_reason)
"""
redis_client = RedisClient.get_client()
error_reason = None
# 2. Check if a stream is already active for this channel
stream_id = redis_client.get(f"channel_stream:{self.id}")
if stream_id:
stream_id = int(stream_id)
profile_id = redis_client.get(f"stream_profile:{stream_id}")
if profile_id:
profile_id = int(profile_id)
return stream_id, profile_id
# Check if this channel has any streams
if not self.streams.exists():
error_reason = "No streams assigned to channel"
return None, None, error_reason
# 3. Iterate through channel streams and their profiles
# Check if a stream is already active for this channel
stream_id_bytes = redis_client.get(f"channel_stream:{self.id}")
if stream_id_bytes:
try:
stream_id = int(stream_id_bytes)
profile_id_bytes = redis_client.get(f"stream_profile:{stream_id}")
if profile_id_bytes:
try:
profile_id = int(profile_id_bytes)
return stream_id, profile_id, None
except (ValueError, TypeError):
logger.debug(f"Invalid profile ID retrieved from Redis: {profile_id_bytes}")
except (ValueError, TypeError):
logger.debug(f"Invalid stream ID retrieved from Redis: {stream_id_bytes}")
# No existing active stream, attempt to assign a new one
has_streams_but_maxed_out = False
has_active_profiles = False
# Iterate through channel streams and their profiles
for stream in self.streams.all().order_by('channelstream__order'):
# Retrieve the M3U account associated with the stream.
m3u_account = stream.m3u_account
if not m3u_account:
logger.debug(f"Stream {stream.id} has no M3U account")
continue
m3u_profiles = m3u_account.profiles.all()
default_profile = next((obj for obj in m3u_profiles if obj.is_default), None)
if not default_profile:
logger.debug(f"M3U account {m3u_account.id} has no default profile")
continue
profiles = [default_profile] + [obj for obj in m3u_profiles if not obj.is_default]
logger.info('profiles')
for profile in profiles:
logger.info(profile)
# Skip inactive profiles
if profile.is_active == False:
if not profile.is_active:
logger.debug(f"Skipping inactive profile {profile.id}")
continue
has_active_profiles = True
profile_connections_key = f"profile_connections:{profile.id}"
current_connections = int(redis_client.get(profile_connections_key) or 0)
@ -234,21 +351,34 @@ class Channel(models.Model):
if profile.max_streams == 0 or current_connections < profile.max_streams:
# Start a new stream
redis_client.set(f"channel_stream:{self.id}", stream.id)
redis_client.set(f"stream_profile:{stream.id}", profile.id) # Store only the matched profile
redis_client.set(f"stream_profile:{stream.id}", profile.id)
# Increment connection count for profiles with limits
if profile.max_streams > 0:
redis_client.incr(profile_connections_key)
return stream.id, profile.id # Return newly assigned stream and matched profile
return stream.id, profile.id, None # Return newly assigned stream and matched profile
else:
# This profile is at max connections
has_streams_but_maxed_out = True
logger.debug(f"Profile {profile.id} at max connections: {current_connections}/{profile.max_streams}")
# 4. No available streams
return None, None
# No available streams - determine specific reason
if has_streams_but_maxed_out:
error_reason = "All M3U profiles have reached maximum connection limits"
elif has_active_profiles:
error_reason = "No compatible profile found for any assigned stream"
else:
error_reason = "No active profiles found for any assigned stream"
return None, None, error_reason
def release_stream(self):
"""
Called when a stream is finished to release the lock.
"""
redis_client = RedisClient.get_client()
stream_id = redis_client.get(f"channel_stream:{self.id}")
if not stream_id:
logger.debug("Invalid stream ID pulled from channel index")
@ -277,6 +407,18 @@ class Channel(models.Model):
if current_count > 0:
redis_client.decr(profile_connections_key)
class ChannelProfile(models.Model):
name = models.CharField(max_length=100, unique=True)
class ChannelProfileMembership(models.Model):
channel_profile = models.ForeignKey(ChannelProfile, on_delete=models.CASCADE)
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
enabled = models.BooleanField(default=True) # Track if the channel is enabled for this group
class Meta:
unique_together = ('channel_profile', 'channel')
class ChannelStream(models.Model):
channel = models.ForeignKey(Channel, on_delete=models.CASCADE)
stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
@ -303,3 +445,21 @@ class ChannelGroupM3UAccount(models.Model):
def __str__(self):
return f"{self.channel_group.name} - {self.m3u_account.name} (Enabled: {self.enabled})"
class Logo(models.Model):
name = models.CharField(max_length=255)
url = models.TextField(unique=True)
def __str__(self):
return self.name
class Recording(models.Model):
channel = models.ForeignKey("Channel", on_delete=models.CASCADE, related_name="recordings")
start_time = models.DateTimeField()
end_time = models.DateTimeField()
task_id = models.CharField(max_length=255, null=True, blank=True)
custom_properties = models.TextField(null=True, blank=True)
def __str__(self):
return f"{self.channel.name} - {self.start_time} to {self.end_time}"

121
apps/channels/serializers.py
View file

@ -1,6 +1,25 @@
from rest_framework import serializers
from .models import Stream, Channel, ChannelGroup, ChannelStream, ChannelGroupM3UAccount
from .models import Stream, Channel, ChannelGroup, ChannelStream, ChannelGroupM3UAccount, Logo, ChannelProfile, ChannelProfileMembership, Recording
from apps.epg.serializers import EPGDataSerializer
from core.models import StreamProfile
from apps.epg.models import EPGData
from django.urls import reverse
from rest_framework import serializers
from django.utils import timezone
class LogoSerializer(serializers.ModelSerializer):
cache_url = serializers.SerializerMethodField()
class Meta:
model = Logo
fields = ['id', 'name', 'url', 'cache_url']
def get_cache_url(self, obj):
# return f"/api/channels/logos/{obj.id}/cache/"
request = self.context.get('request')
if request:
return request.build_absolute_uri(reverse('api:channels:logo-cache', args=[obj.id]))
return reverse('api:channels:logo-cache', args=[obj.id])
#
# Stream
@ -12,7 +31,7 @@ class StreamSerializer(serializers.ModelSerializer):
allow_null=True,
required=False
)
read_only_fields = ['is_custom', 'm3u_account']
read_only_fields = ['is_custom', 'm3u_account', 'stream_hash']
class Meta:
model = Stream
@ -29,6 +48,7 @@ class StreamSerializer(serializers.ModelSerializer):
'stream_profile_id',
'is_custom',
'channel_group',
'stream_hash',
]
def get_fields(self):
@ -55,12 +75,49 @@ class ChannelGroupSerializer(serializers.ModelSerializer):
model = ChannelGroup
fields = ['id', 'name']
class ChannelProfileSerializer(serializers.ModelSerializer):
channels = serializers.SerializerMethodField()
class Meta:
model = ChannelProfile
fields = ['id', 'name', 'channels']
def get_channels(self, obj):
memberships = ChannelProfileMembership.objects.filter(channel_profile=obj)
return [
{
'id': membership.channel.id,
'enabled': membership.enabled
}
for membership in memberships
]
class ChannelProfileMembershipSerializer(serializers.ModelSerializer):
class Meta:
model = ChannelProfileMembership
fields = ['channel', 'enabled']
class ChanneProfilelMembershipUpdateSerializer(serializers.Serializer):
channel_id = serializers.IntegerField() # Ensure channel_id is an integer
enabled = serializers.BooleanField()
class BulkChannelProfileMembershipSerializer(serializers.Serializer):
channels = serializers.ListField(
child=ChanneProfilelMembershipUpdateSerializer(), # Use the nested serializer
allow_empty=False
)
def validate_channels(self, value):
if not value:
raise serializers.ValidationError("At least one channel must be provided.")
return value
#
# Channel
#
class ChannelSerializer(serializers.ModelSerializer):
# Show nested group data, or ID
channel_number = serializers.IntegerField(allow_null=True, required=False)
channel_group = ChannelGroupSerializer(read_only=True)
channel_group_id = serializers.PrimaryKeyRelatedField(
queryset=ChannelGroup.objects.all(),
@ -68,12 +125,20 @@ class ChannelSerializer(serializers.ModelSerializer):
write_only=True,
required=False
)
epg_data = EPGDataSerializer(read_only=True)
epg_data_id = serializers.PrimaryKeyRelatedField(
queryset=EPGData.objects.all(),
source="epg_data",
write_only=True,
required=False,
allow_null=True,
)
stream_profile_id = serializers.PrimaryKeyRelatedField(
queryset=StreamProfile.objects.all(),
source='stream_profile',
allow_null=True,
required=False
required=False,
)
streams = serializers.SerializerMethodField()
@ -81,22 +146,32 @@ class ChannelSerializer(serializers.ModelSerializer):
queryset=Stream.objects.all(), many=True, write_only=True, required=False
)
logo = LogoSerializer(read_only=True)
logo_id = serializers.PrimaryKeyRelatedField(
queryset=Logo.objects.all(),
source='logo',
allow_null=True,
required=False,
write_only=True,
)
class Meta:
model = Channel
fields = [
'id',
'channel_number',
'name',
'logo_url',
'logo_file',
'channel_group',
'channel_group_id',
'tvg_id',
'tvg_name',
'epg_data',
'epg_data_id',
'streams',
'stream_ids',
'stream_profile_id',
'uuid',
'logo',
'logo_id',
]
def get_streams(self, obj):
@ -104,12 +179,17 @@ class ChannelSerializer(serializers.ModelSerializer):
ordered_streams = obj.streams.all().order_by('channelstream__order')
return StreamSerializer(ordered_streams, many=True).data
def get_logo(self, obj):
return LogoSerializer(obj.logo).data
# def get_stream_ids(self, obj):
# """Retrieve ordered stream IDs for GET requests."""
# return list(obj.streams.all().order_by('channelstream__order').values_list('id', flat=True))
def create(self, validated_data):
stream_ids = validated_data.pop('streams', [])
channel_number = validated_data.pop('channel_number', Channel.get_next_available_channel_number())
validated_data["channel_number"] = channel_number
channel = Channel.objects.create(**validated_data)
# Add streams in the specified order
@ -124,15 +204,16 @@ class ChannelSerializer(serializers.ModelSerializer):
# Update the actual Channel fields
instance.channel_number = validated_data.get('channel_number', instance.channel_number)
instance.name = validated_data.get('name', instance.name)
instance.logo_url = validated_data.get('logo_url', instance.logo_url)
instance.tvg_id = validated_data.get('tvg_id', instance.tvg_id)
instance.tvg_name = validated_data.get('tvg_name', instance.tvg_name)
instance.epg_data = validated_data.get('epg_data', None)
# If serializer allows changing channel_group or stream_profile:
if 'channel_group' in validated_data:
instance.channel_group = validated_data['channel_group']
if 'stream_profile' in validated_data:
instance.stream_profile = validated_data['stream_profile']
if 'logo' in validated_data:
instance.logo = validated_data['logo']
instance.save()
@ -156,3 +237,27 @@ class ChannelGroupM3UAccountSerializer(serializers.ModelSerializer):
# Optionally, if you only need the id of the ChannelGroup, you can customize it like this:
# channel_group = serializers.PrimaryKeyRelatedField(queryset=ChannelGroup.objects.all())
class RecordingSerializer(serializers.ModelSerializer):
class Meta:
model = Recording
fields = '__all__'
read_only_fields = ['task_id']
def validate(self, data):
start_time = data.get('start_time')
end_time = data.get('end_time')
now = timezone.now() # timezone-aware current time
if end_time < now:
raise serializers.ValidationError("End time must be in the future.")
if start_time < now:
# Optional: Adjust start_time if it's in the past but end_time is in the future
data['start_time'] = now # or: timezone.now() + timedelta(seconds=1)
if end_time <= data['start_time']:
raise serializers.ValidationError("End time must be after start time.")
return data

107
apps/channels/signals.py
View file

@ -1,17 +1,24 @@
# apps/channels/signals.py
from django.db.models.signals import m2m_changed, pre_save
from django.db.models.signals import m2m_changed, pre_save, post_save, post_delete
from django.dispatch import receiver
from .models import Channel, Stream
from django.utils.timezone import now
from celery.result import AsyncResult
from .models import Channel, Stream, ChannelProfile, ChannelProfileMembership, Recording
from apps.m3u.models import M3UAccount
from apps.epg.tasks import parse_programs_for_tvg_id
import logging, requests, time
from .tasks import run_recording
from django.utils.timezone import now, is_aware, make_aware
from datetime import timedelta
logger = logging.getLogger(__name__)
@receiver(m2m_changed, sender=Channel.streams.through)
def update_channel_tvg_id_and_logo(sender, instance, action, reverse, model, pk_set, **kwargs):
"""
Whenever streams are added to a channel:
1) If the channel doesn't have a tvg_id, fill it from the first newly-added stream that has one.
2) If the channel doesn't have a logo_url, fill it from the first newly-added stream that has one.
This way if an M3U or EPG entry carried a logo, newly created channels automatically get that logo.
"""
# We only care about post_add, i.e. once the new streams are fully associated
if action == "post_add":
@ -23,14 +30,6 @@ def update_channel_tvg_id_and_logo(sender, instance, action, reverse, model, pk_
instance.tvg_id = streams_with_tvg.first().tvg_id
instance.save(update_fields=['tvg_id'])
# --- 2) Populate channel.logo_url if empty ---
if not instance.logo_url:
# Look for newly added streams that have a nonempty logo_url
streams_with_logo = model.objects.filter(pk__in=pk_set).exclude(logo_url__exact='')
if streams_with_logo.exists():
instance.logo_url = streams_with_logo.first().logo_url
instance.save(update_fields=['logo_url'])
@receiver(pre_save, sender=Stream)
def set_default_m3u_account(sender, instance, **kwargs):
"""
@ -45,3 +44,87 @@ def set_default_m3u_account(sender, instance, **kwargs):
instance.m3u_account = default_account
else:
raise ValueError("No default M3UAccount found.")
@receiver(post_save, sender=Channel)
def refresh_epg_programs(sender, instance, created, **kwargs):
if instance.epg_data:
parse_programs_for_tvg_id.delay(instance.epg_data.id)
@receiver(post_save, sender=Channel)
def add_new_channel_to_groups(sender, instance, created, **kwargs):
if created:
profiles = ChannelProfile.objects.all()
ChannelProfileMembership.objects.bulk_create([
ChannelProfileMembership(channel_profile=profile, channel=instance)
for profile in profiles
])
@receiver(post_save, sender=ChannelProfile)
def create_profile_memberships(sender, instance, created, **kwargs):
if created:
channels = Channel.objects.all()
ChannelProfileMembership.objects.bulk_create([
ChannelProfileMembership(channel_profile=instance, channel=channel)
for channel in channels
])
def schedule_recording_task(instance):
eta = instance.start_time
task = run_recording.apply_async(
args=[instance.channel_id, str(instance.start_time), str(instance.end_time)],
eta=eta
)
return task.id
def revoke_task(task_id):
if task_id:
AsyncResult(task_id).revoke()
@receiver(pre_save, sender=Recording)
def revoke_old_task_on_update(sender, instance, **kwargs):
if not instance.pk:
return # New instance
try:
old = Recording.objects.get(pk=instance.pk)
if old.task_id and (
old.start_time != instance.start_time or
old.end_time != instance.end_time or
old.channel_id != instance.channel_id
):
revoke_task(old.task_id)
instance.task_id = None
except Recording.DoesNotExist:
pass
@receiver(post_save, sender=Recording)
def schedule_task_on_save(sender, instance, created, **kwargs):
try:
if not instance.task_id:
start_time = instance.start_time
# Make both datetimes aware (in UTC)
if not is_aware(start_time):
print("Start time was not aware, making aware")
start_time = make_aware(start_time)
current_time = now()
# Debug log
print(f"Start time: {start_time}, Now: {current_time}")
# Optionally allow slight fudge factor (1 second) to ensure scheduling happens
if start_time > current_time - timedelta(seconds=1):
print("Scheduling recording task!")
task_id = schedule_recording_task(instance)
instance.task_id = task_id
instance.save(update_fields=['task_id'])
else:
print("Start time is in the past. Not scheduling.")
except Exception as e:
import traceback
print("Error in post_save signal:", e)
traceback.print_exc()
@receiver(post_delete, sender=Recording)
def revoke_task_on_delete(sender, instance, **kwargs):
revoke_task(instance.task_id)

270
apps/channels/tasks.py Normal file → Executable file
View file

@ -2,38 +2,28 @@
import logging
import os
import re
import requests
import time
import json
import subprocess
from datetime import datetime
from celery import shared_task
from rapidfuzz import fuzz
from sentence_transformers import SentenceTransformer, util
from django.conf import settings
from django.db import transaction
from django.utils.text import slugify
from apps.channels.models import Channel
from apps.epg.models import EPGData, EPGSource
from apps.epg.models import EPGData
from core.models import CoreSettings
from apps.epg.tasks import parse_programs_for_tvg_id # <-- we import our new helper
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
import tempfile
logger = logging.getLogger(__name__)
# Load the sentence-transformers model once at the module level
SENTENCE_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
MODEL_PATH = os.path.join(settings.MEDIA_ROOT, "models", "all-MiniLM-L6-v2")
os.makedirs(MODEL_PATH, exist_ok=True)
# If not present locally, download:
if not os.path.exists(os.path.join(MODEL_PATH, "config.json")):
logger.info(f"Local model not found in {MODEL_PATH}; downloading from {SENTENCE_MODEL_NAME}...")
st_model = SentenceTransformer(SENTENCE_MODEL_NAME, cache_folder=MODEL_PATH)
else:
logger.info(f"Loading local model from {MODEL_PATH}")
st_model = SentenceTransformer(MODEL_PATH)
# Thresholds
BEST_FUZZY_THRESHOLD = 85
LOWER_FUZZY_THRESHOLD = 40
EMBED_SIM_THRESHOLD = 0.65
# Words we remove to help with fuzzy + embedding matching
COMMON_EXTRANEOUS_WORDS = [
"tv", "channel", "network", "television",
@ -70,8 +60,7 @@ def match_epg_channels():
1) If channel.tvg_id is valid in EPGData, skip.
2) If channel has a tvg_id but not found in EPGData, attempt direct EPGData lookup.
3) Otherwise, perform name-based fuzzy matching with optional region-based bonus.
4) If a match is found, we set channel.tvg_id and also parse its programs
from the cached EPG file (parse_programs_for_tvg_id).
4) If a match is found, we set channel.tvg_id
5) Summarize and log results.
"""
logger.info("Starting EPG matching logic...")
@ -83,132 +72,82 @@ def match_epg_channels():
except CoreSettings.DoesNotExist:
region_code = None
# Gather EPGData rows so we can do fuzzy matching in memory
all_epg = list(EPGData.objects.all())
epg_rows = []
for e in all_epg:
epg_rows.append({
"epg_id": e.id,
"tvg_id": e.tvg_id or "",
"raw_name": e.name,
"norm_name": normalize_name(e.name),
})
epg_embeddings = None
if any(row["norm_name"] for row in epg_rows):
epg_embeddings = st_model.encode(
[row["norm_name"] for row in epg_rows],
convert_to_tensor=True
)
matched_channels = []
channels_to_update = []
source = EPGSource.objects.filter(is_active=True).first()
epg_file_path = getattr(source, 'file_path', None) if source else None
channels_json = [{
"id": channel.id,
"name": channel.name,
"tvg_id": channel.tvg_id,
"fallback_name": channel.tvg_id.strip() if channel.tvg_id else channel.name,
"norm_chan": normalize_name(channel.tvg_id.strip() if channel.tvg_id else channel.name)
} for channel in Channel.objects.all() if not channel.epg_data]
with transaction.atomic():
for chan in Channel.objects.all():
epg_json = [{
'id': epg.id,
'tvg_id': epg.tvg_id,
'name': epg.name,
'norm_name': normalize_name(epg.name),
'epg_source_id': epg.epg_source.id,
} for epg in EPGData.objects.all()]
# A) Skip if channel.tvg_id is already valid
if chan.tvg_id and EPGData.objects.filter(tvg_id=chan.tvg_id).exists():
continue
payload = {
"channels": channels_json,
"epg_data": epg_json,
"region_code": region_code,
}
# B) If channel has a tvg_id that doesn't exist in EPGData, do direct check
if chan.tvg_id:
epg_match = EPGData.objects.filter(tvg_id=chan.tvg_id).first()
if epg_match:
logger.info(f"Channel {chan.id} '{chan.name}' => EPG found by tvg_id={chan.tvg_id}")
continue
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_file.write(json.dumps(payload).encode('utf-8'))
temp_file_path = temp_file.name
# C) Perform name-based fuzzy matching
fallback_name = chan.tvg_name.strip() if chan.tvg_name else chan.name
norm_chan = normalize_name(fallback_name)
if not norm_chan:
logger.info(f"Channel {chan.id} '{chan.name}' => empty after normalization, skipping")
continue
process = subprocess.Popen(
['python', '/app/scripts/epg_match.py', temp_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
)
best_score = 0
best_epg = None
for row in epg_rows:
if not row["norm_name"]:
continue
base_score = fuzz.ratio(norm_chan, row["norm_name"])
bonus = 0
# Region-based bonus/penalty
combined_text = row["tvg_id"].lower() + " " + row["raw_name"].lower()
dot_regions = re.findall(r'\.([a-z]{2})', combined_text)
if region_code:
if dot_regions:
if region_code in dot_regions:
bonus = 30 # bigger bonus if .us or .ca matches
else:
bonus = -15
elif region_code in combined_text:
bonus = 15
score = base_score + bonus
# Log stderr in real-time
for line in iter(process.stderr.readline, ''):
if line:
logger.info(line.strip())
logger.debug(
f"Channel {chan.id} '{fallback_name}' => EPG row {row['epg_id']}: "
f"raw_name='{row['raw_name']}', norm_name='{row['norm_name']}', "
f"combined_text='{combined_text}', dot_regions={dot_regions}, "
f"base_score={base_score}, bonus={bonus}, total_score={score}"
)
process.stderr.close()
stdout, stderr = process.communicate()
if score > best_score:
best_score = score
best_epg = row
os.remove(temp_file_path)
# If no best match was found, skip
if not best_epg:
logger.info(f"Channel {chan.id} '{fallback_name}' => no EPG match at all.")
continue
if process.returncode != 0:
return f"Failed to process EPG matching: {stderr}"
# If best_score is above BEST_FUZZY_THRESHOLD => direct accept
if best_score >= BEST_FUZZY_THRESHOLD:
chan.tvg_id = best_epg["tvg_id"]
chan.save()
result = json.loads(stdout)
# This returns lists of dicts, not model objects
channels_to_update_dicts = result["channels_to_update"]
matched_channels = result["matched_channels"]
# Attempt to parse program data for this channel
if epg_file_path:
parse_programs_for_tvg_id(epg_file_path, best_epg["tvg_id"])
logger.info(f"Loaded program data for tvg_id={best_epg['tvg_id']}")
# Convert your dict-based 'channels_to_update' into real Channel objects
if channels_to_update_dicts:
# Extract IDs of the channels that need updates
channel_ids = [d["id"] for d in channels_to_update_dicts]
matched_channels.append((chan.id, fallback_name, best_epg["tvg_id"]))
logger.info(
f"Channel {chan.id} '{fallback_name}' => matched tvg_id={best_epg['tvg_id']} "
f"(score={best_score})"
)
# Fetch them from DB
channels_qs = Channel.objects.filter(id__in=channel_ids)
channels_list = list(channels_qs)
# If best_score is in the “middle range,” do embedding check
elif best_score >= LOWER_FUZZY_THRESHOLD and epg_embeddings is not None:
chan_embedding = st_model.encode(norm_chan, convert_to_tensor=True)
sim_scores = util.cos_sim(chan_embedding, epg_embeddings)[0]
top_index = int(sim_scores.argmax())
top_value = float(sim_scores[top_index])
if top_value >= EMBED_SIM_THRESHOLD:
matched_epg = epg_rows[top_index]
chan.tvg_id = matched_epg["tvg_id"]
chan.save()
# Build a map from channel_id -> epg_data_id (or whatever fields you need)
epg_mapping = {
d["id"]: d["epg_data_id"] for d in channels_to_update_dicts
}
if epg_file_path:
parse_programs_for_tvg_id(epg_file_path, matched_epg["tvg_id"])
logger.info(f"Loaded program data for tvg_id={matched_epg['tvg_id']}")
# Populate each Channel object with the updated epg_data_id
for channel_obj in channels_list:
# The script sets 'epg_data_id' in the returned dict
# We either assign directly, or fetch the EPGData instance if needed.
channel_obj.epg_data_id = epg_mapping.get(channel_obj.id)
matched_channels.append((chan.id, fallback_name, matched_epg["tvg_id"]))
logger.info(
f"Channel {chan.id} '{fallback_name}' => matched EPG tvg_id={matched_epg['tvg_id']} "
f"(fuzzy={best_score}, cos-sim={top_value:.2f})"
)
else:
logger.info(
f"Channel {chan.id} '{fallback_name}' => fuzzy={best_score}, "
f"cos-sim={top_value:.2f} < {EMBED_SIM_THRESHOLD}, skipping"
)
else:
logger.info(
f"Channel {chan.id} '{fallback_name}' => fuzzy={best_score} < "
f"{LOWER_FUZZY_THRESHOLD}, skipping"
)
# Now we have real model objects, so bulk_update will work
Channel.objects.bulk_update(channels_list, ["epg_data"])
total_matched = len(matched_channels)
if total_matched:
@ -219,4 +158,63 @@ def match_epg_channels():
logger.info("No new channels were matched.")
logger.info("Finished EPG matching logic.")
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'updates',
{
'type': 'update',
"data": {"success": True, "type": "epg_match"}
}
)
return f"Done. Matched {total_matched} channel(s)."
@shared_task
def run_recording(channel_id, start_time_str, end_time_str):
channel = Channel.objects.get(id=channel_id)
start_time = datetime.fromisoformat(start_time_str)
end_time = datetime.fromisoformat(end_time_str)
duration_seconds = int((end_time - start_time).total_seconds())
filename = f'{slugify(channel.name)}-{start_time.strftime("%Y-%m-%d_%H-%M-%S")}.mp4'
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
"updates",
{
"type": "update",
"data": {"success": True, "type": "recording_started", "channel": channel.name}
},
)
logger.info(f"Starting recording for channel {channel.name}")
with requests.get(f"http://localhost:5656/proxy/ts/stream/{channel.uuid}", headers={
'User-Agent': 'Dispatcharr-DVR',
}, stream=True) as response:
# Raise an exception for bad responses (4xx, 5xx)
response.raise_for_status()
# Open the file in write-binary mode
with open(f"/data/recordings/{filename}", 'wb') as file:
start_time = time.time() # Start the timer
for chunk in response.iter_content(chunk_size=8192): # 8KB chunks
if time.time() - start_time > duration_seconds:
print(f"Timeout reached: {duration_seconds} seconds")
break
# Write the chunk to the file
file.write(chunk)
async_to_sync(channel_layer.group_send)(
"updates",
{
"type": "update",
"data": {"success": True, "type": "recording_ended", "channel": channel.name}
},
)
# After the loop, the file and response are closed automatically.
logger.info(f"Finished recording for channel {channel.name}")

52
apps/epg/api_views.py
View file

@ -1,8 +1,9 @@
import logging
import logging, os
from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated
from rest_framework.decorators import action
from drf_yasg.utils import swagger_auto_schema
from drf_yasg import openapi
from django.utils import timezone
@ -26,6 +27,29 @@ class EPGSourceViewSet(viewsets.ModelViewSet):
logger.debug("Listing all EPG sources.")
return super().list(request, *args, **kwargs)
@action(detail=False, methods=['post'])
def upload(self, request):
if 'file' not in request.FILES:
return Response({'error': 'No file uploaded'}, status=status.HTTP_400_BAD_REQUEST)
file = request.FILES['file']
file_name = file.name
file_path = os.path.join('/data/uploads/epgs', file_name)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
new_obj_data = request.data.copy()
new_obj_data['file_path'] = file_path
serializer = self.get_serializer(data=new_obj_data)
serializer.is_valid(raise_exception=True)
self.perform_create(serializer)
return Response(serializer.data, status=status.HTTP_201_CREATED)
# ─────────────────────────────
# 2) Program API (CRUD)
# ─────────────────────────────
@ -43,23 +67,28 @@ class ProgramViewSet(viewsets.ModelViewSet):
# 3) EPG Grid View
# ─────────────────────────────
class EPGGridAPIView(APIView):
"""Returns all programs airing in the next 12 hours"""
"""Returns all programs airing in the next 24 hours including currently running ones and recent ones"""
@swagger_auto_schema(
operation_description="Retrieve upcoming EPG programs within the next 12 hours",
operation_description="Retrieve programs from the previous hour, currently running and upcoming for the next 24 hours",
responses={200: ProgramDataSerializer(many=True)}
)
def get(self, request, format=None):
# Get current date and reset time to midnight (00:00)
now = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0)
twelve_hours_later = now + timedelta(hours=24)
logger.debug(f"EPGGridAPIView: Querying programs between {now} and {twelve_hours_later}.")
# Use select_related to prefetch EPGData (no channel relation now)
# Use current time instead of midnight
now = timezone.now()
one_hour_ago = now - timedelta(hours=1)
twenty_four_hours_later = now + timedelta(hours=24)
logger.debug(f"EPGGridAPIView: Querying programs between {one_hour_ago} and {twenty_four_hours_later}.")
# Use select_related to prefetch EPGData and include programs from the last hour
programs = ProgramData.objects.select_related('epg').filter(
start_time__gte=now, start_time__lte=twelve_hours_later
# Programs that end after one hour ago (includes recently ended programs)
end_time__gt=one_hour_ago,
# AND start before the end time window
start_time__lt=twenty_four_hours_later
)
count = programs.count()
logger.debug(f"EPG`Grid`APIView: Found {count} program(s).")
logger.debug(f"EPGGridAPIView: Found {count} program(s), including recently ended, currently running, and upcoming shows.")
serializer = ProgramDataSerializer(programs, many=True)
return Response({'data': serializer.data}, status=status.HTTP_200_OK)
@ -75,7 +104,7 @@ class EPGImportAPIView(APIView):
)
def post(self, request, format=None):
logger.info("EPGImportAPIView: Received request to import EPG data.")
refresh_epg_data.delay() # Trigger Celery task
refresh_epg_data.delay(request.data.get('id', None)) # Trigger Celery task
logger.info("EPGImportAPIView: Task dispatched to refresh EPG data.")
return Response({'success': True, 'message': 'EPG data import initiated.'}, status=status.HTTP_202_ACCEPTED)
@ -90,4 +119,3 @@ class EPGDataViewSet(viewsets.ReadOnlyModelViewSet):
queryset = EPGData.objects.all()
serializer_class = EPGDataSerializer
permission_classes = [IsAuthenticated]

18
apps/epg/migrations/0003_alter_epgdata_tvg_id.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.1.6 on 2025-03-25 19:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('epg', '0002_epgsource_file_path'),
]
operations = [
migrations.AlterField(
model_name='epgdata',
name='tvg_id',
field=models.CharField(blank=True, max_length=255, null=True, unique=True),
),
]

24
apps/epg/migrations/0004_epgdata_epg_source_alter_epgdata_tvg_id.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-03-26 12:44
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('epg', '0003_alter_epgdata_tvg_id'),
]
operations = [
migrations.AddField(
model_name='epgdata',
name='epg_source',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='epgs', to='epg.epgsource'),
),
migrations.AlterField(
model_name='epgdata',
name='tvg_id',
field=models.CharField(blank=True, db_index=True, max_length=255, null=True),
),
]

22
apps/epg/migrations/0005_programdata_custom_properties_and_more.py Normal file
View file

@ -0,0 +1,22 @@
# Generated by Django 5.1.6 on 2025-03-27 17:01
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('epg', '0004_epgdata_epg_source_alter_epgdata_tvg_id'),
]
operations = [
migrations.AddField(
model_name='programdata',
name='custom_properties',
field=models.TextField(blank=True, null=True),
),
migrations.AlterUniqueTogether(
name='epgdata',
unique_together={('tvg_id', 'epg_source')},
),
]

25
apps/epg/migrations/0006_epgsource_refresh_interval_epgsource_refresh_task.py Normal file
View file

@ -0,0 +1,25 @@
# Generated by Django 5.1.6 on 2025-03-29 17:31
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('django_celery_beat', '0019_alter_periodictasks_options'),
('epg', '0005_programdata_custom_properties_and_more'),
]
operations = [
migrations.AddField(
model_name='epgsource',
name='refresh_interval',
field=models.IntegerField(default=24),
),
migrations.AddField(
model_name='epgsource',
name='refresh_task',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_celery_beat.periodictask'),
),
]

52
apps/epg/migrations/0007_populate_periodic_tasks.py Normal file
View file

@ -0,0 +1,52 @@
from django.db import migrations
import json
def create_default_refresh_tasks(apps, schema_editor):
"""
Creates a PeriodicTask for each existing EPGSource that doesn't have one.
"""
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
EPGSource = apps.get_model("epg", "EPGSource")
default_interval, _ = IntervalSchedule.objects.get_or_create(
every=24,
period="hours",
)
for account in EPGSource.objects.all():
if account.refresh_task:
continue
task_name = f"epg_source-refresh-{account.id}"
refresh_task = PeriodicTask.objects.create(
name=task_name,
interval=default_interval,
task="apps.epg.tasks.refresh_epg_data",
kwargs=json.dumps({"account_id": account.id}),
)
account.refresh_task = refresh_task
account.save(update_fields=["refresh_task"])
def reverse_migration(apps, schema_editor):
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
EPGSource = apps.get_model("epg", "EPGSource")
for account in EPGSource.objects.all():
IntervalSchedule.objects.all().delete()
PeriodicTask.objects.all().delete()
class Migration(migrations.Migration):
dependencies = [
("epg", "0006_epgsource_refresh_interval_epgsource_refresh_task"),
("django_celery_beat", "0019_alter_periodictasks_options"),
]
operations = [
migrations.RunPython(create_default_refresh_tasks, reverse_migration),
]

24
apps/epg/migrations/0008_epgsource_created_at_epgsource_updated_at.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.1.6 on 2025-04-07 16:29
import django.utils.timezone
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('epg', '0007_populate_periodic_tasks'),
]
operations = [
migrations.AddField(
model_name='epgsource',
name='created_at',
field=models.DateTimeField(default=django.utils.timezone.now, help_text='Time when this source was created'),
),
migrations.AddField(
model_name='epgsource',
name='updated_at',
field=models.DateTimeField(default=django.utils.timezone.now, help_text='Time when this source was last updated'),
),
]

23
apps/epg/migrations/0009_alter_epgsource_created_at_and_more.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 5.1.6 on 2025-04-07 16:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('epg', '0008_epgsource_created_at_epgsource_updated_at'),
]
operations = [
migrations.AlterField(
model_name='epgsource',
name='created_at',
field=models.DateTimeField(auto_now_add=True, help_text='Time when this source was created'),
),
migrations.AlterField(
model_name='epgsource',
name='updated_at',
field=models.DateTimeField(auto_now=True, help_text='Time when this source was last updated'),
),
]

26
apps/epg/models.py
View file

@ -1,5 +1,6 @@
from django.db import models
from django.utils import timezone
from django_celery_beat.models import PeriodicTask
class EPGSource(models.Model):
SOURCE_TYPE_CHOICES = [
@ -12,6 +13,18 @@ class EPGSource(models.Model):
api_key = models.CharField(max_length=255, blank=True, null=True) # For Schedules Direct
is_active = models.BooleanField(default=True)
file_path = models.CharField(max_length=1024, blank=True, null=True)
refresh_interval = models.IntegerField(default=24)
refresh_task = models.ForeignKey(
PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True
)
created_at = models.DateTimeField(
auto_now_add=True,
help_text="Time when this source was created"
)
updated_at = models.DateTimeField(
auto_now=True,
help_text="Time when this source was last updated"
)
def __str__(self):
return self.name
@ -19,8 +32,18 @@ class EPGSource(models.Model):
class EPGData(models.Model):
# Removed the Channel foreign key. We now just store the original tvg_id
# and a name (which might simply be the tvg_id if no real channel exists).
tvg_id = models.CharField(max_length=255, null=True, blank=True, unique=True)
tvg_id = models.CharField(max_length=255, null=True, blank=True, db_index=True)
name = models.CharField(max_length=255)
epg_source = models.ForeignKey(
EPGSource,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="epgs",
)
class Meta:
unique_together = ('tvg_id', 'epg_source')
def __str__(self):
return f"EPG Data for {self.name}"
@ -34,6 +57,7 @@ class ProgramData(models.Model):
sub_title = models.CharField(max_length=255, blank=True, null=True)
description = models.TextField(blank=True, null=True)
tvg_id = models.CharField(max_length=255, null=True, blank=True)
custom_properties = models.TextField(null=True, blank=True)
def __str__(self):
return f"{self.title} ({self.start_time} - {self.end_time})"

13
apps/epg/serializers.py
View file

@ -3,9 +3,15 @@ from .models import EPGSource, EPGData, ProgramData
from apps.channels.models import Channel
class EPGSourceSerializer(serializers.ModelSerializer):
epg_data_ids = serializers.SerializerMethodField()
read_only_fields = ['created_at', 'updated_at']
class Meta:
model = EPGSource
fields = ['id', 'name', 'source_type', 'url', 'api_key', 'is_active']
fields = ['id', 'name', 'source_type', 'url', 'api_key', 'is_active', 'epg_data_ids', 'refresh_interval', 'created_at', 'updated_at']
def get_epg_data_ids(self, obj):
return list(obj.epgs.values_list('id', flat=True))
class ProgramDataSerializer(serializers.ModelSerializer):
class Meta:
@ -17,10 +23,13 @@ class EPGDataSerializer(serializers.ModelSerializer):
Only returns the tvg_id and the 'name' field from EPGData.
We assume 'name' is effectively the channel name.
"""
read_only_fields = ['epg_source']
class Meta:
model = EPGData
fields = [
'id',
'tvg_id',
'name',
]
'epg_source',
]

50
apps/epg/signals.py
View file

@ -1,10 +1,56 @@
from django.db.models.signals import post_save
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from .models import EPGSource
from .tasks import refresh_epg_data
from django_celery_beat.models import PeriodicTask, IntervalSchedule
import json
@receiver(post_save, sender=EPGSource)
def trigger_refresh_on_new_epg_source(sender, instance, created, **kwargs):
# Trigger refresh only if the source is newly created and active
if created and instance.is_active:
refresh_epg_data.delay()
refresh_epg_data.delay(instance.id)
@receiver(post_save, sender=EPGSource)
def create_or_update_refresh_task(sender, instance, **kwargs):
"""
Create or update a Celery Beat periodic task when an EPGSource is created/updated.
"""
task_name = f"epg_source-refresh-{instance.id}"
interval, _ = IntervalSchedule.objects.get_or_create(
every=int(instance.refresh_interval),
period=IntervalSchedule.HOURS
)
task, created = PeriodicTask.objects.get_or_create(name=task_name, defaults={
"interval": interval,
"task": "apps.epg.tasks.refresh_epg_data",
"kwargs": json.dumps({"source_id": instance.id}),
"enabled": instance.refresh_interval != 0,
})
update_fields = []
if created:
task.interval = interval
if task.interval != interval:
task.interval = interval
update_fields.append("interval")
if task.enabled != (instance.refresh_interval != 0):
task.enabled = instance.refresh_interval != 0
update_fields.append("enabled")
if update_fields:
task.save(update_fields=update_fields)
if instance.refresh_task != task:
instance.refresh_task = task
instance.save(update_fields=update_fields)
@receiver(post_delete, sender=EPGSource)
def delete_refresh_task(sender, instance, **kwargs):
"""
Delete the associated Celery Beat periodic task when a Channel is deleted.
"""
if instance.refresh_task:
instance.refresh_task.delete()

260
apps/epg/tasks.py
View file

@ -14,30 +14,54 @@ from django.db import transaction
from django.utils import timezone
from apps.channels.models import Channel
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from .models import EPGSource, EPGData, ProgramData
from core.utils import acquire_task_lock, release_task_lock
logger = logging.getLogger(__name__)
@shared_task
def refresh_epg_data():
def refresh_all_epg_data():
logger.info("Starting refresh_epg_data task.")
active_sources = EPGSource.objects.filter(is_active=True)
logger.debug(f"Found {active_sources.count()} active EPGSource(s).")
for source in active_sources:
logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})")
if source.source_type == 'xmltv':
fetch_xmltv(source)
elif source.source_type == 'schedules_direct':
fetch_schedules_direct(source)
refresh_epg_data(source.id)
logger.info("Finished refresh_epg_data task.")
return "EPG data refreshed."
@shared_task
def refresh_epg_data(source_id):
if not acquire_task_lock('refresh_epg_data', source_id):
logger.debug(f"EPG refresh for {source_id} already running")
return
source = EPGSource.objects.get(id=source_id)
if not source.is_active:
logger.info(f"EPG source {source_id} is not active. Skipping.")
return
logger.info(f"Processing EPGSource: {source.name} (type: {source.source_type})")
if source.source_type == 'xmltv':
fetch_xmltv(source)
parse_channels_only(source)
parse_programs_for_source(source)
elif source.source_type == 'schedules_direct':
fetch_schedules_direct(source)
source.save(update_fields=['updated_at'])
release_task_lock('refresh_epg_data', source_id)
def fetch_xmltv(source):
if not source.url:
return
logger.info(f"Fetching XMLTV data from source: {source.name}")
try:
response = requests.get(source.url, timeout=30)
@ -62,21 +86,14 @@ def fetch_xmltv(source):
source.file_path = file_path
source.save(update_fields=['file_path'])
epg_entries = EPGData.objects.exclude(tvg_id__isnull=True).exclude(tvg_id__exact='')
for epg in epg_entries:
if Channel.objects.filter(tvg_id=epg.tvg_id).exists():
logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}")
parse_programs_for_tvg_id(file_path, epg.tvg_id)
# Now parse <channel> blocks only
parse_channels_only(file_path)
except Exception as e:
logger.error(f"Error fetching XMLTV from {source.name}: {e}", exc_info=True)
def parse_channels_only(file_path):
def parse_channels_only(source):
file_path = source.file_path
logger.info(f"Parsing channels from EPG file: {file_path}")
existing_epgs = {e.tvg_id: e for e in EPGData.objects.filter(epg_source=source)}
# Read entire file (decompress if .gz)
if file_path.endswith('.gz'):
@ -90,77 +107,190 @@ def parse_channels_only(file_path):
root = ET.fromstring(xml_data)
channels = root.findall('channel')
epgs_to_create = []
epgs_to_update = []
logger.info(f"Found {len(channels)} <channel> entries in {file_path}")
with transaction.atomic():
for channel_elem in channels:
tvg_id = channel_elem.get('id', '').strip()
if not tvg_id:
continue # skip blank/invalid IDs
for channel_elem in channels:
tvg_id = channel_elem.get('id', '').strip()
if not tvg_id:
continue # skip blank/invalid IDs
display_name = channel_elem.findtext('display-name', default=tvg_id).strip()
display_name = channel_elem.findtext('display-name', default=tvg_id).strip()
epg_obj, created = EPGData.objects.get_or_create(
if tvg_id in existing_epgs:
epg_obj = existing_epgs[tvg_id]
if epg_obj.name != display_name:
epg_obj.name = display_name
epgs_to_update.append(epg_obj)
else:
epgs_to_create.append(EPGData(
tvg_id=tvg_id,
defaults={'name': display_name}
)
if not created:
# Optionally update if new name is different
if epg_obj.name != display_name:
epg_obj.name = display_name
epg_obj.save()
logger.debug(f"Channel <{tvg_id}> => EPGData.id={epg_obj.id}, created={created}")
name=display_name,
epg_source=source,
))
parse_programs_for_tvg_id(file_path, tvg_id)
if epgs_to_create:
EPGData.objects.bulk_create(epgs_to_create, ignore_conflicts=True)
if epgs_to_update:
EPGData.objects.bulk_update(epgs_to_update, ["name"])
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'updates',
{
'type': 'update',
"data": {"success": True, "type": "epg_channels"}
}
)
logger.info("Finished parsing channel info.")
@shared_task
def parse_programs_for_tvg_id(epg_id):
if not acquire_task_lock('parse_epg_programs', epg_id):
logger.debug(f"Program parse for {epg_id} already in progress")
return
def parse_programs_for_tvg_id(file_path, tvg_id):
logger.info(f"Parsing <programme> for tvg_id={tvg_id} from {file_path}")
epg = EPGData.objects.get(id=epg_id)
epg_source = epg.epg_source
if not Channel.objects.filter(epg_data=epg).exists():
logger.info(f"No channels matched to EPG {epg.tvg_id}")
release_task_lock('parse_epg_programs', epg_id)
return
logger.info(f"Refreshing program data for tvg_id: {epg.tvg_id}")
# First, remove all existing programs
ProgramData.objects.filter(epg=epg).delete()
# Read entire file (decompress if .gz)
if file_path.endswith('.gz'):
with open(file_path, 'rb') as gz_file:
if epg_source.file_path.endswith('.gz'):
with open(epg_source.file_path, 'rb') as gz_file:
decompressed = gzip.decompress(gz_file.read())
xml_data = decompressed.decode('utf-8')
else:
with open(file_path, 'r', encoding='utf-8') as xml_file:
with open(epg_source.file_path, 'r', encoding='utf-8') as xml_file:
xml_data = xml_file.read()
root = ET.fromstring(xml_data)
# Retrieve the EPGData record
try:
epg_obj = EPGData.objects.get(tvg_id=tvg_id)
except EPGData.DoesNotExist:
logger.warning(f"No EPGData record found for tvg_id={tvg_id}")
return
# Find only <programme> elements for this tvg_id
matched_programmes = [p for p in root.findall('programme') if p.get('channel') == tvg_id]
logger.debug(f"Found {len(matched_programmes)} programmes for tvg_id={tvg_id}")
matched_programmes = [p for p in root.findall('programme') if p.get('channel') == epg.tvg_id]
logger.debug(f"Found {len(matched_programmes)} programmes for tvg_id={epg.tvg_id}")
with transaction.atomic():
for prog in matched_programmes:
start_time = parse_xmltv_time(prog.get('start'))
end_time = parse_xmltv_time(prog.get('stop'))
title = prog.findtext('title', default='No Title')
desc = prog.findtext('desc', default='')
programs_to_create = []
for prog in matched_programmes:
start_time = parse_xmltv_time(prog.get('start'))
end_time = parse_xmltv_time(prog.get('stop'))
title = prog.findtext('title', default='No Title')
desc = prog.findtext('desc', default='')
sub_title = prog.findtext('sub-title', default='')
obj, created = ProgramData.objects.update_or_create(
epg=epg_obj,
start_time=start_time,
title=title,
defaults={
'end_time': end_time,
'description': desc,
'sub_title': '',
'tvg_id': tvg_id,
}
)
if created:
logger.debug(f"Created ProgramData: {title} [{start_time} - {end_time}]")
logger.info(f"Completed program parsing for tvg_id={tvg_id}.")
# Extract custom properties
custom_props = {}
# Extract categories
categories = []
for cat_elem in prog.findall('category'):
if cat_elem.text and cat_elem.text.strip():
categories.append(cat_elem.text.strip())
if categories:
custom_props['categories'] = categories
# Extract episode numbers
for ep_num in prog.findall('episode-num'):
system = ep_num.get('system', '')
if system == 'xmltv_ns' and ep_num.text:
# Parse XMLTV episode-num format (season.episode.part)
parts = ep_num.text.split('.')
if len(parts) >= 2:
if parts[0].strip() != '':
try:
season = int(parts[0]) + 1 # XMLTV format is zero-based
custom_props['season'] = season
except ValueError:
pass
if parts[1].strip() != '':
try:
episode = int(parts[1]) + 1 # XMLTV format is zero-based
custom_props['episode'] = episode
except ValueError:
pass
elif system == 'onscreen' and ep_num.text:
# Just store the raw onscreen format
custom_props['onscreen_episode'] = ep_num.text.strip()
# Extract ratings
for rating_elem in prog.findall('rating'):
if rating_elem.findtext('value'):
custom_props['rating'] = rating_elem.findtext('value').strip()
if rating_elem.get('system'):
custom_props['rating_system'] = rating_elem.get('system')
break # Just use the first rating
# Extract credits (actors, directors, etc.)
credits_elem = prog.find('credits')
if credits_elem is not None:
credits = {}
for credit_type in ['director', 'actor', 'writer', 'presenter', 'producer']:
elements = credits_elem.findall(credit_type)
if elements:
names = [e.text.strip() for e in elements if e.text and e.text.strip()]
if names:
credits[credit_type] = names
if credits:
custom_props['credits'] = credits
# Extract other common program metadata
if prog.findtext('date'):
custom_props['year'] = prog.findtext('date').strip()[:4] # Just the year part
if prog.findtext('country'):
custom_props['country'] = prog.findtext('country').strip()
for icon_elem in prog.findall('icon'):
if icon_elem.get('src'):
custom_props['icon'] = icon_elem.get('src')
break # Just use the first icon
for kw in ['previously-shown', 'premiere', 'new']:
if prog.find(kw) is not None:
custom_props[kw.replace('-', '_')] = True
# Convert custom_props to JSON string if not empty
custom_properties_json = None
if custom_props:
import json
try:
custom_properties_json = json.dumps(custom_props)
except Exception as e:
logger.error(f"Error serializing custom properties to JSON: {e}", exc_info=True)
programs_to_create.append(ProgramData(
epg=epg,
start_time=start_time,
end_time=end_time,
title=title,
description=desc,
sub_title=sub_title,
tvg_id=epg.tvg_id,
custom_properties=custom_properties_json
))
ProgramData.objects.bulk_create(programs_to_create)
release_task_lock('parse_epg_programs', epg_id)
logger.info(f"Completed program parsing for tvg_id={epg.tvg_id}.")
def parse_programs_for_source(epg_source, tvg_id=None):
file_path = epg_source.file_path
epg_entries = EPGData.objects.filter(epg_source=epg_source)
for epg in epg_entries:
if epg.tvg_id:
parse_programs_for_tvg_id(epg.id)
def fetch_schedules_direct(source):
logger.info(f"Fetching Schedules Direct data from source: {source.name}")

30
apps/hdhr/api_views.py
View file

@ -6,7 +6,7 @@ from django.http import JsonResponse, HttpResponseForbidden, HttpResponse
from drf_yasg.utils import swagger_auto_schema
from drf_yasg import openapi
from django.shortcuts import get_object_or_404
from apps.channels.models import Channel
from apps.channels.models import Channel, ChannelProfile
from .models import HDHRDevice
from .serializers import HDHRDeviceSerializer
from django.contrib.auth.decorators import login_required
@ -38,8 +38,12 @@ class DiscoverAPIView(APIView):
operation_description="Retrieve HDHomeRun device discovery information",
responses={200: openapi.Response("HDHR Discovery JSON")}
)
def get(self, request):
base_url = request.build_absolute_uri('/hdhr/').rstrip('/')
def get(self, request, profile=None):
uri_parts = ["hdhr"]
if profile is not None:
uri_parts.append(profile)
base_url = request.build_absolute_uri(f'/{"/".join(uri_parts)}/').rstrip('/')
device = HDHRDevice.objects.first()
if not device:
@ -52,6 +56,7 @@ class DiscoverAPIView(APIView):
"DeviceAuth": "test_auth_token",
"BaseURL": base_url,
"LineupURL": f"{base_url}/lineup.json",
"TunerCount": 10,
}
else:
data = {
@ -63,6 +68,7 @@ class DiscoverAPIView(APIView):
"DeviceAuth": "test_auth_token",
"BaseURL": base_url,
"LineupURL": f"{base_url}/lineup.json",
"TunerCount": 10,
}
return JsonResponse(data)
@ -75,13 +81,23 @@ class LineupAPIView(APIView):
operation_description="Retrieve the available channel lineup",
responses={200: openapi.Response("Channel Lineup JSON")}
)
def get(self, request):
channels = Channel.objects.all().order_by('channel_number')
def get(self, request, profile=None):
if profile is not None:
channel_profile = ChannelProfile.objects.get(name=profile)
channels = Channel.objects.filter(
channelprofilemembership__channel_profile=channel_profile,
channelprofilemembership__enabled=True
).order_by('channel_number')
else:
channels = Channel.objects.all().order_by('channel_number')
lineup = [
{
"GuideNumber": str(ch.channel_number),
"GuideName": ch.name,
"URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}")
"URL": request.build_absolute_uri(f"/proxy/ts/stream/{ch.uuid}"),
"Guide_ID": str(ch.channel_number),
"Station": str(ch.channel_number),
}
for ch in channels
]
@ -96,7 +112,7 @@ class LineupStatusAPIView(APIView):
operation_description="Retrieve the HDHomeRun lineup status",
responses={200: openapi.Response("Lineup Status JSON")}
)
def get(self, request):
def get(self, request, profile=None):
data = {
"ScanInProgress": 0,
"ScanPossible": 0,

9
apps/hdhr/urls.py
View file

@ -10,9 +10,12 @@ router.register(r'devices', HDHRDeviceViewSet, basename='hdhr-device')
urlpatterns = [
path('dashboard/', hdhr_dashboard_view, name='hdhr_dashboard'),
path('', hdhr_dashboard_view, name='hdhr_dashboard'),
path('discover.json', DiscoverAPIView.as_view(), name='discover'),
path('lineup.json', LineupAPIView.as_view(), name='lineup'),
path('lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status'),
path('<str:profile>/discover.json', DiscoverAPIView.as_view(), name='discover_with_profile'),
path('discover.json', DiscoverAPIView.as_view(), name='discover_no_profile'),
path('<str:profile>/lineup.json', LineupAPIView.as_view(), name='lineup_with_profile'),
path('lineup.json', LineupAPIView.as_view(), name='lineup_no_profile'),
path('<str:profile>/lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status_with_profile'),
path('lineup_status.json', LineupStatusAPIView.as_view(), name='lineup_status_no_profile'),
path('device.xml', HDHRDeviceXMLAPIView.as_view(), name='device_xml'),
]

2
apps/hdhr/views.py
View file

@ -52,6 +52,7 @@ class DiscoverAPIView(APIView):
"DeviceAuth": "test_auth_token",
"BaseURL": base_url,
"LineupURL": f"{base_url}/lineup.json",
"TunerCount": "10",
}
else:
data = {
@ -63,6 +64,7 @@ class DiscoverAPIView(APIView):
"DeviceAuth": "test_auth_token",
"BaseURL": base_url,
"LineupURL": f"{base_url}/lineup.json",
"TunerCount": "10",
}
return JsonResponse(data)

75
apps/m3u/api_views.py
View file

@ -7,6 +7,9 @@ from drf_yasg import openapi
from django.shortcuts import get_object_or_404
from django.http import JsonResponse
from django.core.cache import cache
import os
from rest_framework.decorators import action
from django.conf import settings
# Import all models, including UserAgent.
from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile
@ -22,6 +25,8 @@ from .serializers import (
)
from .tasks import refresh_single_m3u_account, refresh_m3u_accounts
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
class M3UAccountViewSet(viewsets.ModelViewSet):
"""Handles CRUD operations for M3U accounts"""
@ -29,38 +34,60 @@ class M3UAccountViewSet(viewsets.ModelViewSet):
serializer_class = M3UAccountSerializer
permission_classes = [IsAuthenticated]
def create(self, request, *args, **kwargs):
# Handle file upload first, if any
file_path = None
if 'file' in request.FILES:
file = request.FILES['file']
file_name = file.name
file_path = os.path.join('/data/uploads/m3us', file_name)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
# Add file_path to the request data so it's available during creation
request.data._mutable = True # Allow modification of the request data
request.data['file_path'] = file_path # Include the file path if a file was uploaded
request.data.pop('server_url')
request.data._mutable = False # Make the request data immutable again
# Now call super().create() to create the instance
response = super().create(request, *args, **kwargs)
# After the instance is created, return the response
return response
def update(self, request, *args, **kwargs):
# Get the M3UAccount instance we're updating
instance = self.get_object()
# Handle updates to the 'enabled' flag of the related ChannelGroupM3UAccount instances
updates = request.data.get('channel_groups', [])
# Handle file upload first, if any
file_path = None
if 'file' in request.FILES:
file = request.FILES['file']
file_name = file.name
file_path = os.path.join('/data/uploads/m3us', file_name)
for update_data in updates:
channel_group_id = update_data.get('channel_group')
enabled = update_data.get('enabled')
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
try:
# Get the specific relationship to update
relationship = ChannelGroupM3UAccount.objects.get(
m3u_account=instance, channel_group_id=channel_group_id
)
relationship.enabled = enabled
relationship.save()
except ChannelGroupM3UAccount.DoesNotExist:
return Response(
{"error": "ChannelGroupM3UAccount not found for the given M3UAccount and ChannelGroup."},
status=status.HTTP_400_BAD_REQUEST
)
# Add file_path to the request data so it's available during creation
request.data._mutable = True # Allow modification of the request data
request.data['file_path'] = file_path # Include the file path if a file was uploaded
request.data.pop('server_url')
request.data._mutable = False # Make the request data immutable again
# After updating the ChannelGroupM3UAccount relationships, reload the M3UAccount instance
instance.refresh_from_db()
if instance.file_path and os.path.exists(instance.file_path):
os.remove(instance.file_path)
refresh_single_m3u_account.delay(instance.id)
# Now call super().create() to create the instance
response = super().update(request, *args, **kwargs)
# Serialize and return the updated M3UAccount data
serializer = self.get_serializer(instance)
return Response(serializer.data)
# After the instance is created, return the response
return response
class M3UFilterViewSet(viewsets.ModelViewSet):
"""Handles CRUD operations for M3U filters"""

30
apps/m3u/migrations/0005_m3uaccount_custom_properties_and_more.py Normal file
View file

@ -0,0 +1,30 @@
# Generated by Django 5.1.6 on 2025-03-29 13:44
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('django_celery_beat', '0019_alter_periodictasks_options'),
('m3u', '0004_m3uaccount_stream_profile'),
]
operations = [
migrations.AddField(
model_name='m3uaccount',
name='custom_properties',
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name='m3uaccount',
name='refresh_interval',
field=models.IntegerField(default=24),
),
migrations.AddField(
model_name='m3uaccount',
name='refresh_task',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_celery_beat.periodictask'),
),
]

52
apps/m3u/migrations/0006_populate_periodic_tasks.py Normal file
View file

@ -0,0 +1,52 @@
from django.db import migrations
import json
def create_default_refresh_tasks(apps, schema_editor):
"""
Creates a PeriodicTask for each existing M3UAccount that doesn't have one.
"""
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
M3UAccount = apps.get_model("m3u", "M3UAccount")
default_interval, _ = IntervalSchedule.objects.get_or_create(
every=24,
period="hours",
)
for account in M3UAccount.objects.all():
if account.refresh_task:
continue
task_name = f"m3u_account-refresh-{account.id}"
refresh_task = PeriodicTask.objects.create(
name=task_name,
interval=default_interval,
task="apps.m3u.tasks.refresh_single_m3u_account",
kwargs=json.dumps({"account_id": account.id}),
)
account.refresh_task = refresh_task
account.save(update_fields=["refresh_task"])
def reverse_migration(apps, schema_editor):
IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
M3UAccount = apps.get_model("m3u", "M3UAccount")
for account in M3UAccount.objects.all():
IntervalSchedule.objects.filter(name=f"m3u_account-refresh-interval-{account.id}").delete()
PeriodicTask.objects.filter(name=f"m3u_account-refresh-{account.id}").delete()
class Migration(migrations.Migration):
dependencies = [
("m3u", "0005_m3uaccount_custom_properties_and_more"),
("django_celery_beat", "0019_alter_periodictasks_options"),
]
operations = [
migrations.RunPython(create_default_refresh_tasks, reverse_migration),
]

22
apps/m3u/migrations/0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path.py Normal file
View file

@ -0,0 +1,22 @@
# Generated by Django 5.1.6 on 2025-04-06 19:09
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('m3u', '0006_populate_periodic_tasks'),
]
operations = [
migrations.RemoveField(
model_name='m3uaccount',
name='uploaded_file',
),
migrations.AddField(
model_name='m3uaccount',
name='file_path',
field=models.CharField(blank=True, max_length=255, null=True),
),
]

18
apps/m3u/models.py
View file

@ -4,6 +4,8 @@ from core.models import UserAgent
import re
from django.dispatch import receiver
from apps.channels.models import StreamProfile
from django_celery_beat.models import PeriodicTask
from core.models import CoreSettings, UserAgent
CUSTOM_M3U_ACCOUNT_NAME="custom"
@ -19,8 +21,8 @@ class M3UAccount(models.Model):
null=True,
help_text="The base URL of the M3U server (optional if a file is uploaded)"
)
uploaded_file = models.FileField(
upload_to='m3u_uploads/',
file_path = models.CharField(
max_length=255,
blank=True,
null=True
)
@ -67,6 +69,11 @@ class M3UAccount(models.Model):
blank=True,
related_name='m3u_accounts'
)
custom_properties = models.TextField(null=True, blank=True)
refresh_interval = models.IntegerField(default=24)
refresh_task = models.ForeignKey(
PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True
)
def __str__(self):
return self.name
@ -94,6 +101,13 @@ class M3UAccount(models.Model):
def get_custom_account(cls):
return cls.objects.get(name=CUSTOM_M3U_ACCOUNT_NAME, locked=True)
def get_user_agent(self):
user_agent = self.user_agent
if not user_agent:
user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id())
return user_agent
# def get_channel_groups(self):
# return ChannelGroup.objects.filter(m3u_account__m3u_account=self)

101
apps/m3u/serializers.py
View file

@ -1,4 +1,5 @@
from rest_framework import serializers
from rest_framework.response import Response
from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile
from core.models import UserAgent
from apps.channels.models import ChannelGroup, ChannelGroupM3UAccount
@ -32,6 +33,19 @@ class M3UAccountProfileSerializer(serializers.ModelSerializer):
return super().create(validated_data)
def update(self, instance, validated_data):
if instance.is_default:
raise serializers.ValidationError("Default profiles cannot be modified.")
return super().update(instance, validated_data)
def destroy(self, request, *args, **kwargs):
instance = self.get_object()
if instance.is_default:
return Response(
{"error": "Default profiles cannot be deleted."},
status=status.HTTP_400_BAD_REQUEST
)
return super().destroy(request, *args, **kwargs)
class M3UAccountSerializer(serializers.ModelSerializer):
"""Serializer for M3U Account"""
@ -39,77 +53,52 @@ class M3UAccountSerializer(serializers.ModelSerializer):
# Include user_agent as a mandatory field using its primary key.
user_agent = serializers.PrimaryKeyRelatedField(
queryset=UserAgent.objects.all(),
required=True
required=False,
allow_null=True,
)
profiles = M3UAccountProfileSerializer(many=True, read_only=True)
read_only_fields = ['locked']
read_only_fields = ['locked', 'created_at', 'updated_at']
# channel_groups = serializers.SerializerMethodField()
channel_groups = ChannelGroupM3UAccountSerializer(source='channel_group.all', many=True, required=False)
channel_groups = ChannelGroupM3UAccountSerializer(source='channel_group', many=True, required=False)
class Meta:
model = M3UAccount
fields = [
'id', 'name', 'server_url', 'uploaded_file', 'server_group',
'id', 'name', 'server_url', 'file_path', 'server_group',
'max_streams', 'is_active', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked',
'channel_groups',
'channel_groups', 'refresh_interval'
]
# def get_channel_groups(self, obj):
# # Retrieve related ChannelGroupM3UAccount records for this M3UAccount
# relations = ChannelGroupM3UAccount.objects.filter(m3u_account=obj).select_related('channel_group')
def update(self, instance, validated_data):
# Pop out channel group memberships so we can handle them manually
channel_group_data = validated_data.pop('channel_group', [])
# # Serialize the channel groups with their enabled status
# return [
# {
# 'channel_group_name': relation.channel_group.name,
# 'channel_group_id': relation.channel_group.id,
# 'enabled': relation.enabled,
# }
# for relation in relations
# ]
# First, update the M3UAccount itself
for attr, value in validated_data.items():
setattr(instance, attr, value)
instance.save()
# def to_representation(self, instance):
# """Override the default to_representation method to include channel_groups"""
# representation = super().to_representation(instance)
# Prepare a list of memberships to update
memberships_to_update = []
for group_data in channel_group_data:
group = group_data.get('channel_group')
enabled = group_data.get('enabled')
# # Manually add the channel_groups to the representation
# channel_groups = ChannelGroupM3UAccount.objects.filter(m3u_account=instance).select_related('channel_group')
# representation['channel_groups'] = [
# {
# 'id': relation.id,
# 'channel_group_name': relation.channel_group.name,
# 'channel_group_id': relation.channel_group.id,
# 'enabled': relation.enabled,
# }
# for relation in channel_groups
# ]
try:
membership = ChannelGroupM3UAccount.objects.get(
m3u_account=instance,
channel_group=group
)
membership.enabled = enabled
memberships_to_update.append(membership)
except ChannelGroupM3UAccount.DoesNotExist:
continue
# return representation
# Perform the bulk update
if memberships_to_update:
ChannelGroupM3UAccount.objects.bulk_update(memberships_to_update, ['enabled'])
# def update(self, instance, validated_data):
# logger.info(validated_data)
# channel_groups_data = validated_data.pop('channel_groups', None)
# instance = super().update(instance, validated_data)
# if channel_groups_data is not None:
# logger.info(json.dumps(channel_groups_data))
# # Remove existing relationships not included in the request
# existing_groups = {cg.channel_group_id: cg for cg in instance.channel_group.all()}
# # for group_id in set(existing_groups.keys()) - sent_group_ids:
# # existing_groups[group_id].delete()
# # Create or update relationships
# for cg_data in channel_groups_data:
# logger.info(json.dumps(cg_data))
# ChannelGroupM3UAccount.objects.update_or_create(
# channel_group=existing_groups[cg_data['channel_group_id']],
# m3u_account=instance,
# defaults={'enabled': cg_data.get('enabled', True)}
# )
# return instance
return instance
class ServerGroupSerializer(serializers.ModelSerializer):
"""Serializer for Server Group"""

51
apps/m3u/signals.py
View file

@ -1,8 +1,10 @@
# apps/m3u/signals.py
from django.db.models.signals import post_save
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from .models import M3UAccount
from .tasks import refresh_single_m3u_account, refresh_m3u_groups
from django_celery_beat.models import PeriodicTask, IntervalSchedule
import json
@receiver(post_save, sender=M3UAccount)
def refresh_account_on_save(sender, instance, created, **kwargs):
@ -12,4 +14,49 @@ def refresh_account_on_save(sender, instance, created, **kwargs):
if it is active or newly created.
"""
if created:
refresh_single_m3u_account.delay(instance.id)
refresh_m3u_groups.delay(instance.id)
@receiver(post_save, sender=M3UAccount)
def create_or_update_refresh_task(sender, instance, **kwargs):
"""
Create or update a Celery Beat periodic task when an M3UAccount is created/updated.
"""
task_name = f"m3u_account-refresh-{instance.id}"
interval, _ = IntervalSchedule.objects.get_or_create(
every=int(instance.refresh_interval),
period=IntervalSchedule.HOURS
)
if not instance.refresh_task:
refresh_task = PeriodicTask.objects.create(
name=task_name,
interval=interval,
task="apps.m3u.tasks.refresh_single_m3u_account",
kwargs=json.dumps({"account_id": instance.id}),
enabled=instance.refresh_interval != 0,
)
M3UAccount.objects.filter(id=instance.id).update(refresh_task=refresh_task)
else:
task = instance.refresh_task
updated_fields = []
if task.enabled != (instance.refresh_interval != 0):
task.enabled = instance.refresh_interval != 0
updated_fields.append("enabled")
if task.interval != interval:
task.interval = interval
updated_fields.append("interval")
if updated_fields:
task.save(update_fields=updated_fields)
@receiver(post_delete, sender=M3UAccount)
def delete_refresh_task(sender, instance, **kwargs):
"""
Delete the associated Celery Beat periodic task when a Channel is deleted.
"""
if instance.refresh_task:
instance.refresh_task.interval.delete()
instance.refresh_task.delete()

246
apps/m3u/tasks.py
View file

@ -3,6 +3,8 @@ import logging
import re
import requests
import os
import gc
import gzip, zipfile
from celery.app.control import Inspect
from celery.result import AsyncResult
from celery import shared_task, current_app, group
@ -15,15 +17,13 @@ from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from django.utils import timezone
import time
from channels.layers import get_channel_layer
import json
from core.utils import redis_client
from core.utils import RedisClient, acquire_task_lock, release_task_lock
from core.models import CoreSettings
from asgiref.sync import async_to_sync
logger = logging.getLogger(__name__)
LOCK_EXPIRE = 300
BATCH_SIZE = 1000
SKIP_EXTS = {}
m3u_dir = os.path.join(settings.MEDIA_ROOT, "cached_m3u")
@ -35,30 +35,74 @@ def fetch_m3u_lines(account, use_cache=False):
"""Fetch M3U file lines efficiently."""
if account.server_url:
if not use_cache or not os.path.exists(file_path):
headers = {"User-Agent": account.user_agent.user_agent}
user_agent = account.get_user_agent()
headers = {"User-Agent": user_agent.user_agent}
logger.info(f"Fetching from URL {account.server_url}")
try:
response = requests.get(account.server_url, headers=headers, stream=True)
response.raise_for_status() # This will raise an HTTPError if the status is not 200
response.raise_for_status()
total_size = int(response.headers.get('Content-Length', 0))
downloaded = 0
start_time = time.time()
last_update_time = start_time
progress = 0
with open(file_path, 'wb') as file:
# Stream the content in chunks and write to the file
for chunk in response.iter_content(chunk_size=8192): # You can adjust the chunk size
if chunk: # Ensure chunk is not empty
send_m3u_update(account.id, "downloading", 0)
for chunk in response.iter_content(chunk_size=8192):
if chunk:
file.write(chunk)
downloaded += len(chunk)
elapsed_time = time.time() - start_time
# Calculate download speed in KB/s
speed = downloaded / elapsed_time / 1024 # in KB/s
# Calculate progress percentage
if total_size and total_size > 0:
progress = (downloaded / total_size) * 100
# Time remaining (in seconds)
time_remaining = (total_size - downloaded) / (speed * 1024)
current_time = time.time()
if current_time - last_update_time >= 0.5:
last_update_time = current_time
if progress > 0:
send_m3u_update(account.id, "downloading", progress, speed=speed, elapsed_time=elapsed_time, time_remaining=time_remaining)
send_m3u_update(account.id, "downloading", 100)
except requests.exceptions.RequestException as e:
logger.error(f"Error fetching M3U from URL {account.server_url}: {e}")
return [] # Return an empty list in case of error
return []
with open(file_path, 'r', encoding='utf-8') as f:
return f.readlines()
elif account.uploaded_file:
elif account.file_path:
try:
# Open the file and return the lines as a list or iterator
with open(account.uploaded_file.path, 'r', encoding='utf-8') as f:
return f.readlines() # Ensure you return lines from the file, not the file object
except IOError as e:
logger.error(f"Error opening file {account.uploaded_file.path}: {e}")
return [] # Return an empty list in case of error
if account.file_path.endswith('.gz'):
with gzip.open(account.file_path, 'rt', encoding='utf-8') as f:
return f.readlines()
elif account.file_path.endswith('.zip'):
with zipfile.ZipFile(account.file_path, 'r') as zip_file:
for name in zip_file.namelist():
if name.endswith('.m3u'):
with zip_file.open(name) as f:
return [line.decode('utf-8') for line in f.readlines()]
logger.warning(f"No .m3u file found in ZIP archive: {account.file_path}")
return []
else:
with open(account.file_path, 'r', encoding='utf-8') as f:
return f.readlines()
except (IOError, OSError, zipfile.BadZipFile, gzip.BadGzipFile) as e:
logger.error(f"Error opening file {account.file_path}: {e}")
return []
# Return an empty list if neither server_url nor uploaded_file is available
return []
@ -105,19 +149,6 @@ def _matches_filters(stream_name: str, group_name: str, filters):
return exclude
return False
def acquire_lock(task_name, account_id):
"""Acquire a lock to prevent concurrent task execution."""
lock_id = f"task_lock_{task_name}_{account_id}"
lock_acquired = cache.add(lock_id, "locked", timeout=LOCK_EXPIRE)
if not lock_acquired:
logger.warning(f"Lock for {task_name} and account_id={account_id} already acquired. Task will not proceed.")
return lock_acquired
def release_lock(task_name, account_id):
"""Release the lock after task execution."""
lock_id = f"task_lock_{task_name}_{account_id}"
cache.delete(lock_id)
@shared_task
def refresh_m3u_accounts():
"""Queue background parse for all active M3UAccounts."""
@ -175,20 +206,15 @@ def process_groups(account, group_names):
)
@shared_task
def process_m3u_batch(account_id, batch, group_names, hash_keys):
def process_m3u_batch(account_id, batch, groups, hash_keys):
"""Processes a batch of M3U streams using bulk operations."""
account = M3UAccount.objects.get(id=account_id)
existing_groups = {group.name: group for group in ChannelGroup.objects.filter(
m3u_account__m3u_account=account, # Filter by the M3UAccount
m3u_account__enabled=True # Filter by the enabled flag in the join table
)}
streams_to_create = []
streams_to_update = []
stream_hashes = {}
# compiled_filters = [(f.filter_type, re.compile(f.regex_pattern, re.IGNORECASE)) for f in filters]
logger.debug(f"Processing batch of {len(batch)}")
for stream_info in batch:
name, url = stream_info["name"], stream_info["url"]
@ -196,7 +222,7 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys):
group_title = stream_info["attributes"].get("group-title", "Default Group")
# Filter out disabled groups for this account
if group_title not in existing_groups:
if group_title not in groups:
logger.debug(f"Skipping stream in disabled group: {group_title}")
continue
@ -213,19 +239,20 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys):
try:
stream_hash = Stream.generate_hash_key(name, url, tvg_id, hash_keys)
if redis_client.exists(f"m3u_refresh:{stream_hash}"):
# duplicate already processed by another batch
continue
# if redis_client.exists(f"m3u_refresh:{stream_hash}"):
# # duplicate already processed by another batch
# continue
redis_client.set(f"m3u_refresh:{stream_hash}", "true")
# redis_client.set(f"m3u_refresh:{stream_hash}", "true")
stream_props = {
"name": name,
"url": url,
"logo_url": tvg_logo,
"tvg_id": tvg_id,
"m3u_account": account,
"channel_group": existing_groups[group_title],
"channel_group_id": int(groups.get(group_title)),
"stream_hash": stream_hash,
"custom_properties": json.dumps(stream_info["attributes"]),
}
if stream_hash not in stream_hashes:
@ -235,20 +262,17 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys):
logger.error(json.dumps(stream_info))
existing_streams = {s.stream_hash: s for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys())}
logger.info(f"Hashed {len(stream_hashes.keys())} unique streams")
for stream_hash, stream_props in stream_hashes.items():
if stream_hash in existing_streams:
obj = existing_streams[stream_hash]
changed = False
for key, value in stream_props.items():
if getattr(obj, key) == value:
continue
changed = True
setattr(obj, key, value)
existing_attr = {field.name: getattr(obj, field.name) for field in Stream._meta.fields if field != 'channel_group_id'}
changed = any(existing_attr[key] != value for key, value in stream_props.items() if key != 'channel_group_id')
obj.last_seen = timezone.now()
if changed:
for key, value in stream_props.items():
setattr(obj, key, value)
obj.last_seen = timezone.now()
streams_to_update.append(obj)
del existing_streams[stream_hash]
else:
@ -258,17 +282,23 @@ def process_m3u_batch(account_id, batch, group_names, hash_keys):
streams_to_create.append(Stream(**stream_props))
try:
if streams_to_create:
Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True)
if streams_to_update:
Stream.objects.bulk_update(streams_to_update, stream_props.keys())
if len(existing_streams.keys()) > 0:
Stream.objects.bulk_update(existing_streams.values(), ["last_seen"])
with transaction.atomic():
if streams_to_create:
Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True)
if streams_to_update:
Stream.objects.bulk_update(streams_to_update, { key for key in stream_props.keys() if key not in ["m3u_account", "stream_hash"] and key not in hash_keys})
# if len(existing_streams.keys()) > 0:
# Stream.objects.bulk_update(existing_streams.values(), ["last_seen"])
except Exception as e:
logger.error(f"Bulk create failed: {str(e)}")
retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated."
return f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated."
# Aggressive garbage collection
del streams_to_create, streams_to_update, stream_hashes, existing_streams
gc.collect()
return retval
def cleanup_streams(account_id):
account = M3UAccount.objects.get(id=account_id, is_active=True)
@ -290,25 +320,21 @@ def cleanup_streams(account_id):
logger.info(f"Cleanup complete")
def refresh_m3u_groups(account_id):
if not acquire_lock('refresh_m3u_account_groups', account_id):
@shared_task
def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False):
if not acquire_task_lock('refresh_m3u_account_groups', account_id):
return f"Task already running for account_id={account_id}.", None
# Record start time
start_time = time.time()
send_progress_update(0, account_id)
try:
account = M3UAccount.objects.get(id=account_id, is_active=True)
except M3UAccount.DoesNotExist:
release_lock('refresh_m3u_account_groups', account_id)
return f"M3UAccount with ID={account_id} not found or inactive."
release_task_lock('refresh_m3u_account_groups', account_id)
return f"M3UAccount with ID={account_id} not found or inactive.", None
lines = fetch_m3u_lines(account)
extinf_data = []
groups = set(["Default Group"])
for line in lines:
for line in fetch_m3u_lines(account, use_cache):
line = line.strip()
if line.startswith("#EXTINF"):
parsed = parse_extinf_line(line)
@ -321,6 +347,8 @@ def refresh_m3u_groups(account_id):
# Associate URL with the last EXTINF line
extinf_data[-1]["url"] = line
send_m3u_update(account_id, "processing_groups", 0)
groups = list(groups)
cache_path = os.path.join(m3u_dir, f"{account_id}.json")
with open(cache_path, 'w', encoding='utf-8') as f:
@ -331,25 +359,41 @@ def refresh_m3u_groups(account_id):
process_groups(account, groups)
release_lock('refresh_m3u_account_groups`', account_id)
release_task_lock('refresh_m3u_account_groups', account_id)
send_m3u_update(account_id, "processing_groups", 100)
if not full_refresh:
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'updates',
{
'type': 'update',
"data": {"success": True, "type": "m3u_group_refresh", "account": account_id}
}
)
return extinf_data, groups
@shared_task
def refresh_single_m3u_account(account_id, use_cache=False):
def refresh_single_m3u_account(account_id):
"""Splits M3U processing into chunks and dispatches them as parallel tasks."""
if not acquire_lock('refresh_single_m3u_account', account_id):
if not acquire_task_lock('refresh_single_m3u_account', account_id):
return f"Task already running for account_id={account_id}."
# redis_client = RedisClient.get_client()
# Record start time
start_time = time.time()
send_progress_update(0, account_id)
try:
account = M3UAccount.objects.get(id=account_id, is_active=True)
if not account.is_active:
logger.info(f"Account {account_id} is not active, skipping.")
return
filters = list(account.filters.all())
except M3UAccount.DoesNotExist:
release_lock('refresh_single_m3u_account', account_id)
release_task_lock('refresh_single_m3u_account', account_id)
return f"M3UAccount with ID={account_id} not found or inactive."
# Fetch M3U lines and handle potential issues
@ -358,7 +402,7 @@ def refresh_single_m3u_account(account_id, use_cache=False):
groups = None
cache_path = os.path.join(m3u_dir, f"{account_id}.json")
if use_cache and os.path.exists(cache_path):
if os.path.exists(cache_path):
with open(cache_path, 'r') as file:
data = json.load(file)
@ -367,15 +411,24 @@ def refresh_single_m3u_account(account_id, use_cache=False):
if not extinf_data:
try:
extinf_data, groups = refresh_m3u_groups(account_id)
extinf_data, groups = refresh_m3u_groups(account_id, full_refresh=True)
if not extinf_data or not groups:
release_task_lock('refresh_single_m3u_account', account_id)
return "Failed to update m3u account, task may already be running"
except:
release_task_lock('refresh_single_m3u_account', account_id)
return "Failed to update m3u account"
hash_keys = CoreSettings.get_m3u_hash_key().split(",")
existing_groups = {group.name: group.id for group in ChannelGroup.objects.filter(
m3u_account__m3u_account=account, # Filter by the M3UAccount
m3u_account__enabled=True # Filter by the enabled flag in the join table
)}
# Break into batches and process in parallel
batches = [extinf_data[i:i + BATCH_SIZE] for i in range(0, len(extinf_data), BATCH_SIZE)]
task_group = group(process_m3u_batch.s(account_id, batch, groups, hash_keys) for batch in batches)
task_group = group(process_m3u_batch.s(account_id, batch, existing_groups, hash_keys) for batch in batches)
total_batches = len(batches)
completed_batches = 0
@ -399,7 +452,7 @@ def refresh_single_m3u_account(account_id, use_cache=False):
if progress == 100:
progress = 99
send_progress_update(progress, account_id)
send_m3u_update(account_id, "parsing", progress)
# Optionally remove completed task from the group to prevent processing it again
result.remove(async_result)
@ -408,33 +461,54 @@ def refresh_single_m3u_account(account_id, use_cache=False):
# Run cleanup
cleanup_streams(account_id)
send_progress_update(100, account_id)
send_m3u_update(account_id, "parsing", 100)
end_time = time.time()
# Calculate elapsed time
elapsed_time = end_time - start_time
account.save(update_fields=['updated_at'])
print(f"Function took {elapsed_time} seconds to execute.")
release_lock('refresh_single_m3u_account', account_id)
# Aggressive garbage collection
del existing_groups, extinf_data, groups, batches
gc.collect()
cursor = 0
while True:
cursor, keys = redis_client.scan(cursor, match=f"m3u_refresh:*", count=BATCH_SIZE)
if keys:
redis_client.delete(*keys) # Delete the matching keys
if cursor == 0:
break
# Clean up cache file since we've fully processed it
if os.path.exists(cache_path):
os.remove(cache_path)
release_task_lock('refresh_single_m3u_account', account_id)
# cursor = 0
# while True:
# cursor, keys = redis_client.scan(cursor, match=f"m3u_refresh:*", count=BATCH_SIZE)
# if keys:
# redis_client.delete(*keys) # Delete the matching keys
# if cursor == 0:
# break
return f"Dispatched jobs complete."
def send_progress_update(progress, account_id):
def send_m3u_update(account_id, action, progress, **kwargs):
# Start with the base data dictionary
data = {
"progress": progress,
"type": "m3u_refresh",
"account": account_id,
"action": action,
}
# Add the additional key-value pairs from kwargs
data.update(kwargs)
# Now, send the updated data dictionary
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'updates',
{
'type': 'update',
"data": {"progress": progress, "type": "m3u_refresh", "account": account_id}
'data': data
}
)

12
apps/output/urls.py
View file

@ -5,12 +5,12 @@ from core.views import stream_view
app_name = 'output'
urlpatterns = [
# Allow both `/m3u` and `/m3u/`
re_path(r'^m3u/?$', generate_m3u, name='generate_m3u'),
# Allow both `/epg` and `/epg/`
re_path(r'^epg/?$', generate_epg, name='generate_epg'),
# Allow `/m3u`, `/m3u/`, `/m3u/profile_name`, and `/m3u/profile_name/`
re_path(r'^m3u(?:/(?P<profile_name>[^/]+))?/?$', generate_m3u, name='generate_m3u'),
# Allow `/epg`, `/epg/`, `/epg/profile_name`, and `/epg/profile_name/`
re_path(r'^epg(?:/(?P<profile_name>[^/]+))?/?$', generate_epg, name='generate_epg'),
# Allow both `/stream/<int:stream_id>` and `/stream/<int:stream_id>/`
re_path(r'^stream/(?P<channel_uuid>[0-9a-fA-F\-]+)/?$', stream_view, name='stream'),
]

193
apps/output/views.py
View file

@ -1,21 +1,32 @@
from django.http import HttpResponse
from django.urls import reverse
from apps.channels.models import Channel
from apps.channels.models import Channel, ChannelProfile
from apps.epg.models import ProgramData
from django.utils import timezone
from datetime import datetime, timedelta
import re
import html # Add this import for XML escaping
def generate_m3u(request):
def generate_m3u(request, profile_name=None):
"""
Dynamically generate an M3U file from channels.
The stream URL now points to the new stream_view that uses StreamProfile.
"""
if profile_name is not None:
channel_profile = ChannelProfile.objects.get(name=profile_name)
channels = Channel.objects.filter(
channelprofilemembership__channel_profile=channel_profile,
channelprofilemembership__enabled=True
).order_by('channel_number')
else:
channels = Channel.objects.order_by('channel_number')
m3u_content = "#EXTM3U\n"
channels = Channel.objects.order_by('channel_number')
for channel in channels:
group_title = channel.channel_group.name if channel.channel_group else "Default"
tvg_id = channel.tvg_id or ""
tvg_name = channel.tvg_name or channel.name
tvg_logo = channel.logo_url or ""
tvg_id = channel.channel_number or channel.id
tvg_name = channel.name
tvg_logo = channel.logo.url if channel.logo else ""
channel_number = channel.channel_number
extinf_line = (
@ -33,44 +44,162 @@ def generate_m3u(request):
response['Content-Disposition'] = 'attachment; filename="channels.m3u"'
return response
def generate_epg(request):
def generate_dummy_epg(name, channel_id, num_days=7, interval_hours=4):
xml_lines = []
# Loop through the number of days
for day_offset in range(num_days):
current_day = datetime.now() + timedelta(days=day_offset)
# Loop through each 4-hour interval in the day
for hour in range(0, 24, interval_hours):
start_time = current_day.replace(hour=hour, minute=0, second=0, microsecond=0)
stop_time = start_time + timedelta(hours=interval_hours)
# Format the times as per the requested format
start_str = start_time.strftime("%Y%m%d%H%M%S") + " 0000"
stop_str = stop_time.strftime("%Y%m%d%H%M%S") + " 0000"
# Create the XML-like programme entry with escaped name
xml_lines.append(f'<programme start="{start_str}" stop="{stop_str}" channel="{channel_id}">')
xml_lines.append(f' <title lang="en">{html.escape(name)}</title>')
xml_lines.append(f'</programme>')
return xml_lines
def generate_epg(request, profile_name=None):
"""
Dynamically generate an XMLTV (EPG) file using the new EPGData/ProgramData models.
Since the EPG data is stored independently of Channels, we group programmes
by their associated EPGData record.
This version does not filter by time, so it includes the entire EPG saved in the DB.
"""
# Retrieve all ProgramData records and join the related EPGData record.
programs = ProgramData.objects.select_related('epg').all().order_by('start_time')
# Group programmes by their EPGData record.
epg_programs = {}
for prog in programs:
epg = prog.epg
epg_programs.setdefault(epg, []).append(prog)
xml_lines = []
xml_lines.append('<?xml version="1.0" encoding="UTF-8"?>')
xml_lines.append('<tv generator-info-name="Dispatcharr" generator-info-url="https://example.com">')
xml_lines.append('<tv generator-info-name="Dispatcharr" generator-info-url="https://github.com/Dispatcharr/Dispatcharr">')
# Output channel definitions based on EPGData.
# Use the EPGData's tvg_id (or a fallback) as the channel identifier.
for epg in epg_programs.keys():
channel_id = epg.tvg_id if epg.tvg_id else f"default-{epg.id}"
if profile_name is not None:
channel_profile = ChannelProfile.objects.get(name=profile_name)
channels = Channel.objects.filter(
channelprofilemembership__channel_profile=channel_profile,
channelprofilemembership__enabled=True
)
else:
channels = Channel.objects.all()
# Retrieve all active channels
for channel in channels:
channel_id = channel.channel_number or channel.id
display_name = channel.epg_data.name if channel.epg_data else channel.name
xml_lines.append(f' <channel id="{channel_id}">')
xml_lines.append(f' <display-name>{epg.name}</display-name>')
xml_lines.append(f' <display-name>{html.escape(display_name)}</display-name>')
# Add channel logo if available
if channel.logo:
logo_url = channel.logo.url
# Convert to absolute URL if it's relative
if logo_url.startswith('/data'):
# Use the full URL for the logo
logo_uri = re.sub(r"^\/data", '', logo_url)
base_url = request.build_absolute_uri('/')[:-1]
logo_url = f"{base_url}{logo_uri}"
xml_lines.append(f' <icon src="{html.escape(logo_url)}" />')
xml_lines.append(' </channel>')
# Output programme entries referencing the channel id from EPGData.
for epg, progs in epg_programs.items():
channel_id = epg.tvg_id if epg.tvg_id else f"default-{epg.id}"
for prog in progs:
start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z")
stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z")
xml_lines.append(f' <programme start="{start_str}" stop="{stop_str}" channel="{channel_id}">')
xml_lines.append(f' <title>{prog.title}</title>')
xml_lines.append(f' <desc>{prog.description}</desc>')
xml_lines.append(' </programme>')
for channel in channels:
channel_id = channel.channel_number or channel.id
display_name = channel.epg_data.name if channel.epg_data else channel.name
if not channel.epg_data:
xml_lines = xml_lines + generate_dummy_epg(display_name, channel_id)
else:
programs = channel.epg_data.programs.all()
for prog in programs:
start_str = prog.start_time.strftime("%Y%m%d%H%M%S %z")
stop_str = prog.end_time.strftime("%Y%m%d%H%M%S %z")
xml_lines.append(f' <programme start="{start_str}" stop="{stop_str}" channel="{channel_id}">')
xml_lines.append(f' <title>{html.escape(prog.title)}</title>')
# Add subtitle if available
if prog.sub_title:
xml_lines.append(f' <sub-title>{html.escape(prog.sub_title)}</sub-title>')
# Add description if available
if prog.description:
xml_lines.append(f' <desc>{html.escape(prog.description)}</desc>')
# Process custom properties if available
if prog.custom_properties:
try:
import json
custom_data = json.loads(prog.custom_properties)
# Add categories if available
if 'categories' in custom_data and custom_data['categories']:
for category in custom_data['categories']:
xml_lines.append(f' <category>{html.escape(category)}</category>')
# Handle episode numbering - multiple formats supported
# Standard episode number if available
if 'episode' in custom_data:
xml_lines.append(f' <episode-num system="onscreen">E{custom_data["episode"]}</episode-num>')
# Handle onscreen episode format (like S06E128)
if 'onscreen_episode' in custom_data:
xml_lines.append(f' <episode-num system="onscreen">{html.escape(custom_data["onscreen_episode"])}</episode-num>')
# Add season and episode numbers in xmltv_ns format if available
if 'season' in custom_data and 'episode' in custom_data:
season = int(custom_data['season']) - 1 if str(custom_data['season']).isdigit() else 0
episode = int(custom_data['episode']) - 1 if str(custom_data['episode']).isdigit() else 0
xml_lines.append(f' <episode-num system="xmltv_ns">{season}.{episode}.</episode-num>')
# Add rating if available
if 'rating' in custom_data:
rating_system = custom_data.get('rating_system', 'TV Parental Guidelines')
xml_lines.append(f' <rating system="{html.escape(rating_system)}">')
xml_lines.append(f' <value>{html.escape(custom_data["rating"])}</value>')
xml_lines.append(f' </rating>')
# Add actors/directors/writers if available
if 'credits' in custom_data:
xml_lines.append(f' <credits>')
for role, people in custom_data['credits'].items():
if isinstance(people, list):
for person in people:
xml_lines.append(f' <{role}>{html.escape(person)}</{role}>')
else:
xml_lines.append(f' <{role}>{html.escape(people)}</{role}>')
xml_lines.append(f' </credits>')
# Add program date/year if available
if 'year' in custom_data:
xml_lines.append(f' <date>{html.escape(custom_data["year"])}</date>')
# Add country if available
if 'country' in custom_data:
xml_lines.append(f' <country>{html.escape(custom_data["country"])}</country>')
# Add icon if available
if 'icon' in custom_data:
xml_lines.append(f' <icon src="{html.escape(custom_data["icon"])}" />')
# Add special flags as proper tags
if custom_data.get('previously_shown', False):
xml_lines.append(f' <previously-shown />')
if custom_data.get('premiere', False):
xml_lines.append(f' <premiere />')
if custom_data.get('new', False):
xml_lines.append(f' <new />')
except Exception as e:
xml_lines.append(f' <!-- Error parsing custom properties: {html.escape(str(e))} -->')
xml_lines.append(' </programme>')
xml_lines.append('</tv>')
xml_content = "\n".join(xml_lines)

14
apps/proxy/apps.py
View file

@ -1,3 +1,4 @@
import sys
from django.apps import AppConfig
class ProxyConfig(AppConfig):
@ -7,9 +8,10 @@ class ProxyConfig(AppConfig):
def ready(self):
"""Initialize proxy servers when Django starts"""
from .hls_proxy.server import ProxyServer as HLSProxyServer
from .ts_proxy.server import ProxyServer as TSProxyServer
# Initialize proxy servers
self.hls_proxy = HLSProxyServer()
self.ts_proxy = TSProxyServer()
if 'manage.py' not in sys.argv:
from .hls_proxy.server import ProxyServer as HLSProxyServer
from .ts_proxy.server import ProxyServer as TSProxyServer
# Initialize proxy servers
self.hls_proxy = HLSProxyServer()
self.ts_proxy = TSProxyServer()

19
apps/proxy/config.py
View file

@ -5,6 +5,10 @@ class BaseConfig:
CHUNK_SIZE = 8192
CLIENT_POLL_INTERVAL = 0.1
MAX_RETRIES = 3
RETRY_WAIT_INTERVAL = 0.5 # seconds to wait between retries
CONNECTION_TIMEOUT = 10 # seconds to wait for initial connection
MAX_STREAM_SWITCHES = 10 # Maximum number of stream switch attempts before giving up
BUFFER_CHUNK_SIZE = 188 * 1361 # ~256KB
# Redis settings
REDIS_CHUNK_TTL = 60 # Number in seconds - Chunks expire after 1 minute
@ -24,10 +28,6 @@ class HLSConfig(BaseConfig):
class TSConfig(BaseConfig):
"""Configuration settings for TS proxy"""
# Connection settings
CONNECTION_TIMEOUT = 10 # seconds to wait for initial connection
MAX_RETRIES = 3 # maximum connection retry attempts
# Buffer settings
INITIAL_BEHIND_CHUNKS = 4 # How many chunks behind to start a client (4 chunks = ~1MB)
CHUNK_BATCH_SIZE = 5 # How many chunks to fetch in one batch
@ -52,8 +52,13 @@ class TSConfig(BaseConfig):
# TS packets are 188 bytes
# Make chunk size a multiple of TS packet size for perfect alignment
# ~1MB is ideal for streaming (matches typical media buffer sizes)
BUFFER_CHUNK_SIZE = 188 * 1361 # ~256KB
# Maximum number of stream switch attempts before giving up
MAX_STREAM_SWITCHES = 10
# Stream health and recovery settings
MAX_HEALTH_RECOVERY_ATTEMPTS = 2 # Maximum times to attempt recovery for a single stream
MAX_RECONNECT_ATTEMPTS = 3 # Maximum reconnects to try before switching streams
MIN_STABLE_TIME_BEFORE_RECONNECT = 30 # Minimum seconds a stream must be stable to try reconnect
FAILOVER_GRACE_PERIOD = 20 # Extra time (seconds) to allow for stream switching before disconnecting clients

4
apps/proxy/tasks.py
View file

@ -6,7 +6,7 @@ import redis
import json
import logging
import re
from core.utils import redis_client
from core.utils import RedisClient
from apps.proxy.ts_proxy.channel_status import ChannelStatus
logger = logging.getLogger(__name__)
@ -16,6 +16,8 @@ last_known_data = {}
@shared_task
def fetch_channel_stats():
redis_client = RedisClient.get_client()
try:
# Basic info for all channels
channel_pattern = "ts_proxy:channel:*:metadata"

9
apps/proxy/ts_proxy/__init__.py
View file

@ -1,9 +0,0 @@
"""Transport Stream proxy module"""
# Only class imports, no instance creation
from .server import ProxyServer
from .stream_manager import StreamManager
from .stream_buffer import StreamBuffer
from .client_manager import ClientManager
proxy_server = ProxyServer()

13
apps/proxy/ts_proxy/apps.py Normal file
View file

@ -0,0 +1,13 @@
import sys
from django.apps import AppConfig
class TSProxyConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'apps.proxy.ts_proxy'
verbose_name = "TS Stream Proxies"
def ready(self):
"""Initialize proxy servers when Django starts"""
if 'manage.py' not in sys.argv:
from .server import ProxyServer
ProxyServer.get_instance()

51
apps/proxy/ts_proxy/channel_status.py
View file

@ -1,9 +1,9 @@
import logging
import time
import re
from . import proxy_server
from .server import ProxyServer
from .redis_keys import RedisKeys
from .constants import TS_PACKET_SIZE
from .constants import TS_PACKET_SIZE, ChannelMetadataField
from redis.exceptions import ConnectionError, TimeoutError
from .utils import get_logger
@ -22,6 +22,8 @@ class ChannelStatus:
return (total_bytes * 8) / duration / 1000
def get_detailed_channel_info(channel_id):
proxy_server = ProxyServer.get_instance()
# Get channel metadata
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
@ -35,28 +37,31 @@ class ChannelStatus:
info = {
'channel_id': channel_id,
'state': metadata.get(b'state', b'unknown').decode('utf-8'),
'url': metadata.get(b'url', b'').decode('utf-8'),
'profile': metadata.get(b'profile', b'unknown').decode('utf-8'),
'started_at': metadata.get(b'init_time', b'0').decode('utf-8'),
'owner': metadata.get(b'owner', b'unknown').decode('utf-8'),
'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'),
'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'),
'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'),
'started_at': metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'),
'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'),
'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0,
}
# Add timing information
if b'state_changed_at' in metadata:
state_changed_at = float(metadata[b'state_changed_at'].decode('utf-8'))
state_changed_field = ChannelMetadataField.STATE_CHANGED_AT.encode('utf-8')
if state_changed_field in metadata:
state_changed_at = float(metadata[state_changed_field].decode('utf-8'))
info['state_changed_at'] = state_changed_at
info['state_duration'] = time.time() - state_changed_at
if b'init_time' in metadata:
created_at = float(metadata[b'init_time'].decode('utf-8'))
init_time_field = ChannelMetadataField.INIT_TIME.encode('utf-8')
if init_time_field in metadata:
created_at = float(metadata[init_time_field].decode('utf-8'))
info['started_at'] = created_at
info['uptime'] = time.time() - created_at
# Add data throughput information
if b'total_bytes' in metadata:
total_bytes = int(metadata[b'total_bytes'].decode('utf-8'))
total_bytes_field = ChannelMetadataField.TOTAL_BYTES.encode('utf-8')
if total_bytes_field in metadata:
total_bytes = int(metadata[total_bytes_field].decode('utf-8'))
info['total_bytes'] = total_bytes
# Format total bytes in human-readable form
@ -87,7 +92,7 @@ class ChannelStatus:
for client_id in client_ids:
client_id_str = client_id.decode('utf-8')
client_key = f"ts_proxy:channel:{channel_id}:clients:{client_id_str}"
client_key = RedisKeys.client_metadata(channel_id, client_id_str)
client_data = proxy_server.redis_client.hgetall(client_key)
if client_data:
@ -227,6 +232,8 @@ class ChannelStatus:
@staticmethod
def _execute_redis_command(command_func):
"""Execute Redis command with error handling"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return None
@ -242,6 +249,8 @@ class ChannelStatus:
@staticmethod
def get_basic_channel_info(channel_id):
"""Get basic channel information with Redis error handling"""
proxy_server = ProxyServer.get_instance()
try:
# Use _execute_redis_command for Redis operations
metadata_key = RedisKeys.channel_metadata(channel_id)
@ -261,23 +270,23 @@ class ChannelStatus:
client_count = proxy_server.redis_client.scard(client_set_key) or 0
# Calculate uptime
created_at = float(metadata.get(b'init_time', b'0').decode('utf-8'))
created_at = float(metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'))
uptime = time.time() - created_at if created_at > 0 else 0
# Simplified info
info = {
'channel_id': channel_id,
'state': metadata.get(b'state', b'unknown').decode('utf-8'),
'url': metadata.get(b'url', b'').decode('utf-8'),
'profile': metadata.get(b'profile', b'unknown').decode('utf-8'),
'owner': metadata.get(b'owner', b'unknown').decode('utf-8'),
'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'),
'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'),
'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'),
'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'),
'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0,
'client_count': client_count,
'uptime': uptime
}
# Add data throughput information to basic info
total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, 'total_bytes')
total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES.encode('utf-8'))
if total_bytes_bytes:
total_bytes = int(total_bytes_bytes.decode('utf-8'))
info['total_bytes'] = total_bytes
@ -307,7 +316,7 @@ class ChannelStatus:
# Get up to 10 clients for the basic view
for client_id in list(client_ids)[:10]:
client_id_str = client_id.decode('utf-8')
client_key = f"ts_proxy:channel:{channel_id}:clients:{client_id_str}"
client_key = RedisKeys.client_metadata(channel_id, client_id_str)
# Efficient way - just retrieve the essentials
client_info = {

17
apps/proxy/ts_proxy/client_manager.py
View file

@ -38,6 +38,11 @@ class ClientManager:
def _start_heartbeat_thread(self):
"""Start thread to regularly refresh client presence in Redis"""
def heartbeat_task():
no_clients_count = 0 # Track consecutive empty cycles
max_empty_cycles = 3 # Exit after this many consecutive empty checks
logger.debug(f"Started heartbeat thread for channel {self.channel_id} (interval: {self.heartbeat_interval}s)")
while True:
try:
# Wait for the interval
@ -46,7 +51,19 @@ class ClientManager:
# Send heartbeat for all local clients
with self.lock:
if not self.clients or not self.redis_client:
# No clients left, increment our counter
no_clients_count += 1
# If we've seen no clients for several consecutive checks, exit the thread
if no_clients_count >= max_empty_cycles:
logger.info(f"No clients for channel {self.channel_id} after {no_clients_count} consecutive checks, exiting heartbeat thread")
return # This exits the thread
# Skip this cycle if we have no clients
continue
else:
# Reset counter when we see clients
no_clients_count = 0
# IMPROVED GHOST DETECTION: Check for stale clients before sending heartbeats
current_time = time.time()

5
apps/proxy/ts_proxy/config_helper.py
View file

@ -70,3 +70,8 @@ class ConfigHelper:
def max_stream_switches():
"""Get maximum number of stream switch attempts"""
return ConfigHelper.get('MAX_STREAM_SWITCHES', 10)
@staticmethod
def retry_wait_interval():
"""Get wait interval between connection retries in seconds"""
return ConfigHelper.get('RETRY_WAIT_INTERVAL', 0.5) # Default to 0.5 second

39
apps/proxy/ts_proxy/constants.py
View file

@ -35,6 +35,45 @@ class StreamType:
TS = "ts"
UNKNOWN = "unknown"
# Channel metadata field names stored in Redis
class ChannelMetadataField:
# Basic fields
URL = "url"
USER_AGENT = "user_agent"
STATE = "state"
OWNER = "owner"
STREAM_ID = "stream_id"
# Profile fields
STREAM_PROFILE = "stream_profile"
M3U_PROFILE = "m3u_profile"
# Status and error fields
ERROR_MESSAGE = "error_message"
ERROR_TIME = "error_time"
STATE_CHANGED_AT = "state_changed_at"
INIT_TIME = "init_time"
CONNECTION_READY_TIME = "connection_ready_time"
# Buffer and data tracking
BUFFER_CHUNKS = "buffer_chunks"
TOTAL_BYTES = "total_bytes"
# Stream switching
STREAM_SWITCH_TIME = "stream_switch_time"
STREAM_SWITCH_REASON = "stream_switch_reason"
# Client metadata fields
CONNECTED_AT = "connected_at"
LAST_ACTIVE = "last_active"
BYTES_SENT = "bytes_sent"
AVG_RATE_KBPS = "avg_rate_KBps"
CURRENT_RATE_KBPS = "current_rate_KBps"
IP_ADDRESS = "ip_address"
WORKER_ID = "worker_id"
CHUNKS_SENT = "chunks_sent"
STATS_UPDATED_AT = "stats_updated_at"
# TS packet constants
TS_PACKET_SIZE = 188
TS_SYNC_BYTE = 0x47

110
apps/proxy/ts_proxy/server.py
View file

@ -17,8 +17,8 @@ import os
import json
from typing import Dict, Optional, Set
from apps.proxy.config import TSConfig as Config
from apps.channels.models import Channel
from core.utils import redis_client as global_redis_client, redis_pubsub_client as global_redis_pubsub_client # Import both global Redis clients
from apps.channels.models import Channel, Stream
from core.utils import RedisClient
from redis.exceptions import ConnectionError, TimeoutError
from .stream_manager import StreamManager
from .stream_buffer import StreamBuffer
@ -32,6 +32,19 @@ logger = get_logger()
class ProxyServer:
"""Manages TS proxy server instance with worker coordination"""
_instance = None
@classmethod
def get_instance(cls):
if cls._instance is None:
from .server import ProxyServer
from .stream_manager import StreamManager
from .stream_buffer import StreamBuffer
from .client_manager import ClientManager
cls._instance = ProxyServer()
return cls._instance
def __init__(self):
"""Initialize proxy server with worker identification"""
@ -46,17 +59,17 @@ class ProxyServer:
hostname = socket.gethostname()
self.worker_id = f"{hostname}:{pid}"
# Connect to Redis - try using global client first
# Connect to Redis - use dedicated client for proxy
self.redis_client = None
self.redis_connection_attempts = 0
self.redis_max_retries = 3
self.redis_retry_interval = 5 # seconds
try:
# First try to use the global client from core.utils
if global_redis_client is not None:
self.redis_client = global_redis_client
logger.info(f"Using global Redis client")
# Use dedicated Redis client for proxy
self.redis_client = RedisClient.get_client()
if self.redis_client is not None:
logger.info(f"Using dedicated Redis client for proxy server")
logger.info(f"Worker ID: {self.worker_id}")
else:
# Fall back to direct connection with retry
@ -75,50 +88,14 @@ class ProxyServer:
def _setup_redis_connection(self):
"""Setup Redis connection with retry logic"""
import redis
from django.conf import settings
while self.redis_connection_attempts < self.redis_max_retries:
try:
logger.info(f"Attempting to connect to Redis ({self.redis_connection_attempts+1}/{self.redis_max_retries})")
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
# Create Redis client with reasonable timeouts
self.redis_client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
socket_timeout=5,
socket_connect_timeout=5,
retry_on_timeout=True,
health_check_interval=30
)
# Test connection
self.redis_client.ping()
logger.info(f"Successfully connected to Redis at {redis_host}:{redis_port}/{redis_db}")
logger.info(f"Worker ID: {self.worker_id}")
break
except (ConnectionError, TimeoutError) as e:
self.redis_connection_attempts += 1
if self.redis_connection_attempts >= self.redis_max_retries:
logger.error(f"Failed to connect to Redis after {self.redis_max_retries} attempts: {e}")
self.redis_client = None
else:
# Exponential backoff with a maximum of 30 seconds
retry_delay = min(self.redis_retry_interval * (2 ** (self.redis_connection_attempts - 1)), 30)
logger.warning(f"Redis connection failed. Retrying in {retry_delay}s... ({self.redis_connection_attempts}/{self.redis_max_retries})")
time.sleep(retry_delay)
except Exception as e:
logger.error(f"Unexpected error connecting to Redis: {e}", exc_info=True)
self.redis_client = None
break
# Try to use get_redis_client utility instead of direct connection
self.redis_client = RedisClient.get_client(max_retries=self.redis_max_retries,
retry_interval=self.redis_retry_interval)
if self.redis_client:
logger.info(f"Successfully connected to Redis using utility function")
logger.info(f"Worker ID: {self.worker_id}")
else:
logger.error(f"Failed to connect to Redis after {self.redis_max_retries} attempts")
def _execute_redis_command(self, command_func, *args, **kwargs):
"""Execute Redis command with error handling and reconnection logic"""
@ -156,12 +133,13 @@ class ProxyServer:
while True:
try:
# Use the global PubSub client if available
if global_redis_pubsub_client:
pubsub_client = global_redis_pubsub_client
logger.info("Using global Redis PubSub client for event listener")
# Use dedicated PubSub client for event listener
pubsub_client = RedisClient.get_pubsub_client()
if pubsub_client:
logger.info("Using dedicated Redis PubSub client for event listener")
else:
# Fall back to creating a dedicated client if global one is unavailable
# Fall back to creating a dedicated client if utility fails
logger.warning("Utility function for PubSub client failed, creating direct connection")
from django.conf import settings
import redis
@ -178,7 +156,7 @@ class ProxyServer:
socket_keepalive=True,
health_check_interval=30
)
logger.info("Created dedicated Redis PubSub client for event listener")
logger.info("Created fallback Redis PubSub client for event listener")
# Test connection before subscribing
pubsub_client.ping()
@ -740,12 +718,16 @@ class ProxyServer:
# Force release resources in the Channel model
try:
from apps.channels.models import Channel
channel = Channel.objects.get(uuid=channel_id)
channel.release_stream()
logger.info(f"Released stream allocation for zombie channel {channel_id}")
except Exception as e:
logger.error(f"Error releasing stream for zombie channel {channel_id}: {e}")
try:
stream = Stream.objects.get(stream_hash=channel_id)
stream.release_stream()
logger.info(f"Released stream allocation for zombie channel {channel_id}")
except Exception as e:
logger.error(f"Error releasing stream for zombie channel {channel_id}: {e}")
return True
except Exception as e:
@ -1067,8 +1049,12 @@ class ProxyServer:
def _clean_redis_keys(self, channel_id):
"""Clean up all Redis keys for a channel more efficiently"""
# Release the channel, stream, and profile keys from the channel
channel = Channel.objects.get(uuid=channel_id)
channel.release_stream()
try:
channel = Channel.objects.get(uuid=channel_id)
channel.release_stream()
except:
stream = Stream.objects.get(stream_hash=channel_id)
stream.release_stream()
if not self.redis_client:
return 0
@ -1149,7 +1135,7 @@ class ProxyServer:
self.redis_client.hset(metadata_key, mapping=update_data)
# Log the transition
logger.info(f"Channel {channel_id} state transition: {current_state or 'None'} → {new_state}")
logger.info(f"Channel {channel_id} state transition: {current_state or 'None'} -> {new_state}")
return True
except Exception as e:
logger.error(f"Error updating channel state: {e}")

95
apps/proxy/ts_proxy/services/channel_service.py
View file

@ -9,9 +9,9 @@ import json
from django.shortcuts import get_object_or_404
from apps.channels.models import Channel
from apps.proxy.config import TSConfig as Config
from .. import proxy_server
from ..server import ProxyServer
from ..redis_keys import RedisKeys
from ..constants import EventType, ChannelState
from ..constants import EventType, ChannelState, ChannelMetadataField
from ..url_utils import get_stream_info_for_switch
logger = logging.getLogger("ts_proxy")
@ -20,7 +20,7 @@ class ChannelService:
"""Service class for channel operations"""
@staticmethod
def initialize_channel(channel_id, stream_url, user_agent, transcode=False, profile_value=None, stream_id=None):
def initialize_channel(channel_id, stream_url, user_agent, transcode=False, stream_profile_value=None, stream_id=None, m3u_profile_id=None):
"""
Initialize a channel with the given parameters.
@ -29,12 +29,14 @@ class ChannelService:
stream_url: URL of the stream
user_agent: User agent for the stream connection
transcode: Whether to transcode the stream
profile_value: Stream profile value to store in metadata
stream_profile_value: Stream profile value to store in metadata
stream_id: ID of the stream being used
m3u_profile_id: ID of the M3U profile being used
Returns:
bool: Success status
"""
proxy_server = ProxyServer.get_instance()
# FIXED: First, ensure that Redis metadata including stream_id is set BEFORE channel initialization
# This ensures the stream ID is available when the StreamManager looks it up
if stream_id and proxy_server.redis_client:
@ -42,19 +44,19 @@ class ChannelService:
# Check if metadata already exists
if proxy_server.redis_client.exists(metadata_key):
# Just update the existing metadata with stream_id
proxy_server.redis_client.hset(metadata_key, "stream_id", str(stream_id))
proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STREAM_ID, str(stream_id))
logger.info(f"Pre-set stream ID {stream_id} in Redis for channel {channel_id}")
else:
# Create initial metadata with essential values
initial_metadata = {
"stream_id": str(stream_id),
ChannelMetadataField.STREAM_ID: str(stream_id),
"temp_init": str(time.time())
}
proxy_server.redis_client.hset(metadata_key, mapping=initial_metadata)
logger.info(f"Created initial metadata with stream_id {stream_id} for channel {channel_id}")
# Verify the stream_id was set
stream_id_value = proxy_server.redis_client.hget(metadata_key, "stream_id")
stream_id_value = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID)
if stream_id_value:
logger.info(f"Verified stream_id {stream_id_value.decode('utf-8')} is now set in Redis")
else:
@ -67,10 +69,12 @@ class ChannelService:
if success and proxy_server.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
update_data = {}
if profile_value:
update_data["profile"] = profile_value
if stream_profile_value:
update_data[ChannelMetadataField.STREAM_PROFILE] = stream_profile_value
if stream_id:
update_data["stream_id"] = str(stream_id)
update_data[ChannelMetadataField.STREAM_ID] = str(stream_id)
if m3u_profile_id:
update_data[ChannelMetadataField.M3U_PROFILE] = str(m3u_profile_id)
if update_data:
proxy_server.redis_client.hset(metadata_key, mapping=update_data)
@ -91,7 +95,10 @@ class ChannelService:
Returns:
dict: Result information including success status and diagnostics
"""
proxy_server = ProxyServer.get_instance()
# If no direct URL is provided but a target stream is, get URL from target stream
stream_id = None
if not new_url and target_stream_id:
stream_info = get_stream_info_for_switch(channel_id, target_stream_id)
if 'error' in stream_info:
@ -101,6 +108,10 @@ class ChannelService:
}
new_url = stream_info['url']
user_agent = stream_info['user_agent']
stream_id = target_stream_id
elif target_stream_id:
# If we have both URL and target_stream_id, use the target_stream_id
stream_id = target_stream_id
# Check if channel exists
in_local_managers = channel_id in proxy_server.stream_managers
@ -152,7 +163,7 @@ class ChannelService:
# Update metadata in Redis regardless of ownership
if proxy_server.redis_client:
try:
ChannelService._update_channel_metadata(channel_id, new_url, user_agent)
ChannelService._update_channel_metadata(channel_id, new_url, user_agent, stream_id)
result['metadata_updated'] = True
except Exception as e:
logger.error(f"Error updating Redis metadata: {e}", exc_info=True)
@ -177,7 +188,7 @@ class ChannelService:
# If we're not the owner, publish an event for the owner to pick up
logger.info(f"Not the owner, requesting URL change via Redis PubSub")
if proxy_server.redis_client:
ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent)
ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent, stream_id)
result.update({
'direct_update': False,
'event_published': True,
@ -203,6 +214,8 @@ class ChannelService:
Returns:
dict: Result information including previous state if available
"""
proxy_server = ProxyServer.get_instance()
# Check if channel exists
channel_exists = proxy_server.check_if_channel_exists(channel_id)
if not channel_exists:
@ -220,8 +233,8 @@ class ChannelService:
channel_info = {"state": state}
# Immediately mark as stopping in metadata so clients detect it faster
proxy_server.redis_client.hset(metadata_key, "state", ChannelState.STOPPING)
proxy_server.redis_client.hset(metadata_key, "state_changed_at", str(time.time()))
proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE, ChannelState.STOPPING)
proxy_server.redis_client.hset(metadata_key, ChannelMetadataField.STATE_CHANGED_AT, str(time.time()))
except Exception as e:
logger.error(f"Error fetching channel state: {e}")
@ -248,8 +261,11 @@ class ChannelService:
logger.info(f"Released channel {channel_id} stream allocation")
model_released = True
except Channel.DoesNotExist:
logger.warning(f"Could not find Channel model for UUID {channel_id}")
model_released = False
logger.warning(f"Could not find Channel model for UUID {channel_id}, attempting stream hash")
stream = Stream.objects.get(stream_hash=channel_id)
stream.release_stream()
logger.info(f"Released stream {channel_id} stream allocation")
model_released = True
except Exception as e:
logger.error(f"Error releasing channel stream: {e}")
model_released = False
@ -276,6 +292,7 @@ class ChannelService:
dict: Result information
"""
logger.info(f"Request to stop client {client_id} on channel {channel_id}")
proxy_server = ProxyServer.get_instance()
# Set a Redis key for immediate detection
key_set = False
@ -339,6 +356,8 @@ class ChannelService:
Returns:
tuple: (valid, state, owner, details) - validity status, current state, owner, and diagnostic info
"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return False, None, None, {"error": "Redis not available"}
@ -350,8 +369,8 @@ class ChannelService:
metadata = proxy_server.redis_client.hgetall(metadata_key)
# Extract state and owner
state = metadata.get(b'state', b'unknown').decode('utf-8')
owner = metadata.get(b'owner', b'unknown').decode('utf-8')
state = metadata.get(ChannelMetadataField.STATE.encode(), b'unknown').decode('utf-8')
owner = metadata.get(ChannelMetadataField.OWNER.encode(), b'unknown').decode('utf-8')
# Valid states indicate channel is running properly
valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING]
@ -360,7 +379,7 @@ class ChannelService:
return False, state, owner, {"error": f"Invalid state: {state}"}
# Check if owner is still active
owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat"
owner_heartbeat_key = RedisKeys.worker_heartbeat(owner)
owner_alive = proxy_server.redis_client.exists(owner_heartbeat_key)
if not owner_alive:
@ -394,8 +413,10 @@ class ChannelService:
# Helper methods for Redis operations
@staticmethod
def _update_channel_metadata(channel_id, url, user_agent=None):
def _update_channel_metadata(channel_id, url, user_agent=None, stream_id=None):
"""Update channel metadata in Redis"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return False
@ -405,23 +426,22 @@ class ChannelService:
key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8')
logger.debug(f"Redis key {metadata_key} is of type: {key_type}")
# Build metadata update dict
metadata = {ChannelMetadataField.URL: url}
if user_agent:
metadata[ChannelMetadataField.USER_AGENT] = user_agent
if stream_id:
metadata[ChannelMetadataField.STREAM_ID] = str(stream_id)
logger.info(f"Updating stream ID to {stream_id} in Redis for channel {channel_id}")
# Use the appropriate method based on the key type
if key_type == 'hash':
proxy_server.redis_client.hset(metadata_key, "url", url)
if user_agent:
proxy_server.redis_client.hset(metadata_key, "user_agent", user_agent)
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
elif key_type == 'none': # Key doesn't exist yet
# Create new hash with all required fields
metadata = {"url": url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
else:
# If key exists with wrong type, delete it and recreate
proxy_server.redis_client.delete(metadata_key)
metadata = {"url": url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
# Set switch request flag to ensure all workers see it
@ -432,16 +452,19 @@ class ChannelService:
return True
@staticmethod
def _publish_stream_switch_event(channel_id, new_url, user_agent=None):
def _publish_stream_switch_event(channel_id, new_url, user_agent=None, stream_id=None):
"""Publish a stream switch event to Redis pubsub"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return False
switch_request = {
"event": EventType.STREAM_SWITCH, # Use constant instead of string
"event": EventType.STREAM_SWITCH,
"channel_id": channel_id,
"url": new_url,
"user_agent": user_agent,
"stream_id": stream_id,
"requester": proxy_server.worker_id,
"timestamp": time.time()
}
@ -455,11 +478,13 @@ class ChannelService:
@staticmethod
def _publish_channel_stop_event(channel_id):
"""Publish a channel stop event to Redis pubsub"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return False
stop_request = {
"event": EventType.CHANNEL_STOP, # Use constant instead of string
"event": EventType.CHANNEL_STOP,
"channel_id": channel_id,
"requester_worker_id": proxy_server.worker_id,
"timestamp": time.time()
@ -476,11 +501,13 @@ class ChannelService:
@staticmethod
def _publish_client_stop_event(channel_id, client_id):
"""Publish a client stop event to Redis pubsub"""
proxy_server = ProxyServer.get_instance()
if not proxy_server.redis_client:
return False
stop_request = {
"event": EventType.CLIENT_STOP, # Use constant instead of string
"event": EventType.CLIENT_STOP,
"channel_id": channel_id,
"client_id": client_id,
"requester_worker_id": proxy_server.worker_id,

2
apps/proxy/ts_proxy/stream_buffer.py
View file

@ -94,7 +94,7 @@ class StreamBuffer:
writes_done += 1
if writes_done > 0:
logger.debug(f"Added {writes_done} optimized chunks ({self.target_chunk_size} bytes each) to Redis")
logger.debug(f"Added {writes_done} chunks ({self.target_chunk_size} bytes each) to Redis for channel {self.channel_id} at index {self.index}")
return True

68
apps/proxy/ts_proxy/stream_generator.py
View file

@ -7,10 +7,11 @@ import time
import logging
import threading
from apps.proxy.config import TSConfig as Config
from . import proxy_server
from .server import ProxyServer
from .utils import create_ts_packet, get_logger
from .redis_keys import RedisKeys
from .utils import get_logger
from .constants import ChannelMetadataField
logger = get_logger()
@ -96,6 +97,7 @@ class StreamGenerator:
max_init_wait = getattr(Config, 'CLIENT_WAIT_TIMEOUT', 30)
keepalive_interval = 0.5
last_keepalive = 0
proxy_server = ProxyServer.get_instance()
# While init is happening, send keepalive packets
while time.time() - initialization_start < max_init_wait:
@ -142,6 +144,8 @@ class StreamGenerator:
def _setup_streaming(self):
"""Setup streaming parameters and check resources."""
proxy_server = ProxyServer.get_instance()
# Get buffer - stream manager may not exist in this worker
buffer = proxy_server.stream_buffers.get(self.channel_id)
stream_manager = proxy_server.stream_managers.get(self.channel_id)
@ -217,6 +221,8 @@ class StreamGenerator:
def _check_resources(self):
"""Check if required resources still exist."""
proxy_server = ProxyServer.get_instance()
# Enhanced resource checks
if self.channel_id not in proxy_server.stream_buffers:
logger.info(f"[{self.client_id}] Channel buffer no longer exists, terminating stream")
@ -263,6 +269,7 @@ class StreamGenerator:
# Process and send chunks
total_size = sum(len(c) for c in chunks)
logger.debug(f"[{self.client_id}] Retrieved {len(chunks)} chunks ({total_size} bytes) from index {self.local_index+1} to {next_index}")
proxy_server = ProxyServer.get_instance()
# Send the chunks to the client
for chunk in chunks:
@ -298,11 +305,11 @@ class StreamGenerator:
try:
client_key = RedisKeys.client_metadata(self.channel_id, self.client_id)
stats = {
"chunks_sent": str(self.chunks_sent),
"bytes_sent": str(self.bytes_sent),
"avg_rate_KBps": str(round(avg_rate, 1)),
"current_rate_KBps": str(round(self.current_rate, 1)),
"stats_updated_at": str(current_time)
ChannelMetadataField.CHUNKS_SENT: str(self.chunks_sent),
ChannelMetadataField.BYTES_SENT: str(self.bytes_sent),
ChannelMetadataField.AVG_RATE_KBPS: str(round(avg_rate, 1)),
ChannelMetadataField.CURRENT_RATE_KBPS: str(round(self.current_rate, 1)),
ChannelMetadataField.STATS_UPDATED_AT: str(current_time)
}
proxy_server.redis_client.hset(client_key, mapping=stats)
# No need to set expiration as client heartbeat will refresh this key
@ -328,14 +335,24 @@ class StreamGenerator:
def _is_timeout(self):
"""Check if the stream has timed out."""
# Get a more generous timeout for stream switching
stream_timeout = getattr(Config, 'STREAM_TIMEOUT', 10)
failover_grace_period = getattr(Config, 'FAILOVER_GRACE_PERIOD', 20)
total_timeout = stream_timeout + failover_grace_period
# Disconnect after long inactivity
if time.time() - self.last_yield_time > Config.STREAM_TIMEOUT:
if time.time() - self.last_yield_time > total_timeout:
if self.stream_manager and not self.stream_manager.healthy:
logger.warning(f"[{self.client_id}] No data for {Config.STREAM_TIMEOUT}s and stream unhealthy, disconnecting")
# Check if stream manager is actively switching or reconnecting
if (hasattr(self.stream_manager, 'url_switching') and self.stream_manager.url_switching):
logger.info(f"[{self.client_id}] Stream switching in progress, giving more time")
return False
logger.warning(f"[{self.client_id}] No data for {total_timeout}s and stream unhealthy, disconnecting")
return True
elif not self.is_owner_worker and self.consecutive_empty > 100:
# Non-owner worker without data for too long
logger.warning(f"[{self.client_id}] Non-owner worker with no data for {Config.STREAM_TIMEOUT}s, disconnecting")
logger.warning(f"[{self.client_id}] Non-owner worker with no data for {total_timeout}s, disconnecting")
return True
return False
@ -345,6 +362,34 @@ class StreamGenerator:
elapsed = time.time() - self.stream_start_time
local_clients = 0
total_clients = 0
proxy_server = ProxyServer.get_instance()
# Release M3U profile stream allocation if this is the last client
stream_released = False
if proxy_server.redis_client:
try:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata:
stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID)
if stream_id_bytes:
stream_id = int(stream_id_bytes.decode('utf-8'))
# Check if we're the last client
if self.channel_id in proxy_server.client_managers:
client_count = proxy_server.client_managers[self.channel_id].get_total_client_count()
# Only the last client or owner should release the stream
if client_count <= 1 and proxy_server.am_i_owner(self.channel_id):
from apps.channels.models import Stream
try:
stream = Stream.objects.get(pk=stream_id)
stream.release_stream()
stream_released = True
logger.debug(f"[{self.client_id}] Released stream {stream_id} for channel {self.channel_id}")
except Exception as e:
logger.error(f"[{self.client_id}] Error releasing stream {stream_id}: {e}")
except Exception as e:
logger.error(f"[{self.client_id}] Error checking stream data for release: {e}")
if self.channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[self.channel_id]
@ -353,12 +398,15 @@ class StreamGenerator:
logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})")
# Schedule channel shutdown if no clients left
self._schedule_channel_shutdown_if_needed(local_clients)
if not stream_released: # Only if we haven't already released the stream
self._schedule_channel_shutdown_if_needed(local_clients)
def _schedule_channel_shutdown_if_needed(self, local_clients):
"""
Schedule channel shutdown if there are no clients left and we're the owner.
"""
proxy_server = ProxyServer.get_instance()
# If no clients left and we're the owner, schedule shutdown using the config value
if local_clients == 0 and proxy_server.am_i_owner(self.channel_id):
logger.info(f"No local clients left for channel {self.channel_id}, scheduling shutdown")

165
apps/proxy/ts_proxy/stream_manager.py
View file

@ -15,9 +15,9 @@ from core.models import UserAgent, CoreSettings
from .stream_buffer import StreamBuffer
from .utils import detect_stream_type, get_logger
from .redis_keys import RedisKeys
from .constants import ChannelState, EventType, StreamType, TS_PACKET_SIZE
from .constants import ChannelState, EventType, StreamType, ChannelMetadataField, TS_PACKET_SIZE
from .config_helper import ConfigHelper
from .url_utils import get_alternate_streams, get_stream_info_for_switch
from .url_utils import get_alternate_streams, get_stream_info_for_switch, get_stream_object
logger = get_logger()
@ -284,10 +284,10 @@ class StreamManager:
# Update metadata to indicate error state
update_data = {
"state": ChannelState.ERROR,
"state_changed_at": str(time.time()),
"error_message": error_message,
"error_time": str(time.time())
ChannelMetadataField.STATE: ChannelState.ERROR,
ChannelMetadataField.STATE_CHANGED_AT: str(time.time()),
ChannelMetadataField.ERROR_MESSAGE: error_message,
ChannelMetadataField.ERROR_TIME: str(time.time())
}
self.buffer.redis_client.hset(metadata_key, mapping=update_data)
logger.info(f"Updated channel {self.channel_id} state to ERROR in Redis after stream failure")
@ -304,7 +304,7 @@ class StreamManager:
"""Establish a connection using transcoding"""
try:
logger.debug(f"Building transcode command for channel {self.channel_id}")
channel = get_object_or_404(Channel, uuid=self.channel_id)
channel = get_stream_object(self.channel_id)
# Use FFmpeg specifically for HLS streams
if hasattr(self, 'force_ffmpeg') and self.force_ffmpeg:
@ -336,6 +336,9 @@ class StreamManager:
self.socket = self.transcode_process.stdout # Read from std output
self.connected = True
# Set connection start time for stability tracking
self.connection_start_time = time.time()
# Set channel state to waiting for clients
self._set_waiting_for_clients()
@ -367,6 +370,9 @@ class StreamManager:
self.healthy = True
logger.info(f"Successfully connected to stream source")
# Store connection start time for stability tracking
self.connection_start_time = time.time()
# Set channel state to waiting for clients
self._set_waiting_for_clients()
@ -398,13 +404,13 @@ class StreamManager:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
# Use hincrby to atomically increment the total_bytes field
self.buffer.redis_client.hincrby(metadata_key, "total_bytes", self.bytes_processed)
self.buffer.redis_client.hincrby(metadata_key, ChannelMetadataField.TOTAL_BYTES, self.bytes_processed)
# Reset local counter after updating Redis
self.bytes_processed = 0
self.last_bytes_update = now
logger.debug(f"Updated total_bytes in Redis for channel {self.channel_id}")
logger.debug(f"Updated {ChannelMetadataField.TOTAL_BYTES} in Redis for channel {self.channel_id}")
except Exception as e:
logger.error(f"Error updating bytes processed: {e}")
@ -490,6 +496,21 @@ class StreamManager:
# Add at the beginning of your stop method
self.stopping = True
# Release stream resources if we're the owner
if self.current_stream_id and hasattr(self, 'worker_id') and self.worker_id:
if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client:
owner_key = RedisKeys.channel_owner(self.channel_id)
current_owner = self.buffer.redis_client.get(owner_key)
if current_owner and current_owner.decode('utf-8') == self.worker_id:
try:
from apps.channels.models import Stream
stream = Stream.objects.get(pk=self.current_stream_id)
stream.release_stream()
logger.info(f"Released stream {self.current_stream_id} for channel {self.channel_id}")
except Exception as e:
logger.error(f"Error releasing stream {self.current_stream_id}: {e}")
# Cancel all buffer check timers
for timer in list(self._buffer_check_timers):
try:
@ -573,24 +594,112 @@ class StreamManager:
def _monitor_health(self):
"""Monitor stream health and attempt recovery if needed"""
consecutive_unhealthy_checks = 0
health_recovery_attempts = 0
reconnect_attempts = 0
max_health_recovery_attempts = ConfigHelper.get('MAX_HEALTH_RECOVERY_ATTEMPTS', 2)
max_reconnect_attempts = ConfigHelper.get('MAX_RECONNECT_ATTEMPTS', 3)
min_stable_time = ConfigHelper.get('MIN_STABLE_TIME_BEFORE_RECONNECT', 30) # seconds
while self.running:
try:
now = time.time()
if now - self.last_data_time > getattr(Config, 'CONNECTION_TIMEOUT', 10) and self.connected:
inactivity_duration = now - self.last_data_time
timeout_threshold = getattr(Config, 'CONNECTION_TIMEOUT', 10)
if inactivity_duration > timeout_threshold and self.connected:
# Mark unhealthy if no data for too long
if self.healthy:
logger.warning(f"Stream unhealthy - no data for {now - self.last_data_time:.1f}s")
logger.warning(f"Stream unhealthy - no data for {inactivity_duration:.1f}s")
self.healthy = False
# Track consecutive unhealthy checks
consecutive_unhealthy_checks += 1
# After several unhealthy checks in a row, try recovery
if consecutive_unhealthy_checks >= 3 and health_recovery_attempts < max_health_recovery_attempts:
# Calculate how long the stream was stable before failing
connection_start_time = getattr(self, 'connection_start_time', 0)
stable_time = self.last_data_time - connection_start_time if connection_start_time > 0 else 0
if stable_time >= min_stable_time and reconnect_attempts < max_reconnect_attempts:
# Stream was stable for a while, try reconnecting first
logger.warning(f"Stream was stable for {stable_time:.1f}s before failing. "
f"Attempting reconnect {reconnect_attempts + 1}/{max_reconnect_attempts}")
reconnect_attempts += 1
threading.Thread(target=self._attempt_reconnect, daemon=True).start()
else:
# Stream was not stable long enough, or reconnects failed too many times
# Try switching to another stream
if reconnect_attempts > 0:
logger.warning(f"Reconnect attempts exhausted ({reconnect_attempts}/{max_reconnect_attempts}). "
f"Attempting stream switch recovery")
else:
logger.warning(f"Stream was only stable for {stable_time:.1f}s (<{min_stable_time}s). "
f"Skipping reconnect, attempting stream switch")
health_recovery_attempts += 1
reconnect_attempts = 0 # Reset for next time
threading.Thread(target=self._attempt_health_recovery, daemon=True).start()
elif self.connected and not self.healthy:
# Auto-recover health when data resumes
logger.info(f"Stream health restored")
self.healthy = True
consecutive_unhealthy_checks = 0
health_recovery_attempts = 0
reconnect_attempts = 0
# If healthy, reset unhealthy counter (but keep other state)
if self.healthy:
consecutive_unhealthy_checks = 0
except Exception as e:
logger.error(f"Error in health monitor: {e}")
time.sleep(self.health_check_interval)
def _attempt_reconnect(self):
"""Attempt to reconnect to the current stream"""
try:
logger.info(f"Attempting reconnect to current stream for channel {self.channel_id}")
# Don't try to reconnect if we're already switching URLs
if self.url_switching:
logger.info("URL switching already in progress, skipping reconnect")
return
# Close existing connection
if self.transcode or self.socket:
self._close_socket()
else:
self._close_connection()
self.connected = False
# Attempt to establish a new connection using the same URL
connection_result = False
try:
if self.transcode:
connection_result = self._establish_transcode_connection()
else:
connection_result = self._establish_http_connection()
if connection_result:
# Store connection start time to measure stability
self.connection_start_time = time.time()
logger.info(f"Reconnect successful for channel {self.channel_id}")
return True
else:
logger.warning(f"Reconnect failed for channel {self.channel_id}")
return False
except Exception as e:
logger.error(f"Error during reconnect: {e}", exc_info=True)
return False
except Exception as e:
logger.error(f"Error in reconnect attempt: {e}", exc_info=True)
return False
def _close_connection(self):
"""Close HTTP connection resources"""
# Close response if it exists
@ -743,8 +852,9 @@ class StreamManager:
current_state = None
try:
metadata = redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
current_state = metadata[b'state'].decode('utf-8')
state_field = ChannelMetadataField.STATE.encode('utf-8')
if metadata and state_field in metadata:
current_state = metadata[state_field].decode('utf-8')
except Exception as e:
logger.error(f"Error checking current state: {e}")
@ -758,8 +868,8 @@ class StreamManager:
# Not enough buffer yet - set to connecting state if not already
if current_state != ChannelState.CONNECTING:
update_data = {
"state": ChannelState.CONNECTING,
"state_changed_at": current_time
ChannelMetadataField.STATE: ChannelState.CONNECTING,
ChannelMetadataField.STATE_CHANGED_AT: current_time
}
redis_client.hset(metadata_key, mapping=update_data)
logger.info(f"Channel {channel_id} connected but waiting for buffer to fill: {current_buffer_index}/{initial_chunks_needed} chunks")
@ -772,16 +882,16 @@ class StreamManager:
# We have enough buffer, proceed with state change
update_data = {
"state": ChannelState.WAITING_FOR_CLIENTS,
"connection_ready_time": current_time,
"state_changed_at": current_time,
"buffer_chunks": str(current_buffer_index)
ChannelMetadataField.STATE: ChannelState.WAITING_FOR_CLIENTS,
ChannelMetadataField.CONNECTION_READY_TIME: current_time,
ChannelMetadataField.STATE_CHANGED_AT: current_time,
ChannelMetadataField.BUFFER_CHUNKS: str(current_buffer_index)
}
redis_client.hset(metadata_key, mapping=update_data)
# Get configured grace period or default
grace_period = ConfigHelper.get('CHANNEL_INIT_GRACE_PERIOD', 20)
logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'} → {ChannelState.WAITING_FOR_CLIENTS} with {current_buffer_index} buffer chunks")
logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'} -> {ChannelState.WAITING_FOR_CLIENTS} with {current_buffer_index} buffer chunks")
logger.info(f"Started initial connection grace period ({grace_period}s) for channel {channel_id}")
else:
logger.debug(f"Not changing state: channel {channel_id} already in {current_state} state")
@ -885,12 +995,13 @@ class StreamManager:
if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
self.buffer.redis_client.hset(metadata_key, mapping={
"url": new_url,
"user_agent": new_user_agent,
"profile": stream_info['profile'],
"stream_id": str(stream_id),
"stream_switch_time": str(time.time()),
"stream_switch_reason": "max_retries_exceeded"
ChannelMetadataField.URL: new_url,
ChannelMetadataField.USER_AGENT: new_user_agent,
ChannelMetadataField.STREAM_PROFILE: stream_info['stream_profile'],
ChannelMetadataField.M3U_PROFILE: stream_info['m3u_profile_id'],
ChannelMetadataField.STREAM_ID: str(stream_id),
ChannelMetadataField.STREAM_SWITCH_TIME: str(time.time()),
ChannelMetadataField.STREAM_SWITCH_REASON: "max_retries_exceeded"
})
# Log the switch
@ -908,5 +1019,3 @@ class StreamManager:
except Exception as e:
logger.error(f"Error trying next stream for channel {self.channel_id}: {e}", exc_info=True)
return False

123
apps/proxy/ts_proxy/url_utils.py
View file

@ -10,10 +10,21 @@ from apps.channels.models import Channel, Stream
from apps.m3u.models import M3UAccount, M3UAccountProfile
from core.models import UserAgent, CoreSettings
from .utils import get_logger
from uuid import UUID
logger = get_logger()
def generate_stream_url(channel_id: str) -> Tuple[str, str, bool]:
def get_stream_object(id: str):
try:
uuid_obj = UUID(id, version=4)
logger.info(f"Fetching channel ID {id}")
return get_object_or_404(Channel, uuid=id)
except:
# UUID check failed, assume stream hash
logger.info(f"Fetching stream hash {id}")
return get_object_or_404(Stream, stream_hash=id)
def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]]:
"""
Generate the appropriate stream URL for a channel based on its profile settings.
@ -21,43 +32,55 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool]:
channel_id: The UUID of the channel
Returns:
Tuple[str, str, bool]: (stream_url, user_agent, transcode_flag)
Tuple[str, str, bool, Optional[int]]: (stream_url, user_agent, transcode_flag, profile_id)
"""
# Get channel and related objects
channel = get_object_or_404(Channel, uuid=channel_id)
stream_id, profile_id = channel.get_stream()
try:
channel = get_stream_object(channel_id)
if stream_id is None or profile_id is None:
logger.error(f"No stream assigned to channel {channel_id}")
return None, None, False
# Get stream and profile for this channel
# Note: get_stream now returns 3 values (stream_id, profile_id, error_reason)
stream_id, profile_id, error_reason = channel.get_stream()
# Get the M3U account profile for URL pattern
stream = get_object_or_404(Stream, pk=stream_id)
profile = get_object_or_404(M3UAccountProfile, pk=profile_id)
if not stream_id or not profile_id:
logger.error(f"No stream available for channel {channel_id}: {error_reason}")
return None, None, False, None
# Get the appropriate user agent
m3u_account = M3UAccount.objects.get(id=profile.m3u_account.id)
stream_user_agent = UserAgent.objects.get(id=m3u_account.user_agent.id).user_agent
# Look up the Stream and Profile objects
try:
stream = Stream.objects.get(id=stream_id)
profile = M3UAccountProfile.objects.get(id=profile_id)
except (Stream.DoesNotExist, M3UAccountProfile.DoesNotExist) as e:
logger.error(f"Error getting stream or profile: {e}")
return None, None, False, None
if stream_user_agent is None:
stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id())
logger.debug(f"No user agent found for account, using default: {stream_user_agent}")
# Get the M3U account profile for URL pattern
m3u_profile = profile
# Generate stream URL based on the selected profile
input_url = stream.url
stream_url = transform_url(input_url, profile.search_pattern, profile.replace_pattern)
# Get the appropriate user agent
m3u_account = M3UAccount.objects.get(id=m3u_profile.m3u_account.id)
stream_user_agent = UserAgent.objects.get(id=m3u_account.user_agent.id).user_agent
# Check if transcoding is needed
stream_profile = channel.get_stream_profile()
if stream_profile.is_proxy() or stream_profile is None:
transcode = False
else:
transcode = True
if stream_user_agent is None:
stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id())
logger.debug(f"No user agent found for account, using default: {stream_user_agent}")
# Get profile name as string
profile_value = stream_profile.id
# Generate stream URL based on the selected profile
input_url = stream.url
stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern)
return stream_url, stream_user_agent, transcode, profile_value
# Check if transcoding is needed
stream_profile = channel.get_stream_profile()
if stream_profile.is_proxy() or stream_profile is None:
transcode = False
else:
transcode = True
stream_profile_id = stream_profile.id
return stream_url, stream_user_agent, transcode, stream_profile_id
except Exception as e:
logger.error(f"Error generating stream URL: {e}")
return None, None, False, None
def transform_url(input_url: str, search_pattern: str, replace_pattern: str) -> str:
"""
@ -72,18 +95,18 @@ def transform_url(input_url: str, search_pattern: str, replace_pattern: str) ->
str: The transformed URL
"""
try:
logger.debug("Executing URL pattern replacement:")
logger.debug(f" base URL: {input_url}")
logger.debug(f" search: {search_pattern}")
logger.info("Executing URL pattern replacement:")
logger.info(f" base URL: {input_url}")
logger.info(f" search: {search_pattern}")
# Handle backreferences in the replacement pattern
safe_replace_pattern = re.sub(r'\$(\d+)', r'\\\1', replace_pattern)
logger.debug(f" replace: {replace_pattern}")
logger.debug(f" safe replace: {safe_replace_pattern}")
logger.info(f" replace: {replace_pattern}")
logger.info(f" safe replace: {safe_replace_pattern}")
# Apply the transformation
stream_url = re.sub(search_pattern, safe_replace_pattern, input_url)
logger.debug(f"Generated stream url: {stream_url}")
logger.info(f"Generated stream url: {stream_url}")
return stream_url
except Exception as e:
@ -122,21 +145,21 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int]
).first()
if default_profile:
profile_id = default_profile.id
m3u_profile_id = default_profile.id
else:
logger.error(f"No profile found for stream {stream_id}")
return {'error': 'No profile found for stream'}
else:
# Use first available profile
profile_id = profiles.first().id
m3u_profile_id = profiles.first().id
else:
stream_id, profile_id = channel.get_stream()
if stream_id is None or profile_id is None:
return {'error': 'No stream assigned to channel'}
stream_id, m3u_profile_id, error_reason = channel.get_stream()
if stream_id is None or m3u_profile_id is None:
return {'error': error_reason or 'No stream assigned to channel'}
# Get the stream and profile objects directly
stream = get_object_or_404(Stream, pk=stream_id)
profile = get_object_or_404(M3UAccountProfile, pk=profile_id)
profile = get_object_or_404(M3UAccountProfile, pk=m3u_profile_id)
# Get the user agent from the M3U account
m3u_account = M3UAccount.objects.get(id=profile.m3u_account.id)
@ -156,9 +179,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int]
'url': stream_url,
'user_agent': user_agent,
'transcode': transcode,
'profile': profile_value,
'stream_profile': profile_value,
'stream_id': stream_id,
'profile_id': profile_id
'm3u_profile_id': m3u_profile_id
}
except Exception as e:
logger.error(f"Error getting stream info for switch: {e}", exc_info=True)
@ -177,11 +200,15 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No
"""
try:
# Get channel object
channel = get_object_or_404(Channel, uuid=channel_id)
channel = get_stream_object(channel_id)
if isinstance(channel, Stream):
logger.error(f"Stream is not a channel")
return []
logger.debug(f"Looking for alternate streams for channel {channel_id}, current stream ID: {current_stream_id}")
# Get all assigned streams for this channel
streams = channel.streams.all()
# Get all assigned streams for this channel using the correct ordering from the channelstream table
streams = channel.streams.all().order_by('channelstream__order')
logger.debug(f"Channel {channel_id} has {streams.count()} total assigned streams")
if not streams.exists():
@ -190,7 +217,7 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No
alternate_streams = []
# Process each stream
# Process each stream in the user-defined order
for stream in streams:
# Log each stream we're checking
logger.debug(f"Checking stream ID {stream.id} ({stream.name}) for channel {channel_id}")
@ -201,8 +228,6 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No
continue
# Find compatible profiles for this stream
# FIX: Looking at the error message, M3UAccountProfile doesn't have a 'stream' field
# We need to find which field relates M3UAccountProfile to Stream
try:
# Check if we can find profiles via m3u_account
profiles = M3UAccountProfile.objects.filter(m3u_account=stream.m3u_account)

1
apps/proxy/ts_proxy/urls.py
View file

@ -10,4 +10,5 @@ urlpatterns = [
path('status/<str:channel_id>', views.channel_status, name='channel_status_detail'),
path('stop/<str:channel_id>', views.stop_channel, name='stop_channel'),
path('stop_client/<str:channel_id>', views.stop_client, name='stop_client'),
path('next_stream/<str:channel_id>', views.next_stream, name='next_stream'),
]

204
apps/proxy/ts_proxy/views.py
View file

@ -7,7 +7,7 @@ from django.http import StreamingHttpResponse, JsonResponse, HttpResponseRedirec
from django.views.decorators.csrf import csrf_exempt
from django.shortcuts import get_object_or_404
from apps.proxy.config import TSConfig as Config
from . import proxy_server
from .server import ProxyServer
from .channel_status import ChannelStatus
from .stream_generator import create_stream_generator
from .utils import get_client_ip
@ -18,11 +18,12 @@ from apps.m3u.models import M3UAccount, M3UAccountProfile
from core.models import UserAgent, CoreSettings, PROXY_PROFILE_NAME
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from .constants import ChannelState, EventType, StreamType
from .constants import ChannelState, EventType, StreamType, ChannelMetadataField
from .config_helper import ConfigHelper
from .services.channel_service import ChannelService
from .url_utils import generate_stream_url, transform_url, get_stream_info_for_switch
from .url_utils import generate_stream_url, transform_url, get_stream_info_for_switch, get_stream_object, get_alternate_streams
from .utils import get_logger
from uuid import UUID
logger = get_logger()
@ -30,9 +31,10 @@ logger = get_logger()
@api_view(['GET'])
def stream_ts(request, channel_id):
"""Stream TS data to client with immediate response and keep-alive packets during initialization"""
channel = get_stream_object(channel_id)
client_user_agent = None
logger.info(f"Fetching channel ID {channel_id}")
channel = get_object_or_404(Channel, uuid=channel_id)
proxy_server = ProxyServer.get_instance()
try:
# Generate a unique client ID
@ -56,15 +58,17 @@ def stream_ts(request, channel_id):
metadata_key = RedisKeys.channel_metadata(channel_id)
if proxy_server.redis_client.exists(metadata_key):
metadata = proxy_server.redis_client.hgetall(metadata_key)
if b'state' in metadata:
channel_state = metadata[b'state'].decode('utf-8')
state_field = ChannelMetadataField.STATE.encode('utf-8')
if state_field in metadata:
channel_state = metadata[state_field].decode('utf-8')
# Only skip initialization if channel is in a healthy state
valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS]
if channel_state in valid_states:
# Verify the owner is still active
if b'owner' in metadata:
owner = metadata[b'owner'].decode('utf-8')
owner_field = ChannelMetadataField.OWNER.encode('utf-8')
if owner_field in metadata:
owner = metadata[owner_field].decode('utf-8')
owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat"
if proxy_server.redis_client.exists(owner_heartbeat_key):
# Owner is active and channel is in good state
@ -82,14 +86,62 @@ def stream_ts(request, channel_id):
# Initialize the channel (but don't wait for completion)
logger.info(f"[{client_id}] Starting channel {channel_id} initialization")
# Use the utility function to get stream URL and settings
stream_url, stream_user_agent, transcode, profile_value = generate_stream_url(channel_id)
# Use max retry attempts and connection timeout from config
max_retries = ConfigHelper.max_retries()
retry_timeout = ConfigHelper.connection_timeout()
wait_start_time = time.time()
stream_url = None
stream_user_agent = None
transcode = False
profile_value = None
error_reason = None
# Try to get a stream with configured retries
for attempt in range(max_retries):
stream_url, stream_user_agent, transcode, profile_value = generate_stream_url(channel_id)
if stream_url is not None:
logger.info(f"[{client_id}] Successfully obtained stream for channel {channel_id}")
break
# If we failed because there are no streams assigned, don't retry
_, _, error_reason = channel.get_stream()
if error_reason and 'maximum connection limits' not in error_reason:
logger.warning(f"[{client_id}] Can't retry - error not related to connection limits: {error_reason}")
break
# Don't exceed the overall connection timeout
if time.time() - wait_start_time > retry_timeout:
logger.warning(f"[{client_id}] Connection wait timeout exceeded ({retry_timeout}s)")
break
# Wait before retrying (using exponential backoff with a cap)
wait_time = min(0.5 * (2 ** attempt), 2.0) # Caps at 2 seconds
logger.info(f"[{client_id}] Waiting {wait_time:.1f}s for a connection to become available (attempt {attempt+1}/{max_retries})")
time.sleep(wait_time)
if stream_url is None:
return JsonResponse({'error': 'Channel not available'}, status=404)
# Make sure to release any stream locks that might have been acquired
if hasattr(channel, 'streams') and channel.streams.exists():
for stream in channel.streams.all():
try:
stream.release_stream()
logger.info(f"[{client_id}] Released stream {stream.id} for channel {channel_id}")
except Exception as e:
logger.error(f"[{client_id}] Error releasing stream: {e}")
# Get the specific error message if available
wait_duration = f"{int(time.time() - wait_start_time)}s"
error_msg = error_reason if error_reason else 'No available streams for this channel'
return JsonResponse({
'error': error_msg,
'waited': wait_duration
}, status=503) # 503 Service Unavailable is appropriate here
# Get the stream ID from the channel
stream_id, profile_id = channel.get_stream()
logger.info(f"Channel {channel_id} using stream ID {stream_id}, profile ID {profile_id}")
stream_id, m3u_profile_id, _ = channel.get_stream()
logger.info(f"Channel {channel_id} using stream ID {stream_id}, m3u account profile ID {m3u_profile_id}")
# Generate transcode command if needed
stream_profile = channel.get_stream_profile()
@ -98,7 +150,7 @@ def stream_ts(request, channel_id):
# Initialize channel with the stream's user agent (not the client's)
success = ChannelService.initialize_channel(
channel_id, stream_url, stream_user_agent, transcode, profile_value, stream_id
channel_id, stream_url, stream_user_agent, transcode, profile_value, stream_id, m3u_profile_id
)
if not success:
@ -134,9 +186,9 @@ def stream_ts(request, channel_id):
if proxy_server.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
url_bytes = proxy_server.redis_client.hget(metadata_key, "url")
ua_bytes = proxy_server.redis_client.hget(metadata_key, "user_agent")
profile_bytes = proxy_server.redis_client.hget(metadata_key, "profile")
url_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.URL)
ua_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.USER_AGENT)
profile_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_PROFILE)
if url_bytes:
url = url_bytes.decode('utf-8')
@ -189,6 +241,8 @@ def stream_ts(request, channel_id):
@permission_classes([IsAuthenticated])
def change_stream(request, channel_id):
"""Change stream URL for existing channel with enhanced diagnostics"""
proxy_server = ProxyServer.get_instance()
try:
data = json.loads(request.body)
new_url = data.get('url')
@ -240,6 +294,8 @@ def channel_status(request, channel_id=None):
- /status/ returns basic summary of all channels
- /status/{channel_id} returns detailed info about specific channel
"""
proxy_server = ProxyServer.get_instance()
try:
# Check if Redis is available
if not proxy_server.redis_client:
@ -334,3 +390,115 @@ def stop_client(request, channel_id):
except Exception as e:
logger.error(f"Failed to stop client: {e}", exc_info=True)
return JsonResponse({'error': str(e)}, status=500)
@csrf_exempt
@api_view(['POST'])
@permission_classes([IsAuthenticated])
def next_stream(request, channel_id):
"""Switch to the next available stream for a channel"""
proxy_server = ProxyServer.get_instance()
try:
logger.info(f"Request to switch to next stream for channel {channel_id} received")
# Check if the channel exists
channel = get_stream_object(channel_id)
# First check if channel is active in Redis
current_stream_id = None
profile_id = None
if proxy_server.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
if proxy_server.redis_client.exists(metadata_key):
# Get current stream ID from Redis
stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID)
if stream_id_bytes:
current_stream_id = int(stream_id_bytes.decode('utf-8'))
logger.info(f"Found current stream ID {current_stream_id} in Redis for channel {channel_id}")
# Get M3U profile from Redis if available
profile_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.M3U_PROFILE)
if profile_id_bytes:
profile_id = int(profile_id_bytes.decode('utf-8'))
logger.info(f"Found M3U profile ID {profile_id} in Redis for channel {channel_id}")
if not current_stream_id:
# Channel is not running
return JsonResponse({'error': 'No current stream found for channel'}, status=404)
# Get all streams for this channel in their defined order
streams = list(channel.streams.all().order_by('channelstream__order'))
if len(streams) <= 1:
return JsonResponse({
'error': 'No alternate streams available for this channel',
'current_stream_id': current_stream_id
}, status=404)
# Find the current stream's position in the list
current_index = None
for i, stream in enumerate(streams):
if stream.id == current_stream_id:
current_index = i
break
if current_index is None:
logger.warning(f"Current stream ID {current_stream_id} not found in channel's streams list")
# Fall back to the first stream that's not the current one
next_stream = next((s for s in streams if s.id != current_stream_id), None)
if not next_stream:
return JsonResponse({
'error': 'Could not find current stream in channel list',
'current_stream_id': current_stream_id
}, status=404)
else:
# Get the next stream in the rotation (with wrap-around)
next_index = (current_index + 1) % len(streams)
next_stream = streams[next_index]
next_stream_id = next_stream.id
logger.info(f"Rotating to next stream ID {next_stream_id} for channel {channel_id}")
# Get full stream info including URL for the next stream
stream_info = get_stream_info_for_switch(channel_id, next_stream_id)
if 'error' in stream_info:
return JsonResponse({
'error': stream_info['error'],
'current_stream_id': current_stream_id,
'next_stream_id': next_stream_id
}, status=404)
# Now use the ChannelService to change the stream URL
result = ChannelService.change_stream_url(
channel_id,
stream_info['url'],
stream_info['user_agent'],
next_stream_id # Pass the stream_id to be stored in Redis
)
if result.get('status') == 'error':
return JsonResponse({
'error': result.get('message', 'Unknown error'),
'diagnostics': result.get('diagnostics', {}),
'current_stream_id': current_stream_id,
'next_stream_id': next_stream_id
}, status=404)
# Format success response
response_data = {
'message': 'Stream switched to next available',
'channel': channel_id,
'previous_stream_id': current_stream_id,
'new_stream_id': next_stream_id,
'new_url': stream_info['url'],
'owner': result.get('direct_update', False),
'worker_id': proxy_server.worker_id
}
return JsonResponse(response_data)
except Exception as e:
logger.error(f"Failed to switch to next stream: {e}", exc_info=True)
return JsonResponse({'error': str(e)}, status=500)

5
core/api_urls.py
View file

@ -2,14 +2,15 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from .api_views import UserAgentViewSet, StreamProfileViewSet, CoreSettingsViewSet, environment
from .api_views import UserAgentViewSet, StreamProfileViewSet, CoreSettingsViewSet, environment, version
router = DefaultRouter()
router.register(r'useragents', UserAgentViewSet, basename='useragent')
router.register(r'streamprofiles', StreamProfileViewSet, basename='streamprofile')
router.register(r'settings', CoreSettingsViewSet, basename='coresettings')
router.register(r'settings', CoreSettingsViewSet, basename='settings')
urlpatterns = [
path('settings/env/', environment, name='token_refresh'),
path('version/', version, name='version'),
path('', include(router.urls)),
]

16
core/api_views.py
View file

@ -42,6 +42,8 @@ class CoreSettingsViewSet(viewsets.ModelViewSet):
@api_view(['GET'])
@permission_classes([IsAuthenticated])
def environment(request):
public_ip = None
local_ip = None
country_code = None
@ -84,3 +86,17 @@ def environment(request):
'country_name': country_name,
'env_mode': "dev" if os.getenv('DISPATCHARR_ENV') == "dev" else "prod",
})
@swagger_auto_schema(
method='get',
operation_description="Get application version information",
responses={200: "Version information"}
)
@api_view(['GET'])
def version(request):
# Import version information
from version import __version__, __build__
return Response({
'version': __version__,
'build': __build__,
})

3
core/apps.py
View file

@ -1,5 +1,6 @@
from django.apps import AppConfig
from django.conf import settings
import os, logging
class CoreConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'

22
core/migrations/0010_reload_additional_settings.py Normal file
View file

@ -0,0 +1,22 @@
# Generated by Django 5.1.6 on 2025-03-01 14:01
from django.db import migrations
from django.utils.text import slugify
def preload_core_settings(apps, schema_editor):
CoreSettings = apps.get_model("core", "CoreSettings")
CoreSettings.objects.create(
key=slugify("Preferred Region"),
name="Preferred Region",
value="us",
)
class Migration(migrations.Migration):
dependencies = [
('core', '0009_m3u_hash_settings'),
]
operations = [
migrations.RunPython(preload_core_settings),
]

27
core/migrations/0011_fix_stream_profiles_and_user_agents.py Normal file
View file

@ -0,0 +1,27 @@
# Generated by Django 5.1.6 on 2025-04-04
from django.db import migrations
def fix_stream_profiles_and_user_agents(apps, schema_editor):
# Get the model
StreamProfile = apps.get_model("core", "StreamProfile")
streamlink_profile = StreamProfile.objects.get(name="streamlink", locked=True)
streamlink_profile.parameters = "{streamUrl} --http-header User-Agent={userAgent} best --stdout"
streamlink_profile.save()
UserAgent = apps.get_model("core", "UserAgent")
tivimate = UserAgent.objects.get(name="TiviMate", user_agent="TiviMate/5.16 (Android 12)")
if tivimate:
tivimate.user_agent = "TiviMate/5.1.6 (Android 12)"
tivimate.save()
class Migration(migrations.Migration):
dependencies = [
('core', '0010_reload_additional_settings'),
]
operations = [
migrations.RunPython(fix_stream_profiles_and_user_agents),
]

22
core/migrations/0012_default_active_m3u_accounts.py Normal file
View file

@ -0,0 +1,22 @@
# Generated by Django 5.1.6 on 2025-03-01 14:01
from django.db import migrations
from django.utils.text import slugify
def preload_core_settings(apps, schema_editor):
CoreSettings = apps.get_model("core", "CoreSettings")
CoreSettings.objects.create(
key=slugify("Auto-Import Mapped Files"),
name="Auto-Import Mapped Files",
value=True,
)
class Migration(migrations.Migration):
dependencies = [
('core', '0011_fix_stream_profiles_and_user_agents'),
]
operations = [
migrations.RunPython(preload_core_settings),
]

10
core/models.py
View file

@ -144,6 +144,7 @@ DEFAULT_USER_AGENT_KEY= slugify("Default User-Agent")
DEFAULT_STREAM_PROFILE_KEY = slugify("Default Stream Profile")
STREAM_HASH_KEY = slugify("M3U Hash Key")
PREFERRED_REGION_KEY = slugify("Preferred Region")
AUTO_IMPORT_MAPPED_FILES = slugify("Auto-Import Mapped Files")
class CoreSettings(models.Model):
key = models.CharField(
@ -173,9 +174,18 @@ class CoreSettings(models.Model):
def get_m3u_hash_key(cls):
return cls.objects.get(key=STREAM_HASH_KEY).value
@classmethod
def get_preferred_region(cls):
"""Retrieve the preferred region setting (or return None if not found)."""
try:
return cls.objects.get(key=PREFERRED_REGION_KEY).value
except cls.DoesNotExist:
return None
@classmethod
def get_auto_import_mapped_files(cls):
"""Retrieve the preferred region setting (or return None if not found)."""
try:
return cls.objects.get(key=AUTO_IMPORT_MAPPED_FILES).value
except cls.DoesNotExist:
return None

178
core/tasks.py Normal file
View file

@ -0,0 +1,178 @@
# yourapp/tasks.py
from celery import shared_task
from channels.layers import get_channel_layer
from asgiref.sync import async_to_sync
import redis
import json
import logging
import re
import time
import os
from core.utils import RedisClient
from apps.proxy.ts_proxy.channel_status import ChannelStatus
from apps.m3u.models import M3UAccount
from apps.epg.models import EPGSource
from apps.m3u.tasks import refresh_single_m3u_account
from apps.epg.tasks import refresh_epg_data
from .models import CoreSettings
logger = logging.getLogger(__name__)
EPG_WATCH_DIR = '/data/epgs'
M3U_WATCH_DIR = '/data/m3us'
MIN_AGE_SECONDS = 6
STARTUP_SKIP_AGE = 30
REDIS_PREFIX = "processed_file:"
REDIS_TTL = 60 * 60 * 24 * 3 # expire keys after 3 days (optional)
# Store the last known value to compare with new data
last_known_data = {}
@shared_task
def beat_periodic_task():
fetch_channel_stats()
scan_and_process_files()
@shared_task
def scan_and_process_files():
redis_client = RedisClient.get_client()
now = time.time()
for filename in os.listdir(M3U_WATCH_DIR):
filepath = os.path.join(M3U_WATCH_DIR, filename)
if not os.path.isfile(filepath):
continue
if not filename.endswith('.m3u') and not filename.endswith('.m3u8'):
continue
mtime = os.path.getmtime(filepath)
age = now - mtime
redis_key = REDIS_PREFIX + filepath
stored_mtime = redis_client.get(redis_key)
# Startup safety: skip old untracked files
if not stored_mtime and age > STARTUP_SKIP_AGE:
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
continue # Assume already processed before startup
# File too new — probably still being written
if age < MIN_AGE_SECONDS:
continue
# Skip if we've already processed this mtime
if stored_mtime and float(stored_mtime) >= mtime:
continue
m3u_account, _ = M3UAccount.objects.get_or_create(file_path=filepath, defaults={
"name": filename,
"is_active": True if CoreSettings.get_auto_import_mapped_files() == "true" else False,
})
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
if not m3u_account.is_active:
logger.info("M3U account is inactive, skipping.")
continue
refresh_single_m3u_account.delay(m3u_account.id)
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
"updates",
{
"type": "update",
"data": {"success": True, "type": "m3u_file", "filename": filename}
},
)
for filename in os.listdir(EPG_WATCH_DIR):
filepath = os.path.join(EPG_WATCH_DIR, filename)
if not os.path.isfile(filepath):
continue
if not filename.endswith('.xml') and not filename.endswith('.gz'):
continue
mtime = os.path.getmtime(filepath)
age = now - mtime
redis_key = REDIS_PREFIX + filepath
stored_mtime = redis_client.get(redis_key)
# Startup safety: skip old untracked files
if not stored_mtime and age > STARTUP_SKIP_AGE:
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
continue # Assume already processed before startup
# File too new — probably still being written
if age < MIN_AGE_SECONDS:
continue
# Skip if we've already processed this mtime
if stored_mtime and float(stored_mtime) >= mtime:
continue
epg_source, _ = EPGSource.objects.get_or_create(file_path=filepath, defaults={
"name": filename,
"source_type": "xmltv",
"is_active": True if CoreSettings.get_auto_import_mapped_files() == "true" else False,
})
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
redis_client.set(redis_key, mtime, ex=REDIS_TTL)
if not epg_source.is_active:
logger.info("EPG source is inactive, skipping.")
continue
refresh_epg_data.delay(epg_source.id) # Trigger Celery task
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
"updates",
{
"type": "update",
"data": {"success": True, "type": "epg_file", "filename": filename}
},
)
def fetch_channel_stats():
redis_client = RedisClient.get_client()
try:
# Basic info for all channels
channel_pattern = "ts_proxy:channel:*:metadata"
all_channels = []
# Extract channel IDs from keys
cursor = 0
while True:
cursor, keys = redis_client.scan(cursor, match=channel_pattern)
for key in keys:
channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8'))
if channel_id_match:
ch_id = channel_id_match.group(1)
channel_info = ChannelStatus.get_basic_channel_info(ch_id)
if channel_info:
all_channels.append(channel_info)
if cursor == 0:
break
except Exception as e:
logger.error(f"Error in channel_status: {e}", exc_info=True)
return
# return JsonResponse({'error': str(e)}, status=500)
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
"updates",
{
"type": "update",
"data": {"success": True, "type": "channel_stats", "stats": json.dumps({'channels': all_channels, 'count': len(all_channels)})}
},
)

261
core/utils.py
View file

@ -5,161 +5,158 @@ import os
import threading
from django.conf import settings
from redis.exceptions import ConnectionError, TimeoutError
from django.core.cache import cache
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
import gc
logger = logging.getLogger(__name__)
# Import the command detector
from .command_utils import is_management_command
def get_redis_client(max_retries=5, retry_interval=1):
"""Get Redis client with connection validation and retry logic"""
# Skip Redis connection for management commands like collectstatic
if is_management_command():
logger.info("Running as management command - skipping Redis initialization")
return None
class RedisClient:
_client = None
_pubsub_client = None
retry_count = 0
while retry_count < max_retries:
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
@classmethod
def get_client(cls, max_retries=5, retry_interval=1):
if cls._client is None:
retry_count = 0
while retry_count < max_retries:
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
# Use standardized settings
socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5)
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
# Use standardized settings
socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5)
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
# Create Redis client with better defaults
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
socket_timeout=socket_timeout,
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
)
# Create Redis client with better defaults
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
socket_timeout=socket_timeout,
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
)
# Validate connection with ping
client.ping()
logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}")
return client
# Validate connection with ping
client.ping()
client.flushdb()
logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}")
except (ConnectionError, TimeoutError) as e:
retry_count += 1
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}")
return None
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
cls._client = client
break
except Exception as e:
logger.error(f"Unexpected error connecting to Redis: {e}")
return None
except (ConnectionError, TimeoutError) as e:
retry_count += 1
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}")
return None
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
def get_redis_pubsub_client(max_retries=5, retry_interval=1):
"""Get Redis client optimized for PubSub operations"""
# Skip Redis connection for management commands like collectstatic
if is_management_command():
logger.info("Running as management command - skipping Redis PubSub initialization")
return None
except Exception as e:
logger.error(f"Unexpected error connecting to Redis: {e}")
return None
retry_count = 0
while retry_count < max_retries:
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
return cls._client
# Use standardized settings but without socket timeouts for PubSub
# Important: socket_timeout is None for PubSub operations
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
@classmethod
def get_pubsub_client(cls, max_retries=5, retry_interval=1):
"""Get Redis client optimized for PubSub operations"""
if cls._pubsub_client is None:
retry_count = 0
while retry_count < max_retries:
try:
# Get connection parameters from settings or environment
redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost'))
redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379)))
redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0)))
# Create Redis client with PubSub-optimized settings - no timeout
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
socket_timeout=None, # Critical: No timeout for PubSub operations
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
)
# Use standardized settings but without socket timeouts for PubSub
# Important: socket_timeout is None for PubSub operations
socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5)
socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True)
health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30)
retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True)
# Validate connection with ping
client.ping()
logger.info(f"Connected to Redis for PubSub at {redis_host}:{redis_port}/{redis_db}")
# Create Redis client with PubSub-optimized settings - no timeout
client = redis.Redis(
host=redis_host,
port=redis_port,
db=redis_db,
socket_timeout=None, # Critical: No timeout for PubSub operations
socket_connect_timeout=socket_connect_timeout,
socket_keepalive=socket_keepalive,
health_check_interval=health_check_interval,
retry_on_timeout=retry_on_timeout
)
# We don't need the keepalive thread anymore since we're using proper PubSub handling
return client
# Validate connection with ping
client.ping()
logger.info(f"Connected to Redis for PubSub at {redis_host}:{redis_port}/{redis_db}")
except (ConnectionError, TimeoutError) as e:
retry_count += 1
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}")
return None
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis PubSub connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
# We don't need the keepalive thread anymore since we're using proper PubSub handling
cls._pubsub_client = client
break
except Exception as e:
logger.error(f"Unexpected error connecting to Redis for PubSub: {e}")
return None
except (ConnectionError, TimeoutError) as e:
retry_count += 1
if retry_count >= max_retries:
logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}")
return None
else:
# Use exponential backoff for retries
wait_time = retry_interval * (2 ** (retry_count - 1))
logger.warning(f"Redis PubSub connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})")
time.sleep(wait_time)
def execute_redis_command(redis_client, command_func, default_return=None):
"""
Execute a Redis command with proper error handling
except Exception as e:
logger.error(f"Unexpected error connecting to Redis for PubSub: {e}")
return None
Args:
redis_client: The Redis client instance
command_func: Lambda function containing the Redis command to execute
default_return: Value to return if command fails
return cls._pubsub_client
Returns:
Command result or default_return on failure
"""
if redis_client is None:
return default_return
def acquire_task_lock(task_name, id):
"""Acquire a lock to prevent concurrent task execution."""
redis_client = RedisClient.get_client()
lock_id = f"task_lock_{task_name}_{id}"
try:
return command_func()
except (ConnectionError, TimeoutError) as e:
logger.warning(f"Redis connection error: {e}")
return default_return
except Exception as e:
logger.error(f"Redis command error: {e}")
return default_return
# Use the Redis SET command with NX (only set if not exists) and EX (set expiration)
lock_acquired = redis_client.set(lock_id, "locked", ex=300, nx=True)
# Initialize the global clients with retry logic
# Skip Redis initialization if running as a management command
if is_management_command():
redis_client = None
redis_pubsub_client = None
logger.info("Running as management command - Redis clients set to None")
else:
redis_client = get_redis_client()
redis_pubsub_client = get_redis_pubsub_client()
if not lock_acquired:
logger.warning(f"Lock for {task_name} and id={id} already acquired. Task will not proceed.")
# Import and initialize the PubSub manager
# Skip if running as management command or if Redis client is None
if not is_management_command() and redis_client is not None:
from .redis_pubsub import get_pubsub_manager
pubsub_manager = get_pubsub_manager(redis_client)
else:
logger.info("PubSub manager not initialized (running as management command or Redis not available)")
pubsub_manager = None
return lock_acquired
def release_task_lock(task_name, id):
"""Release the lock after task execution."""
redis_client = RedisClient.get_client()
lock_id = f"task_lock_{task_name}_{id}"
# Remove the lock
redis_client.delete(lock_id)
def send_websocket_event(event, success, data):
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'updates',
{
'type': 'update',
"data": {"success": True, "type": "epg_channels"}
}
)

27
dispatcharr/consumers.py
View file

@ -1,5 +1,8 @@
import json
from channels.generic.websocket import AsyncWebsocketConsumer
import re, logging
logger = logging.getLogger(__name__)
class MyWebSocketConsumer(AsyncWebsocketConsumer):
async def connect(self):
@ -12,7 +15,29 @@ class MyWebSocketConsumer(AsyncWebsocketConsumer):
async def receive(self, text_data):
data = json.loads(text_data)
print("Received:", data)
if data["type"] == "m3u_profile_test":
from apps.proxy.ts_proxy.url_utils import transform_url
def replace_with_mark(match):
# Wrap the match in <mark> tags
return f"<mark>{match.group(0)}</mark>"
# Apply the transformation using the replace_with_mark function
try:
search_preview = re.sub(data["search"], replace_with_mark, data["url"])
except Exception as e:
search_preview = data["search"]
logger.error(f"Failed to generate replace preview: {e}")
result = transform_url(data["url"], data["search"], data["replace"])
await self.send(text_data=json.dumps({
"data": {
'type': 'm3u_profile_test',
'search_preview': search_preview,
'result': result,
}
}))
async def update(self, event):
await self.send(text_data=json.dumps(event))

22
dispatcharr/settings.py
View file

@ -1,7 +1,6 @@
import os
from pathlib import Path
from datetime import timedelta
from celery.schedules import crontab
BASE_DIR = Path(__file__).resolve().parent.parent
@ -9,7 +8,12 @@ SECRET_KEY = 'REPLACE_ME_WITH_A_REAL_SECRET'
REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
REDIS_DB = os.environ.get("REDIS_DB", "0")
DEBUG = True
# Set DEBUG to True for development, False for production
if os.environ.get('DISPATCHARR_DEBUG', 'False').lower() == 'true':
DEBUG = True
else:
DEBUG = False
ALLOWED_HOSTS = ["*"]
INSTALLED_APPS = [
@ -22,9 +26,10 @@ INSTALLED_APPS = [
'apps.m3u',
'apps.output',
'apps.proxy.apps.ProxyConfig',
'apps.proxy.ts_proxy',
'core',
'drf_yasg',
'daphne',
'drf_yasg',
'channels',
'django.contrib.admin',
'django.contrib.auth',
@ -35,10 +40,9 @@ INSTALLED_APPS = [
'rest_framework',
'corsheaders',
'django_filters',
'django_celery_beat',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
@ -153,7 +157,7 @@ CELERY_RESULT_BACKEND = CELERY_BROKER_URL
# Configure Redis key prefix
CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = {
'prefix': 'celery-task:', # Set the Redis key prefix for Celery
'global_keyprefix': 'celery-tasks:', # Set the Redis key prefix for Celery
}
# Set TTL (Time-to-Live) for task results (in seconds)
@ -164,9 +168,13 @@ CELERY_BROKER_TRANSPORT_OPTIONS = {
'visibility_timeout': 3600, # Time in seconds that a task remains invisible during retries
}
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
CELERY_BEAT_SCHEDULE = {
'fetch-channel-statuses': {
'task': 'apps.proxy.tasks.fetch_channel_stats',
'task': 'core.tasks.beat_periodic_task',
'schedule': 2.0,
},
}

57
docker/Dockerfile
View file

@ -14,30 +14,27 @@ ENV PATH="/dispatcharrpy/bin:$PATH" \
RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential \
curl \
gcc \
git \
libpcre3 \
libpcre3-dev \
python3-dev \
wget && \
echo "=== setting up nodejs ===" && \
curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh && \
bash /tmp/nodesource_setup.sh && \
apt-get update && \
apt-get install -y --no-install-recommends \
nodejs && \
build-essential \
curl \
gcc \
git \
libpcre3 \
libpcre3-dev \
python3-dev \
wget && \
python -m pip install virtualenv && \
virtualenv /dispatcharrpy && \
git clone -b ${BRANCH} ${REPO_URL} /app && \
cd /app && \
rm -rf .git && \
cd /app && \
pip install --no-cache-dir -r requirements.txt && \
python manage.py collectstatic --noinput && \
cd /app/frontend && \
npm install --legacy-peer-deps && \
pip install --no-cache-dir -r requirements.txt
# Use a dedicated Node.js stage for frontend building
FROM node:20-slim AS frontend-builder
WORKDIR /app/frontend
COPY --from=builder /app /app
RUN npm install --legacy-peer-deps && \
npm run build && \
find . -maxdepth 1 ! -name '.' ! -name 'dist' -exec rm -rf '{}' \;
@ -51,20 +48,24 @@ ENV PATH="/dispatcharrpy/bin:$PATH" \
# Copy the virtual environment and application from the builder stage
COPY --from=builder /dispatcharrpy /dispatcharrpy
COPY --from=builder /app /app
COPY --from=frontend-builder /app/frontend/dist /app/frontend/dist
# Run collectstatic after frontend assets are copied
RUN cd /app && python manage.py collectstatic --noinput
# Install base dependencies with memory optimization
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
curl \
ffmpeg \
libpcre3 \
libpq-dev \
nginx \
procps \
streamlink \
wget \
gnupg2 \
lsb-release && \
curl \
ffmpeg \
libpcre3 \
libpq-dev \
nginx \
procps \
streamlink \
wget \
gnupg2 \
lsb-release && \
cp /app/docker/nginx.conf /etc/nginx/sites-enabled/default && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

12
docker/build-dev.sh
View file

@ -1,3 +1,13 @@
#!/bin/bash
docker build --build-arg BRANCH=dev -t dispatcharr/dispatcharr:dev -f Dockerfile ..
# Get version information
VERSION=$(python -c "import sys; sys.path.append('..'); import version; print(version.__version__)")
BUILD=$(python -c "import sys; sys.path.append('..'); import version; print(version.__build__)")
# Build with version tags
docker build --build-arg BRANCH=dev \
-t dispatcharr/dispatcharr:dev \
-t dispatcharr/dispatcharr:${VERSION}-${BUILD} \
-f Dockerfile ..
.

6
docker/docker-compose.aio.yml
View file

@ -3,16 +3,16 @@ services:
# build:
# context: .
# dockerfile: Dockerfile
image: dispatcharr/dispatcharr:latest
image: ghcr.io/dispatcharr/dispatcharr:latest
container_name: dispatcharr
ports:
- 9191:9191
volumes:
- dispatcharr_db:/data
- dispatcharr_data:/data
environment:
- DISPATCHARR_ENV=aio
- REDIS_HOST=localhost
- CELERY_BROKER_URL=redis://localhost:6379/0
volumes:
dispatcharr_db:
dispatcharr_data:

19
docker/docker-compose.debug.yml Normal file
View file

@ -0,0 +1,19 @@
services:
dispatcharr:
# build:
# context: ..
# dockerfile: docker/Dockerfile.dev
image: dispatcharr/dispatcharr
container_name: dispatcharr_debug
ports:
- 5656:5656 # API port
- 9193:9191 # Web UI port
- 8001:8001 # Socket port
- 5678:5678 # Debugging port
volumes:
- ../:/app
environment:
- DISPATCHARR_ENV=dev
- DISPATCHARR_DEBUG=true
- REDIS_HOST=localhost
- CELERY_BROKER_URL=redis://localhost:6379/0

25
docker/docker-compose.dev.yml
View file

@ -3,7 +3,7 @@ services:
# build:
# context: ..
# dockerfile: docker/Dockerfile.dev
image: dispatcharr/dispatcharr
image: ghcr.io/dispatcharr/dispatcharr:dev
container_name: dispatcharr_dev
ports:
- 5656:5656
@ -11,7 +11,30 @@ services:
- 8001:8001
volumes:
- ../:/app
# - ./data/db:/data
environment:
- DISPATCHARR_ENV=dev
- REDIS_HOST=localhost
- CELERY_BROKER_URL=redis://localhost:6379/0
pgadmin:
image: dpage/pgadmin4
environment:
PGADMIN_DEFAULT_EMAIL: admin@admin.com
PGADMIN_DEFAULT_PASSWORD: admin
volumes:
- dispatcharr_dev_pgadmin:/var/lib/pgadmin
ports:
- 8082:80
redis-commander:
image: rediscommander/redis-commander:latest
environment:
- REDIS_HOSTS=dispatcharr:dispatcharr:6379:0
- TRUST_PROXY=true
- ADDRESS=0.0.0.0
ports:
- 8081:8081
volumes:
dispatcharr_dev_pgadmin:

72
docker/entrypoint.sh
View file

@ -36,6 +36,15 @@ export POSTGRES_PORT=${POSTGRES_PORT:-5432}
export REDIS_HOST=${REDIS_HOST:-localhost}
export REDIS_DB=${REDIS_DB:-0}
export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191}
# Extract version information from version.py
export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)")
export DISPATCHARR_BUILD=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__build__)")
echo "📦 Dispatcharr version: ${DISPATCHARR_VERSION}-${DISPATCHARR_BUILD}"
# READ-ONLY - don't let users change these
export POSTGRES_DIR=/data/db
# Global variables, stored so other users inherit them
if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then
@ -49,8 +58,13 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then
echo "export POSTGRES_HOST=$POSTGRES_HOST" >> /etc/profile.d/dispatcharr.sh
echo "export POSTGRES_PORT=$POSTGRES_PORT" >> /etc/profile.d/dispatcharr.sh
echo "export DISPATCHARR_ENV=$DISPATCHARR_ENV" >> /etc/profile.d/dispatcharr.sh
echo "export DISPATCHARR_DEBUG=$DISPATCHARR_DEBUG" >> /etc/profile.d/dispatcharr.sh
echo "export REDIS_HOST=$REDIS_HOST" >> /etc/profile.d/dispatcharr.sh
echo "export REDIS_DB=$REDIS_DB" >> /etc/profile.d/dispatcharr.sh
echo "export POSTGRES_DIR=$POSTGRES_DIR" >> /etc/profile.d/dispatcharr.sh
echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh
echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh
echo "export DISPATCHARR_BUILD=$DISPATCHARR_BUILD" >> /etc/profile.d/dispatcharr.sh
fi
chmod +x /etc/profile.d/dispatcharr.sh
@ -65,18 +79,23 @@ echo "Starting init process..."
# Start PostgreSQL
echo "Starting Postgres..."
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data start -w -t 300 -o '-c port=${POSTGRES_PORT}'"
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} start -w -t 300 -o '-c port=${POSTGRES_PORT}'"
# Wait for PostgreSQL to be ready
until su - postgres -c "/usr/lib/postgresql/14/bin/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do
echo_with_timestamp "Waiting for PostgreSQL to be ready..."
sleep 1
done
postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p')
postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p')
echo "✅ Postgres started with PID $postgres_pid"
pids+=("$postgres_pid")
if [ "$DISPATCHARR_ENV" = "dev" ]; then
if [[ "$DISPATCHARR_ENV" = "dev" ]]; then
. /app/docker/init/99-init-dev.sh
echo "Starting frontend dev environment"
su - $POSTGRES_USER -c "cd /app/frontend && npm run dev &"
npm_pid=$(pgrep vite | sort | head -n1)
echo "✅ vite started with PID $npm_pid"
pids+=("$npm_pid")
else
echo "🚀 Starting nginx..."
nginx
@ -85,22 +104,57 @@ else
pids+=("$nginx_pid")
fi
uwsgi_file="/app/docker/uwsgi.ini"
if [ "$DISPATCHARR_ENV" = "dev" ]; then
cd /app
python manage.py migrate --noinput
python manage.py collectstatic --noinput
# Select proper uwsgi config based on environment
if [ "$DISPATCHARR_ENV" = "dev" ] && [ "$DISPATCHARR_DEBUG" != "true" ]; then
echo "🚀 Starting uwsgi in dev mode..."
uwsgi_file="/app/docker/uwsgi.dev.ini"
elif [ "$DISPATCHARR_DEBUG" = "true" ]; then
echo "🚀 Starting uwsgi in debug mode..."
uwsgi_file="/app/docker/uwsgi.debug.ini"
else
echo "🚀 Starting uwsgi in production mode..."
uwsgi_file="/app/docker/uwsgi.ini"
fi
echo "🚀 Starting uwsgi..."
su - $POSTGRES_USER -c "cd /app && uwsgi --ini $uwsgi_file &"
uwsgi_pid=$(pgrep uwsgi | sort | head -n1)
echo "✅ uwsgi started with PID $uwsgi_pid"
pids+=("$uwsgi_pid")
# sed -i 's/protected-mode yes/protected-mode no/g' /etc/redis/redis.conf
# su - $POSTGRES_USER -c "redis-server --protected-mode no &"
# redis_pid=$(pgrep redis)
# echo "✅ redis started with PID $redis_pid"
# pids+=("$redis_pid")
# echo "🚀 Starting gunicorn..."
# su - $POSTGRES_USER -c "cd /app && gunicorn dispatcharr.asgi:application \
# --bind 0.0.0.0:5656 \
# --worker-class uvicorn.workers.UvicornWorker \
# --workers 2 \
# --threads 1 \
# --timeout 0 \
# --keep-alive 30 \
# --access-logfile - \
# --error-logfile - &"
# gunicorn_pid=$(pgrep gunicorn | sort | head -n1)
# echo "✅ gunicorn started with PID $gunicorn_pid"
# pids+=("$gunicorn_pid")
cd /app
python manage.py migrate --noinput
python manage.py collectstatic --noinput
# echo "Starting celery and beat..."
# su - $POSTGRES_USER -c "cd /app && celery -A dispatcharr worker -l info --autoscale=8,2 &"
# celery_pid=$(pgrep celery | sort | head -n1)
# echo "✅ celery started with PID $celery_pid"
# pids+=("$celery_pid")
# su - $POSTGRES_USER -c "cd /app && celery -A dispatcharr beat -l info &"
# beat_pid=$(pgrep beat | sort | head -n1)
# echo "✅ celery beat started with PID $beat_pid"
# pids+=("$beat_pid")
# Wait for at least one process to exit and log the process that exited first
if [ ${#pids[@]} -gt 0 ]; then

59
docker/init/02-postgres.sh
View file

@ -1,37 +1,64 @@
#!/bin/bash
# Inwitialize PostgreSQL database
if [ -z "$(ls -A "/data")" ]; then
echo_with_timestamp "Initializing PostgreSQL database..."
mkdir -p "/data"
chown -R postgres:postgres "/data"
chmod 700 "/data"
# Temporary migration from postgres in /data to $POSTGRES_DIR. Can likely remove
# some time in the future.
if [ -e "/data/postgresql.conf" ]; then
echo "Migrating PostgreSQL data from /data to $POSTGRES_DIR..."
# Create a temporary directory outside of /data
mkdir -p /tmp/postgres_migration
# Move the PostgreSQL files to the temporary directory
mv /data/* /tmp/postgres_migration/
# Create the target directory
mkdir -p $POSTGRES_DIR
# Move the files from temporary directory to the final location
mv /tmp/postgres_migration/* $POSTGRES_DIR/
# Clean up the temporary directory
rmdir /tmp/postgres_migration
# Set proper ownership and permissions for PostgreSQL data directory
chown -R postgres:postgres $POSTGRES_DIR
chmod 700 $POSTGRES_DIR
echo "Migration completed successfully."
fi
# Initialize PostgreSQL database
if [ -z "$(ls -A $POSTGRES_DIR)" ]; then
echo "Initializing PostgreSQL database..."
mkdir -p $POSTGRES_DIR
chown -R postgres:postgres $POSTGRES_DIR
chmod 700 $POSTGRES_DIR
# Initialize PostgreSQL
su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D /data"
su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D ${POSTGRES_DIR}"
# Configure PostgreSQL
echo "host all all 0.0.0.0/0 md5" >> "/data/pg_hba.conf"
echo "listen_addresses='*'" >> "/data/postgresql.conf"
echo "host all all 0.0.0.0/0 md5" >> "${POSTGRES_DIR}/pg_hba.conf"
echo "listen_addresses='*'" >> "${POSTGRES_DIR}/postgresql.conf"
# Start PostgreSQL
echo "Starting Postgres..."
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data start -w -t 300 -o '-c port=${POSTGRES_PORT}'"
su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} start -w -t 300 -o '-c port=${POSTGRES_PORT}'"
# Wait for PostgreSQL to be ready
until su - postgres -c "/usr/lib/postgresql/14/bin/pg_isready -h ${POSTGRES_HOST} -p ${POSTGRES_PORT}" >/dev/null 2>&1; do
echo_with_timestamp "Waiting for PostgreSQL to be ready..."
echo "Waiting for PostgreSQL to be ready..."
sleep 1
done
postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p')
postgres_pid=$(su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D ${POSTGRES_DIR} status" | sed -n 's/.*PID: \([0-9]\+\).*/\1/p')
# Setup database if needed
if ! su - postgres -c "psql -p ${POSTGRES_PORT} -tAc \"SELECT 1 FROM pg_database WHERE datname = '$POSTGRES_DB';\"" | grep -q 1; then
# Create PostgreSQL database
echo_with_timestamp "Creating PostgreSQL database..."
echo "Creating PostgreSQL database..."
su - postgres -c "createdb -p ${POSTGRES_PORT} ${POSTGRES_DB}"
# Create user, set ownership, and grant privileges
echo_with_timestamp "Creating PostgreSQL user..."
echo "Creating PostgreSQL user..."
su - postgres -c "psql -p ${POSTGRES_PORT} -d ${POSTGRES_DB}" <<EOF
DO \$\$
BEGIN
@ -41,11 +68,11 @@ BEGIN
END
\$\$;
EOF
echo_with_timestamp "Setting PostgreSQL user privileges..."
echo "Setting PostgreSQL user privileges..."
su postgres -c "/usr/lib/postgresql/14/bin/psql -p ${POSTGRES_PORT} -c \"ALTER DATABASE ${POSTGRES_DB} OWNER TO $POSTGRES_USER;\""
su postgres -c "/usr/lib/postgresql/14/bin/psql -p ${POSTGRES_PORT} -c \"GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO $POSTGRES_USER;\""
# Finished setting up PosgresSQL database
echo_with_timestamp "PostgreSQL database setup complete."
echo "PostgreSQL database setup complete."
fi
kill $postgres_pid

23
docker/init/03-init-dispatcharr.sh
View file

@ -1,15 +1,24 @@
#!/bin/bash
mkdir -p /data/logos
mkdir -p /data/recordings
mkdir -p /data/uploads/m3us
mkdir -p /data/uploads/epgs
mkdir -p /data/m3us
mkdir -p /data/epgs
mkdir -p /app/logo_cache
mkdir -p /app/media
sed -i "s/NGINX_PORT/${DISPATCHARR_PORT}/g" /etc/nginx/sites-enabled/default
# NOTE: mac doesn't run as root, so only manage permissions
# if this script is running as root
if [ "$(id -u)" = "0" ]; then
touch /app/uwsgi.sock
# Needs to own ALL of /data except db, we handle that below
chown -R $PUID:$PGID /data
chown -R $PUID:$PGID /app
chown $PUID:$PGID /app/uwsgi.sock
# Create and set permissions for the media / cache directories
mkdir -p /app/media
chown -R $PUID:$PGID /app
echo "Created and set permissions for cached_m3u directory"
# Permissions
chown -R postgres:postgres /data/db
chmod +x /data
fi

8
docker/init/99-init-dev.sh
View file

@ -15,5 +15,11 @@ fi
# Install frontend dependencies
cd /app/frontend && npm install
# Install pip dependencies
cd /app && pip install -r requirements.txt
# Install debugpy for remote debugging
if [ "$DISPATCHARR_DEBUG" = "true" ]; then
echo "=== setting up debugpy ==="
pip install debugpy
fi

54
docker/nginx.conf
View file

@ -1,12 +1,20 @@
proxy_cache_path /app/logo_cache levels=1:2 keys_zone=logo_cache:10m
inactive=24h use_temp_path=off;
server {
listen 9191;
listen NGINX_PORT;
proxy_connect_timeout 75;
proxy_send_timeout 300;
proxy_read_timeout 300;
# Serve Django via uWSGI
location / {
include uwsgi_params;
uwsgi_pass unix:/app/uwsgi.sock;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
location /assets/ {
@ -17,6 +25,26 @@ server {
root /app;
}
location /logos/ {
root /data;
}
location /api/logos/(?<logo_id>\d+)/cache/ {
proxy_pass http://127.0.0.1:5656;
proxy_cache logo_cache;
proxy_cache_key "$scheme$request_uri"; # Cache per logo URL
proxy_cache_valid 200 24h; # Cache for 24 hours
proxy_cache_use_stale error timeout updating; # Serve stale if Django is slow
}
location ~ ^/api/channels/logos/(?<logo_id>\d+)/cache/ {
proxy_pass http://127.0.0.1:5656;
proxy_cache logo_cache;
proxy_cache_key "$scheme$request_uri"; # Cache per logo URL
proxy_cache_valid 200 24h; # Cache for 24 hours
proxy_cache_use_stale error timeout updating; # Serve stale if Django is slow
}
# admin disabled when not in dev mode
location /admin {
return 301 /login;
@ -24,7 +52,13 @@ server {
# Route HDHR request to Django
location /hdhr {
proxy_pass http://127.0.0.1:5656;
include uwsgi_params;
uwsgi_pass unix:/app/uwsgi.sock;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host:$server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $host;
}
# Serve FFmpeg streams efficiently
@ -32,8 +66,9 @@ server {
proxy_pass http://127.0.0.1:5656;
proxy_buffering off;
proxy_set_header Connection keep-alive;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
# WebSockets for real-time communication
@ -42,6 +77,8 @@ server {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
@ -55,7 +92,8 @@ server {
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
client_max_body_size 0;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
}
}

81
docker/uwsgi.debug.ini Normal file
View file

@ -0,0 +1,81 @@
[uwsgi]
; exec-before = python manage.py collectstatic --noinput
; exec-before = python manage.py migrate --noinput
; First run Redis availability check script once
exec-before = python /app/scripts/wait_for_redis.py
; Start Redis first
attach-daemon = redis-server
; Then start other services
attach-daemon = celery -A dispatcharr worker -l info
attach-daemon = celery -A dispatcharr beat -l info
attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application
attach-daemon = cd /app/frontend && npm run dev
# Core settings
chdir = /app
module = scripts.debug_wrapper:application
virtualenv = /dispatcharrpy
master = true
env = DJANGO_SETTINGS_MODULE=dispatcharr.settings
socket = /app/uwsgi.sock
chmod-socket = 777
vacuum = true
die-on-term = true
static-map = /static=/app/static
# Worker configuration
workers = 1
threads = 8
enable-threads = true
lazy-apps = true
# HTTP server
http = 0.0.0.0:5656
http-keepalive = 1
buffer-size = 65536
http-timeout = 600
# Async mode (use gevent for high concurrency)
gevent = 100
async = 100
# Performance tuning
thunder-lock = true
log-4xx = true
log-5xx = true
disable-logging = false
; Longer timeouts for debugging sessions
harakiri = 3600
socket-timeout = 3600
http-timeout = 3600
# Ignore unknown options
ignore-sigpipe = true
ignore-write-errors = true
disable-write-exception = true
# Explicitly disable for-server option that confuses debugpy
for-server = false
# Debugging settings
py-autoreload = 1
honour-stdin = true
# Environment variables
env = PYTHONPATH=/app
env = PYTHONUNBUFFERED=1
env = PYDEVD_DISABLE_FILE_VALIDATION=1
env = PYTHONUTF8=1
env = PYTHONXOPT=-Xfrozen_modules=off
env = PYDEVD_DEBUG=1
env = DEBUGPY_LOG_DIR=/app/debugpy_logs
# Debugging control variables
env = WAIT_FOR_DEBUGGER=false
env = DEBUG_TIMEOUT=30

2
docker/uwsgi.dev.ini
View file

@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py
; Start Redis first
attach-daemon = redis-server
; Then start other services
attach-daemon = celery -A dispatcharr worker -l info
attach-daemon = celery -A dispatcharr worker -l info --concurrency=4
attach-daemon = celery -A dispatcharr beat -l info
attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application
attach-daemon = cd /app/frontend && npm run dev

7
docker/uwsgi.ini
View file

@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py
; Start Redis first
attach-daemon = redis-server
; Then start other services
attach-daemon = celery -A dispatcharr worker -l error
attach-daemon = celery -A dispatcharr worker -l error --concurrency=4
attach-daemon = celery -A dispatcharr beat -l error
attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application
@ -25,9 +25,8 @@ die-on-term = true
static-map = /static=/app/static
# Worker management (Optimize for I/O bound tasks)
workers = 4
threads = 2
enable-threads = true
workers = 2
enable-threads = false
# Optimize for streaming
http = 0.0.0.0:5656

8
frontend/index.html
View file

@ -2,7 +2,13 @@
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/logo.png" />
<link rel="icon" type="image/png" href="/logo.png" />
<link rel="apple-touch-icon" sizes="180x180" href="/static/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/static/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/static/favicon-16x16.png">
<link rel="manifest" href="/static/site.webmanifest">
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Dispatcharr</title>
</head>

358
frontend/package-lock.json generated
View file

@ -8,9 +8,11 @@
"name": "vite",
"version": "0.0.0",
"dependencies": {
"@mantine/charts": "^7.17.2",
"@mantine/core": "^7.17.2",
"@mantine/dates": "^7.17.2",
"@mantine/dropzone": "^7.17.2",
"@mantine/form": "^7.17.3",
"@mantine/hooks": "^7.17.2",
"@mantine/notifications": "^7.17.2",
"@tabler/icons-react": "^3.31.0",
@ -29,6 +31,8 @@
"react-draggable": "^4.4.6",
"react-pro-sidebar": "^1.1.0",
"react-router-dom": "^7.3.0",
"react-window": "^1.8.11",
"recharts": "^2.15.1",
"video.js": "^8.21.0",
"yup": "^1.6.1",
"zustand": "^5.0.3"
@ -1082,6 +1086,19 @@
"integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==",
"license": "Apache-2.0"
},
"node_modules/@mantine/charts": {
"version": "7.17.2",
"resolved": "https://registry.npmjs.org/@mantine/charts/-/charts-7.17.2.tgz",
"integrity": "sha512-ckB23pIqRjzysUz2EiWZD9AVyf7t0r7o7zfJbl01nzOezFgYq5RGeRoxvpcsfBC+YoSbB/43rjNcXtYhtA7QzA==",
"license": "MIT",
"peerDependencies": {
"@mantine/core": "7.17.2",
"@mantine/hooks": "7.17.2",
"react": "^18.x || ^19.x",
"react-dom": "^18.x || ^19.x",
"recharts": "^2.13.3"
}
},
"node_modules/@mantine/core": {
"version": "7.17.2",
"resolved": "https://registry.npmjs.org/@mantine/core/-/core-7.17.2.tgz",
@ -1132,6 +1149,19 @@
"react-dom": "^18.x || ^19.x"
}
},
"node_modules/@mantine/form": {
"version": "7.17.3",
"resolved": "https://registry.npmjs.org/@mantine/form/-/form-7.17.3.tgz",
"integrity": "sha512-ktERldD8f9lrjjz6wIbwMnNbAZq8XEWPx4K5WuFyjXaK0PI8D+gsXIGKMtA5rVrAUFHCWCdbK3yLgtjJNki8ew==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.3",
"klona": "^2.0.6"
},
"peerDependencies": {
"react": "^18.x || ^19.x"
}
},
"node_modules/@mantine/hooks": {
"version": "7.17.2",
"resolved": "https://registry.npmjs.org/@mantine/hooks/-/hooks-7.17.2.tgz",
@ -1776,6 +1806,69 @@
"integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==",
"license": "MIT"
},
"node_modules/@types/d3-array": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
"integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
"license": "MIT"
},
"node_modules/@types/d3-color": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==",
"license": "MIT"
},
"node_modules/@types/d3-ease": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==",
"license": "MIT"
},
"node_modules/@types/d3-interpolate": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
"license": "MIT",
"dependencies": {
"@types/d3-color": "*"
}
},
"node_modules/@types/d3-path": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz",
"integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==",
"license": "MIT"
},
"node_modules/@types/d3-scale": {
"version": "4.0.9",
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
"license": "MIT",
"dependencies": {
"@types/d3-time": "*"
}
},
"node_modules/@types/d3-shape": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz",
"integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==",
"license": "MIT",
"dependencies": {
"@types/d3-path": "*"
}
},
"node_modules/@types/d3-time": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==",
"license": "MIT"
},
"node_modules/@types/d3-timer": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==",
"license": "MIT"
},
"node_modules/@types/estree": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@ -2205,6 +2298,127 @@
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"license": "MIT"
},
"node_modules/d3-array": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
"license": "ISC",
"dependencies": {
"internmap": "1 - 2"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-color": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-ease": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-format": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz",
"integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-interpolate": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
"license": "ISC",
"dependencies": {
"d3-color": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-scale": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
"license": "ISC",
"dependencies": {
"d3-array": "2.10.0 - 3",
"d3-format": "1 - 3",
"d3-interpolate": "1.2.0 - 3",
"d3-time": "2.1.1 - 3",
"d3-time-format": "2 - 4"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"license": "ISC",
"dependencies": {
"d3-path": "^3.1.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
"license": "ISC",
"dependencies": {
"d3-array": "2 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time-format": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
"license": "ISC",
"dependencies": {
"d3-time": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-timer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/dayjs": {
"version": "1.11.13",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz",
@ -2228,6 +2442,12 @@
}
}
},
"node_modules/decimal.js-light": {
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
"license": "MIT"
},
"node_modules/deep-is": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@ -2589,9 +2809,17 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"dev": true,
"license": "MIT"
},
"node_modules/fast-equals": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz",
"integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==",
"license": "MIT",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@ -2947,6 +3175,15 @@
"node": ">=0.8.19"
}
},
"node_modules/internmap": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/is-arrayish": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
@ -3072,6 +3309,15 @@
"json-buffer": "3.0.1"
}
},
"node_modules/klona": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz",
"integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==",
"license": "MIT",
"engines": {
"node": ">= 8"
}
},
"node_modules/levn": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@ -3215,6 +3461,12 @@
"node": ">= 0.4"
}
},
"node_modules/memoize-one": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
"integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==",
"license": "MIT"
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@ -3640,6 +3892,12 @@
"integrity": "sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw==",
"license": "MIT"
},
"node_modules/react-is": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
"license": "MIT"
},
"node_modules/react-number-format": {
"version": "5.4.3",
"resolved": "https://registry.npmjs.org/react-number-format/-/react-number-format-5.4.3.tgz",
@ -3753,6 +4011,21 @@
"react-dom": ">=18"
}
},
"node_modules/react-smooth": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz",
"integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==",
"license": "MIT",
"dependencies": {
"fast-equals": "^5.0.1",
"prop-types": "^15.8.1",
"react-transition-group": "^4.4.5"
},
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/react-style-singleton": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz",
@ -3808,6 +4081,61 @@
"react-dom": ">=16.6.0"
}
},
"node_modules/react-window": {
"version": "1.8.11",
"resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.11.tgz",
"integrity": "sha512-+SRbUVT2scadgFSWx+R1P754xHPEqvcfSfVX10QYg6POOz+WNgkN48pS+BtZNIMGiL1HYrSEiCkwsMS15QogEQ==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.0.0",
"memoize-one": ">=3.1.1 <6"
},
"engines": {
"node": ">8.0.0"
},
"peerDependencies": {
"react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/recharts": {
"version": "2.15.1",
"resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.1.tgz",
"integrity": "sha512-v8PUTUlyiDe56qUj82w/EDVuzEFXwEHp9/xOowGAZwfLjB9uAy3GllQVIYMWF6nU+qibx85WF75zD7AjqoT54Q==",
"license": "MIT",
"dependencies": {
"clsx": "^2.0.0",
"eventemitter3": "^4.0.1",
"lodash": "^4.17.21",
"react-is": "^18.3.1",
"react-smooth": "^4.0.4",
"recharts-scale": "^0.4.4",
"tiny-invariant": "^1.3.1",
"victory-vendor": "^36.6.8"
},
"engines": {
"node": ">=14"
},
"peerDependencies": {
"react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/recharts-scale": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz",
"integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==",
"license": "MIT",
"dependencies": {
"decimal.js-light": "^2.4.1"
}
},
"node_modules/recharts/node_modules/eventemitter3": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
"license": "MIT"
},
"node_modules/regenerator-runtime": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
@ -3998,6 +4326,12 @@
"integrity": "sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==",
"license": "MIT"
},
"node_modules/tiny-invariant": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==",
"license": "MIT"
},
"node_modules/tiny-warning": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz",
@ -4145,6 +4479,28 @@
}
}
},
"node_modules/victory-vendor": {
"version": "36.9.2",
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz",
"integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==",
"license": "MIT AND ISC",
"dependencies": {
"@types/d3-array": "^3.0.3",
"@types/d3-ease": "^3.0.0",
"@types/d3-interpolate": "^3.0.1",
"@types/d3-scale": "^4.0.2",
"@types/d3-shape": "^3.1.0",
"@types/d3-time": "^3.0.0",
"@types/d3-timer": "^3.0.0",
"d3-array": "^3.1.6",
"d3-ease": "^3.0.1",
"d3-interpolate": "^3.0.1",
"d3-scale": "^4.0.2",
"d3-shape": "^3.1.0",
"d3-time": "^3.0.0",
"d3-timer": "^3.0.1"
}
},
"node_modules/video.js": {
"version": "8.22.0",
"resolved": "https://registry.npmjs.org/video.js/-/video.js-8.22.0.tgz",

4
frontend/package.json
View file

@ -10,9 +10,11 @@
"preview": "vite preview"
},
"dependencies": {
"@mantine/charts": "^7.17.2",
"@mantine/core": "^7.17.2",
"@mantine/dates": "^7.17.2",
"@mantine/dropzone": "^7.17.2",
"@mantine/form": "^7.17.3",
"@mantine/hooks": "^7.17.2",
"@mantine/notifications": "^7.17.2",
"@tabler/icons-react": "^3.31.0",
@ -31,6 +33,8 @@
"react-draggable": "^4.4.6",
"react-pro-sidebar": "^1.1.0",
"react-router-dom": "^7.3.0",
"react-window": "^1.8.11",
"recharts": "^2.15.1",
"video.js": "^8.21.0",
"yup": "^1.6.1",
"zustand": "^5.0.3"

BIN
frontend/public/android-chrome-192x192.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
frontend/public/android-chrome-512x512.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 32 KiB

BIN
frontend/public/apple-touch-icon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
frontend/public/favicon-16x16.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 772 B

BIN
frontend/public/favicon-32x32.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.7 KiB

BIN
frontend/public/favicon.ico Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 15 KiB

1
frontend/public/site.webmanifest Normal file
View file

@ -0,0 +1 @@
{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

128
frontend/src/App.jsx
View file

@ -9,12 +9,11 @@ import {
import Sidebar from './components/Sidebar';
import Login from './pages/Login';
import Channels from './pages/Channels';
import M3U from './pages/M3U';
import EPG from './pages/EPG';
import ContentSources from './pages/ContentSources';
import Guide from './pages/Guide';
import Stats from './pages/Stats';
import DVR from './pages/DVR';
import Settings from './pages/Settings';
import StreamProfiles from './pages/StreamProfiles';
import useAuthStore from './store/auth';
import FloatingVideo from './components/FloatingVideo';
import { WebsocketProvider } from './WebSocket';
@ -23,11 +22,13 @@ import '@mantine/core/styles.css'; // Ensure Mantine global styles load
import '@mantine/notifications/styles.css';
import 'mantine-react-table/styles.css';
import '@mantine/dropzone/styles.css';
import '@mantine/dates/styles.css';
import './index.css';
import mantineTheme from './mantineTheme';
import API from './api';
import { Notifications } from '@mantine/notifications';
import M3URefreshNotification from './components/M3URefreshNotification';
import 'allotment/dist/style.css';
const drawerWidth = 240;
const miniDrawerWidth = 60;
@ -84,72 +85,67 @@ const App = () => {
withGlobalStyles
withNormalizeCSS
>
<WebsocketProvider>
<Notifications containerWidth={350} />
<Router>
<AppShell
header={{
height: 0,
}}
navbar={{
width: open ? drawerWidth : miniDrawerWidth,
}}
>
<Sidebar
drawerWidth
miniDrawerWidth
collapsed={!open}
toggleDrawer={toggleDrawer}
/>
<Router>
<AppShell
header={{
height: 0,
}}
navbar={{
width: open ? drawerWidth : miniDrawerWidth,
}}
>
<Sidebar
drawerWidth
miniDrawerWidth
collapsed={!open}
toggleDrawer={toggleDrawer}
/>
<AppShell.Main>
<Box
style={{
display: 'flex',
flexDirection: 'column',
// transition: 'margin-left 0.3s',
backgroundColor: '#18181b',
height: '100vh',
color: 'white',
}}
>
<Box sx={{ p: 2, flex: 1, overflow: 'auto' }}>
<Routes>
{isAuthenticated ? (
<>
<Route path="/channels" element={<Channels />} />
<Route path="/m3u" element={<M3U />} />
<Route path="/epg" element={<EPG />} />
<Route
path="/stream-profiles"
element={<StreamProfiles />}
/>
<Route path="/guide" element={<Guide />} />
<Route path="/stats" element={<Stats />} />
<Route path="/settings" element={<Settings />} />
</>
) : (
<Route path="/login" element={<Login needsSuperuser />} />
)}
<Route
path="*"
element={
<Navigate
to={isAuthenticated ? defaultRoute : '/login'}
replace
/>
}
/>
</Routes>
</Box>
<AppShell.Main>
<Box
style={{
display: 'flex',
flexDirection: 'column',
// transition: 'margin-left 0.3s',
backgroundColor: '#18181b',
height: '100vh',
color: 'white',
}}
>
<Box sx={{ p: 2, flex: 1, overflow: 'auto' }}>
<Routes>
{isAuthenticated ? (
<>
<Route path="/channels" element={<Channels />} />
<Route path="/sources" element={<ContentSources />} />
<Route path="/guide" element={<Guide />} />
<Route path="/dvr" element={<DVR />} />
<Route path="/stats" element={<Stats />} />
<Route path="/settings" element={<Settings />} />
</>
) : (
<Route path="/login" element={<Login needsSuperuser />} />
)}
<Route
path="*"
element={
<Navigate
to={isAuthenticated ? defaultRoute : '/login'}
replace
/>
}
/>
</Routes>
</Box>
</AppShell.Main>
</AppShell>
<M3URefreshNotification />
</Router>
</Box>
</AppShell.Main>
</AppShell>
<M3URefreshNotification />
<Notifications containerWidth={350} />
<WebsocketProvider />
</Router>
<FloatingVideo />
</WebsocketProvider>
<FloatingVideo />
</MantineProvider>
);
};

110
frontend/src/WebSocket.jsx
View file

@ -4,12 +4,15 @@ import React, {
useRef,
createContext,
useContext,
useMemo,
} from 'react';
import useStreamsStore from './store/streams';
import { notifications } from '@mantine/notifications';
import useChannelsStore from './store/channels';
import usePlaylistsStore from './store/playlists';
import useEPGsStore from './store/epgs';
import { Box, Button, Stack } from '@mantine/core';
import API from './api';
export const WebsocketContext = createContext(false, null, () => {});
@ -18,9 +21,12 @@ export const WebsocketProvider = ({ children }) => {
const [val, setVal] = useState(null);
const { fetchStreams } = useStreamsStore();
const { setChannelStats, fetchChannelGroups } = useChannelsStore();
const { fetchPlaylists, setRefreshProgress } = usePlaylistsStore();
const { fetchEPGData } = useEPGsStore();
const { fetchChannels, setChannelStats, fetchChannelGroups } =
useChannelsStore();
const { fetchPlaylists, setRefreshProgress, setProfilePreview } =
usePlaylistsStore();
const { fetchEPGData, fetchEPGs } = useEPGsStore();
const { playlists } = usePlaylistsStore();
const ws = useRef(null);
@ -55,29 +61,91 @@ export const WebsocketProvider = ({ children }) => {
socket.onmessage = async (event) => {
event = JSON.parse(event.data);
switch (event.data.type) {
case 'epg_file':
fetchEPGs();
notifications.show({
title: 'EPG File Detected',
message: `Processing ${event.data.filename}`,
});
break;
case 'm3u_file':
fetchPlaylists();
notifications.show({
title: 'M3U File Detected',
message: `Processing ${event.data.filename}`,
});
break;
case 'm3u_group_refresh':
fetchChannelGroups();
fetchPlaylists();
notifications.show({
title: 'Group processing finished!',
autoClose: 5000,
message: (
<Stack>
Refresh M3U or filter out groups to pull in streams.
<Button
size="xs"
variant="default"
onClick={() => {
API.refreshPlaylist(event.data.account);
setRefreshProgress(event.data.account, 0);
}}
>
Refresh Now
</Button>
</Stack>
),
color: 'green.5',
});
break;
case 'm3u_refresh':
console.log('inside m3u_refresh event');
if (event.data.success) {
fetchStreams();
notifications.show({
message: event.data.message,
color: 'green.5',
});
} else if (event.data.progress) {
if (event.data.progress == 100) {
fetchStreams();
fetchChannelGroups();
fetchEPGData();
fetchPlaylists();
}
setRefreshProgress(event.data.account, event.data.progress);
}
setRefreshProgress(event.data);
break;
case 'channel_stats':
setChannelStats(JSON.parse(event.data.stats));
break;
case 'epg_channels':
notifications.show({
message: 'EPG channels updated!',
color: 'green.5',
});
fetchEPGData();
break;
case 'epg_match':
notifications.show({
message: 'EPG match is complete!',
color: 'green.5',
});
fetchChannels();
fetchEPGData();
break;
case 'm3u_profile_test':
setProfilePreview(event.data.search_preview, event.data.result);
break;
case 'recording_started':
notifications.show({
title: 'Recording started!',
message: `Started recording channel ${event.data.channel}`,
});
break;
case 'recording_ended':
notifications.show({
title: 'Recording finished!',
message: `Stopped recording channel ${event.data.channel}`,
});
break;
default:
console.error(`Unknown websocket event type: ${event.type}`);
break;
@ -91,7 +159,9 @@ export const WebsocketProvider = ({ children }) => {
};
}, []);
const ret = [isReady, val, ws.current?.send.bind(ws.current)];
const ret = useMemo(() => {
return [isReady, ws.current?.send.bind(ws.current), val];
}, [isReady, val]);
return (
<WebsocketContext.Provider value={ret}>

1479
frontend/src/api.js
View file

File diff suppressed because it is too large Load diff

BIN
frontend/src/assets/android-chrome-192x192.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
frontend/src/assets/android-chrome-512x512.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 32 KiB

Some files were not shown because too many files have changed in this diff Show more