From 78fc7d9f2b1e4221c3a6d68459551b80d1284942 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 12:42:48 -0500 Subject: [PATCH 1/7] Use proper user agents when downloading epgs. And a little more robust user agent selection for m3u downloads. --- apps/epg/tasks.py | 32 +++++++++++++++++++++++++++++++- apps/m3u/tasks.py | 13 +++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 74411bdb..cb985a51 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -13,6 +13,7 @@ from django.conf import settings from django.db import transaction from django.utils import timezone from apps.channels.models import Channel +from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -67,7 +68,22 @@ def fetch_xmltv(source): logger.info(f"Fetching XMLTV data from source: {source.name}") try: - response = requests.get(source.url, timeout=30) + # Get default user agent from settings + default_user_agent_setting = CoreSettings.objects.filter(key='default-user-agent').first() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + if default_user_agent_setting and default_user_agent_setting.value: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_setting.value)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + headers = { + 'User-Agent': user_agent + } + + response = requests.get(source.url, headers=headers, timeout=30) response.raise_for_status() logger.debug("XMLTV data fetched successfully.") @@ -296,10 +312,24 @@ def parse_programs_for_source(epg_source, tvg_id=None): def fetch_schedules_direct(source): logger.info(f"Fetching Schedules Direct data from source: {source.name}") try: + # Get default user agent from settings + default_user_agent_setting = CoreSettings.objects.filter(key='default-user-agent').first() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + + if default_user_agent_setting and default_user_agent_setting.value: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_setting.value)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + api_url = '' headers = { 'Content-Type': 'application/json', 'Authorization': f'Bearer {source.api_key}', + 'User-Agent': user_agent } logger.debug(f"Requesting subscriptions from Schedules Direct using URL: {api_url}") response = requests.get(api_url, headers=headers, timeout=30) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index beacaaa2..c6393123 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -35,10 +35,15 @@ def fetch_m3u_lines(account, use_cache=False): """Fetch M3U file lines efficiently.""" if account.server_url: if not use_cache or not os.path.exists(file_path): - user_agent = account.get_user_agent() - headers = {"User-Agent": user_agent.user_agent} - logger.info(f"Fetching from URL {account.server_url}") try: + # Try to get account-specific user agent first + user_agent_obj = account.get_user_agent() + user_agent = user_agent_obj.user_agent if user_agent_obj else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + + logger.debug(f"Using user agent: {user_agent} for M3U account: {account.name}") + headers = {"User-Agent": user_agent} + logger.info(f"Fetching from URL {account.server_url}") + response = requests.get(account.server_url, headers=headers, stream=True) response.raise_for_status() @@ -74,7 +79,7 @@ def fetch_m3u_lines(account, use_cache=False): send_m3u_update(account.id, "downloading", progress, speed=speed, elapsed_time=elapsed_time, time_remaining=time_remaining) send_m3u_update(account.id, "downloading", 100) - except requests.exceptions.RequestException as e: + except Exception as e: logger.error(f"Error fetching M3U from URL {account.server_url}: {e}") return [] From 90c1c3d2eddce265922d5db31e7e55cc226b5f7b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 13:10:49 -0500 Subject: [PATCH 2/7] uwsgi config tuning. --- docker/uwsgi.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 326f4b5d..726730bf 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -24,11 +24,8 @@ vacuum = true die-on-term = true static-map = /static=/app/static -# Worker management (Optimize for I/O bound tasks) +# Worker management workers = 4 -threads = 4 -enable-threads = true -thread-stacksize=512 # Optimize for streaming http = 0.0.0.0:5656 @@ -39,8 +36,9 @@ http-timeout = 600 # Prevent disconnects from long streams lazy-apps = true # Improve memory efficiency # Async mode (use gevent for high concurrency) -gevent = 100 -async = 100 +gevent = 400 # Each unused greenlet costs ~2-4KB of memory +# Higher values have minimal performance impact when idle, but provide capacity for traffic spikes +# If memory usage becomes an issue, reduce this value # Performance tuning thunder-lock = true From 4cb2cb7b206ac0166493f46b97370c0d65aa3721 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 1 May 2025 18:49:59 +0000 Subject: [PATCH 3/7] Release v0.4.1 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 25e27d60..171eaa80 100644 --- a/version.py +++ b/version.py @@ -1,5 +1,5 @@ """ Dispatcharr version information. """ -__version__ = '0.4.0' # Follow semantic versioning (MAJOR.MINOR.PATCH) +__version__ = '0.4.1' # Follow semantic versioning (MAJOR.MINOR.PATCH) __timestamp__ = None # Set during CI/CD build process From d26944a7a51528a2db75abdebaec75c90e1f83bf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 16:01:08 -0500 Subject: [PATCH 4/7] Add stale_stream_days field to M3UAccount model and update related logic - Introduced stale_stream_days field to M3UAccount to specify the retention period for streams. - Updated cleanup_streams task to remove streams not seen within the specified stale_stream_days. - Enhanced M3U form to include stale_stream_days input for user configuration. --- .../0008_m3uaccount_stale_stream_days.py | 18 +++++++++++++++ apps/m3u/models.py | 4 ++++ apps/m3u/serializers.py | 2 +- apps/m3u/tasks.py | 23 +++++++++++++++---- frontend/src/components/forms/M3U.jsx | 12 +++++++++- 5 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py diff --git a/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py b/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py new file mode 100644 index 00000000..69a1397d --- /dev/null +++ b/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('m3u', '0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path'), + ] + + operations = [ + migrations.AddField( + model_name='m3uaccount', + name='stale_stream_days', + field=models.PositiveIntegerField(default=7, help_text='Number of days after which a stream will be removed if not seen in the M3U source.'), + ), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 25a332c6..06c206f6 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -74,6 +74,10 @@ class M3UAccount(models.Model): refresh_task = models.ForeignKey( PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True ) + stale_stream_days = models.PositiveIntegerField( + default=7, + help_text="Number of days after which a stream will be removed if not seen in the M3U source." + ) def __str__(self): return self.name diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index d79b0117..43015713 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -65,7 +65,7 @@ class M3UAccountSerializer(serializers.ModelSerializer): model = M3UAccount fields = [ 'id', 'name', 'server_url', 'file_path', 'server_group', - 'max_streams', 'is_active', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', + 'max_streams', 'is_active', 'stale_stream_days', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', 'channel_groups', 'refresh_interval' ] diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index c6393123..20ed1acb 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -312,18 +312,31 @@ def cleanup_streams(account_id): m3u_account__enabled=True, ).values_list('id', flat=True) logger.info(f"Found {len(existing_groups)} active groups") - streams = Stream.objects.filter(m3u_account=account) + # Calculate cutoff date for stale streams + stale_cutoff = timezone.now() - timezone.timedelta(days=account.stale_stream_days) + logger.info(f"Removing streams not seen since {stale_cutoff}") + + # Delete streams that are not in active groups streams_to_delete = Stream.objects.filter( m3u_account=account ).exclude( - channel_group__in=existing_groups # Exclude products having any of the excluded tags + channel_group__in=existing_groups ) - # Delete the filtered products - streams_to_delete.delete() + # Also delete streams that haven't been seen for longer than stale_stream_days + stale_streams = Stream.objects.filter( + m3u_account=account, + last_seen__lt=stale_cutoff + ) - logger.info(f"Cleanup complete") + deleted_count = streams_to_delete.count() + stale_count = stale_streams.count() + + streams_to_delete.delete() + stale_streams.delete() + + logger.info(f"Cleanup complete: {deleted_count} streams removed due to group filter, {stale_count} removed as stale") @shared_task def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index bd9cf3c8..8fee0d9b 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -52,6 +52,7 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { is_active: true, max_streams: 0, refresh_interval: 24, + stale_stream_days: 7, }, validate: { @@ -65,10 +66,11 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { if (playlist) { form.setValues({ name: playlist.name, - server_url: playlist.server_url, + server_url: playlist.server_url || '', max_streams: playlist.max_streams, user_agent: playlist.user_agent ? `${playlist.user_agent}` : '0', is_active: playlist.is_active, + stale_stream_days: playlist.stale_stream_days || 7, refresh_interval: playlist.refresh_interval, }); } else { @@ -202,6 +204,14 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { key={form.key('refresh_interval')} /> + + Date: Thu, 1 May 2025 17:30:19 -0400 Subject: [PATCH 5/7] updated last_seen of existing streams: --- apps/m3u/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 20ed1acb..90992dbc 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -292,8 +292,8 @@ def process_m3u_batch(account_id, batch, groups, hash_keys): Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) if streams_to_update: Stream.objects.bulk_update(streams_to_update, { key for key in stream_props.keys() if key not in ["m3u_account", "stream_hash"] and key not in hash_keys}) - # if len(existing_streams.keys()) > 0: - # Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) + if len(existing_streams.keys()) > 0: + Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) except Exception as e: logger.error(f"Bulk create failed: {str(e)}") From 0e54062c7358cf284c1394c853098b16aa9d8a12 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 17:42:23 -0500 Subject: [PATCH 6/7] Added drivers for hardware acceleration. --- docker/Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 26b54975..92705403 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -76,7 +76,15 @@ RUN apt-get update && \ streamlink \ wget \ gnupg2 \ - lsb-release && \ + lsb-release \ + libva-drm2 \ + libva-x11-2 \ + libva-dev \ + libva-wayland2 \ + vainfo \ + i965-va-driver \ + intel-media-va-driver \ + mesa-va-drivers && \ cp /app/docker/nginx.conf /etc/nginx/sites-enabled/default && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From 6f1bae819501de06d6084e65a0c607e068554ced Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 2 May 2025 08:23:11 -0500 Subject: [PATCH 7/7] Fixes stream preview in stream table. --- frontend/src/components/FloatingVideo.jsx | 64 ++++++++++++++----- .../src/components/tables/StreamsTable.jsx | 3 +- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 0be42029..9d8fb09e 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -18,23 +18,57 @@ export default function FloatingVideo() { return; } - // If the browser supports MSE for live playback, initialize mpegts.js - if (mpegts.getFeatureList().mseLivePlayback) { - const player = mpegts.createPlayer({ - type: 'mpegts', - url: streamUrl, - isLive: true, - // You can include other custom MPEGTS.js config fields here, e.g.: - // cors: true, - // withCredentials: false, - }); + // Check if we have an existing player and clean it up + if (playerRef.current) { + playerRef.current.destroy(); + playerRef.current = null; + } - player.attachMediaElement(videoRef.current); - player.load(); - player.play(); + // Debug log to help diagnose stream issues + console.log("Attempting to play stream:", streamUrl); - // Store player instance so we can clean up later - playerRef.current = player; + try { + // If the browser supports MSE for live playback, initialize mpegts.js + if (mpegts.getFeatureList().mseLivePlayback) { + const player = mpegts.createPlayer({ + type: 'mpegts', // MPEG-TS format + url: streamUrl, + isLive: true, + enableWorker: true, + enableStashBuffer: false, // Try disabling stash buffer for live streams + liveBufferLatencyChasing: true, + liveSync: true, + cors: true, // Enable CORS for cross-domain requests + }); + + player.attachMediaElement(videoRef.current); + + // Add error event handler + player.on(mpegts.Events.ERROR, (errorType, errorDetail) => { + console.error('Player error:', errorType, errorDetail); + // If it's a format issue, show a helpful message + if (errorDetail.includes('Unsupported media type')) { + const message = document.createElement('div'); + message.textContent = "Unsupported stream format. Please try a different stream."; + message.style.position = 'absolute'; + message.style.top = '50%'; + message.style.left = '50%'; + message.style.transform = 'translate(-50%, -50%)'; + message.style.color = 'white'; + message.style.textAlign = 'center'; + message.style.width = '100%'; + videoRef.current.parentNode.appendChild(message); + } + }); + + player.load(); + player.play(); + + // Store player instance so we can clean up later + playerRef.current = player; + } + } catch (error) { + console.error("Error initializing player:", error); } // Cleanup when component unmounts or streamUrl changes diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index 078e24d9..00d09b67 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -89,8 +89,9 @@ const StreamRowActions = ({ }, []); const onPreview = useCallback(() => { + console.log('Previewing stream:', row.original.name, 'ID:', row.original.id, 'Hash:', row.original.stream_hash); handleWatchStream(row.original.stream_hash); - }, []); + }, [row.original.id]); // Add proper dependency to ensure correct stream return ( <>