From 3054cf2ae9fa9ed5bea21cde7401374e45faef3f Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 27 Apr 2025 10:32:29 -0400 Subject: [PATCH 0001/1290] initial xtreamcodes support --- ...upm3uaccount_custom_properties_and_more.py | 23 +++ apps/channels/models.py | 1 + ...count_type_m3uaccount_password_and_more.py | 28 +++ apps/m3u/models.py | 7 + apps/m3u/serializers.py | 8 +- apps/m3u/tasks.py | 164 +++++++++++++----- core/xtream_codes.py | 26 +++ frontend/src/api.js | 31 +++- frontend/src/components/forms/M3U.jsx | 84 +++++++-- 9 files changed, 305 insertions(+), 67 deletions(-) create mode 100644 apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py create mode 100644 apps/m3u/migrations/0008_m3uaccount_account_type_m3uaccount_password_and_more.py create mode 100644 core/xtream_codes.py diff --git a/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py b/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py new file mode 100644 index 00000000..7d2dafb4 --- /dev/null +++ b/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.6 on 2025-04-27 14:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcharr_channels', '0017_alter_channelgroup_name'), + ] + + operations = [ + migrations.AddField( + model_name='channelgroupm3uaccount', + name='custom_properties', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='channel', + name='channel_number', + field=models.IntegerField(db_index=True), + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 0b66c468..13172e36 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -441,6 +441,7 @@ class ChannelGroupM3UAccount(models.Model): on_delete=models.CASCADE, related_name='channel_group' ) + custom_properties = models.TextField(null=True, blank=True) enabled = models.BooleanField(default=True) class Meta: diff --git a/apps/m3u/migrations/0008_m3uaccount_account_type_m3uaccount_password_and_more.py b/apps/m3u/migrations/0008_m3uaccount_account_type_m3uaccount_password_and_more.py new file mode 100644 index 00000000..02d2937f --- /dev/null +++ b/apps/m3u/migrations/0008_m3uaccount_account_type_m3uaccount_password_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.6 on 2025-04-27 12:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('m3u', '0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path'), + ] + + operations = [ + migrations.AddField( + model_name='m3uaccount', + name='account_type', + field=models.CharField(choices=[('STD', 'Standard'), ('XC', 'Xtream Codes')], default='STD'), + ), + migrations.AddField( + model_name='m3uaccount', + name='password', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='m3uaccount', + name='username', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 25a332c6..99ead627 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -10,6 +10,10 @@ from core.models import CoreSettings, UserAgent CUSTOM_M3U_ACCOUNT_NAME="custom" class M3UAccount(models.Model): + class Types(models.TextChoices): + STADNARD = "STD", "Standard" + XC = "XC", "Xtream Codes" + """Represents an M3U Account for IPTV streams.""" name = models.CharField( max_length=255, @@ -69,6 +73,9 @@ class M3UAccount(models.Model): blank=True, related_name='m3u_accounts' ) + account_type = models.CharField(choices=Types.choices, default=Types.STADNARD) + username = models.CharField(max_length=255, null=True, blank=True) + password = models.CharField(max_length=255, null=True, blank=True) custom_properties = models.TextField(null=True, blank=True) refresh_interval = models.IntegerField(default=24) refresh_task = models.ForeignKey( diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index d79b0117..dd9b0e7a 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -66,8 +66,14 @@ class M3UAccountSerializer(serializers.ModelSerializer): fields = [ 'id', 'name', 'server_url', 'file_path', 'server_group', 'max_streams', 'is_active', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', - 'channel_groups', 'refresh_interval' + 'channel_groups', 'refresh_interval', 'custom_properties', 'account_type', 'username', 'password' ] + extra_kwargs = { + 'password': { + 'required': False, + 'allow_blank': True, + }, + } def update(self, instance, validated_data): # Pop out channel group memberships so we can handle them manually diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index beacaaa2..978f9763 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -21,6 +21,7 @@ import json from core.utils import RedisClient, acquire_task_lock, release_task_lock from core.models import CoreSettings from asgiref.sync import async_to_sync +from core.xtream_codes import Client as XCClient logger = logging.getLogger(__name__) @@ -172,32 +173,33 @@ def check_field_lengths(streams_to_create): print("") @shared_task -def process_groups(account, group_names): - existing_groups = {group.name: group for group in ChannelGroup.objects.filter(name__in=group_names)} +def process_groups(account, groups): + existing_groups = {group.name: group for group in ChannelGroup.objects.filter(name__in=groups.keys())} logger.info(f"Currently {len(existing_groups)} existing groups") - groups = [] + group_objs = [] groups_to_create = [] - for group_name in group_names: + for group_name, custom_props in groups.items(): logger.info(f"Handling group: {group_name}") - if group_name in existing_groups: - groups.append(existing_groups[group_name]) - else: + if group_name not in existing_groups: groups_to_create.append(ChannelGroup( name=group_name, )) + else: + group_objs.append(existing_groups[group_name]) if groups_to_create: logger.info(f"Creating {len(groups_to_create)} groups") created = ChannelGroup.bulk_create_and_fetch(groups_to_create) logger.info(f"Created {len(created)} groups") - groups.extend(created) + group_objs.extend(created) relations = [] - for group in groups: + for group in group_objs: relations.append(ChannelGroupM3UAccount( channel_group=group, m3u_account=account, + custom_properties=json.dumps(groups[group.name]), )) ChannelGroupM3UAccount.objects.bulk_create( @@ -205,6 +207,78 @@ def process_groups(account, group_names): ignore_conflicts=True ) +@shared_task +def process_xc_category(account_id, batch, groups, hash_keys): + account = M3UAccount.objects.get(id=account_id) + + streams_to_create = [] + streams_to_update = [] + stream_hashes = {} + + xc_client = XCClient(account.server_url, account.username, account.password) + for group_name, props in batch.items(): + streams = xc_client.get_live_category_streams(props['xc_id']) + for stream in streams: + name = stream["name"] + url = xc_client.get_stream_url(stream["stream_id"]) + tvg_id = stream["epg_channel_id"] + tvg_logo = stream["stream_icon"] + group_title = group_name + + stream_hash = Stream.generate_hash_key(name, url, tvg_id, hash_keys) + stream_props = { + "name": name, + "url": url, + "logo_url": tvg_logo, + "tvg_id": tvg_id, + "m3u_account": account, + "channel_group_id": int(groups.get(group_title)), + "stream_hash": stream_hash, + "custom_properties": json.dumps(stream), + } + + if stream_hash not in stream_hashes: + stream_hashes[stream_hash] = stream_props + + existing_streams = {s.stream_hash: s for s in Stream.objects.filter(stream_hash__in=stream_hashes.keys())} + + for stream_hash, stream_props in stream_hashes.items(): + if stream_hash in existing_streams: + obj = existing_streams[stream_hash] + existing_attr = {field.name: getattr(obj, field.name) for field in Stream._meta.fields if field != 'channel_group_id'} + changed = any(existing_attr[key] != value for key, value in stream_props.items() if key != 'channel_group_id') + + if changed: + for key, value in stream_props.items(): + setattr(obj, key, value) + obj.last_seen = timezone.now() + streams_to_update.append(obj) + del existing_streams[stream_hash] + else: + existing_streams[stream_hash] = obj + else: + stream_props["last_seen"] = timezone.now() + streams_to_create.append(Stream(**stream_props)) + + try: + with transaction.atomic(): + if streams_to_create: + Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) + if streams_to_update: + Stream.objects.bulk_update(streams_to_update, { key for key in stream_props.keys() if key not in ["m3u_account", "stream_hash"] and key not in hash_keys}) + # if len(existing_streams.keys()) > 0: + # Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) + except Exception as e: + logger.error(f"Bulk create failed: {str(e)}") + + retval = f"Batch processed: {len(streams_to_create)} created, {len(streams_to_update)} updated." + + # Aggressive garbage collection + del streams_to_create, streams_to_update, stream_hashes, existing_streams + gc.collect() + + return retval + @shared_task def process_m3u_batch(account_id, batch, groups, hash_keys): """Processes a batch of M3U streams using bulk operations.""" @@ -227,23 +301,7 @@ def process_m3u_batch(account_id, batch, groups, hash_keys): logger.debug(f"Skipping stream in disabled group: {group_title}") continue - # if any(url.lower().endswith(ext) for ext in SKIP_EXTS) or len(url) > 2000: - # continue - - # if _matches_filters(name, group_title, account.filters.all()): - # continue - - # if any(compiled_pattern.search(current_info['name']) for ftype, compiled_pattern in compiled_filters if ftype == 'name'): - # excluded_count += 1 - # current_info = None - # continue - stream_hash = Stream.generate_hash_key(name, url, tvg_id, hash_keys) - # if redis_client.exists(f"m3u_refresh:{stream_hash}"): - # # duplicate already processed by another batch - # continue - - # redis_client.set(f"m3u_refresh:{stream_hash}", "true") stream_props = { "name": name, "url": url, @@ -332,24 +390,38 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): return f"M3UAccount with ID={account_id} not found or inactive.", None extinf_data = [] - groups = set(["Default Group"]) + groups = {"Default Group": {}} - for line in fetch_m3u_lines(account, use_cache): - line = line.strip() - if line.startswith("#EXTINF"): - parsed = parse_extinf_line(line) - if parsed: - if "group-title" in parsed["attributes"]: - groups.add(parsed["attributes"]["group-title"]) + xc_client = None + if account.account_type == M3UAccount.Types.XC: + xc_client = XCClient(account.server_url, account.username, account.password) + try: + xc_client.authenticate() + except Exception as e: + release_task_lock('refresh_m3u_account_groups', account_id) + return f"M3UAccount with ID={account_id} failed to authenticate with XC server.", None - extinf_data.append(parsed) - elif extinf_data and line.startswith("http"): - # Associate URL with the last EXTINF line - extinf_data[-1]["url"] = line + xc_categories = xc_client.get_live_categories() + for category in xc_categories: + groups[category["category_name"]] = { + "xc_id": category["category_id"], + } + else: + for line in fetch_m3u_lines(account, use_cache): + line = line.strip() + if line.startswith("#EXTINF"): + parsed = parse_extinf_line(line) + if parsed: + if "group-title" in parsed["attributes"]: + groups[parsed["attributes"]["group-title"]] = {} + + extinf_data.append(parsed) + elif extinf_data and line.startswith("http"): + # Associate URL with the last EXTINF line + extinf_data[-1]["url"] = line send_m3u_update(account_id, "processing_groups", 0) - groups = list(groups) cache_path = os.path.join(m3u_dir, f"{account_id}.json") with open(cache_path, 'w', encoding='utf-8') as f: json.dump({ @@ -412,7 +484,7 @@ def refresh_single_m3u_account(account_id): if not extinf_data: try: extinf_data, groups = refresh_m3u_groups(account_id, full_refresh=True) - if not extinf_data or not groups: + if not groups: release_task_lock('refresh_single_m3u_account', account_id) return "Failed to update m3u account, task may already be running" except: @@ -426,9 +498,17 @@ def refresh_single_m3u_account(account_id): m3u_account__enabled=True # Filter by the enabled flag in the join table )} - # Break into batches and process in parallel - batches = [extinf_data[i:i + BATCH_SIZE] for i in range(0, len(extinf_data), BATCH_SIZE)] - task_group = group(process_m3u_batch.s(account_id, batch, existing_groups, hash_keys) for batch in batches) + if account.account_type == M3UAccount.Types.STADNARD: + # Break into batches and process in parallel + batches = [extinf_data[i:i + BATCH_SIZE] for i in range(0, len(extinf_data), BATCH_SIZE)] + task_group = group(process_m3u_batch.s(account_id, batch, existing_groups, hash_keys) for batch in batches) + else: + filtered_groups = [(k, v) for k, v in groups.items() if k in existing_groups] + batches = [ + dict(filtered_groups[i:i + 2]) + for i in range(0, len(filtered_groups), 2) + ] + task_group = group(process_xc_category.s(account_id, batch, existing_groups, hash_keys) for batch in batches) total_batches = len(batches) completed_batches = 0 diff --git a/core/xtream_codes.py b/core/xtream_codes.py new file mode 100644 index 00000000..e79ec5e9 --- /dev/null +++ b/core/xtream_codes.py @@ -0,0 +1,26 @@ +import requests + +class Client: + host = "" + username = "" + password = "" + + def __init__(self, host, username, password): + self.host = host + self.username = username + self.password = password + + def authenticate(self): + response = requests.get(f"{self.host}/player_api.php?username={self.username}&password={self.password}") + return response.json() + + def get_live_categories(self): + response = requests.get(f"{self.host}/player_api.php?username={self.username}&password={self.password}&action=get_live_categories") + return response.json() + + def get_live_category_streams(self, category_id): + response = requests.get(f"{self.host}/player_api.php?username={self.username}&password={self.password}&action=get_live_streams&category_id={category_id}") + return response.json() + + def get_stream_url(self, stream_id): + return f"{self.host}/{self.username}/{self.password}/{stream_id}" diff --git a/frontend/src/api.js b/frontend/src/api.js index 38aac846..2c7f3a05 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -650,6 +650,10 @@ export default class API { } static async addPlaylist(values) { + if (values.custom_properties) { + values.custom_properties = JSON.stringify(values.custom_properties); + } + try { let body = null; if (values.file) { @@ -717,6 +721,10 @@ export default class API { static async updatePlaylist(values) { const { id, ...payload } = values; + if (payload.custom_properties) { + payload.custom_properties = JSON.stringify(payload.custom_properties); + } + try { let body = null; if (payload.file) { @@ -735,6 +743,7 @@ export default class API { body = { ...payload }; delete body.file; } + console.log(body); const response = await request(`${host}/api/m3u/accounts/${id}/`, { method: 'PATCH', @@ -1241,10 +1250,13 @@ export default class API { static async switchStream(channelId, streamId) { try { - const response = await request(`${host}/proxy/ts/change_stream/${channelId}`, { - method: 'POST', - body: { stream_id: streamId }, - }); + const response = await request( + `${host}/proxy/ts/change_stream/${channelId}`, + { + method: 'POST', + body: { stream_id: streamId }, + } + ); return response; } catch (e) { @@ -1255,10 +1267,13 @@ export default class API { static async nextStream(channelId, streamId) { try { - const response = await request(`${host}/proxy/ts/next_stream/${channelId}`, { - method: 'POST', - body: { stream_id: streamId }, - }); + const response = await request( + `${host}/proxy/ts/next_stream/${channelId}`, + { + method: 'POST', + body: { stream_id: streamId }, + } + ); return response; } catch (e) { diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index bd9cf3c8..2d0f29ba 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -18,6 +18,8 @@ import { Stack, Group, Switch, + Box, + PasswordInput, } from '@mantine/core'; import M3UGroupFilter from './M3UGroupFilter'; import useChannelsStore from '../../store/channels'; @@ -35,13 +37,7 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { const [profileModalOpen, setProfileModalOpen] = useState(false); const [groupFilterModalOpen, setGroupFilterModalOpen] = useState(false); const [loadingText, setLoadingText] = useState(''); - - const handleFileChange = (file) => { - console.log(file); - if (file) { - setFile(file); - } - }; + const [showCredentialFields, setShowCredentialFields] = useState(false); const form = useForm({ mode: 'uncontrolled', @@ -52,6 +48,9 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { is_active: true, max_streams: 0, refresh_interval: 24, + is_xc: false, + username: '', + password: '', }, validate: { @@ -63,6 +62,7 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { useEffect(() => { if (playlist) { + const customProperties = JSON.parse(playlist.custom_properties || '{}'); form.setValues({ name: playlist.name, server_url: playlist.server_url, @@ -70,14 +70,43 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { user_agent: playlist.user_agent ? `${playlist.user_agent}` : '0', is_active: playlist.is_active, refresh_interval: playlist.refresh_interval, + is_xc: playlist.account_type == 'XC', + username: customProperties.username ?? '', + password: '', }); + + if (customProperties.is_xc) { + setShowCredentialFields(true); + } else { + setShowCredentialFields(false); + } } else { form.reset(); } }, [playlist]); + useEffect(() => { + if (form.values.is_xc) { + setShowCredentialFields(true); + } + }, [form.values.is_xc]); + const onSubmit = async () => { - const values = form.getValues(); + const { ...values } = form.getValues(); + + if (values.is_xc && values.password == '') { + // If account XC and no password input, assuming no password change + // from previously stored value. + delete values.password; + } + + if (values.is_xc) { + values.account_type = 'XC'; + } else { + values.account_type = 'STD'; + } + + delete values.is_xc; if (values.user_agent == '0') { values.user_agent = null; @@ -150,7 +179,6 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { {...form.getInputProps('name')} key={form.key('name')} /> - { {...form.getInputProps('server_url')} key={form.key('server_url')} /> - - + + {form.getValues().is_xc && ( + + + + + )} + + {!form.getValues().is_xc && ( + + )} From bfaa52ea133bf67554a75b62e8a141848130d821 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 27 Apr 2025 10:38:30 -0400 Subject: [PATCH 0002/1290] fixed username field --- frontend/src/components/forms/M3U.jsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 2d0f29ba..51370f27 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -62,7 +62,6 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { useEffect(() => { if (playlist) { - const customProperties = JSON.parse(playlist.custom_properties || '{}'); form.setValues({ name: playlist.name, server_url: playlist.server_url, @@ -71,11 +70,11 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { is_active: playlist.is_active, refresh_interval: playlist.refresh_interval, is_xc: playlist.account_type == 'XC', - username: customProperties.username ?? '', + username: playlist.username ?? '', password: '', }); - if (customProperties.is_xc) { + if (playlist.account_type == 'XC') { setShowCredentialFields(true); } else { setShowCredentialFields(false); From f295ee219cc5a7524cdb86a4f80e232dc19d39f8 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 27 Apr 2025 10:54:39 -0400 Subject: [PATCH 0003/1290] removed db index here, I don't think it's needed --- ...0018_channelgroupm3uaccount_custom_properties_and_more.py | 5 ----- apps/channels/models.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py b/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py index 7d2dafb4..51507843 100644 --- a/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py +++ b/apps/channels/migrations/0018_channelgroupm3uaccount_custom_properties_and_more.py @@ -15,9 +15,4 @@ class Migration(migrations.Migration): name='custom_properties', field=models.TextField(blank=True, null=True), ), - migrations.AlterField( - model_name='channel', - name='channel_number', - field=models.IntegerField(db_index=True), - ), ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 13172e36..ee92a43b 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -210,7 +210,7 @@ class ChannelManager(models.Manager): class Channel(models.Model): - channel_number = models.IntegerField(db_index=True) + channel_number = models.IntegerField() name = models.CharField(max_length=255) logo = models.ForeignKey( 'Logo', From 1ccf24fe5f936ccdd3dacb72125408c1f33cecb8 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 27 Apr 2025 11:08:12 -0400 Subject: [PATCH 0004/1290] Fixed caching path for non-xc only --- apps/m3u/tasks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 978f9763..ce536234 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -420,14 +420,14 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): # Associate URL with the last EXTINF line extinf_data[-1]["url"] = line - send_m3u_update(account_id, "processing_groups", 0) + cache_path = os.path.join(m3u_dir, f"{account_id}.json") + with open(cache_path, 'w', encoding='utf-8') as f: + json.dump({ + "extinf_data": extinf_data, + "groups": groups, + }, f) - cache_path = os.path.join(m3u_dir, f"{account_id}.json") - with open(cache_path, 'w', encoding='utf-8') as f: - json.dump({ - "extinf_data": extinf_data, - "groups": groups, - }, f) + send_m3u_update(account_id, "processing_groups", 0) process_groups(account, groups) From a86ae715b9d41f0fe99dc783857c1342c1a2fb4b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 30 Apr 2025 21:11:41 -0500 Subject: [PATCH 0005/1290] Fixes add streams to channel to follow correct logic of being disabled. --- .../src/components/tables/StreamsTable.jsx | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index 276ce189..078e24d9 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -156,7 +156,7 @@ const StreamRowActions = ({ ); }; -const StreamsTable = ({}) => { +const StreamsTable = ({ }) => { const theme = useMantineTheme(); /** @@ -606,23 +606,22 @@ const StreamsTable = ({}) => { {/* Top toolbar with Remove, Assign, Auto-match, and Add buttons */} - {selectedStreamIds.length > 0 && ( - - )} + Date: Thu, 1 May 2025 09:05:51 -0500 Subject: [PATCH 0006/1290] More sleep events. --- apps/proxy/ts_proxy/views.py | 5 +++-- docker/uwsgi.ini | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index 35ca3648..ef232fd2 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -24,6 +24,7 @@ from .services.channel_service import ChannelService from .url_utils import generate_stream_url, transform_url, get_stream_info_for_switch, get_stream_object, get_alternate_streams from .utils import get_logger from uuid import UUID +import gevent logger = get_logger() @@ -119,7 +120,7 @@ def stream_ts(request, channel_id): # Wait before retrying (using exponential backoff with a cap) wait_time = min(0.5 * (2 ** attempt), 2.0) # Caps at 2 seconds logger.info(f"[{client_id}] Waiting {wait_time:.1f}s for a connection to become available (attempt {attempt+1}/{max_retries})") - time.sleep(wait_time) + gevent.sleep(wait_time) # FIXED: Using gevent.sleep instead of time.sleep if stream_url is None: # Make sure to release any stream locks that might have been acquired @@ -258,7 +259,7 @@ def stream_ts(request, channel_id): proxy_server.stop_channel(channel_id) return JsonResponse({'error': 'Failed to connect'}, status=502) - time.sleep(0.1) + gevent.sleep(0.1) # FIXED: Using gevent.sleep instead of time.sleep logger.info(f"[{client_id}] Successfully initialized channel {channel_id}") channel_initializing = True diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 326f4b5d..b1ff362b 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -41,6 +41,7 @@ lazy-apps = true # Improve memory efficiency # Async mode (use gevent for high concurrency) gevent = 100 async = 100 +gevent-monkey-patch = true ; Ensure all blocking operations are patched (especially important for Ryzen CPUs) # Performance tuning thunder-lock = true From c11ce048c7fc60676a2e83cf8d5d149c36fb6e1c Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 09:24:16 -0500 Subject: [PATCH 0007/1290] Disable monkey patching. --- docker/uwsgi.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index b1ff362b..b40259c2 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -41,7 +41,7 @@ lazy-apps = true # Improve memory efficiency # Async mode (use gevent for high concurrency) gevent = 100 async = 100 -gevent-monkey-patch = true ; Ensure all blocking operations are patched (especially important for Ryzen CPUs) +#gevent-monkey-patch = true ; Ensure all blocking operations are patched (especially important for Ryzen CPUs) # Performance tuning thunder-lock = true From e8ee59cf00fd68e86f653efc59890a671cd31867 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 09:31:26 -0500 Subject: [PATCH 0008/1290] Not sure why it didn't push. --- docker/uwsgi.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index b40259c2..326f4b5d 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -41,7 +41,6 @@ lazy-apps = true # Improve memory efficiency # Async mode (use gevent for high concurrency) gevent = 100 async = 100 -#gevent-monkey-patch = true ; Ensure all blocking operations are patched (especially important for Ryzen CPUs) # Performance tuning thunder-lock = true From 091d9a6823e0b4c9fc35a0afcc37c12556249b51 Mon Sep 17 00:00:00 2001 From: dekzter Date: Thu, 1 May 2025 11:22:38 -0400 Subject: [PATCH 0009/1290] immediately prompt for group filtering when using an XC account --- apps/m3u/api_views.py | 5 ++ apps/m3u/signals.py | 2 +- .../src/components/M3URefreshNotification.jsx | 4 +- frontend/src/components/forms/M3U.jsx | 76 ++++++++++++------- .../src/components/forms/M3UGroupFilter.jsx | 2 +- frontend/src/components/tables/M3UsTable.jsx | 2 +- 6 files changed, 61 insertions(+), 30 deletions(-) diff --git a/apps/m3u/api_views.py b/apps/m3u/api_views.py index 054bdaa9..ddccbc11 100644 --- a/apps/m3u/api_views.py +++ b/apps/m3u/api_views.py @@ -10,6 +10,7 @@ from django.core.cache import cache import os from rest_framework.decorators import action from django.conf import settings +from .tasks import refresh_m3u_groups # Import all models, including UserAgent. from .models import M3UAccount, M3UFilter, ServerGroup, M3UAccountProfile @@ -56,6 +57,10 @@ class M3UAccountViewSet(viewsets.ModelViewSet): # Now call super().create() to create the instance response = super().create(request, *args, **kwargs) + print(response.data.get('account_type')) + if response.data.get('account_type') == M3UAccount.Types.XC: + refresh_m3u_groups(response.data.get('id')) + # After the instance is created, return the response return response diff --git a/apps/m3u/signals.py b/apps/m3u/signals.py index 6e46a0ff..dc96ed57 100644 --- a/apps/m3u/signals.py +++ b/apps/m3u/signals.py @@ -13,7 +13,7 @@ def refresh_account_on_save(sender, instance, created, **kwargs): call a Celery task that fetches & parses that single account if it is active or newly created. """ - if created: + if created and instance.account_type != M3UAccount.Types.XC: refresh_m3u_groups.delay(instance.id) @receiver(post_save, sender=M3UAccount) diff --git a/frontend/src/components/M3URefreshNotification.jsx b/frontend/src/components/M3URefreshNotification.jsx index 9e469f43..b1c1984f 100644 --- a/frontend/src/components/M3URefreshNotification.jsx +++ b/frontend/src/components/M3URefreshNotification.jsx @@ -24,8 +24,10 @@ export default function M3URefreshNotification() { return; } - console.log(data); const playlist = playlists.find((pl) => pl.id == data.account); + if (!playlist) { + return; + } setNotificationStatus({ ...notificationStatus, diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 51370f27..28ad382b 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -27,12 +27,19 @@ import usePlaylistsStore from '../../store/playlists'; import { notifications } from '@mantine/notifications'; import { isNotEmpty, useForm } from '@mantine/form'; -const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { +const M3U = ({ + m3uAccount = null, + isOpen, + onClose, + playlistCreated = false, +}) => { const theme = useMantineTheme(); const userAgents = useUserAgentsStore((s) => s.userAgents); const fetchChannelGroups = useChannelsStore((s) => s.fetchChannelGroups); + const fetchPlaylists = usePlaylistsStore((s) => s.fetchPlaylists); + const [playlist, setPlaylist] = useState(null); const [file, setFile] = useState(null); const [profileModalOpen, setProfileModalOpen] = useState(false); const [groupFilterModalOpen, setGroupFilterModalOpen] = useState(false); @@ -61,28 +68,31 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { }); useEffect(() => { - if (playlist) { + console.log(m3uAccount); + if (m3uAccount) { + setPlaylist(m3uAccount); form.setValues({ - name: playlist.name, - server_url: playlist.server_url, - max_streams: playlist.max_streams, - user_agent: playlist.user_agent ? `${playlist.user_agent}` : '0', - is_active: playlist.is_active, - refresh_interval: playlist.refresh_interval, - is_xc: playlist.account_type == 'XC', - username: playlist.username ?? '', + name: m3uAccount.name, + server_url: m3uAccount.server_url, + max_streams: m3uAccount.max_streams, + user_agent: m3uAccount.user_agent ? `${m3uAccount.user_agent}` : '0', + is_active: m3uAccount.is_active, + refresh_interval: m3uAccount.refresh_interval, + is_xc: m3uAccount.account_type == 'XC', + username: m3uAccount.username ?? '', password: '', }); - if (playlist.account_type == 'XC') { + if (m3uAccount.account_type == 'XC') { setShowCredentialFields(true); } else { setShowCredentialFields(false); } } else { + setPlaylist(null); form.reset(); } - }, [playlist]); + }, [m3uAccount]); useEffect(() => { if (form.values.is_xc) { @@ -124,15 +134,26 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { file, }); - notifications.show({ - title: 'Fetching M3U Groups', - message: 'Filter out groups or refresh M3U once complete.', - // color: 'green.5', - }); + if (values.account_type != 'XC') { + notifications.show({ + title: 'Fetching M3U Groups', + message: 'Filter out groups or refresh M3U once complete.', + // color: 'green.5', + }); - // Don't prompt for group filters, but keeping this here - // in case we want to revive it - newPlaylist = null; + // Don't prompt for group filters, but keeping this here + // in case we want to revive it + newPlaylist = null; + close(); + return; + } + + const updatedPlaylist = await API.getPlaylist(newPlaylist.id); + await Promise.all([fetchChannelGroups(), fetchPlaylists()]); + console.log('opening group options'); + setPlaylist(updatedPlaylist); + setGroupFilterModalOpen(true); + return; } form.reset(); @@ -140,13 +161,16 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { onClose(newPlaylist); }; + const close = () => { + form.reset(); + setFile(null); + setPlaylist(null); + onClose(); + }; + const closeGroupFilter = () => { setGroupFilterModalOpen(false); - if (playlistCreated) { - form.reset(); - setFile(null); - onClose(); - } + close(); }; useEffect(() => { @@ -160,7 +184,7 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { } return ( - + { name: channelGroups[group.channel_group].name, })) ); - }, [channelGroups]); + }, [playlist, channelGroups]); const toggleGroupEnabled = (id) => { setGroupStates( diff --git a/frontend/src/components/tables/M3UsTable.jsx b/frontend/src/components/tables/M3UsTable.jsx index 63a25118..75edba17 100644 --- a/frontend/src/components/tables/M3UsTable.jsx +++ b/frontend/src/components/tables/M3UsTable.jsx @@ -358,7 +358,7 @@ const M3UTable = () => { Date: Thu, 1 May 2025 10:43:07 -0500 Subject: [PATCH 0010/1290] Finding more timers that can be converted to gevents. --- apps/hdhr/ssdp.py | 3 ++- apps/proxy/ts_proxy/stream_buffer.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/hdhr/ssdp.py b/apps/hdhr/ssdp.py index 660d9c2f..d794799a 100644 --- a/apps/hdhr/ssdp.py +++ b/apps/hdhr/ssdp.py @@ -2,6 +2,7 @@ import os import socket import threading import time +import gevent # Add this import from django.conf import settings # SSDP Multicast Address and Port @@ -59,7 +60,7 @@ def ssdp_broadcaster(host_ip): sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2) while True: sock.sendto(notify.encode("utf-8"), (SSDP_MULTICAST, SSDP_PORT)) - time.sleep(30) + gevent.sleep(30) # Replace time.sleep with gevent.sleep def start_ssdp(): host_ip = get_host_ip() diff --git a/apps/proxy/ts_proxy/stream_buffer.py b/apps/proxy/ts_proxy/stream_buffer.py index f0be1c52..a5169c3a 100644 --- a/apps/proxy/ts_proxy/stream_buffer.py +++ b/apps/proxy/ts_proxy/stream_buffer.py @@ -12,6 +12,7 @@ from .config_helper import ConfigHelper from .constants import TS_PACKET_SIZE from .utils import get_logger import gevent.event +import gevent # Make sure this import is at the top logger = get_logger() @@ -236,8 +237,8 @@ class StreamBuffer: timers_cancelled = 0 for timer in list(self.fill_timers): try: - if timer and timer.is_alive(): - timer.cancel() + if timer and not timer.dead: # Changed from timer.is_alive() + timer.kill() # Changed from timer.cancel() timers_cancelled += 1 except Exception as e: logger.error(f"Error canceling timer: {e}") @@ -325,8 +326,7 @@ class StreamBuffer: if self.stopping: return None - timer = threading.Timer(delay, callback, args=args, kwargs=kwargs) - timer.daemon = True - timer.start() + # Replace threading.Timer with gevent.spawn_later for better compatibility + timer = gevent.spawn_later(delay, callback, *args, **kwargs) self.fill_timers.append(timer) return timer From 78fc7d9f2b1e4221c3a6d68459551b80d1284942 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 12:42:48 -0500 Subject: [PATCH 0011/1290] Use proper user agents when downloading epgs. And a little more robust user agent selection for m3u downloads. --- apps/epg/tasks.py | 32 +++++++++++++++++++++++++++++++- apps/m3u/tasks.py | 13 +++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 74411bdb..cb985a51 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -13,6 +13,7 @@ from django.conf import settings from django.db import transaction from django.utils import timezone from apps.channels.models import Channel +from core.models import UserAgent, CoreSettings from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -67,7 +68,22 @@ def fetch_xmltv(source): logger.info(f"Fetching XMLTV data from source: {source.name}") try: - response = requests.get(source.url, timeout=30) + # Get default user agent from settings + default_user_agent_setting = CoreSettings.objects.filter(key='default-user-agent').first() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + if default_user_agent_setting and default_user_agent_setting.value: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_setting.value)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + headers = { + 'User-Agent': user_agent + } + + response = requests.get(source.url, headers=headers, timeout=30) response.raise_for_status() logger.debug("XMLTV data fetched successfully.") @@ -296,10 +312,24 @@ def parse_programs_for_source(epg_source, tvg_id=None): def fetch_schedules_direct(source): logger.info(f"Fetching Schedules Direct data from source: {source.name}") try: + # Get default user agent from settings + default_user_agent_setting = CoreSettings.objects.filter(key='default-user-agent').first() + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" # Fallback default + + if default_user_agent_setting and default_user_agent_setting.value: + try: + user_agent_obj = UserAgent.objects.filter(id=int(default_user_agent_setting.value)).first() + if user_agent_obj and user_agent_obj.user_agent: + user_agent = user_agent_obj.user_agent + logger.debug(f"Using default user agent: {user_agent}") + except (ValueError, Exception) as e: + logger.warning(f"Error retrieving default user agent, using fallback: {e}") + api_url = '' headers = { 'Content-Type': 'application/json', 'Authorization': f'Bearer {source.api_key}', + 'User-Agent': user_agent } logger.debug(f"Requesting subscriptions from Schedules Direct using URL: {api_url}") response = requests.get(api_url, headers=headers, timeout=30) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index beacaaa2..c6393123 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -35,10 +35,15 @@ def fetch_m3u_lines(account, use_cache=False): """Fetch M3U file lines efficiently.""" if account.server_url: if not use_cache or not os.path.exists(file_path): - user_agent = account.get_user_agent() - headers = {"User-Agent": user_agent.user_agent} - logger.info(f"Fetching from URL {account.server_url}") try: + # Try to get account-specific user agent first + user_agent_obj = account.get_user_agent() + user_agent = user_agent_obj.user_agent if user_agent_obj else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + + logger.debug(f"Using user agent: {user_agent} for M3U account: {account.name}") + headers = {"User-Agent": user_agent} + logger.info(f"Fetching from URL {account.server_url}") + response = requests.get(account.server_url, headers=headers, stream=True) response.raise_for_status() @@ -74,7 +79,7 @@ def fetch_m3u_lines(account, use_cache=False): send_m3u_update(account.id, "downloading", progress, speed=speed, elapsed_time=elapsed_time, time_remaining=time_remaining) send_m3u_update(account.id, "downloading", 100) - except requests.exceptions.RequestException as e: + except Exception as e: logger.error(f"Error fetching M3U from URL {account.server_url}: {e}") return [] From 90c1c3d2eddce265922d5db31e7e55cc226b5f7b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 13:10:49 -0500 Subject: [PATCH 0012/1290] uwsgi config tuning. --- docker/uwsgi.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 326f4b5d..726730bf 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -24,11 +24,8 @@ vacuum = true die-on-term = true static-map = /static=/app/static -# Worker management (Optimize for I/O bound tasks) +# Worker management workers = 4 -threads = 4 -enable-threads = true -thread-stacksize=512 # Optimize for streaming http = 0.0.0.0:5656 @@ -39,8 +36,9 @@ http-timeout = 600 # Prevent disconnects from long streams lazy-apps = true # Improve memory efficiency # Async mode (use gevent for high concurrency) -gevent = 100 -async = 100 +gevent = 400 # Each unused greenlet costs ~2-4KB of memory +# Higher values have minimal performance impact when idle, but provide capacity for traffic spikes +# If memory usage becomes an issue, reduce this value # Performance tuning thunder-lock = true From 4cb2cb7b206ac0166493f46b97370c0d65aa3721 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 1 May 2025 18:49:59 +0000 Subject: [PATCH 0013/1290] Release v0.4.1 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 25e27d60..171eaa80 100644 --- a/version.py +++ b/version.py @@ -1,5 +1,5 @@ """ Dispatcharr version information. """ -__version__ = '0.4.0' # Follow semantic versioning (MAJOR.MINOR.PATCH) +__version__ = '0.4.1' # Follow semantic versioning (MAJOR.MINOR.PATCH) __timestamp__ = None # Set during CI/CD build process From d26944a7a51528a2db75abdebaec75c90e1f83bf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 16:01:08 -0500 Subject: [PATCH 0014/1290] Add stale_stream_days field to M3UAccount model and update related logic - Introduced stale_stream_days field to M3UAccount to specify the retention period for streams. - Updated cleanup_streams task to remove streams not seen within the specified stale_stream_days. - Enhanced M3U form to include stale_stream_days input for user configuration. --- .../0008_m3uaccount_stale_stream_days.py | 18 +++++++++++++++ apps/m3u/models.py | 4 ++++ apps/m3u/serializers.py | 2 +- apps/m3u/tasks.py | 23 +++++++++++++++---- frontend/src/components/forms/M3U.jsx | 12 +++++++++- 5 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py diff --git a/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py b/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py new file mode 100644 index 00000000..69a1397d --- /dev/null +++ b/apps/m3u/migrations/0008_m3uaccount_stale_stream_days.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('m3u', '0007_remove_m3uaccount_uploaded_file_m3uaccount_file_path'), + ] + + operations = [ + migrations.AddField( + model_name='m3uaccount', + name='stale_stream_days', + field=models.PositiveIntegerField(default=7, help_text='Number of days after which a stream will be removed if not seen in the M3U source.'), + ), + ] diff --git a/apps/m3u/models.py b/apps/m3u/models.py index 25a332c6..06c206f6 100644 --- a/apps/m3u/models.py +++ b/apps/m3u/models.py @@ -74,6 +74,10 @@ class M3UAccount(models.Model): refresh_task = models.ForeignKey( PeriodicTask, on_delete=models.SET_NULL, null=True, blank=True ) + stale_stream_days = models.PositiveIntegerField( + default=7, + help_text="Number of days after which a stream will be removed if not seen in the M3U source." + ) def __str__(self): return self.name diff --git a/apps/m3u/serializers.py b/apps/m3u/serializers.py index d79b0117..43015713 100644 --- a/apps/m3u/serializers.py +++ b/apps/m3u/serializers.py @@ -65,7 +65,7 @@ class M3UAccountSerializer(serializers.ModelSerializer): model = M3UAccount fields = [ 'id', 'name', 'server_url', 'file_path', 'server_group', - 'max_streams', 'is_active', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', + 'max_streams', 'is_active', 'stale_stream_days', 'created_at', 'updated_at', 'filters', 'user_agent', 'profiles', 'locked', 'channel_groups', 'refresh_interval' ] diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index c6393123..20ed1acb 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -312,18 +312,31 @@ def cleanup_streams(account_id): m3u_account__enabled=True, ).values_list('id', flat=True) logger.info(f"Found {len(existing_groups)} active groups") - streams = Stream.objects.filter(m3u_account=account) + # Calculate cutoff date for stale streams + stale_cutoff = timezone.now() - timezone.timedelta(days=account.stale_stream_days) + logger.info(f"Removing streams not seen since {stale_cutoff}") + + # Delete streams that are not in active groups streams_to_delete = Stream.objects.filter( m3u_account=account ).exclude( - channel_group__in=existing_groups # Exclude products having any of the excluded tags + channel_group__in=existing_groups ) - # Delete the filtered products - streams_to_delete.delete() + # Also delete streams that haven't been seen for longer than stale_stream_days + stale_streams = Stream.objects.filter( + m3u_account=account, + last_seen__lt=stale_cutoff + ) - logger.info(f"Cleanup complete") + deleted_count = streams_to_delete.count() + stale_count = stale_streams.count() + + streams_to_delete.delete() + stale_streams.delete() + + logger.info(f"Cleanup complete: {deleted_count} streams removed due to group filter, {stale_count} removed as stale") @shared_task def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index bd9cf3c8..8fee0d9b 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -52,6 +52,7 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { is_active: true, max_streams: 0, refresh_interval: 24, + stale_stream_days: 7, }, validate: { @@ -65,10 +66,11 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { if (playlist) { form.setValues({ name: playlist.name, - server_url: playlist.server_url, + server_url: playlist.server_url || '', max_streams: playlist.max_streams, user_agent: playlist.user_agent ? `${playlist.user_agent}` : '0', is_active: playlist.is_active, + stale_stream_days: playlist.stale_stream_days || 7, refresh_interval: playlist.refresh_interval, }); } else { @@ -202,6 +204,14 @@ const M3U = ({ playlist = null, isOpen, onClose, playlistCreated = false }) => { key={form.key('refresh_interval')} /> + + Date: Thu, 1 May 2025 17:30:19 -0400 Subject: [PATCH 0015/1290] updated last_seen of existing streams: --- apps/m3u/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 20ed1acb..90992dbc 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -292,8 +292,8 @@ def process_m3u_batch(account_id, batch, groups, hash_keys): Stream.objects.bulk_create(streams_to_create, ignore_conflicts=True) if streams_to_update: Stream.objects.bulk_update(streams_to_update, { key for key in stream_props.keys() if key not in ["m3u_account", "stream_hash"] and key not in hash_keys}) - # if len(existing_streams.keys()) > 0: - # Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) + if len(existing_streams.keys()) > 0: + Stream.objects.bulk_update(existing_streams.values(), ["last_seen"]) except Exception as e: logger.error(f"Bulk create failed: {str(e)}") From 0e54062c7358cf284c1394c853098b16aa9d8a12 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 1 May 2025 17:42:23 -0500 Subject: [PATCH 0016/1290] Added drivers for hardware acceleration. --- docker/Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 26b54975..92705403 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -76,7 +76,15 @@ RUN apt-get update && \ streamlink \ wget \ gnupg2 \ - lsb-release && \ + lsb-release \ + libva-drm2 \ + libva-x11-2 \ + libva-dev \ + libva-wayland2 \ + vainfo \ + i965-va-driver \ + intel-media-va-driver \ + mesa-va-drivers && \ cp /app/docker/nginx.conf /etc/nginx/sites-enabled/default && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From 6f1bae819501de06d6084e65a0c607e068554ced Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 2 May 2025 08:23:11 -0500 Subject: [PATCH 0017/1290] Fixes stream preview in stream table. --- frontend/src/components/FloatingVideo.jsx | 64 ++++++++++++++----- .../src/components/tables/StreamsTable.jsx | 3 +- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 0be42029..9d8fb09e 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -18,23 +18,57 @@ export default function FloatingVideo() { return; } - // If the browser supports MSE for live playback, initialize mpegts.js - if (mpegts.getFeatureList().mseLivePlayback) { - const player = mpegts.createPlayer({ - type: 'mpegts', - url: streamUrl, - isLive: true, - // You can include other custom MPEGTS.js config fields here, e.g.: - // cors: true, - // withCredentials: false, - }); + // Check if we have an existing player and clean it up + if (playerRef.current) { + playerRef.current.destroy(); + playerRef.current = null; + } - player.attachMediaElement(videoRef.current); - player.load(); - player.play(); + // Debug log to help diagnose stream issues + console.log("Attempting to play stream:", streamUrl); - // Store player instance so we can clean up later - playerRef.current = player; + try { + // If the browser supports MSE for live playback, initialize mpegts.js + if (mpegts.getFeatureList().mseLivePlayback) { + const player = mpegts.createPlayer({ + type: 'mpegts', // MPEG-TS format + url: streamUrl, + isLive: true, + enableWorker: true, + enableStashBuffer: false, // Try disabling stash buffer for live streams + liveBufferLatencyChasing: true, + liveSync: true, + cors: true, // Enable CORS for cross-domain requests + }); + + player.attachMediaElement(videoRef.current); + + // Add error event handler + player.on(mpegts.Events.ERROR, (errorType, errorDetail) => { + console.error('Player error:', errorType, errorDetail); + // If it's a format issue, show a helpful message + if (errorDetail.includes('Unsupported media type')) { + const message = document.createElement('div'); + message.textContent = "Unsupported stream format. Please try a different stream."; + message.style.position = 'absolute'; + message.style.top = '50%'; + message.style.left = '50%'; + message.style.transform = 'translate(-50%, -50%)'; + message.style.color = 'white'; + message.style.textAlign = 'center'; + message.style.width = '100%'; + videoRef.current.parentNode.appendChild(message); + } + }); + + player.load(); + player.play(); + + // Store player instance so we can clean up later + playerRef.current = player; + } + } catch (error) { + console.error("Error initializing player:", error); } // Cleanup when component unmounts or streamUrl changes diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index 078e24d9..00d09b67 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -89,8 +89,9 @@ const StreamRowActions = ({ }, []); const onPreview = useCallback(() => { + console.log('Previewing stream:', row.original.name, 'ID:', row.original.id, 'Hash:', row.original.stream_hash); handleWatchStream(row.original.stream_hash); - }, []); + }, [row.original.id]); // Add proper dependency to ensure correct stream return ( <> From e81b6e3189726b2e2efb6d4dfc36647077d22cf7 Mon Sep 17 00:00:00 2001 From: dekzter Date: Fri, 2 May 2025 09:30:24 -0400 Subject: [PATCH 0018/1290] option to add epg source with xc account --- frontend/src/components/forms/M3U.jsx | 73 +++++++++++++++++++-------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index 28ad382b..a080b401 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -26,6 +26,7 @@ import useChannelsStore from '../../store/channels'; import usePlaylistsStore from '../../store/playlists'; import { notifications } from '@mantine/notifications'; import { isNotEmpty, useForm } from '@mantine/form'; +import useEPGsStore from '../../store/epgs'; const M3U = ({ m3uAccount = null, @@ -38,6 +39,7 @@ const M3U = ({ const userAgents = useUserAgentsStore((s) => s.userAgents); const fetchChannelGroups = useChannelsStore((s) => s.fetchChannelGroups); const fetchPlaylists = usePlaylistsStore((s) => s.fetchPlaylists); + const fetchEPGs = useEPGsStore((s) => s.fetchEPGs); const [playlist, setPlaylist] = useState(null); const [file, setFile] = useState(null); @@ -55,7 +57,8 @@ const M3U = ({ is_active: true, max_streams: 0, refresh_interval: 24, - is_xc: false, + account_type: 'STD', + create_epg: false, username: '', password: '', }, @@ -78,7 +81,7 @@ const M3U = ({ user_agent: m3uAccount.user_agent ? `${m3uAccount.user_agent}` : '0', is_active: m3uAccount.is_active, refresh_interval: m3uAccount.refresh_interval, - is_xc: m3uAccount.account_type == 'XC', + account_type: m3uAccount.account_type, username: m3uAccount.username ?? '', password: '', }); @@ -95,28 +98,20 @@ const M3U = ({ }, [m3uAccount]); useEffect(() => { - if (form.values.is_xc) { + if (form.values.account_type == 'XC') { setShowCredentialFields(true); } - }, [form.values.is_xc]); + }, [form.values.account_type]); const onSubmit = async () => { - const { ...values } = form.getValues(); + const { create_epg, ...values } = form.getValues(); - if (values.is_xc && values.password == '') { + if (values.account_type == 'XC' && values.password == '') { // If account XC and no password input, assuming no password change // from previously stored value. delete values.password; } - if (values.is_xc) { - values.account_type = 'XC'; - } else { - values.account_type = 'STD'; - } - - delete values.is_xc; - if (values.user_agent == '0') { values.user_agent = null; } @@ -134,6 +129,17 @@ const M3U = ({ file, }); + if (create_epg) { + API.addEPG({ + name: values.name, + source_type: 'xmltv', + url: `${values.server_url}/xmltv.php?username=${values.username}&password=${values.password}`, + api_key: '', + is_active: true, + refresh_interval: 24, + }); + } + if (values.account_type != 'XC') { notifications.show({ title: 'Fetching M3U Groups', @@ -149,7 +155,7 @@ const M3U = ({ } const updatedPlaylist = await API.getPlaylist(newPlaylist.id); - await Promise.all([fetchChannelGroups(), fetchPlaylists()]); + await Promise.all([fetchChannelGroups(), fetchPlaylists(), fetchEPGs()]); console.log('opening group options'); setPlaylist(updatedPlaylist); setGroupFilterModalOpen(true); @@ -210,21 +216,44 @@ const M3U = ({ {...form.getInputProps('server_url')} key={form.key('server_url')} /> - - {form.getValues().is_xc && ( + {form.getValues().account_type == 'XC' && ( + + Create EPG + + + + )} - {!form.getValues().is_xc && ( + {form.getValues().account_type != 'XC' && ( Date: Fri, 2 May 2025 10:02:35 -0500 Subject: [PATCH 0019/1290] Fixes console error when playing a stream preview due to channelID being null. --- frontend/src/store/channels.jsx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/frontend/src/store/channels.jsx b/frontend/src/store/channels.jsx index 503cb1cc..6d946f94 100644 --- a/frontend/src/store/channels.jsx +++ b/frontend/src/store/channels.jsx @@ -323,11 +323,17 @@ const useChannelsStore = create((set, get) => ({ acc[ch.channel_id] = ch; if (currentStats.channels) { if (oldChannels[ch.channel_id] === undefined) { - notifications.show({ - title: 'New channel streaming', - message: channels[channelsByUUID[ch.channel_id]].name, - color: 'blue.5', - }); + // Add null checks to prevent accessing properties on undefined + const channelId = channelsByUUID[ch.channel_id]; + const channel = channelId ? channels[channelId] : null; + + if (channel) { + notifications.show({ + title: 'New channel streaming', + message: channel.name, + color: 'blue.5', + }); + } } } ch.clients.map((client) => { From e67f31465606415e1fa361b373670453e718833e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 2 May 2025 10:22:19 -0500 Subject: [PATCH 0020/1290] Better error handling for web video player. --- frontend/src/components/FloatingVideo.jsx | 106 +++++++++++++++++++--- 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index 9d8fb09e..cbb5e1a8 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -12,17 +12,55 @@ export default function FloatingVideo() { const videoRef = useRef(null); const playerRef = useRef(null); const videoContainerRef = useRef(null); + const isLoadingRef = useRef(false); + + // Safely destroy the player to prevent errors + const safeDestroyPlayer = () => { + try { + if (playerRef.current) { + // Set a flag to ignore abort errors + isLoadingRef.current = false; + + // First unload the source to stop any in-progress fetches + if (videoRef.current) { + // Remove src attribute and force a load to clear any pending requests + videoRef.current.removeAttribute('src'); + videoRef.current.load(); + } + + // Pause the player first + try { + playerRef.current.pause(); + } catch (e) { + // Ignore pause errors + } + + // Use a try-catch block specifically for the destroy call + try { + playerRef.current.destroy(); + } catch (error) { + // Ignore expected abort errors + if (error.name !== 'AbortError' && !error.message?.includes('aborted')) { + console.log("Error during player destruction:", error.message); + } + } finally { + playerRef.current = null; + } + } + } catch (error) { + console.log("Error during player cleanup:", error); + playerRef.current = null; + } + }; useEffect(() => { if (!isVisible || !streamUrl) { + safeDestroyPlayer(); return; } // Check if we have an existing player and clean it up - if (playerRef.current) { - playerRef.current.destroy(); - playerRef.current = null; - } + safeDestroyPlayer(); // Debug log to help diagnose stream issues console.log("Attempting to play stream:", streamUrl); @@ -30,6 +68,9 @@ export default function FloatingVideo() { try { // If the browser supports MSE for live playback, initialize mpegts.js if (mpegts.getFeatureList().mseLivePlayback) { + // Set loading flag + isLoadingRef.current = true; + const player = mpegts.createPlayer({ type: 'mpegts', // MPEG-TS format url: streamUrl, @@ -39,15 +80,35 @@ export default function FloatingVideo() { liveBufferLatencyChasing: true, liveSync: true, cors: true, // Enable CORS for cross-domain requests + // Add error recovery options + autoCleanupSourceBuffer: true, + autoCleanupMaxBackwardDuration: 10, + autoCleanupMinBackwardDuration: 5, + reuseRedirectedURL: true, }); player.attachMediaElement(videoRef.current); + // Add events to track loading state + player.on(mpegts.Events.LOADING_COMPLETE, () => { + isLoadingRef.current = false; + }); + + player.on(mpegts.Events.METADATA_ARRIVED, () => { + isLoadingRef.current = false; + }); + // Add error event handler player.on(mpegts.Events.ERROR, (errorType, errorDetail) => { - console.error('Player error:', errorType, errorDetail); + isLoadingRef.current = false; + + // Filter out aborted errors + if (errorType !== 'NetworkError' || !errorDetail?.includes('aborted')) { + console.error('Player error:', errorType, errorDetail); + } + // If it's a format issue, show a helpful message - if (errorDetail.includes('Unsupported media type')) { + if (errorDetail?.includes('Unsupported media type')) { const message = document.createElement('div'); message.textContent = "Unsupported stream format. Please try a different stream."; message.style.position = 'absolute'; @@ -57,29 +118,48 @@ export default function FloatingVideo() { message.style.color = 'white'; message.style.textAlign = 'center'; message.style.width = '100%'; - videoRef.current.parentNode.appendChild(message); + if (videoRef.current?.parentNode) { + videoRef.current.parentNode.appendChild(message); + } } }); player.load(); - player.play(); + + // Don't auto-play until we've loaded properly + player.on(mpegts.Events.MEDIA_INFO, () => { + try { + player.play().catch(e => { + console.log("Auto-play prevented:", e); + }); + } catch (e) { + console.log("Error during play:", e); + } + }); // Store player instance so we can clean up later playerRef.current = player; } } catch (error) { + isLoadingRef.current = false; console.error("Error initializing player:", error); } // Cleanup when component unmounts or streamUrl changes return () => { - if (playerRef.current) { - playerRef.current.destroy(); - playerRef.current = null; - } + safeDestroyPlayer(); }; }, [isVisible, streamUrl]); + // Modified hideVideo handler to clean up player first + const handleClose = () => { + safeDestroyPlayer(); + // Small delay before hiding the video component to ensure cleanup is complete + setTimeout(() => { + hideVideo(); + }, 50); + }; + // If the floating video is hidden or no URL is selected, do not render if (!isVisible || !streamUrl) { return null; @@ -103,7 +183,7 @@ export default function FloatingVideo() { > {/* Simple header row with a close button */} - + {/* The ); From 35579e79fbfe5c8cd533598090b532fa6446279a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 2 May 2025 13:02:34 -0500 Subject: [PATCH 0025/1290] Enhance FloatingVideo component with loading and error handling overlays --- frontend/src/components/FloatingVideo.jsx | 111 +++++++++++++++------- 1 file changed, 79 insertions(+), 32 deletions(-) diff --git a/frontend/src/components/FloatingVideo.jsx b/frontend/src/components/FloatingVideo.jsx index cbb5e1a8..46c191eb 100644 --- a/frontend/src/components/FloatingVideo.jsx +++ b/frontend/src/components/FloatingVideo.jsx @@ -1,9 +1,9 @@ // frontend/src/components/FloatingVideo.js -import React, { useEffect, useRef } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import Draggable from 'react-draggable'; import useVideoStore from '../store/useVideoStore'; import mpegts from 'mpegts.js'; -import { CloseButton, Flex } from '@mantine/core'; +import { CloseButton, Flex, Loader, Text, Box } from '@mantine/core'; export default function FloatingVideo() { const isVisible = useVideoStore((s) => s.isVisible); @@ -12,14 +12,17 @@ export default function FloatingVideo() { const videoRef = useRef(null); const playerRef = useRef(null); const videoContainerRef = useRef(null); - const isLoadingRef = useRef(false); + // Convert ref to state so we can use it for rendering + const [isLoading, setIsLoading] = useState(false); + const [loadError, setLoadError] = useState(null); // Safely destroy the player to prevent errors const safeDestroyPlayer = () => { try { if (playerRef.current) { - // Set a flag to ignore abort errors - isLoadingRef.current = false; + // Set loading to false when destroying player + setIsLoading(false); + setLoadError(null); // First unload the source to stop any in-progress fetches if (videoRef.current) { @@ -62,6 +65,10 @@ export default function FloatingVideo() { // Check if we have an existing player and clean it up safeDestroyPlayer(); + // Set loading state to true when starting a new stream + setIsLoading(true); + setLoadError(null); + // Debug log to help diagnose stream issues console.log("Attempting to play stream:", streamUrl); @@ -69,7 +76,7 @@ export default function FloatingVideo() { // If the browser supports MSE for live playback, initialize mpegts.js if (mpegts.getFeatureList().mseLivePlayback) { // Set loading flag - isLoadingRef.current = true; + setIsLoading(true); const player = mpegts.createPlayer({ type: 'mpegts', // MPEG-TS format @@ -91,36 +98,21 @@ export default function FloatingVideo() { // Add events to track loading state player.on(mpegts.Events.LOADING_COMPLETE, () => { - isLoadingRef.current = false; + setIsLoading(false); }); player.on(mpegts.Events.METADATA_ARRIVED, () => { - isLoadingRef.current = false; + setIsLoading(false); }); // Add error event handler player.on(mpegts.Events.ERROR, (errorType, errorDetail) => { - isLoadingRef.current = false; + setIsLoading(false); // Filter out aborted errors if (errorType !== 'NetworkError' || !errorDetail?.includes('aborted')) { console.error('Player error:', errorType, errorDetail); - } - - // If it's a format issue, show a helpful message - if (errorDetail?.includes('Unsupported media type')) { - const message = document.createElement('div'); - message.textContent = "Unsupported stream format. Please try a different stream."; - message.style.position = 'absolute'; - message.style.top = '50%'; - message.style.left = '50%'; - message.style.transform = 'translate(-50%, -50%)'; - message.style.color = 'white'; - message.style.textAlign = 'center'; - message.style.width = '100%'; - if (videoRef.current?.parentNode) { - videoRef.current.parentNode.appendChild(message); - } + setLoadError(`Error: ${errorType}${errorDetail ? ` - ${errorDetail}` : ''}`); } }); @@ -128,12 +120,15 @@ export default function FloatingVideo() { // Don't auto-play until we've loaded properly player.on(mpegts.Events.MEDIA_INFO, () => { + setIsLoading(false); try { player.play().catch(e => { console.log("Auto-play prevented:", e); + setLoadError("Auto-play was prevented. Click play to start."); }); } catch (e) { console.log("Error during play:", e); + setLoadError(`Playback error: ${e.message}`); } }); @@ -141,7 +136,8 @@ export default function FloatingVideo() { playerRef.current = player; } } catch (error) { - isLoadingRef.current = false; + setIsLoading(false); + setLoadError(`Initialization error: ${error.message}`); console.error("Error initializing player:", error); } @@ -186,12 +182,63 @@ export default function FloatingVideo() { - {/* The