From 50e9075bb50f99bff878266ebf8648da5bb72f51 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 25 Oct 2025 08:15:39 -0400 Subject: [PATCH 001/503] initial run of a binary and encoded redis client - no more encoding / decoding data into redis, huge PITA (still some outstanding spots I need to patch) --- apps/channels/api_views.py | 6 +- apps/channels/tasks.py | 4 +- apps/proxy/tasks.py | 2 +- apps/proxy/ts_proxy/channel_status.py | 208 ++++++------------ apps/proxy/ts_proxy/server.py | 75 +++---- .../ts_proxy/services/channel_service.py | 20 +- apps/proxy/ts_proxy/stream_manager.py | 14 +- apps/proxy/ts_proxy/url_utils.py | 8 +- .../multi_worker_connection_manager.py | 14 +- apps/proxy/vod_proxy/views.py | 16 +- core/redis_pubsub.py | 4 - core/tasks.py | 2 +- core/utils.py | 192 ++++++++-------- dispatcharr/persistent_lock.py | 2 +- 14 files changed, 253 insertions(+), 314 deletions(-) diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 862de7f9..aa25ca85 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -1849,14 +1849,12 @@ class RecordingViewSet(viewsets.ModelViewSet): client_set_key = RedisKeys.clients(channel_uuid) client_ids = r.smembers(client_set_key) or [] stopped = 0 - for raw_id in client_ids: + for cid in client_ids: try: - cid = raw_id.decode("utf-8") if isinstance(raw_id, (bytes, bytearray)) else str(raw_id) meta_key = RedisKeys.client_metadata(channel_uuid, cid) ua = r.hget(meta_key, "user_agent") - ua_s = ua.decode("utf-8") if isinstance(ua, (bytes, bytearray)) else (ua or "") # Identify DVR recording client by its user agent - if ua_s and "Dispatcharr-DVR" in ua_s: + if ua and "Dispatcharr-DVR" in ua: try: ChannelService.stop_client(channel_uuid, cid) stopped += 1 diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 3943cf16..28da31b3 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1875,14 +1875,14 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str): md = r.hgetall(metadata_key) if md: def _gv(bkey): - return md.get(bkey.encode('utf-8')) + return md.get(bkey) def _d(bkey, cast=str): v = _gv(bkey) try: if v is None: return None - s = v.decode('utf-8') + s = v return cast(s) if cast is not str else s except Exception: return None diff --git a/apps/proxy/tasks.py b/apps/proxy/tasks.py index 68843712..d42e4d9a 100644 --- a/apps/proxy/tasks.py +++ b/apps/proxy/tasks.py @@ -31,7 +31,7 @@ def fetch_channel_stats(): while True: cursor, keys = redis_client.scan(cursor, match=channel_pattern) for key in keys: - channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8')) + channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key) if channel_id_match: ch_id = channel_id_match.group(1) channel_info = ChannelStatus.get_basic_channel_info(ch_id) diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 8f1d0649..015d5ca2 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -38,19 +38,19 @@ class ChannelStatus: info = { 'channel_id': channel_id, - 'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'), - 'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'), - 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'), - 'started_at': metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'), - 'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'), - 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, + 'state': metadata.get(ChannelMetadataField.STATE, 'unknown'), + 'url': metadata.get(ChannelMetadataField.URL, ''), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ''), + 'started_at': metadata.get(ChannelMetadataField.INIT_TIME, '0'), + 'owner': metadata.get(ChannelMetadataField.OWNER, 'unknown'), + 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, } # Add stream ID and name information - stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8')) + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: - stream_id = int(stream_id_bytes.decode('utf-8')) + stream_id = int(stream_id_bytes) info['stream_id'] = stream_id # Look up stream name from database @@ -65,10 +65,10 @@ class ChannelStatus: logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}") # Add M3U profile information - m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8')) + m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE) if m3u_profile_id_bytes: try: - m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8')) + m3u_profile_id = int(m3u_profile_id_bytes) info['m3u_profile_id'] = m3u_profile_id # Look up M3U profile name from database @@ -83,22 +83,22 @@ class ChannelStatus: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}") # Add timing information - state_changed_field = ChannelMetadataField.STATE_CHANGED_AT.encode('utf-8') + state_changed_field = ChannelMetadataField.STATE_CHANGED_AT if state_changed_field in metadata: - state_changed_at = float(metadata[state_changed_field].decode('utf-8')) + state_changed_at = float(metadata[state_changed_field]) info['state_changed_at'] = state_changed_at info['state_duration'] = time.time() - state_changed_at - init_time_field = ChannelMetadataField.INIT_TIME.encode('utf-8') + init_time_field = ChannelMetadataField.INIT_TIME if init_time_field in metadata: - created_at = float(metadata[init_time_field].decode('utf-8')) + created_at = float(metadata[init_time_field]) info['started_at'] = created_at info['uptime'] = time.time() - created_at # Add data throughput information - total_bytes_field = ChannelMetadataField.TOTAL_BYTES.encode('utf-8') + total_bytes_field = ChannelMetadataField.TOTAL_BYTES if total_bytes_field in metadata: - total_bytes = int(metadata[total_bytes_field].decode('utf-8')) + total_bytes = int(metadata[total_bytes_field]) info['total_bytes'] = total_bytes # Format total bytes in human-readable form @@ -128,40 +128,40 @@ class ChannelStatus: clients = [] for client_id in client_ids: - client_id_str = client_id.decode('utf-8') + client_id_str = client_id client_key = RedisKeys.client_metadata(channel_id, client_id_str) client_data = proxy_server.redis_client.hgetall(client_key) if client_data: client_info = { 'client_id': client_id_str, - 'user_agent': client_data.get(b'user_agent', b'unknown').decode('utf-8'), - 'worker_id': client_data.get(b'worker_id', b'unknown').decode('utf-8'), + 'user_agent': client_data.get('user_agent', 'unknown'), + 'worker_id': client_data.get('worker_id', 'unknown'), } - if b'connected_at' in client_data: - connected_at = float(client_data[b'connected_at'].decode('utf-8')) + if 'connected_at' in client_data: + connected_at = float(client_data['connected_at']) client_info['connected_at'] = connected_at client_info['connection_duration'] = time.time() - connected_at - if b'last_active' in client_data: - last_active = float(client_data[b'last_active'].decode('utf-8')) + if 'last_active' in client_data: + last_active = float(client_data['last_active']) client_info['last_active'] = last_active client_info['last_active_ago'] = time.time() - last_active # Add transfer rate statistics - if b'bytes_sent' in client_data: - client_info['bytes_sent'] = int(client_data[b'bytes_sent'].decode('utf-8')) + if 'bytes_sent' in client_data: + client_info['bytes_sent'] = int(client_data['bytes_sent']) # Add average transfer rate - if b'avg_rate_KBps' in client_data: - client_info['avg_rate_KBps'] = float(client_data[b'avg_rate_KBps'].decode('utf-8')) - elif b'transfer_rate_KBps' in client_data: # For backward compatibility - client_info['avg_rate_KBps'] = float(client_data[b'transfer_rate_KBps'].decode('utf-8')) + if 'avg_rate_KBps' in client_data: + client_info['avg_rate_KBps'] = float(client_data['avg_rate_KBps']) + elif 'transfer_rate_KBps' in client_data: # For backward compatibility + client_info['avg_rate_KBps'] = float(client_data['transfer_rate_KBps']) # Add current transfer rate - if b'current_rate_KBps' in client_data: - client_info['current_rate_KBps'] = float(client_data[b'current_rate_KBps'].decode('utf-8')) + if 'current_rate_KBps' in client_data: + client_info['current_rate_KBps'] = float(client_data['current_rate_KBps']) clients.append(client_info) @@ -235,7 +235,7 @@ class ChannelStatus: while True: cursor, keys = proxy_server.redis_client.scan(cursor, match=buffer_key_pattern, count=100) if keys: - all_buffer_keys.extend([k.decode('utf-8') for k in keys]) + all_buffer_keys.extend([k for k in keys]) if cursor == 0 or len(all_buffer_keys) >= 20: # Limit to 20 keys break @@ -265,61 +265,22 @@ class ChannelStatus: } # Add FFmpeg stream information - video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8')) - if video_codec: - info['video_codec'] = video_codec.decode('utf-8') - - resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8')) - if resolution: - info['resolution'] = resolution.decode('utf-8') - - source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8')) - if source_fps: - info['source_fps'] = float(source_fps.decode('utf-8')) - - pixel_format = metadata.get(ChannelMetadataField.PIXEL_FORMAT.encode('utf-8')) - if pixel_format: - info['pixel_format'] = pixel_format.decode('utf-8') - - source_bitrate = metadata.get(ChannelMetadataField.SOURCE_BITRATE.encode('utf-8')) - if source_bitrate: - info['source_bitrate'] = float(source_bitrate.decode('utf-8')) - - audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8')) - if audio_codec: - info['audio_codec'] = audio_codec.decode('utf-8') - - sample_rate = metadata.get(ChannelMetadataField.SAMPLE_RATE.encode('utf-8')) - if sample_rate: - info['sample_rate'] = int(sample_rate.decode('utf-8')) - - audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8')) - if audio_channels: - info['audio_channels'] = audio_channels.decode('utf-8') - - audio_bitrate = metadata.get(ChannelMetadataField.AUDIO_BITRATE.encode('utf-8')) - if audio_bitrate: - info['audio_bitrate'] = float(audio_bitrate.decode('utf-8')) + info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) + info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) + info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) + info['pixel_format'] = metadata.get(ChannelMetadataField.PIXEL_FORMAT) + info['source_bitrate'] = metadata.get(ChannelMetadataField.SOURCE_BITRATE) + info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) + info['sample_rate'] = metadata.get(ChannelMetadataField.SAMPLE_RATE) + info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + info['audio_bitrate'] = metadata.get(ChannelMetadataField.AUDIO_BITRATE) # Add FFmpeg performance stats - ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8')) - if ffmpeg_speed: - info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8')) - - ffmpeg_fps = metadata.get(ChannelMetadataField.FFMPEG_FPS.encode('utf-8')) - if ffmpeg_fps: - info['ffmpeg_fps'] = float(ffmpeg_fps.decode('utf-8')) - - actual_fps = metadata.get(ChannelMetadataField.ACTUAL_FPS.encode('utf-8')) - if actual_fps: - info['actual_fps'] = float(actual_fps.decode('utf-8')) - - ffmpeg_bitrate = metadata.get(ChannelMetadataField.FFMPEG_BITRATE.encode('utf-8')) - if ffmpeg_bitrate: - info['ffmpeg_bitrate'] = float(ffmpeg_bitrate.decode('utf-8')) - stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8')) - if stream_type: - info['stream_type'] = stream_type.decode('utf-8') + info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + info['ffmpeg_fps'] = metadata.get(ChannelMetadataField.FFMPEG_FPS) + info['actual_fps'] = metadata.get(ChannelMetadataField.ACTUAL_FPS) + info['ffmpeg_bitrate'] = metadata.get(ChannelMetadataField.FFMPEG_BITRATE) + info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) return info @@ -364,33 +325,27 @@ class ChannelStatus: client_count = proxy_server.redis_client.scard(client_set_key) or 0 # Calculate uptime - init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0') - created_at = float(init_time_bytes.decode('utf-8')) + init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME, '0') + created_at = float(init_time_bytes) uptime = time.time() - created_at if created_at > 0 else 0 - # Safely decode bytes or use defaults - def safe_decode(bytes_value, default="unknown"): - if bytes_value is None: - return default - return bytes_value.decode('utf-8') - # Simplified info info = { 'channel_id': channel_id, - 'state': safe_decode(metadata.get(ChannelMetadataField.STATE.encode('utf-8'))), - 'url': safe_decode(metadata.get(ChannelMetadataField.URL.encode('utf-8')), ""), - 'stream_profile': safe_decode(metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8')), ""), - 'owner': safe_decode(metadata.get(ChannelMetadataField.OWNER.encode('utf-8'))), - 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, + 'state': metadata.get(ChannelMetadataField.STATE), + 'url': metadata.get(ChannelMetadataField.URL, ""), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""), + 'owner': metadata.get(ChannelMetadataField.OWNER), + 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, 'client_count': client_count, 'uptime': uptime } # Add stream ID and name information - stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8')) + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: - stream_id = int(stream_id_bytes.decode('utf-8')) + stream_id = int(stream_id_bytes) info['stream_id'] = stream_id # Look up stream name from database @@ -405,9 +360,9 @@ class ChannelStatus: logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}") # Add data throughput information to basic info - total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES.encode('utf-8')) + total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES) if total_bytes_bytes: - total_bytes = int(total_bytes_bytes.decode('utf-8')) + total_bytes = int(total_bytes_bytes) info['total_bytes'] = total_bytes # Calculate and add bitrate @@ -434,26 +389,25 @@ class ChannelStatus: if client_ids: # Get up to 10 clients for the basic view for client_id in list(client_ids)[:10]: - client_id_str = client_id.decode('utf-8') - client_key = RedisKeys.client_metadata(channel_id, client_id_str) + client_key = RedisKeys.client_metadata(channel_id, client_id) # Efficient way - just retrieve the essentials client_info = { - 'client_id': client_id_str, + 'client_id': client_id, } # Safely get user_agent and ip_address user_agent_bytes = proxy_server.redis_client.hget(client_key, 'user_agent') - client_info['user_agent'] = safe_decode(user_agent_bytes) + client_info['user_agent'] = user_agent_bytes ip_address_bytes = proxy_server.redis_client.hget(client_key, 'ip_address') if ip_address_bytes: - client_info['ip_address'] = safe_decode(ip_address_bytes) + client_info['ip_address'] = ip_address_bytes # Just get connected_at for client age connected_at_bytes = proxy_server.redis_client.hget(client_key, 'connected_at') if connected_at_bytes: - connected_at = float(connected_at_bytes.decode('utf-8')) + connected_at = float(connected_at_bytes) client_info['connected_since'] = time.time() - connected_at clients.append(client_info) @@ -462,10 +416,10 @@ class ChannelStatus: info['clients'] = clients # Add M3U profile information - m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8')) - if m3u_profile_id_bytes: + m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE) + if m3u_profile_id: try: - m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8')) + m3u_profile_id = int(m3u_profile_id) info['m3u_profile_id'] = m3u_profile_id # Look up M3U profile name from database @@ -477,32 +431,16 @@ class ChannelStatus: except (ImportError, DatabaseError) as e: logger.warning(f"Failed to get M3U profile name for ID {m3u_profile_id}: {e}") except ValueError: - logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}") + logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") # Add stream info to basic info as well - video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8')) - if video_codec: - info['video_codec'] = video_codec.decode('utf-8') - - resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8')) - if resolution: - info['resolution'] = resolution.decode('utf-8') - - source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8')) - if source_fps: - info['source_fps'] = float(source_fps.decode('utf-8')) - ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8')) - if ffmpeg_speed: - info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8')) - audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8')) - if audio_codec: - info['audio_codec'] = audio_codec.decode('utf-8') - audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8')) - if audio_channels: - info['audio_channels'] = audio_channels.decode('utf-8') - stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8')) - if stream_type: - info['stream_type'] = stream_type.decode('utf-8') + info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) + info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) + info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) + info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) + info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) return info except Exception as e: diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index cca827a9..7a59f530 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -157,7 +157,8 @@ class ProxyServer: socket_timeout=60, socket_connect_timeout=10, socket_keepalive=True, - health_check_interval=30 + health_check_interval=30, + decode_responses=True ) logger.info("Created fallback Redis PubSub client for event listener") @@ -178,8 +179,8 @@ class ProxyServer: continue try: - channel = message["channel"].decode("utf-8") - data = json.loads(message["data"].decode("utf-8")) + channel = message["channel"] + data = json.loads(message["data"]) event_type = data.get("event") channel_id = data.get("channel_id") @@ -374,7 +375,7 @@ class ProxyServer: try: lock_key = RedisKeys.channel_owner(channel_id) return self._execute_redis_command( - lambda: self.redis_client.get(lock_key).decode('utf-8') if self.redis_client.get(lock_key) else None + lambda: self.redis_client.get(lock_key) if self.redis_client.get(lock_key) else None ) except Exception as e: logger.error(f"Error getting channel owner: {e}") @@ -415,7 +416,7 @@ class ProxyServer: current_owner = self._execute_redis_command( lambda: self.redis_client.get(lock_key) ) - if current_owner and current_owner.decode('utf-8') == self.worker_id: + if current_owner and current_owner == self.worker_id: # Refresh TTL self._execute_redis_command( lambda: self.redis_client.expire(lock_key, ttl) @@ -440,7 +441,7 @@ class ProxyServer: # Only delete if we're the current owner to prevent race conditions current = self.redis_client.get(lock_key) - if current and current.decode('utf-8') == self.worker_id: + if current and current == self.worker_id: self.redis_client.delete(lock_key) logger.info(f"Released ownership of channel {channel_id}") @@ -462,7 +463,7 @@ class ProxyServer: current = self.redis_client.get(lock_key) # Only extend if we're still the owner - if current and current.decode('utf-8') == self.worker_id: + if current and current == self.worker_id: self.redis_client.expire(lock_key, ttl) return True return False @@ -478,15 +479,15 @@ class ProxyServer: metadata_key = RedisKeys.channel_metadata(channel_id) if self.redis_client.exists(metadata_key): metadata = self.redis_client.hgetall(metadata_key) - if b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if 'state' in metadata: + state = metadata['state'] active_states = [ChannelState.INITIALIZING, ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS, ChannelState.ACTIVE, ChannelState.BUFFERING] if state in active_states: logger.info(f"Channel {channel_id} already being initialized with state {state}") # Create buffer and client manager only if we don't have them if channel_id not in self.stream_buffers: - self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=self.redis_client) + self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer()) if channel_id not in self.client_managers: self.client_managers[channel_id] = ClientManager( channel_id, @@ -497,7 +498,7 @@ class ProxyServer: # Create buffer and client manager instances (or reuse if they exist) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer if channel_id not in self.client_managers: @@ -536,18 +537,18 @@ class ProxyServer: # If no url was passed, try to get from Redis if not url and existing_metadata: - url_bytes = existing_metadata.get(b'url') + url_bytes = existing_metadata.get('url') if url_bytes: - channel_url = url_bytes.decode('utf-8') + channel_url = url_bytes - ua_bytes = existing_metadata.get(b'user_agent') + ua_bytes = existing_metadata.get('user_agent') if ua_bytes: - channel_user_agent = ua_bytes.decode('utf-8') + channel_user_agent = ua_bytes # Get stream ID from metadata if not provided - if not channel_stream_id and b'stream_id' in existing_metadata: + if not channel_stream_id and 'stream_id' in existing_metadata: try: - channel_stream_id = int(existing_metadata[b'stream_id'].decode('utf-8')) + channel_stream_id = int(existing_metadata['stream_id']) logger.debug(f"Found stream_id {channel_stream_id} in metadata for channel {channel_id}") except (ValueError, TypeError) as e: logger.debug(f"Could not parse stream_id from metadata: {e}") @@ -562,7 +563,7 @@ class ProxyServer: # Create buffer but not stream manager (only if not already exists) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer # Create client manager with channel_id and redis_client (only if not already exists) @@ -585,7 +586,7 @@ class ProxyServer: # Create buffer but not stream manager (only if not already exists) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer # Create client manager with channel_id and redis_client (only if not already exists) @@ -624,12 +625,12 @@ class ProxyServer: # Verify the stream_id was set correctly in Redis stream_id_value = self.redis_client.hget(metadata_key, "stream_id") if stream_id_value: - logger.info(f"Verified stream_id {stream_id_value.decode('utf-8')} is set in Redis for channel {channel_id}") + logger.info(f"Verified stream_id {stream_id_value} is set in Redis for channel {channel_id}") else: logger.warning(f"Failed to set stream_id in Redis for channel {channel_id}") # Create stream buffer - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) logger.debug(f"Created StreamBuffer for channel {channel_id}") self.stream_buffers[channel_id] = buffer @@ -700,8 +701,8 @@ class ProxyServer: metadata = self.redis_client.hgetall(metadata_key) # Get channel state and owner - state = metadata.get(b'state', b'unknown').decode('utf-8') - owner = metadata.get(b'owner', b'').decode('utf-8') + state = metadata.get('state', 'unknown') + owner = metadata.get('owner', '') # States that indicate the channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, @@ -726,8 +727,8 @@ class ProxyServer: return True else: # Unknown or initializing state, check how long it's been in this state - if b'state_changed_at' in metadata: - state_changed_at = float(metadata[b'state_changed_at'].decode('utf-8')) + if 'state_changed_at' in metadata: + state_changed_at = float(metadata['state_changed_at']) state_age = time.time() - state_changed_at # If in initializing state for too long, consider it stale @@ -762,8 +763,8 @@ class ProxyServer: # If we have metadata, log details for debugging if metadata: - state = metadata.get(b'state', b'unknown').decode('utf-8') - owner = metadata.get(b'owner', b'unknown').decode('utf-8') + state = metadata.get('state', 'unknown') + owner = metadata.get('owner', 'unknown') logger.info(f"Zombie channel details - state: {state}, owner: {owner}") # Clean up Redis keys @@ -931,8 +932,8 @@ class ProxyServer: if self.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) metadata = self.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - channel_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + channel_state = metadata['state'] # Check if channel has any clients left total_clients = 0 @@ -948,9 +949,9 @@ class ProxyServer: if channel_state in [ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS]: # Get connection ready time from metadata connection_ready_time = None - if metadata and b'connection_ready_time' in metadata: + if metadata and 'connection_ready_time' in metadata: try: - connection_ready_time = float(metadata[b'connection_ready_time'].decode('utf-8')) + connection_ready_time = float(metadata['connection_ready_time']) except (ValueError, TypeError): pass @@ -981,8 +982,8 @@ class ProxyServer: # Grace period expired but we have clients - mark channel as active logger.info(f"Grace period expired with {total_clients} clients - marking channel {channel_id} as active") old_state = "unknown" - if metadata and b'state' in metadata: - old_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + old_state = metadata['state'] if self.update_channel_state(channel_id, ChannelState.ACTIVE, { "grace_period_ended_at": str(time.time()), "clients_at_activation": str(total_clients) @@ -998,7 +999,7 @@ class ProxyServer: disconnect_value = self.redis_client.get(disconnect_key) if disconnect_value: try: - disconnect_time = float(disconnect_value.decode('utf-8')) + disconnect_time = float(disconnect_value) except (ValueError, TypeError) as e: logger.error(f"Invalid disconnect time for channel {channel_id}: {e}") @@ -1076,7 +1077,7 @@ class ProxyServer: for key in channel_keys: try: - channel_id = key.decode('utf-8').split(':')[2] + channel_id = key.split(':')[2] # Skip channels we already have locally if channel_id in self.stream_buffers: @@ -1170,8 +1171,8 @@ class ProxyServer: # Get current state for logging current_state = None metadata = self.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - current_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + current_state = metadata['state'] # Only update if state is actually changing if current_state == new_state: diff --git a/apps/proxy/ts_proxy/services/channel_service.py b/apps/proxy/ts_proxy/services/channel_service.py index 551e2d27..6e7d3878 100644 --- a/apps/proxy/ts_proxy/services/channel_service.py +++ b/apps/proxy/ts_proxy/services/channel_service.py @@ -59,7 +59,7 @@ class ChannelService: # Verify the stream_id was set stream_id_value = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) if stream_id_value: - logger.debug(f"Verified stream_id {stream_id_value.decode('utf-8')} is now set in Redis") + logger.debug(f"Verified stream_id {stream_id_value} is now set in Redis") else: logger.error(f"Failed to set stream_id {stream_id} in Redis before initialization") @@ -129,7 +129,7 @@ class ChannelService: try: # This is inefficient but used for diagnostics - in production would use more targeted checks redis_keys = proxy_server.redis_client.keys(f"ts_proxy:*:{channel_id}*") - redis_keys = [k.decode('utf-8') for k in redis_keys] if redis_keys else [] + redis_keys = [k for k in redis_keys] if redis_keys else [] except Exception as e: logger.error(f"Error checking Redis keys: {e}") @@ -234,8 +234,8 @@ class ChannelService: metadata_key = RedisKeys.channel_metadata(channel_id) try: metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] channel_info = {"state": state} # Immediately mark as stopping in metadata so clients detect it faster @@ -375,8 +375,8 @@ class ChannelService: metadata = proxy_server.redis_client.hgetall(metadata_key) # Extract state and owner - state = metadata.get(ChannelMetadataField.STATE.encode(), b'unknown').decode('utf-8') - owner = metadata.get(ChannelMetadataField.OWNER.encode(), b'unknown').decode('utf-8') + state = metadata.get(ChannelMetadataField.STATE.encode(), 'unknown') + owner = metadata.get(ChannelMetadataField.OWNER.encode(), 'unknown') # Valid states indicate channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING] @@ -402,7 +402,7 @@ class ChannelService: } if last_data: - last_data_time = float(last_data.decode('utf-8')) + last_data_time = float(last_data) data_age = time.time() - last_data_time details["last_data_age"] = data_age @@ -598,7 +598,7 @@ class ChannelService: def _update_stream_stats_in_db(stream_id, **stats): """Update stream stats in database""" from django.db import connection - + try: from apps.channels.models import Stream from django.utils import timezone @@ -624,7 +624,7 @@ class ChannelService: except Exception as e: logger.error(f"Error updating stream stats in database for stream {stream_id}: {e}") return False - + finally: # Always close database connection after update try: @@ -645,7 +645,7 @@ class ChannelService: metadata_key = RedisKeys.channel_metadata(channel_id) # First check if the key exists and what type it is - key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8') + key_type = proxy_server.redis_client.type(metadata_key) logger.debug(f"Redis key {metadata_key} is of type: {key_type}") # Build metadata update dict diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index 99ae8027..73118768 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -89,7 +89,7 @@ class StreamManager: # Try to get stream_id specifically stream_id_bytes = buffer.redis_client.hget(metadata_key, "stream_id") if stream_id_bytes: - self.current_stream_id = int(stream_id_bytes.decode('utf-8')) + self.current_stream_id = int(stream_id_bytes) self.tried_stream_ids.add(self.current_stream_id) logger.info(f"Loaded stream ID {self.current_stream_id} from Redis for channel {buffer.channel_id}") else: @@ -362,7 +362,7 @@ class StreamManager: current_owner = self.buffer.redis_client.get(owner_key) # Use the worker_id that was passed in during initialization - if current_owner and self.worker_id and current_owner.decode('utf-8') == self.worker_id: + if current_owner and self.worker_id and current_owner == self.worker_id: # Determine the appropriate error message based on retry failures if self.tried_stream_ids and len(self.tried_stream_ids) > 0: error_message = f"All {len(self.tried_stream_ids)} stream options failed" @@ -948,10 +948,10 @@ class StreamManager: logger.debug(f"Updated m3u profile for channel {self.channel_id} to use profile from stream {stream_id}") else: logger.warning(f"Failed to update stream profile for channel {self.channel_id}") - + except Exception as e: logger.error(f"Error updating stream profile for channel {self.channel_id}: {e}") - + finally: # Always close database connection after profile update try: @@ -1348,9 +1348,9 @@ class StreamManager: current_state = None try: metadata = redis_client.hgetall(metadata_key) - state_field = ChannelMetadataField.STATE.encode('utf-8') + state_field = ChannelMetadataField.STATE if metadata and state_field in metadata: - current_state = metadata[state_field].decode('utf-8') + current_state = metadata[state_field] except Exception as e: logger.error(f"Error checking current state: {e}") @@ -1555,4 +1555,4 @@ class StreamManager: """Safely reset the URL switching state if it gets stuck""" self.url_switching = False self.url_switch_start_time = 0 - logger.info(f"Reset URL switching state for channel {self.channel_id}") \ No newline at end of file + logger.info(f"Reset URL switching state for channel {self.channel_id}") diff --git a/apps/proxy/ts_proxy/url_utils.py b/apps/proxy/ts_proxy/url_utils.py index db53cc74..0bfde312 100644 --- a/apps/proxy/ts_proxy/url_utils.py +++ b/apps/proxy/ts_proxy/url_utils.py @@ -215,9 +215,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id.decode('utf-8') + existing_stream_id = existing_stream_id existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") - if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id: + if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True logger.debug(f"Channel {channel.id} already using profile {profile.id}") @@ -353,9 +353,9 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id.decode('utf-8') + existing_stream_id = existing_stream_id existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") - if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id: + if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True logger.debug(f"Channel {channel.id} already using profile {profile.id}") diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index fefc8739..e1fe4c4b 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -219,7 +219,7 @@ class RedisBackedVODConnection: # Convert bytes keys/values to strings if needed if isinstance(list(data.keys())[0], bytes): - data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()} + data = {k: v for k, v in data.items()} return SerializableConnectionState.from_dict(data) except Exception as e: @@ -1115,14 +1115,14 @@ class MultiWorkerVODConnectionManager: # Convert bytes to strings if needed if isinstance(list(data.keys())[0], bytes): - data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()} + data = {k: v for k, v in data.items()} last_activity = float(data.get('last_activity', 0)) active_streams = int(data.get('active_streams', 0)) # Clean up if stale and no active streams if (current_time - last_activity > max_age_seconds) and active_streams == 0: - session_id = key.decode('utf-8').replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') logger.info(f"Cleaning up stale connection: {session_id}") # Clean up connection and related keys @@ -1219,7 +1219,7 @@ class MultiWorkerVODConnectionManager: if connection_data: # Convert bytes to strings if needed if isinstance(list(connection_data.keys())[0], bytes): - connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()} + connection_data = {k: v for k, v in connection_data.items()} profile_id = connection_data.get('m3u_profile_id') if profile_id: @@ -1279,7 +1279,7 @@ class MultiWorkerVODConnectionManager: # Convert bytes keys/values to strings if needed if isinstance(list(connection_data.keys())[0], bytes): - connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()} + connection_data = {k: v for k, v in connection_data.items()} # Check if content matches (using consolidated data) stored_content_type = connection_data.get('content_obj_type', '') @@ -1289,7 +1289,7 @@ class MultiWorkerVODConnectionManager: continue # Extract session ID - session_id = key.decode('utf-8').replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') # Check if Redis-backed connection exists and has no active streams redis_connection = RedisBackedVODConnection(session_id, self.redis_client) @@ -1367,4 +1367,4 @@ class MultiWorkerVODConnectionManager: return redis_connection.get_session_metadata() except Exception as e: logger.error(f"Error getting session info for {session_id}: {e}") - return None \ No newline at end of file + return None diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 00ed8a10..12e1d071 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -550,14 +550,7 @@ class VODStreamView(View): connection_data = redis_client.hgetall(persistent_connection_key) if connection_data: - # Decode Redis hash data - decoded_data = {} - for k, v in connection_data.items(): - k_str = k.decode('utf-8') if isinstance(k, bytes) else k - v_str = v.decode('utf-8') if isinstance(v, bytes) else v - decoded_data[k_str] = v_str - - existing_profile_id = decoded_data.get('m3u_profile_id') + existing_profile_id = connection_data.get('m3u_profile_id') if existing_profile_id: try: existing_profile = M3UAccountProfile.objects.get( @@ -770,19 +763,16 @@ class VODStatsView(View): for key in keys: try: - key_str = key.decode('utf-8') if isinstance(key, bytes) else key connection_data = redis_client.hgetall(key) if connection_data: # Extract session ID from key - session_id = key_str.replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') # Decode Redis hash data combined_data = {} for k, v in connection_data.items(): - k_str = k.decode('utf-8') if isinstance(k, bytes) else k - v_str = v.decode('utf-8') if isinstance(v, bytes) else v - combined_data[k_str] = v_str + combined_data[k] = v # Get content info from the connection data (using correct field names) content_type = combined_data.get('content_obj_type', 'unknown') diff --git a/core/redis_pubsub.py b/core/redis_pubsub.py index 5d0032b0..b1f48b1f 100644 --- a/core/redis_pubsub.py +++ b/core/redis_pubsub.py @@ -201,10 +201,6 @@ class RedisPubSubManager: channel = message.get('channel') if channel: - # Decode binary channel name if needed - if isinstance(channel, bytes): - channel = channel.decode('utf-8') - # Find and call the appropriate handler handler = self.message_handlers.get(channel) if handler: diff --git a/core/tasks.py b/core/tasks.py index f757613b..257d8660 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -404,7 +404,7 @@ def fetch_channel_stats(): while True: cursor, keys = redis_client.scan(cursor, match=channel_pattern) for key in keys: - channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8')) + channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key) if channel_id_match: ch_id = channel_id_match.group(1) channel_info = ChannelStatus.get_basic_channel_info(ch_id) diff --git a/core/utils.py b/core/utils.py index 36ac5fef..9cc4c26f 100644 --- a/core/utils.py +++ b/core/utils.py @@ -42,101 +42,116 @@ def natural_sort_key(text): return [convert(c) for c in re.split('([0-9]+)', text)] class RedisClient: + _initialized = False _client = None + _buffer = None _pubsub_client = None + @classmethod + def _init_client(cls, decode_responses=True, max_retries=5, retry_interval=1): + retry_count = 0 + while retry_count < max_retries: + try: + # Get connection parameters from settings or environment + redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) + redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) + redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) + + # Use standardized settings + socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) + socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) + health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) + socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) + retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + + # Create Redis client with better defaults + client = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + socket_timeout=socket_timeout, + socket_connect_timeout=socket_connect_timeout, + socket_keepalive=socket_keepalive, + health_check_interval=health_check_interval, + retry_on_timeout=retry_on_timeout, + decode_responses=decode_responses + ) + + # Validate connection with ping + client.ping() + if cls._initialized is False: + client.flushdb() + cls._initialized = True + + # Disable persistence on first connection - improves performance + # Only try to disable if not in a read-only environment + try: + client.config_set('save', '') # Disable RDB snapshots + client.config_set('appendonly', 'no') # Disable AOF logging + + # Set optimal memory settings with environment variable support + # Get max memory from environment or use a larger default (512MB instead of 256MB) + #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') + #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') + + # Apply memory settings + #client.config_set('maxmemory-policy', eviction_policy) + #client.config_set('maxmemory', max_memory) + + #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") + + # Disable protected mode when in debug mode + if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': + client.config_set('protected-mode', 'no') # Disable protected mode in debug + logger.warning("Redis protected mode disabled for debug environment") + + logger.trace("Redis persistence disabled for better performance") + except redis.exceptions.ResponseError as e: + # Improve error handling for Redis configuration errors + if "OOM" in str(e): + logger.error(f"Redis OOM during configuration: {e}") + # Try to increase maxmemory as an emergency measure + try: + client.config_set('maxmemory', '768mb') + logger.warning("Applied emergency Redis memory increase to 768MB") + except: + pass + else: + logger.error(f"Redis configuration error: {e}") + + logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") + break + + except (ConnectionError, TimeoutError) as e: + retry_count += 1 + if retry_count >= max_retries: + logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") + return None + else: + # Use exponential backoff for retries + wait_time = retry_interval * (2 ** (retry_count - 1)) + logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") + time.sleep(wait_time) + + except Exception as e: + logger.error(f"Unexpected error connecting to Redis: {e}") + return None + + return client + @classmethod def get_client(cls, max_retries=5, retry_interval=1): if cls._client is None: - retry_count = 0 - while retry_count < max_retries: - try: - # Get connection parameters from settings or environment - redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) - redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) - redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) - - # Use standardized settings - socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) - socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) - health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) - socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) - retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) - - # Create Redis client with better defaults - client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - socket_timeout=socket_timeout, - socket_connect_timeout=socket_connect_timeout, - socket_keepalive=socket_keepalive, - health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout - ) - - # Validate connection with ping - client.ping() - client.flushdb() - - # Disable persistence on first connection - improves performance - # Only try to disable if not in a read-only environment - try: - client.config_set('save', '') # Disable RDB snapshots - client.config_set('appendonly', 'no') # Disable AOF logging - - # Set optimal memory settings with environment variable support - # Get max memory from environment or use a larger default (512MB instead of 256MB) - #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') - #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') - - # Apply memory settings - #client.config_set('maxmemory-policy', eviction_policy) - #client.config_set('maxmemory', max_memory) - - #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") - - # Disable protected mode when in debug mode - if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': - client.config_set('protected-mode', 'no') # Disable protected mode in debug - logger.warning("Redis protected mode disabled for debug environment") - - logger.trace("Redis persistence disabled for better performance") - except redis.exceptions.ResponseError as e: - # Improve error handling for Redis configuration errors - if "OOM" in str(e): - logger.error(f"Redis OOM during configuration: {e}") - # Try to increase maxmemory as an emergency measure - try: - client.config_set('maxmemory', '768mb') - logger.warning("Applied emergency Redis memory increase to 768MB") - except: - pass - else: - logger.error(f"Redis configuration error: {e}") - - logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") - - cls._client = client - break - - except (ConnectionError, TimeoutError) as e: - retry_count += 1 - if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") - return None - else: - # Use exponential backoff for retries - wait_time = retry_interval * (2 ** (retry_count - 1)) - logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") - time.sleep(wait_time) - - except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}") - return None - + cls._client = cls._init_client(decode_responses=True, max_retries=max_retries, retry_interval=retry_interval) return cls._client + @classmethod + def get_buffer(cls, max_retries=5, retry_interval=1): + """Get Redis client optimized for binary data (no decoding)""" + if cls._buffer is None: + cls._buffer = cls._init_client(decode_responses=False, max_retries=max_retries, retry_interval=retry_interval) + return cls._buffer + @classmethod def get_pubsub_client(cls, max_retries=5, retry_interval=1): """Get Redis client optimized for PubSub operations""" @@ -165,7 +180,8 @@ class RedisClient: socket_connect_timeout=socket_connect_timeout, socket_keepalive=socket_keepalive, health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout + retry_on_timeout=retry_on_timeout, + decode_responses=True ) # Validate connection with ping diff --git a/dispatcharr/persistent_lock.py b/dispatcharr/persistent_lock.py index 360c9b5d..fe62be43 100644 --- a/dispatcharr/persistent_lock.py +++ b/dispatcharr/persistent_lock.py @@ -48,7 +48,7 @@ class PersistentLock: Returns True if the expiration was successfully extended. """ current_value = self.redis_client.get(self.lock_key) - if current_value and current_value.decode("utf-8") == self.lock_token: + if current_value and current_value == self.lock_token: self.redis_client.expire(self.lock_key, self.lock_timeout) self.has_lock = False return True From 16183329623e163ec01145b721e2197571c12097 Mon Sep 17 00:00:00 2001 From: dekzter Date: Wed, 12 Nov 2025 17:52:11 -0500 Subject: [PATCH 002/503] byte to string fix --- apps/proxy/vod_proxy/connection_manager.py | 55 ++++++++++------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/apps/proxy/vod_proxy/connection_manager.py b/apps/proxy/vod_proxy/connection_manager.py index dea5759b..00ca77aa 100644 --- a/apps/proxy/vod_proxy/connection_manager.py +++ b/apps/proxy/vod_proxy/connection_manager.py @@ -329,15 +329,15 @@ class VODConnectionManager: continue # Extract session info - stored_content_type = session_data.get(b'content_type', b'').decode('utf-8') - stored_content_uuid = session_data.get(b'content_uuid', b'').decode('utf-8') + stored_content_type = session_data.get('content_type', '') + stored_content_uuid = session_data.get('content_uuid', '') # Check if content matches if stored_content_type != content_type or stored_content_uuid != content_uuid: continue # Extract session ID from key - session_id = key.decode('utf-8').replace('vod_session:', '') + session_id = key.replace('vod_session:', '') # Check if session has an active persistent connection persistent_conn = self._persistent_connections.get(session_id) @@ -351,13 +351,13 @@ class VODConnectionManager: continue # Get stored client info for comparison - stored_client_ip = session_data.get(b'client_ip', b'').decode('utf-8') - stored_user_agent = session_data.get(b'user_agent', b'').decode('utf-8') + stored_client_ip = session_data.get('client_ip', '') + stored_user_agent = session_data.get('user_agent', '') # Check timeshift parameters match - stored_utc_start = session_data.get(b'utc_start', b'').decode('utf-8') - stored_utc_end = session_data.get(b'utc_end', b'').decode('utf-8') - stored_offset = session_data.get(b'offset', b'').decode('utf-8') + stored_utc_start = session_data.get('utc_start', '') + stored_utc_end = session_data.get('utc_end', '') + stored_offset = session_data.get('offset', '') current_utc_start = utc_start or "" current_utc_end = utc_end or "" @@ -394,7 +394,7 @@ class VODConnectionManager: 'session_id': session_id, 'score': score, 'reasons': match_reasons, - 'last_activity': float(session_data.get(b'last_activity', b'0').decode('utf-8')) + 'last_activity': float(session_data.get('last_activity', '0')) }) except Exception as e: @@ -536,7 +536,7 @@ class VODConnectionManager: # Get current bytes and add to it current_bytes = self.redis_client.hget(connection_key, "bytes_sent") if current_bytes: - total_bytes = int(current_bytes.decode('utf-8')) + bytes_sent + total_bytes = int(current_bytes) + bytes_sent else: total_bytes = bytes_sent update_data["bytes_sent"] = str(total_bytes) @@ -571,7 +571,7 @@ class VODConnectionManager: profile_id = None if b"m3u_profile_id" in connection_data: try: - profile_id = int(connection_data[b"m3u_profile_id"].decode('utf-8')) + profile_id = int(connection_data["m3u_profile_id"]) except ValueError: pass @@ -617,16 +617,14 @@ class VODConnectionManager: # Convert bytes to strings and parse numbers info = {} for key, value in connection_data.items(): - key_str = key.decode('utf-8') - value_str = value.decode('utf-8') # Parse numeric fields - if key_str in ['connected_at', 'last_activity']: - info[key_str] = float(value_str) - elif key_str in ['bytes_sent', 'position_seconds', 'm3u_profile_id']: - info[key_str] = int(value_str) + if key in ['connected_at', 'last_activity']: + info[key] = float(value) + elif key in ['bytes_sent', 'position_seconds', 'm3u_profile_id']: + info[key] = int(valuvaluee_str) else: - info[key_str] = value_str + info[key] = value return info @@ -676,14 +674,13 @@ class VODConnectionManager: for key in keys: try: - key_str = key.decode('utf-8') last_activity = self.redis_client.hget(key, "last_activity") if last_activity: - last_activity_time = float(last_activity.decode('utf-8')) + last_activity_time = float(last_activity) if current_time - last_activity_time > max_age_seconds: # Extract info for cleanup - parts = key_str.split(':') + parts = key.split(':') if len(parts) >= 5: content_type = parts[2] content_uuid = parts[3] @@ -1342,9 +1339,9 @@ class VODConnectionManager: session_data = self.redis_client.hgetall(session_key) if session_data: # Get session details for connection cleanup - content_type = session_data.get(b'content_type', b'').decode('utf-8') - content_uuid = session_data.get(b'content_uuid', b'').decode('utf-8') - profile_id = session_data.get(b'profile_id') + content_type = session_data.get('content_type', '') + content_uuid = session_data.get('content_uuid', '') + profile_id = session_data.get('profile_id') # Generate client_id from session_id (matches what's used during streaming) client_id = session_id @@ -1355,12 +1352,12 @@ class VODConnectionManager: self.remove_connection(content_type, content_uuid, client_id) # Remove from profile connections if counted (additional safety check) - if session_data.get(b'connection_counted') == b'True' and profile_id: - profile_key = self._get_profile_connections_key(int(profile_id.decode('utf-8'))) + if session_data.get('connection_counted') == 'True' and profile_id: + profile_key = self._get_profile_connections_key(int(profile_id)) current_count = int(self.redis_client.get(profile_key) or 0) if current_count > 0: self.redis_client.decr(profile_key) - logger.info(f"[{session_id}] Decremented profile {profile_id.decode('utf-8')} connections") + logger.info(f"[{session_id}] Decremented profile {profile_id} connections") # Remove session tracking key self.redis_client.delete(session_key) @@ -1380,7 +1377,7 @@ class VODConnectionManager: if session_related_keys: # Filter out keys we already deleted - remaining_keys = [k for k in session_related_keys if k.decode('utf-8') != session_key] + remaining_keys = [k for k in session_related_keys if k != session_key] if remaining_keys: self.redis_client.delete(*remaining_keys) logger.info(f"[{session_id}] Cleaned up {len(remaining_keys)} additional session-related keys") @@ -1410,7 +1407,7 @@ class VODConnectionManager: if self.redis_client: session_data = self.redis_client.hgetall(session_key) if session_data: - created_at = float(session_data.get(b'created_at', b'0').decode('utf-8')) + created_at = float(session_data.get('created_at', '0')) if current_time - created_at > max_age_seconds: logger.info(f"[{session_id}] Session older than {max_age_seconds}s") stale_sessions.append(session_id) From cd35e671e56a29e57e256351010147a4c8145c88 Mon Sep 17 00:00:00 2001 From: dekzter Date: Wed, 12 Nov 2025 17:52:48 -0500 Subject: [PATCH 003/503] byte to string fix --- apps/proxy/ts_proxy/stream_generator.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/proxy/ts_proxy/stream_generator.py b/apps/proxy/ts_proxy/stream_generator.py index 368691b8..25122d74 100644 --- a/apps/proxy/ts_proxy/stream_generator.py +++ b/apps/proxy/ts_proxy/stream_generator.py @@ -108,13 +108,13 @@ class StreamGenerator: metadata_key = RedisKeys.channel_metadata(self.channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] if state in ['waiting_for_clients', 'active']: logger.info(f"[{self.client_id}] Channel {self.channel_id} now ready (state={state})") return True elif state in ['error', 'stopped', 'stopping']: # Added 'stopping' to error states - error_message = metadata.get(b'error_message', b'Unknown error').decode('utf-8') + error_message = metadata.get('error_message', 'Unknown error') logger.error(f"[{self.client_id}] Channel {self.channel_id} in error state: {state}, message: {error_message}") # Send error packet before giving up yield create_ts_packet('error', f"Error: {error_message}") @@ -122,9 +122,9 @@ class StreamGenerator: else: # Improved logging to track initialization progress init_time = "unknown" - if b'init_time' in metadata: + if 'init_time' in metadata: try: - init_time_float = float(metadata[b'init_time'].decode('utf-8')) + init_time_float = float(metadata['init_time']) init_duration = time.time() - init_time_float init_time = f"{init_duration:.1f}s ago" except: @@ -267,8 +267,8 @@ class StreamGenerator: # Also check channel state in metadata metadata_key = RedisKeys.channel_metadata(self.channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] if state in ['error', 'stopped', 'stopping']: logger.info(f"[{self.client_id}] Channel in {state} state, terminating stream") return False @@ -397,8 +397,6 @@ class StreamGenerator: if metadata: stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) if stream_id_bytes: - stream_id = int(stream_id_bytes.decode('utf-8')) - # Check if we're the last client if self.channel_id in proxy_server.client_managers: client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() @@ -459,4 +457,4 @@ def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, Returns a function that can be passed to StreamingHttpResponse. """ generator = StreamGenerator(channel_id, client_id, client_ip, client_user_agent, channel_initializing) - return generator.generate \ No newline at end of file + return generator.generate From 2ca2a51e2e63dcdc8c8ab7942472caaf2b9f4a0d Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 6 Dec 2025 09:36:50 -0500 Subject: [PATCH 004/503] reverted fix so only updated values are modified --- apps/proxy/ts_proxy/channel_status.py | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 015d5ca2..4ac8f63f 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -434,13 +434,33 @@ class ChannelStatus: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") # Add stream info to basic info as well - info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) - info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) - info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) - info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) - info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) - info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) - info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) + video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC) + if video_codec: + info['video_codec'] = video_codec + + resolution = metadata.get(ChannelMetadataField.RESOLUTION) + if resolution: + info['resolution'] = resolution + + source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS) + if source_fps: + info['source_fps'] = float(source_fps) + + ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + if ffmpeg_speed: + info['ffmpeg_speed'] = float(ffmpeg_speed) + + audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC) + if audio_codec: + info['audio_codec'] = audio_codec + + audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + if audio_channels: + info['audio_channels'] = audio_channels + + stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE) + if stream_type: + info['stream_type'] = stream_type return info except Exception as e: From 3746b614a28e2020d058c03058ac9f5bfc249654 Mon Sep 17 00:00:00 2001 From: Damien Date: Wed, 7 Jan 2026 19:56:09 +0000 Subject: [PATCH 005/503] Updated UsersTable Component to add Channel Profiles Column --- frontend/src/components/tables/UsersTable.jsx | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/tables/UsersTable.jsx b/frontend/src/components/tables/UsersTable.jsx index 3e9e4971..a5ba02b2 100644 --- a/frontend/src/components/tables/UsersTable.jsx +++ b/frontend/src/components/tables/UsersTable.jsx @@ -2,6 +2,7 @@ import React, { useMemo, useCallback, useState } from 'react'; import API from '../../api'; import UserForm from '../forms/User'; import useUsersStore from '../../store/users'; +import useChannelsStore from '../../store/channels'; import useAuthStore from '../../store/auth'; import { USER_LEVELS, USER_LEVEL_LABELS } from '../../constants'; import useWarningsStore from '../../store/warnings'; @@ -26,6 +27,7 @@ import { UnstyledButton, LoadingOverlay, Stack, + Badge, } from '@mantine/core'; import { CustomTable, useTable } from './CustomTable'; import ConfirmationDialog from '../ConfirmationDialog'; @@ -83,6 +85,7 @@ const UsersTable = () => { * STORES */ const users = useUsersStore((s) => s.users); + const profiles = useChannelsStore((s) => s.profiles); const authUser = useAuthStore((s) => s.user); const isWarningSuppressed = useWarningsStore((s) => s.isWarningSuppressed); const suppressWarning = useWarningsStore((s) => s.suppressWarning); @@ -259,6 +262,37 @@ const UsersTable = () => { ); }, }, + { + header: 'Channel Profiles', + accessorKey: 'channel_profiles', + grow: true, + cell: ({ getValue }) => { + const userProfiles = getValue() || []; + const profileNames = Object.values(profiles) + .filter((profile) => userProfiles.includes(profile.id)) + .map((profile) => profile.name); + return ( + + {profileNames.length > 0 ? ( + profileNames.map((name, index) => ( + + {name} + + )) + ) : ( + + All + + )} + + ); + }, + }, { id: 'actions', size: 80, @@ -313,6 +347,7 @@ const UsersTable = () => { user_level: renderHeaderCell, last_login: renderHeaderCell, date_joined: renderHeaderCell, + channel_profiles: renderHeaderCell, custom_properties: renderHeaderCell, }, }); @@ -327,7 +362,7 @@ const UsersTable = () => { minHeight: '100vh', }} > - + Date: Wed, 7 Jan 2026 20:10:10 +0000 Subject: [PATCH 006/503] Added Vertical Spacing to ensure badges look like they're within the table row. --- frontend/src/components/tables/UsersTable.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/tables/UsersTable.jsx b/frontend/src/components/tables/UsersTable.jsx index a5ba02b2..bd4f17f6 100644 --- a/frontend/src/components/tables/UsersTable.jsx +++ b/frontend/src/components/tables/UsersTable.jsx @@ -272,7 +272,7 @@ const UsersTable = () => { .filter((profile) => userProfiles.includes(profile.id)) .map((profile) => profile.name); return ( - + {profileNames.length > 0 ? ( profileNames.map((name, index) => ( Date: Wed, 7 Jan 2026 20:24:51 +0000 Subject: [PATCH 007/503] Second Thought.. We should useMemo same as we do for the other data to prevent unnecessary compute. --- frontend/src/components/tables/UsersTable.jsx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/tables/UsersTable.jsx b/frontend/src/components/tables/UsersTable.jsx index bd4f17f6..eecb8fc5 100644 --- a/frontend/src/components/tables/UsersTable.jsx +++ b/frontend/src/components/tables/UsersTable.jsx @@ -141,6 +141,15 @@ const UsersTable = () => { /** * useMemo */ + // Create a profile ID to name lookup map for efficient rendering + const profileIdToName = useMemo(() => { + const map = {}; + Object.values(profiles).forEach((profile) => { + map[profile.id] = profile.name; + }); + return map; + }, [profiles]); + const columns = useMemo( () => [ { @@ -268,9 +277,9 @@ const UsersTable = () => { grow: true, cell: ({ getValue }) => { const userProfiles = getValue() || []; - const profileNames = Object.values(profiles) - .filter((profile) => userProfiles.includes(profile.id)) - .map((profile) => profile.name); + const profileNames = userProfiles + .map((id) => profileIdToName[id]) + .filter(Boolean); // Filter out any undefined values return ( {profileNames.length > 0 ? ( @@ -308,7 +317,7 @@ const UsersTable = () => { ), }, ], - [theme, editUser, deleteUser, visiblePasswords, togglePasswordVisibility] + [theme, editUser, deleteUser, visiblePasswords, togglePasswordVisibility, profileIdToName] ); const closeUserForm = () => { From dba70b3c585e0f2dce6cdcf157045992e617b236 Mon Sep 17 00:00:00 2001 From: dekzter Date: Tue, 27 Jan 2026 15:23:07 -0500 Subject: [PATCH 008/503] outstanding commits, may revisit later --- .../migrations/0004_user_stream_priority.py | 18 +++ apps/channels/models.py | 153 +++++++++++++++++- apps/channels/tasks.py | 5 +- apps/proxy/ts_proxy/client_manager.py | 16 +- apps/proxy/ts_proxy/server.py | 6 - apps/proxy/ts_proxy/utils.py | 4 +- apps/proxy/ts_proxy/views.py | 26 +-- docker/init/02-postgres.sh | 2 - docker/init/03-init-dispatcharr.sh | 2 +- docker/init/99-init-dev.sh | 50 +++--- docker/uwsgi.dev.ini | 4 +- frontend/prettier.config.js | 4 +- frontend/src/components/forms/User.jsx | 40 +++-- frontend/src/pages/Users.jsx | 35 ---- 14 files changed, 242 insertions(+), 123 deletions(-) create mode 100644 apps/accounts/migrations/0004_user_stream_priority.py diff --git a/apps/accounts/migrations/0004_user_stream_priority.py b/apps/accounts/migrations/0004_user_stream_priority.py new file mode 100644 index 00000000..35c8bbae --- /dev/null +++ b/apps/accounts/migrations/0004_user_stream_priority.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.4 on 2025-10-21 19:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0003_alter_user_custom_properties'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='stream_priority', + field=models.IntegerField(default=0, help_text='Priority level for streaming tasks. Lower values indicate higher priority.'), + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 3dfb392b..7282d025 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -166,7 +166,7 @@ class Stream(models.Model): return stream_profile - def get_stream(self): + def get_stream(self, requester=None): """ Finds an available stream for the requested channel and returns the selected stream and profile. """ @@ -349,7 +349,145 @@ class Channel(models.Model): return stream_profile - def get_stream(self): + def _pick_channel_to_preempt( + self, + profile_id, + requester_level, + redis_client, + exclude_channel_ids=None, + cooldown_seconds=30, + ): + """ + Pick the lowest-impact channel to terminate on the given profile. + Returns: Optional[int] channel_id to preempt + """ + exclude_channel_ids = set(exclude_channel_ids or []) + candidates = [] + + # 1) Try to get active channel IDs for this profile from an index set if available + ch_set_key = f"ts_proxy:profile:{profile_id}:channels" + try: + ch_ids = { (int(x) if not isinstance(x, int) else x) for x in (redis_client.smembers(ch_set_key) or set()) } + except Exception: + ch_ids = set() + + logger.debug("Candidate channels for preemption:") + logger.debug(ch_ids) + + # 2) Fallback: scan metadata keys and filter by m3u_profile == profile_id + if not ch_ids: + cursor = 0 + pattern = "ts_proxy:channel:*:metadata" + while True: + cursor, keys = redis_client.scan(cursor=cursor, match=pattern, count=500) + if keys: + # Prefer HGET m3u_profile if metadata is a hash + pipe = redis_client.pipeline() + for k in keys: + pipe.hget(k, "m3u_profile") + prof_vals = pipe.execute() + for k, prof_val in zip(keys, prof_vals): + try: + pid = int(prof_val) if prof_val is not None else None + except Exception: + pid = None + + if pid == profile_id: + parts = k.split(":") # ts_proxy:channel:{id}:metadata + if len(parts) >= 4: + try: + ch_ids.add(int(parts[2])) + except Exception: + pass + if cursor == 0: + break + + logger.debug("Candidate channels for preemption:") + logger.debug(ch_ids) + + if not ch_ids: + return None + + # 3) Score candidates + for ch_id in ch_ids: + if ch_id in exclude_channel_ids: + continue + + # Skip if recently preempted + last_preempt_key = f"ts_proxy:channel:{ch_id}:last_preempt" + try: + last_preempt = float(redis_client.get(last_preempt_key) or 0.0) + except Exception: + last_preempt = 0.0 + if last_preempt and (time.time() - last_preempt) < cooldown_seconds: + continue + + # Clients and their levels + clients_key = f"ts_proxy:channel:{ch_id}:clients" + member_ids = list(redis_client.smembers(clients_key) or []) + viewer_count = len(member_ids) + max_viewer_level = 0 + if viewer_count: + pipe = redis_client.pipeline() + for cid in member_ids: + pipe.hget(f"ts_proxy:channel:{ch_id}:clients:{cid}", "user_level") + levels_raw = pipe.execute() + levels = [] + for lv in levels_raw: + try: + levels.append(int(lv or 0)) + except Exception: + levels.append(0) + max_viewer_level = max(levels or [0]) + + # Only preempt if requester strictly outranks this channel's viewers + if requester_level <= max_viewer_level: + continue + + # Metadata (protected/recording/started_at_ts) + meta_key = f"ts_proxy:channel:{ch_id}:metadata" + try: + protected, recording, started_at_ts = redis_client.hmget( + meta_key, "protected", "recording", "started_at_ts" + ) + except Exception: + protected = recording = started_at_ts = None + + protected = str(protected or "0") in ("1", "true", "True") + recording = str(recording or "0") in ("1", "true", "True") + if protected or recording: + continue + + try: + started_at_ts = float(started_at_ts) if started_at_ts is not None else None + except Exception: + started_at_ts = None + if started_at_ts is None: + started_at_ts = time.time() # treat unknown as newest + + # Score: lower is safer to terminate + has_viewers = 1 if viewer_count > 0 else 0 + score = (has_viewers, max_viewer_level, viewer_count, started_at_ts) + candidates.append((score, ch_id)) + + logger.debug("Candidate channels after scoring:") + logger.debug(candidates) + + if not candidates: + return None + + candidates.sort(key=lambda x: x[0]) + victim_id = candidates[0][1] + + # Mark preempt timestamp to avoid thrashing + try: + redis_client.set(f"ts_proxy:channel:{victim_id}:last_preempt", str(time.time()), ex=3600) + except Exception: + pass + + return victim_id + + def get_stream(self, requester=None): """ Finds an available stream for the requested channel and returns the selected stream and profile. @@ -438,6 +576,17 @@ class Channel(models.Model): None, ) # Return newly assigned stream and matched profile else: + # At capacity: try to preempt a lower-impact channel on this profile + victim_channel_id = self._pick_channel_to_preempt( + profile_id=profile.id, + requester_level=requester.user_level if requester else 100, + redis_client=redis_client, + exclude_channel_ids=None, + ) + if victim_channel_id: + logger.info(f"Preempting channel {victim_channel_id} for new stream on profile {profile.id}") + # return self.id, profile.id, victim_channel_id + # This profile is at max connections has_streams_but_maxed_out = True logger.debug( diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index ffb2b97a..b63577d2 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1900,11 +1900,8 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str): metadata_key = RedisKeys.channel_metadata(str(channel.uuid)) md = r.hgetall(metadata_key) if md: - def _gv(bkey): - return md.get(bkey) - def _d(bkey, cast=str): - v = _gv(bkey) + v = md.get(key) try: if v is None: return None diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index bffecdde..80be26f1 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -1,10 +1,8 @@ """Client connection management for TS streams""" import threading -import logging import time import json -import gevent from typing import Set, Optional from apps.proxy.config import TSConfig as Config from redis.exceptions import ConnectionError, TimeoutError @@ -123,7 +121,7 @@ class ClientManager: # Check for stale activity using last_active field last_active = self.redis_client.hget(client_key, "last_active") if last_active: - last_active_time = float(last_active.decode('utf-8')) + last_active_time = float(last_active) ghost_timeout = self.heartbeat_interval * getattr(Config, 'GHOST_CLIENT_MULTIPLIER', 5.0) if current_time - last_active_time > ghost_timeout: @@ -224,7 +222,7 @@ class ClientManager: except Exception as e: logger.error(f"Error notifying owner of client activity: {e}") - def add_client(self, client_id, client_ip, user_agent=None): + def add_client(self, client_id, client_ip, user_agent=None, user=None): """Add a client with duplicate prevention""" if client_id in self._registered_clients: logger.debug(f"Client {client_id} already registered, skipping") @@ -242,7 +240,8 @@ class ClientManager: "ip_address": client_ip, "connected_at": current_time, "last_active": current_time, - "worker_id": self.worker_id or "unknown" + "worker_id": self.worker_id or "unknown", + "user_level": user.user_level if user is not None else 100, # default to a high value since no user means the non-user specific M3U/HDHR } try: @@ -303,8 +302,6 @@ class ClientManager: def remove_client(self, client_id): """Remove a client from this channel and Redis""" - client_ip = None - with self.lock: if client_id in self.clients: self.clients.remove(client_id) @@ -317,11 +314,6 @@ class ClientManager: if self.redis_client: # Get client IP before removing the data client_key = f"ts_proxy:channel:{self.channel_id}:clients:{client_id}" - client_data = self.redis_client.hgetall(client_key) - if client_data and b'ip_address' in client_data: - client_ip = client_data[b'ip_address'].decode('utf-8') - elif client_data and 'ip_address' in client_data: - client_ip = client_data['ip_address'] # Remove from channel's client set self.redis_client.srem(self.client_set_key, client_id) diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index 9773e487..7d4a9f89 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -1121,12 +1121,6 @@ class ProxyServer: else: # Grace period expired with clients - mark channel as active logger.info(f"Grace period expired with {total_clients} clients - marking channel {channel_id} as active") -<<<<<<< HEAD - old_state = "unknown" - if metadata and 'state' in metadata: - old_state = metadata['state'] -======= ->>>>>>> origin/dev if self.update_channel_state(channel_id, ChannelState.ACTIVE, { "grace_period_ended_at": str(time.time()), "clients_at_activation": str(total_clients) diff --git a/apps/proxy/ts_proxy/utils.py b/apps/proxy/ts_proxy/utils.py index 20a6e140..ff9e1780 100644 --- a/apps/proxy/ts_proxy/utils.py +++ b/apps/proxy/ts_proxy/utils.py @@ -83,7 +83,7 @@ def create_ts_packet(packet_type='null', message=None): # Add message to payload if provided if message: - msg_bytes = message.encode('utf-8') + msg_bytes = message packet[4:4+min(len(msg_bytes), 180)] = msg_bytes[:180] return bytes(packet) @@ -113,4 +113,4 @@ def get_logger(component_name=None): # Default if detection fails logger_name = "ts_proxy" - return logging.getLogger(logger_name) \ No newline at end of file + return logging.getLogger(logger_name) diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index 91f254a7..0f9c1228 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -45,7 +45,7 @@ logger = get_logger() @api_view(["GET"]) -def stream_ts(request, channel_id): +def stream_ts(request, channel_id, user=None): if not network_access_allowed(request, "STREAMS"): return JsonResponse({"error": "Forbidden"}, status=403) @@ -80,9 +80,9 @@ def stream_ts(request, channel_id): metadata_key = RedisKeys.channel_metadata(channel_id) if proxy_server.redis_client.exists(metadata_key): metadata = proxy_server.redis_client.hgetall(metadata_key) - state_field = ChannelMetadataField.STATE.encode("utf-8") + state_field = ChannelMetadataField.STATE if state_field in metadata: - channel_state = metadata[state_field].decode("utf-8") + channel_state = metadata[state_field] # Active/running states - channel is operational, don't reinitialize if channel_state in [ @@ -120,7 +120,7 @@ def stream_ts(request, channel_id): else: owner_field = ChannelMetadataField.OWNER.encode("utf-8") if owner_field in metadata: - owner = metadata[owner_field].decode("utf-8") + owner = metadata[owner_field] owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat" if proxy_server.redis_client.exists(owner_heartbeat_key): # Owner is still active with unknown state - don't reinitialize @@ -373,7 +373,7 @@ def stream_ts(request, channel_id): metadata_key, ChannelMetadataField.STATE ) if state_bytes: - current_state = state_bytes.decode("utf-8") + current_state = state_bytes logger.debug( f"[{client_id}] Current state of channel {channel_id}: {current_state}" ) @@ -449,12 +449,12 @@ def stream_ts(request, channel_id): ) if url_bytes: - url = url_bytes.decode("utf-8") + url = url_bytes if ua_bytes: - stream_user_agent = ua_bytes.decode("utf-8") + stream_user_agent = ua_bytes # Extract transcode setting from Redis if profile_bytes: - profile_str = profile_bytes.decode("utf-8") + profile_str = profile_bytes use_transcode = ( profile_str == PROXY_PROFILE_NAME or profile_str == "None" ) @@ -490,7 +490,7 @@ def stream_ts(request, channel_id): # Register client buffer = proxy_server.stream_buffers[channel_id] client_manager = proxy_server.client_managers[channel_id] - client_manager.add_client(client_id, client_ip, client_user_agent) + client_manager.add_client(client_id, client_ip, client_user_agent, user) logger.info(f"[{client_id}] Client registered with channel {channel_id}") # Create a stream generator for this client @@ -553,7 +553,7 @@ def stream_xc(request, username, password, channel_id): channel = get_object_or_404(Channel, id=channel_id) # @TODO: we've got the file 'type' via extension, support this when we support multiple outputs - return stream_ts(request._request, str(channel.uuid)) + return stream_ts(request._request, str(channel.uuid), user) @csrf_exempt @@ -681,7 +681,7 @@ def channel_status(request, channel_id=None): ) for key in keys: channel_id_match = re.search( - r"ts_proxy:channel:(.*):metadata", key.decode("utf-8") + r"ts_proxy:channel:(.*):metadata", key ) if channel_id_match: ch_id = channel_id_match.group(1) @@ -802,7 +802,7 @@ def next_stream(request, channel_id): metadata_key, ChannelMetadataField.STREAM_ID ) if stream_id_bytes: - current_stream_id = int(stream_id_bytes.decode("utf-8")) + current_stream_id = int(stream_id_bytes) logger.info( f"Found current stream ID {current_stream_id} in Redis for channel {channel_id}" ) @@ -812,7 +812,7 @@ def next_stream(request, channel_id): metadata_key, ChannelMetadataField.M3U_PROFILE ) if profile_id_bytes: - profile_id = int(profile_id_bytes.decode("utf-8")) + profile_id = int(profile_id_bytes) logger.info( f"Found M3U profile ID {profile_id} in Redis for channel {channel_id}" ) diff --git a/docker/init/02-postgres.sh b/docker/init/02-postgres.sh index e36dd744..233a5fb8 100644 --- a/docker/init/02-postgres.sh +++ b/docker/init/02-postgres.sh @@ -162,5 +162,3 @@ ensure_utf8_encoding() { echo "Database $POSTGRES_DB converted to UTF8 and permissions set." fi } - - diff --git a/docker/init/03-init-dispatcharr.sh b/docker/init/03-init-dispatcharr.sh index 5fbef23d..bfe081e2 100644 --- a/docker/init/03-init-dispatcharr.sh +++ b/docker/init/03-init-dispatcharr.sh @@ -64,4 +64,4 @@ if [ "$(id -u)" = "0" ]; then fi chmod +x /data -fi \ No newline at end of file +fi diff --git a/docker/init/99-init-dev.sh b/docker/init/99-init-dev.sh index 861c8307..fc8ef18a 100644 --- a/docker/init/99-init-dev.sh +++ b/docker/init/99-init-dev.sh @@ -1,25 +1,33 @@ #!/bin/bash -echo "🚀 Development Mode - Setting up Frontend..." +if [ ! -e "/tmp/init" ]; then + echo "🚀 Development Mode - Setting up Frontend..." -# Install Node.js -if ! command -v node 2>&1 >/dev/null -then - echo "=== setting up nodejs ===" - curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh - bash /tmp/nodesource_setup.sh - apt-get update - apt-get install -y --no-install-recommends \ - nodejs -fi - -# Install frontend dependencies -cd /app/frontend && npm install -# Install pip dependencies -cd /app && pip install -r requirements.txt - -# Install debugpy for remote debugging -if [ "$DISPATCHARR_DEBUG" = "true" ]; then - echo "=== setting up debugpy ===" - pip install debugpy + # Install Node.js + if ! command -v node 2>&1 >/dev/null + then + echo "=== setting up nodejs ===" + curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh + bash /tmp/nodesource_setup.sh + apt-get update + apt-get install -y --no-install-recommends \ + nodejs + fi + + # Install frontend dependencies + cd /app/frontend && npm install + # Install pip dependencies + cd /app && pip install -r requirements.txt + + # Install debugpy for remote debugging + if [ "$DISPATCHARR_DEBUG" = "true" ]; then + echo "=== setting up debugpy ===" + pip install debugpy + fi + + if [[ "$DISPATCHARR_ENV" = "dev" ]]; then + touch /tmp/init + fi +else + echo "Development mode initialization already done. Skipping dev setup." fi diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index e476e216..e6d0806d 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first -attach-daemon = redis-server +attach-daemon = redis-server --protected-mode no ; Then start other services with configurable nice level (default: 5 for low priority) ; Users can override via CELERY_NICE_LEVEL environment variable in docker-compose attach-daemon = nice -n $(CELERY_NICE_LEVEL) celery -A dispatcharr worker --autoscale=6,1 @@ -59,4 +59,4 @@ logformat-strftime = true log-date = %%Y-%%m-%%d %%H:%%M:%%S,000 # Use formatted time with environment variable for log level log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms -log-buffering = 1024 # Add buffer size limit for logging \ No newline at end of file +log-buffering = 1024 # Add buffer size limit for logging diff --git a/frontend/prettier.config.js b/frontend/prettier.config.js index d2b86878..e143dfef 100644 --- a/frontend/prettier.config.js +++ b/frontend/prettier.config.js @@ -3,8 +3,8 @@ export default { semi: true, // Add semicolons at the end of statements singleQuote: true, // Use single quotes instead of double tabWidth: 2, // Set the indentation width - trailingComma: "es5", // Add trailing commas where valid in ES5 + trailingComma: 'es5', // Add trailing commas where valid in ES5 printWidth: 80, // Wrap lines at 80 characters bracketSpacing: true, // Add spaces inside object braces - arrowParens: "always", // Always include parentheses around arrow function parameters + arrowParens: 'always', // Always include parentheses around arrow function parameters }; diff --git a/frontend/src/components/forms/User.jsx b/frontend/src/components/forms/User.jsx index 619b156f..6ec8705e 100644 --- a/frontend/src/components/forms/User.jsx +++ b/frontend/src/components/forms/User.jsx @@ -12,8 +12,9 @@ import { Stack, MultiSelect, ActionIcon, + NumberInput, } from '@mantine/core'; -import { RotateCcwKey, X } from 'lucide-react'; +import { RotateCcwKey } from 'lucide-react'; import { useForm } from '@mantine/form'; import useChannelsStore from '../../store/channels'; import { USER_LEVELS, USER_LEVEL_LABELS } from '../../constants'; @@ -209,26 +210,23 @@ const User = ({ user = null, isOpen, onClose }) => { key={form.key('last_name')} /> - - - - - } - /> - + + + + } + /> {showPermissions && ( { const authUser = useAuthStore((s) => s.user); - const [selectedUser, setSelectedUser] = useState(null); - const [userModalOpen, setUserModalOpen] = useState(false); - const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false); - const [deleteTarget, setDeleteTarget] = useState(null); - const [userToDelete, setUserToDelete] = useState(null); - if (!authUser.id) { return <>; } - const closeUserModal = () => { - setSelectedUser(null); - setUserModalOpen(false); - }; - const editUser = (user) => { - setSelectedUser(user); - setUserModalOpen(true); - }; - - const deleteUser = (id) => { - // Get user details for the confirmation dialog - const user = users.find((u) => u.id === id); - setUserToDelete(user); - setDeleteTarget(id); - - // Skip warning if it's been suppressed - if (isWarningSuppressed('delete-user')) { - return executeDeleteUser(id); - } - - setConfirmDeleteOpen(true); - }; - - const executeDeleteUser = async (id) => { - await API.deleteUser(id); - setConfirmDeleteOpen(false); - }; - return ( From 4796184a67e0eb32e5448c221346956c8b5ba884 Mon Sep 17 00:00:00 2001 From: Jeff Casimir Date: Sun, 1 Feb 2026 13:42:48 -0700 Subject: [PATCH 009/503] Fix uWSGI segfaults by disabling threads when using gevent Mixing threading and gevent concurrency models causes uWSGI workers to crash with segmentation faults, particularly on ARM64/Python 3.13. The production uwsgi.ini was already correct (gevent without threads). This fixes the dev and debug configs to match. References: - https://github.com/gevent/gevent/issues/1784 - https://github.com/unbit/uwsgi/issues/2457 Co-Authored-By: Claude Opus 4.5 --- docker/uwsgi.debug.ini | 2 -- docker/uwsgi.dev.ini | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 69c040f2..d2847335 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -28,8 +28,6 @@ static-map = /static=/app/static # Worker configuration workers = 1 -threads = 8 -enable-threads = true lazy-apps = true # HTTP server diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index e476e216..c4c5d0aa 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -28,10 +28,8 @@ vacuum = true die-on-term = true static-map = /static=/app/static -# Worker management (Optimize for I/O bound tasks) +# Worker management workers = 4 -threads = 2 -enable-threads = true # Optimize for streaming http = 0.0.0.0:5656 From a170663407b688507e17ace3e9d809d53cf6d626 Mon Sep 17 00:00:00 2001 From: None Date: Wed, 4 Feb 2026 21:09:35 -0600 Subject: [PATCH 010/503] feat: Add sorting by Group and EPG columns to Channels and Streams tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Backend: Add epg_data__name to ChannelViewSet ordering_fields - ChannelsTable: Add sort icons to EPG and Group column headers using rightSection prop - ChannelsTable: Add field mapping for channel_group → channel_group__name and epg → epg_data__name - StreamsTable: Add sort icon to Group column header for consistency Adds the ability to sort channels by Group and EPG columns, and streams by Group column. Closes #854 --- apps/channels/api_views.py | 2 +- .../src/components/tables/ChannelsTable.jsx | 85 +++++++++++++------ .../src/components/tables/StreamsTable.jsx | 9 ++ 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index ad5a270f..b1ea9a45 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -461,7 +461,7 @@ class ChannelViewSet(viewsets.ModelViewSet): filter_backends = [DjangoFilterBackend, SearchFilter, OrderingFilter] filterset_class = ChannelFilter search_fields = ["name", "channel_group__name"] - ordering_fields = ["channel_number", "name", "channel_group__name"] + ordering_fields = ["channel_number", "name", "channel_group__name", "epg_data__name"] ordering = ["-channel_number"] def create(self, request, *args, **kwargs): diff --git a/frontend/src/components/tables/ChannelsTable.jsx b/frontend/src/components/tables/ChannelsTable.jsx index 371cb77f..8cc33904 100644 --- a/frontend/src/components/tables/ChannelsTable.jsx +++ b/frontend/src/components/tables/ChannelsTable.jsx @@ -443,7 +443,15 @@ const ChannelsTable = ({ onReady }) => { // Apply sorting if (sorting.length > 0) { - const sortField = sorting[0].id; + let sortField = sorting[0].id; + // Map frontend column ids to backend ordering field names + const fieldMapping = { + channel_group: 'channel_group__name', + epg: 'epg_data__name', + }; + if (fieldMapping[sortField]) { + sortField = fieldMapping[sortField]; + } const sortDirection = sorting[0].desc ? '-' : ''; params.append('ordering', `${sortDirection}${sortField}`); } @@ -931,7 +939,7 @@ const ChannelsTable = ({ onReady }) => { /> ), size: columnSizing.epg || 200, - minSize: 80, + minSize: 120, }, { id: 'channel_group', @@ -942,8 +950,8 @@ const ChannelsTable = ({ onReady }) => { cell: (props) => ( ), - size: columnSizing.channel_group || 175, - minSize: 100, + size: columnSizing.channel_group || 200, + minSize: 120, }, { id: 'logo', @@ -1025,6 +1033,15 @@ const ChannelsTable = ({ onReady }) => { : [] } style={{ width: '100%' }} + rightSectionPointerEvents="auto" + rightSection={React.createElement(sortingIcon, { + onClick: (e) => { + e.stopPropagation(); + onSortingChange('epg'); + }, + size: 14, + style: { cursor: 'pointer' }, + })} /> ); case 'enabled': @@ -1036,11 +1053,16 @@ const ChannelsTable = ({ onReady }) => { case 'channel_number': return ( - + # -
+
{ + e.stopPropagation(); + onSortingChange('channel_number'); + }} + style={{ cursor: 'pointer' }} + > {React.createElement(sortingIcon, { - onClick: () => onSortingChange('channel_number'), size: 14, })}
@@ -1049,25 +1071,27 @@ const ChannelsTable = ({ onReady }) => { case 'name': return ( - - e.stopPropagation()} - onChange={handleFilterChange} - size="xs" - variant="unstyled" - className="table-input-header" - leftSection={} - /> -
- {React.createElement(sortingIcon, { - onClick: () => onSortingChange('name'), - size: 14, - })} -
- + e.stopPropagation()} + onChange={handleFilterChange} + size="xs" + variant="unstyled" + className="table-input-header" + leftSection={} + style={{ width: '100%' }} + rightSectionPointerEvents="auto" + rightSection={React.createElement(sortingIcon, { + onClick: (e) => { + e.stopPropagation(); + onSortingChange('name'); + }, + size: 14, + style: { cursor: 'pointer' }, + })} + /> ); case 'channel_group': @@ -1090,6 +1114,15 @@ const ChannelsTable = ({ onReady }) => { : [] } style={{ width: '100%' }} + rightSectionPointerEvents="auto" + rightSection={React.createElement(sortingIcon, { + onClick: (e) => { + e.stopPropagation(); + onSortingChange('channel_group'); + }, + size: 14, + style: { cursor: 'pointer' }, + })} /> ); } diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index d7c1b059..cd66f74f 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -1102,6 +1102,15 @@ const StreamsTable = ({ onReady }) => { className="table-input-header custom-multiselect" clearable style={{ width: '100%' }} + rightSectionPointerEvents="auto" + rightSection={React.createElement(sortingIcon, { + onClick: (e) => { + e.stopPropagation(); + onSortingChange('group'); + }, + size: 14, + style: { cursor: 'pointer' }, + })} /> ); } From 8de23eec35057e4c0620b7fa3cb6149126df47dc Mon Sep 17 00:00:00 2001 From: None Date: Thu, 5 Feb 2026 20:55:52 -0600 Subject: [PATCH 011/503] Fix XC URL sub-path handling in profile refresh and EPG creation - Fix credential extraction in get_transformed_credentials() using negative indices anchored to the known tail structure instead of hardcoded indices that break when server URLs contain sub-paths - Fix EPG URL construction in M3U form to normalize server URL to origin before appending xmltv.php endpoint XC accounts with sub-paths in their server URL (e.g., http://server.com/portal/a/) caused two failures: profile refresh extracted wrong credentials from the URL path due to hardcoded array indices, and EPG auto-creation appended xmltv.php under the sub-path instead of at the domain root. Both fixes normalize away the sub-path using the same approach the backend's XCClient already uses for API calls. --- apps/m3u/tasks.py | 10 ++++++---- frontend/src/components/forms/M3U.jsx | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index f929a208..0b035702 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -2305,10 +2305,12 @@ def get_transformed_credentials(account, profile=None): parsed_url = urllib.parse.urlparse(transformed_complete_url) path_parts = [part for part in parsed_url.path.split('/') if part] - if len(path_parts) >= 2: - # Extract username and password from path - transformed_username = path_parts[1] - transformed_password = path_parts[2] + if len(path_parts) >= 4 and path_parts[-1] == '1234.ts': + # Extract username and password from the known structure: + # .../{live}/{username}/{password}/1234.ts + # Using negative indices so sub-paths in the server URL don't shift extraction + transformed_username = path_parts[-3] + transformed_password = path_parts[-2] # Rebuild server URL without the username/password path transformed_url = f"{parsed_url.scheme}://{parsed_url.netloc}" diff --git a/frontend/src/components/forms/M3U.jsx b/frontend/src/components/forms/M3U.jsx index a13f4fef..c92ce823 100644 --- a/frontend/src/components/forms/M3U.jsx +++ b/frontend/src/components/forms/M3U.jsx @@ -148,7 +148,7 @@ const M3U = ({ API.addEPG({ name: values.name, source_type: 'xmltv', - url: `${values.server_url}/xmltv.php?username=${values.username}&password=${values.password}`, + url: `${new URL(values.server_url).origin}/xmltv.php?username=${values.username}&password=${values.password}`, api_key: '', is_active: true, refresh_interval: 24, From 49b7b9e21b0dd896764d9d12abb61983bd7896e9 Mon Sep 17 00:00:00 2001 From: None Date: Thu, 5 Feb 2026 23:12:30 -0600 Subject: [PATCH 012/503] Fix: Connection capacity leak - failed stream initialization permanently consumes connection slot Fixed four leak paths in the TS proxy streaming initialization: 1. url_utils.py - generate_stream_url(): Wrapped post-get_stream() code in try/except that calls release_stream() on any failure 2. views.py - stream_ts() retry loop: Added release_stream() before returning 503 to clean up the error-checking get_stream() call at line 181 which could INCR without a matching release when the retry loop times out 3. views.py - stream_ts() init failure: Added guarded release_stream() when initialize_channel() returns False, using a connection_allocated flag to prevent incorrect decrements when joining an existing channel 4. views.py - stream_ts() outer except: Added guarded release_stream() as a safety net for unexpected exceptions, only releasing when the flag confirms the slot was allocated and the channel hasn't been initialized yet The connection_allocated flag prevents double-decrements by tracking whether request flow actually INCR'd the counter (fresh initialization) vs returned cached results (joining an existing channel on another worker) --- apps/proxy/ts_proxy/url_utils.py | 60 +++++++++++++++++--------------- apps/proxy/ts_proxy/views.py | 23 ++++++++++-- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/apps/proxy/ts_proxy/url_utils.py b/apps/proxy/ts_proxy/url_utils.py index 8b467b7f..9daaf4ef 100644 --- a/apps/proxy/ts_proxy/url_utils.py +++ b/apps/proxy/ts_proxy/url_utils.py @@ -129,39 +129,41 @@ def generate_stream_url(channel_id: str) -> Tuple[str, str, bool, Optional[int]] logger.error(f"No stream available for channel {channel_id}: {error_reason}") return None, None, False, None - # Look up the Stream and Profile objects + # get_stream() allocated a connection slot - ensure it's released on any error try: + # Look up the Stream and Profile objects stream = Stream.objects.get(id=stream_id) profile = M3UAccountProfile.objects.get(id=profile_id) - except (Stream.DoesNotExist, M3UAccountProfile.DoesNotExist) as e: - logger.error(f"Error getting stream or profile: {e}") + + # Get the M3U account profile for URL pattern + m3u_profile = profile + + # Get the appropriate user agent + m3u_account = M3UAccount.objects.get(id=m3u_profile.m3u_account.id) + stream_user_agent = m3u_account.get_user_agent().user_agent + + if stream_user_agent is None: + stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) + logger.debug(f"No user agent found for account, using default: {stream_user_agent}") + + # Generate stream URL based on the selected profile + input_url = stream.url + stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern) + + # Check if transcoding is needed + stream_profile = channel.get_stream_profile() + if stream_profile.is_proxy() or stream_profile is None: + transcode = False + else: + transcode = True + + stream_profile_id = stream_profile.id + + return stream_url, stream_user_agent, transcode, stream_profile_id + except Exception as e: + logger.error(f"Error generating stream URL for channel {channel_id}: {e}") + channel.release_stream() return None, None, False, None - - # Get the M3U account profile for URL pattern - m3u_profile = profile - - # Get the appropriate user agent - m3u_account = M3UAccount.objects.get(id=m3u_profile.m3u_account.id) - stream_user_agent = m3u_account.get_user_agent().user_agent - - if stream_user_agent is None: - stream_user_agent = UserAgent.objects.get(id=CoreSettings.get_default_user_agent_id()) - logger.debug(f"No user agent found for account, using default: {stream_user_agent}") - - # Generate stream URL based on the selected profile - input_url = stream.url - stream_url = transform_url(input_url, m3u_profile.search_pattern, m3u_profile.replace_pattern) - - # Check if transcoding is needed - stream_profile = channel.get_stream_profile() - if stream_profile.is_proxy() or stream_profile is None: - transcode = False - else: - transcode = True - - stream_profile_id = stream_profile.id - - return stream_url, stream_user_agent, transcode, stream_profile_id except Exception as e: logger.error(f"Error generating stream URL: {e}") return None, None, False, None diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index 91f254a7..6980f957 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -54,6 +54,7 @@ def stream_ts(request, channel_id): client_user_agent = None proxy_server = ProxyServer.get_instance() + connection_allocated = False # Track if connection slot was allocated via get_stream() try: # Generate a unique client ID @@ -219,9 +220,9 @@ def stream_ts(request, channel_id): ) if stream_url is None: - # Release the channel's stream lock if one was acquired - # Note: Only call this if get_stream() actually assigned a stream - # In our case, if stream_url is None, no stream was ever assigned, so don't release + # Release any connection slot that may have been allocated + # by the error-checking get_stream() call during retries + channel.release_stream() # Get the specific error message if available wait_duration = f"{int(time.time() - wait_start_time)}s" @@ -237,6 +238,11 @@ def stream_ts(request, channel_id): {"error": error_msg, "waited": wait_duration}, status=503 ) # 503 Service Unavailable is appropriate here + # generate_stream_url() called get_stream() which allocated a connection + # slot (INCR'd profile_connections) - track this for cleanup on error + if needs_initialization: + connection_allocated = True + # Get the stream ID from the channel stream_id, m3u_profile_id, _ = channel.get_stream() logger.info( @@ -344,10 +350,16 @@ def stream_ts(request, channel_id): ) if not success: + if connection_allocated: + channel.release_stream() + connection_allocated = False return JsonResponse( {"error": "Failed to initialize channel"}, status=500 ) + # Channel initialized - cleanup lifecycle now owns the connection release + connection_allocated = False + # If we're the owner, wait for connection to establish if proxy_server.am_i_owner(channel_id): manager = proxy_server.stream_managers.get(channel_id) @@ -507,6 +519,11 @@ def stream_ts(request, channel_id): except Exception as e: logger.error(f"Error in stream_ts: {e}", exc_info=True) + if connection_allocated: + try: + channel.release_stream() + except Exception: + pass return JsonResponse({"error": str(e)}, status=500) From 24f812dc4d39b7881c72a77da32967f6a5ffb662 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 8 Feb 2026 09:29:22 -0500 Subject: [PATCH 013/503] initial connect feature --- apps/accounts/api_views.py | 13 + apps/api/urls.py | 1 + apps/connect/__init__.py | 0 apps/connect/api_urls.py | 17 + apps/connect/api_views.py | 133 +++++++ apps/connect/apps.py | 8 + apps/connect/handlers/__init__.py | 0 apps/connect/handlers/api.py | 0 apps/connect/handlers/base.py | 12 + apps/connect/handlers/script.py | 28 ++ apps/connect/handlers/webhook.py | 10 + apps/connect/migrations/0001_initial.py | 52 +++ apps/connect/migrations/__init__.py | 0 apps/connect/models.py | 36 ++ apps/connect/serializers.py | 46 +++ apps/connect/utils.py | 73 ++++ core/utils.py | 74 +++- dispatcharr/settings.py | 1 + frontend/src/App.jsx | 7 + frontend/src/api.js | 367 ++++++++++++++----- frontend/src/components/Sidebar.jsx | 164 +++++++-- frontend/src/components/forms/Connection.jsx | 189 ++++++++++ frontend/src/components/forms/Recording.jsx | 80 +++- frontend/src/components/sidebar.css | 20 +- frontend/src/constants.js | 22 +- frontend/src/pages/Connect.jsx | 204 +++++++++++ frontend/src/pages/ConnectLogs.jsx | 299 +++++++++++++++ frontend/src/store/connect.jsx | 46 +++ 28 files changed, 1761 insertions(+), 141 deletions(-) create mode 100644 apps/connect/__init__.py create mode 100644 apps/connect/api_urls.py create mode 100644 apps/connect/api_views.py create mode 100644 apps/connect/apps.py create mode 100644 apps/connect/handlers/__init__.py create mode 100644 apps/connect/handlers/api.py create mode 100644 apps/connect/handlers/base.py create mode 100644 apps/connect/handlers/script.py create mode 100644 apps/connect/handlers/webhook.py create mode 100644 apps/connect/migrations/0001_initial.py create mode 100644 apps/connect/migrations/__init__.py create mode 100644 apps/connect/models.py create mode 100644 apps/connect/serializers.py create mode 100644 apps/connect/utils.py create mode 100644 frontend/src/components/forms/Connection.jsx create mode 100644 frontend/src/pages/Connect.jsx create mode 100644 frontend/src/pages/ConnectLogs.jsx create mode 100644 frontend/src/store/connect.jsx diff --git a/apps/accounts/api_views.py b/apps/accounts/api_views.py index 607ce199..188cd8cb 100644 --- a/apps/accounts/api_views.py +++ b/apps/accounts/api_views.py @@ -1,4 +1,5 @@ from django.contrib.auth import authenticate, login, logout +import logging from django.contrib.auth.models import Group, Permission from django.http import JsonResponse, HttpResponse from django.views.decorators.csrf import csrf_exempt @@ -15,6 +16,8 @@ from .models import User from .serializers import UserSerializer, GroupSerializer, PermissionSerializer from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView +logger = logging.getLogger(__name__) + class TokenObtainPairView(TokenObtainPairView): def post(self, request, *args, **kwargs): @@ -25,6 +28,7 @@ class TokenObtainPairView(TokenObtainPairView): username = request.data.get("username", 'unknown') client_ip = request.META.get('REMOTE_ADDR', 'unknown') user_agent = request.META.get('HTTP_USER_AGENT', 'unknown') + logger.info(f"Login blocked by network policy: user={username} ip={client_ip} ua={user_agent}") log_system_event( event_type='login_failed', user=username, @@ -43,6 +47,7 @@ class TokenObtainPairView(TokenObtainPairView): user_agent = request.META.get('HTTP_USER_AGENT', 'unknown') try: + logger.debug(f"Attempting JWT login for user={username}") response = super().post(request, *args, **kwargs) # If login was successful, update last_login and log success @@ -61,6 +66,7 @@ class TokenObtainPairView(TokenObtainPairView): client_ip=client_ip, user_agent=user_agent, ) + logger.info(f"Login success: user={username} ip={client_ip}") except User.DoesNotExist: pass # User doesn't exist, but login somehow succeeded else: @@ -72,6 +78,7 @@ class TokenObtainPairView(TokenObtainPairView): user_agent=user_agent, reason='Invalid credentials', ) + logger.info(f"Login failed: user={username} ip={client_ip}") return response @@ -84,6 +91,7 @@ class TokenObtainPairView(TokenObtainPairView): user_agent=user_agent, reason=f'Authentication error: {str(e)[:100]}', ) + logger.error(f"Login error for user={username}: {e}") raise # Re-raise the exception to maintain normal error flow @@ -95,6 +103,7 @@ class TokenRefreshView(TokenRefreshView): from core.utils import log_system_event client_ip = request.META.get('REMOTE_ADDR', 'unknown') user_agent = request.META.get('HTTP_USER_AGENT', 'unknown') + logger.info(f"Token refresh blocked by network policy: ip={client_ip} ua={user_agent}") log_system_event( event_type='login_failed', user='token_refresh', @@ -167,6 +176,7 @@ class AuthViewSet(viewsets.ViewSet): from core.utils import log_system_event client_ip = request.META.get('REMOTE_ADDR', 'unknown') user_agent = request.META.get('HTTP_USER_AGENT', 'unknown') + logger.debug(f"Login attempt via session: user={username} ip={client_ip}") if user: login(request, user) @@ -182,6 +192,7 @@ class AuthViewSet(viewsets.ViewSet): client_ip=client_ip, user_agent=user_agent, ) + logger.info(f"Login success via session: user={username} ip={client_ip}") return Response( { @@ -203,6 +214,7 @@ class AuthViewSet(viewsets.ViewSet): user_agent=user_agent, reason='Invalid credentials', ) + logger.info(f"Login failed via session: user={username} ip={client_ip}") return Response({"error": "Invalid credentials"}, status=400) @extend_schema( @@ -222,6 +234,7 @@ class AuthViewSet(viewsets.ViewSet): client_ip=client_ip, user_agent=user_agent, ) + logger.info(f"Logout: user={username} ip={client_ip}") logout(request) return Response({"message": "Logout successful"}) diff --git a/apps/api/urls.py b/apps/api/urls.py index 5a688778..b176bc1b 100644 --- a/apps/api/urls.py +++ b/apps/api/urls.py @@ -13,6 +13,7 @@ urlpatterns = [ path('plugins/', include(('apps.plugins.api_urls', 'plugins'), namespace='plugins')), path('vod/', include(('apps.vod.api_urls', 'vod'), namespace='vod')), path('backups/', include(('apps.backups.api_urls', 'backups'), namespace='backups')), + path('connect/', include(('apps.connect.api_urls', 'connect'), namespace='connect')), # path('output/', include(('apps.output.api_urls', 'output'), namespace='output')), #path('player/', include(('apps.player.api_urls', 'player'), namespace='player')), #path('settings/', include(('apps.settings.api_urls', 'settings'), namespace='settings')), diff --git a/apps/connect/__init__.py b/apps/connect/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/connect/api_urls.py b/apps/connect/api_urls.py new file mode 100644 index 00000000..664c437b --- /dev/null +++ b/apps/connect/api_urls.py @@ -0,0 +1,17 @@ +from django.urls import path +from rest_framework.routers import DefaultRouter +from .api_views import ( + IntegrationViewSet, + EventSubscriptionViewSet, + DeliveryLogViewSet, +) + +app_name = 'connect' + +router = DefaultRouter() +router.register(r'integrations', IntegrationViewSet, basename='integration') +router.register(r'subscriptions', EventSubscriptionViewSet, basename='subscription') +router.register(r'logs', DeliveryLogViewSet, basename='delivery-log') + +urlpatterns = [] +urlpatterns += router.urls diff --git a/apps/connect/api_views.py b/apps/connect/api_views.py new file mode 100644 index 00000000..1402de04 --- /dev/null +++ b/apps/connect/api_views.py @@ -0,0 +1,133 @@ +from rest_framework import viewsets, status +from rest_framework.pagination import PageNumberPagination +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.response import Response +from rest_framework.decorators import action +from .models import Integration, EventSubscription, DeliveryLog +from .serializers import ( + IntegrationSerializer, + EventSubscriptionSerializer, + DeliveryLogSerializer, +) +from apps.accounts.permissions import ( + Authenticated, + permission_classes_by_action, +) + + +class IntegrationViewSet(viewsets.ModelViewSet): + queryset = Integration.objects.all() + serializer_class = IntegrationSerializer + + def get_permissions(self): + try: + return [perm() for perm in permission_classes_by_action[self.action]] + except KeyError: + return [Authenticated()] + + @action(detail=True, methods=["get"], url_path="subscriptions") + def list_subscriptions(self, request, pk=None): + qs = EventSubscription.objects.filter(integration_id=pk) + serializer = EventSubscriptionSerializer(qs, many=True) + return Response(serializer.data) + + @action(detail=True, methods=["put"], url_path=r"subscriptions/set") + def set_subscriptions(self, request, pk=None): + """ + Replace the integration's subscriptions with the provided list. + Body format: [{"event": "channel_start", "enabled": true, "payload_template": "..."}, ...] + Any existing subscriptions not in the list will be deleted; missing ones will be created/updated. + """ + try: + integration = Integration.objects.get(pk=pk) + except Integration.DoesNotExist: + return Response( + {"detail": "Integration not found"}, status=status.HTTP_404_NOT_FOUND + ) + + data = request.data + if not isinstance(data, list): + return Response( + {"detail": "Expected a list of subscriptions"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Validate incoming items using serializer (without integration field) + # We'll attach the integration explicitly + valid_events = set(evt for evt, _ in EventSubscription.EVENT_CHOICES) + incoming = [] + for item in data: + if not isinstance(item, dict): + return Response( + {"detail": "Each subscription must be an object"}, + status=status.HTTP_400_BAD_REQUEST, + ) + event = item.get("event") + if event not in valid_events: + return Response( + {"detail": f"Invalid event: {event}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + incoming.append( + { + "event": event, + "enabled": bool(item.get("enabled", True)), + "payload_template": item.get("payload_template"), + } + ) + + incoming_events = {s["event"] for s in incoming} + + # Delete subscriptions that are no longer present + EventSubscription.objects.filter(integration=integration).exclude( + event__in=incoming_events + ).delete() + + # Upsert incoming subscriptions + updated = [] + for sub in incoming: + obj, _created = EventSubscription.objects.update_or_create( + integration=integration, + event=sub["event"], + defaults={ + "enabled": sub["enabled"], + "payload_template": sub.get("payload_template"), + }, + ) + updated.append(obj) + + serializer = EventSubscriptionSerializer(updated, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + +class EventSubscriptionViewSet(viewsets.ModelViewSet): + queryset = EventSubscription.objects.all() + serializer_class = EventSubscriptionSerializer + + +class DeliveryLogViewSet(viewsets.ReadOnlyModelViewSet): + queryset = DeliveryLog.objects.all().order_by("-created_at") + serializer_class = DeliveryLogSerializer + filter_backends = [DjangoFilterBackend] + + # Support server-side pagination with page_size query param + class ConnectLogsPagination(PageNumberPagination): + page_size = 50 + page_size_query_param = "page_size" + max_page_size = 250 + + pagination_class = ConnectLogsPagination + + def get_queryset(self): + qs = super().get_queryset() + + # Optional filters: integration id and type + integration_id = self.request.query_params.get("integration") + if integration_id: + qs = qs.filter(subscription__integration_id=integration_id) + + integration_type = self.request.query_params.get("type") + if integration_type: + qs = qs.filter(subscription__integration__type=integration_type) + + return qs diff --git a/apps/connect/apps.py b/apps/connect/apps.py new file mode 100644 index 00000000..db063fa4 --- /dev/null +++ b/apps/connect/apps.py @@ -0,0 +1,8 @@ +from django.apps import AppConfig + + +class ConnectConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.connect' + verbose_name = "Connect Integrations" + label = 'dispatcharr_connect' diff --git a/apps/connect/handlers/__init__.py b/apps/connect/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/connect/handlers/api.py b/apps/connect/handlers/api.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/connect/handlers/base.py b/apps/connect/handlers/base.py new file mode 100644 index 00000000..d61b7c53 --- /dev/null +++ b/apps/connect/handlers/base.py @@ -0,0 +1,12 @@ +# connect/handlers/base.py +import abc + +class IntegrationHandler(abc.ABC): + def __init__(self, integration, subscription, payload): + self.integration = integration + self.subscription = subscription + self.payload = payload + + @abc.abstractmethod + def execute(self): + pass diff --git a/apps/connect/handlers/script.py b/apps/connect/handlers/script.py new file mode 100644 index 00000000..3fa8215b --- /dev/null +++ b/apps/connect/handlers/script.py @@ -0,0 +1,28 @@ +# connect/handlers/script.py +import os +import subprocess +from .base import IntegrationHandler + +class ScriptHandler(IntegrationHandler): + def execute(self): + script_path = self.integration.config.get("path") + + # Build environment variables from payload (prefixed for clarity) + env = os.environ.copy() + for key, value in (self.payload or {}).items(): + # Convert keys to upper snake case and prefix + env_key = f"DISPATCHARR_{str(key).upper()}" + env[env_key] = str(value) if value is not None else "" + + result = subprocess.run( + [script_path], + capture_output=True, + text=True, + env=env, + ) + return { + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "success": result.returncode == 0, + } diff --git a/apps/connect/handlers/webhook.py b/apps/connect/handlers/webhook.py new file mode 100644 index 00000000..ccca530a --- /dev/null +++ b/apps/connect/handlers/webhook.py @@ -0,0 +1,10 @@ +# connect/handlers/webhook.py +import requests +from .base import IntegrationHandler + +class WebhookHandler(IntegrationHandler): + def execute(self): + url = self.integration.config.get("url") + headers = self.integration.config.get("headers", {}) + response = requests.post(url, json=self.payload, headers=headers, timeout=10) + return {"status_code": response.status_code, "body": response.text, "success": response.ok} diff --git a/apps/connect/migrations/0001_initial.py b/apps/connect/migrations/0001_initial.py new file mode 100644 index 00000000..cd8f6b42 --- /dev/null +++ b/apps/connect/migrations/0001_initial.py @@ -0,0 +1,52 @@ +# Generated by Django 5.2.9 on 2026-01-27 21:05 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='EventSubscription', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('event', models.CharField(choices=[('channel_start', 'Channel Started'), ('channel_stop', 'Channel Stopped'), ('movie_added', 'Movie Added'), ('series_added', 'Series Added'), ('download_complete', 'Download Complete')], max_length=100)), + ('enabled', models.BooleanField(default=True)), + ('payload_template', models.TextField(blank=True, help_text='Optional Jinja2/Django template for customizing payload', null=True)), + ], + ), + migrations.CreateModel( + name='Integration', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('type', models.CharField(choices=[('webhook', 'Webhook'), ('api', 'API'), ('script', 'Custom Script')], max_length=50)), + ('config', models.JSONField(default=dict)), + ('enabled', models.BooleanField(default=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='DeliveryLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('success', 'Success'), ('failed', 'Failed')], max_length=50)), + ('request_payload', models.JSONField(blank=True, default=dict)), + ('response_payload', models.JSONField(blank=True, default=dict)), + ('error_message', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('subscription', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='dispatcharr_connect.eventsubscription')), + ], + ), + migrations.AddField( + model_name='eventsubscription', + name='integration', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subscriptions', to='dispatcharr_connect.integration'), + ), + ] diff --git a/apps/connect/migrations/__init__.py b/apps/connect/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/connect/models.py b/apps/connect/models.py new file mode 100644 index 00000000..99e89ac8 --- /dev/null +++ b/apps/connect/models.py @@ -0,0 +1,36 @@ +from django.db import models + + +class Integration(models.Model): + TYPE_CHOICES = [ + ("webhook", "Webhook"), + ("api", "API"), + ("script", "Custom Script"), + ] + name = models.CharField(max_length=255) + type = models.CharField(max_length=50, choices=TYPE_CHOICES) + config = models.JSONField(default=dict) + enabled = models.BooleanField(default=True) + created_at = models.DateTimeField(auto_now_add=True) + + +class EventSubscription(models.Model): + EVENT_CHOICES = [ + ("channel_start", "Channel Started"), + ("channel_stop", "Channel Stopped"), + ("movie_added", "Movie Added"), + ("series_added", "Series Added"), + ("download_complete", "Download Complete"), + ] + event = models.CharField(max_length=100, choices=EVENT_CHOICES) + integration = models.ForeignKey(Integration, on_delete=models.CASCADE, related_name="subscriptions") + enabled = models.BooleanField(default=True) + payload_template = models.TextField(blank=True, null=True, help_text="Optional Jinja2/Django template for customizing payload") + +class DeliveryLog(models.Model): + subscription = models.ForeignKey(EventSubscription, on_delete=models.CASCADE, related_name="logs") + status = models.CharField(max_length=50, choices=[("success", "Success"), ("failed", "Failed")]) + request_payload = models.JSONField(default=dict, blank=True) + response_payload = models.JSONField(default=dict, blank=True) + error_message = models.TextField(blank=True, null=True) + created_at = models.DateTimeField(auto_now_add=True) diff --git a/apps/connect/serializers.py b/apps/connect/serializers.py new file mode 100644 index 00000000..859295f3 --- /dev/null +++ b/apps/connect/serializers.py @@ -0,0 +1,46 @@ +from rest_framework import serializers +from .models import Integration, EventSubscription, DeliveryLog + + +class EventSubscriptionSerializer(serializers.ModelSerializer): + class Meta: + model = EventSubscription + fields = [ + "id", + "event", + "enabled", + "payload_template", + "integration", + ] + + +class IntegrationSerializer(serializers.ModelSerializer): + subscriptions = EventSubscriptionSerializer(many=True, read_only=True) + + class Meta: + model = Integration + fields = [ + "id", + "name", + "type", + "config", + "enabled", + "created_at", + "subscriptions", + ] + + +class DeliveryLogSerializer(serializers.ModelSerializer): + subscription = EventSubscriptionSerializer(read_only=True) + + class Meta: + model = DeliveryLog + fields = [ + "id", + "subscription", + "status", + "request_payload", + "response_payload", + "error_message", + "created_at", + ] diff --git a/apps/connect/utils.py b/apps/connect/utils.py new file mode 100644 index 00000000..dcd44132 --- /dev/null +++ b/apps/connect/utils.py @@ -0,0 +1,73 @@ +# connect/utils.py +import logging +from django.template import Template, Context +from .models import EventSubscription, DeliveryLog +from .handlers.webhook import WebhookHandler +from .handlers.script import ScriptHandler + +logger = logging.getLogger(__name__) + +HANDLERS = { + "webhook": WebhookHandler, + "script": ScriptHandler, +} + + +def trigger_event(event_name, payload): + logger.debug(f"Triggering connect event: {event_name} payload_keys={list((payload or {}).keys())}") + subscriptions = ( + EventSubscription.objects.filter(event=event_name, enabled=True) + .select_related("integration") + ) + + count = subscriptions.count() + logger.info(f"Found {count} connect subscription(s) for event '{event_name}'") + + for sub in subscriptions: + integration = sub.integration + if not integration.enabled: + logger.debug(f"Skipping disabled integration id={integration.id} name={integration.name}") + continue + + # apply optional payload template + final_payload = payload + if sub.payload_template: + try: + template = Template(sub.payload_template) + rendered = template.render(Context(payload)) + final_payload = {"message": rendered} + except Exception as e: + logger.error(f"Payload template render failed for subscription id={sub.id}: {e}") + final_payload = payload + + handler_cls = HANDLERS.get(integration.type) + if not handler_cls: + DeliveryLog.objects.create( + subscription=sub, + status="failed", + request_payload=final_payload, + error_message=f"No handler for integration type '{integration.type}'", + ) + logger.error(f"No handler for integration type '{integration.type}' (integration id={integration.id})") + continue + + handler = handler_cls(integration, sub, final_payload) + logger.debug(f"Executing handler type={integration.type} integration_id={integration.id} subscription_id={sub.id}") + + try: + result = handler.execute() + DeliveryLog.objects.create( + subscription=sub, + status="success" if result.get("success") else "failed", + request_payload=final_payload, + response_payload=result, + ) + logger.info(f"Connect delivery succeeded for subscription id={sub.id} integration '{integration.name}'") + except Exception as e: + DeliveryLog.objects.create( + subscription=sub, + status="failed", + request_payload=final_payload, + error_message=str(e), + ) + logger.error(f"Connect delivery failed for subscription id={sub.id} integration '{integration.name}': {e}") diff --git a/core/utils.py b/core/utils.py index 5c5a0045..cdd9f0bd 100644 --- a/core/utils.py +++ b/core/utils.py @@ -415,6 +415,79 @@ def log_system_event(event_type, channel_id=None, channel_name=None, **details): details=details ) + # Trigger connect integrations for specific events + if event_type in ("channel_start", "channel_stop"): + try: + from apps.connect.utils import trigger_event + from apps.channels.models import Channel, Stream + from core.models import StreamProfile + from core.utils import RedisClient + + payload = {} + + channel_obj = None + if channel_id: + try: + channel_obj = Channel.objects.get(uuid=channel_id) + payload["channel_name"] = channel_obj.name + except Exception: + payload["channel_name"] = channel_name or None + else: + payload["channel_name"] = channel_name or None + + # Resolve current stream info + stream_id = details.get("stream_id") + stream_obj = None + if not stream_id and channel_obj: + try: + redis = RedisClient.get_client() + sid = redis.get(f"channel_stream:{channel_obj.id}") + if sid: + stream_id = int(sid) + except Exception: + stream_id = None + + if stream_id: + try: + stream_obj = Stream.objects.get(id=stream_id) + except Exception: + stream_obj = None + + # Populate stream details + payload["stream_name"] = getattr(stream_obj, "name", None) + payload["stream_url"] = getattr(stream_obj, "url", None) + + # Channel URL: use stream URL as best-effort + payload["channel_url"] = payload.get("stream_url") + + # Provider name from M3U account + provider_name = None + try: + if stream_obj and stream_obj.m3u_account: + provider_name = stream_obj.m3u_account.name + except Exception: + provider_name = None + payload["provider_name"] = provider_name + + # Profile used + profile_used = None + try: + if stream_id: + redis = RedisClient.get_client() + pid = redis.get(f"stream_profile:{stream_id}") + if pid: + profile = StreamProfile.objects.filter(id=int(pid)).first() + profile_used = profile.name if profile else None + except Exception: + profile_used = None + + payload["profile_used"] = profile_used + + trigger_event(event_type, payload) + except Exception as e: + # Don't fail main path if connect dispatch fails + pass + # Get max events from settings (default 100) try: from .models import CoreSettings @@ -509,4 +582,3 @@ def send_notification_dismissed(notification_key): ) except Exception as e: logger.error(f"Failed to send notification dismissed event: {e}") - diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index cfc499da..7afa75c5 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -31,6 +31,7 @@ INSTALLED_APPS = [ "apps.proxy.apps.ProxyConfig", "apps.proxy.ts_proxy", "apps.vod.apps.VODConfig", + "apps.connect.apps.ConnectConfig", "core", "daphne", "drf_spectacular", diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 3869740e..86e8a7d7 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -14,6 +14,8 @@ import Stats from './pages/Stats'; import DVR from './pages/DVR'; import Settings from './pages/Settings'; import PluginsPage from './pages/Plugins'; +import ConnectPage from './pages/Connect'; +import ConnectLogsPage from './pages/ConnectLogs'; import Users from './pages/Users'; import LogosPage from './pages/Logos'; import VODsPage from './pages/VODs'; @@ -152,6 +154,11 @@ const App = () => { } /> } /> } /> + } /> + } + /> } /> } /> } /> diff --git a/frontend/src/api.js b/frontend/src/api.js index 9cc5494c..fe8bf275 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -12,6 +12,7 @@ import { notifications } from '@mantine/notifications'; import useChannelsTableStore from './store/channelsTable'; import useStreamsTableStore from './store/streamsTable'; import useUsersStore from './store/users'; +import useConnectStore from './store/connect'; // If needed, you can set a base host or keep it empty if relative requests const host = import.meta.env.DEV @@ -171,7 +172,7 @@ export default class API { static async logout() { return await request(`${host}/api/accounts/auth/logout/`, { - auth: true, // Send JWT token so backend can identify the user + auth: true, // Send JWT token so backend can identify the user method: 'POST', }); } @@ -261,15 +262,11 @@ export default class API { API.lastQueryParams = newParams; const [response, ids] = await Promise.all([ - request( - `${host}/api/channels/channels/?${newParams.toString()}` - ), + request(`${host}/api/channels/channels/?${newParams.toString()}`), API.getAllChannelIds(newParams), ]); - useChannelsTableStore - .getState() - .queryChannels(response, newParams); + useChannelsTableStore.getState().queryChannels(response, newParams); useChannelsTableStore.getState().setAllQueryIds(ids); return response; @@ -390,7 +387,8 @@ export default class API { channelData.channel_number === '' || channelData.channel_number === null || channelData.channel_number === undefined || - (typeof channelData.channel_number === 'string' && channelData.channel_number.trim() === '') + (typeof channelData.channel_number === 'string' && + channelData.channel_number.trim() === '') ) { delete channelData.channel_number; } @@ -719,7 +717,11 @@ export default class API { } } - static async createChannelsFromStreamsAsync(streamIds, channelProfileIds = null, startingChannelNumber = null) { + static async createChannelsFromStreamsAsync( + streamIds, + channelProfileIds = null, + startingChannelNumber = null + ) { try { const requestBody = { stream_ids: streamIds, @@ -815,15 +817,11 @@ export default class API { try { const [response, ids] = await Promise.all([ - request( - `${host}/api/channels/streams/?${params.toString()}` - ), + request(`${host}/api/channels/streams/?${params.toString()}`), API.getAllStreamIds(params), ]); - useStreamsTableStore - .getState() - .queryStreams(response, params); + useStreamsTableStore.getState().queryStreams(response, params); useStreamsTableStore.getState().setAllQueryIds(ids); return response; @@ -1179,13 +1177,10 @@ export default class API { static async getCurrentPrograms(channelIds = null) { try { - const response = await request( - `${host}/api/epg/current-programs/`, - { - method: 'POST', - body: { channel_ids: channelIds }, - } - ); + const response = await request(`${host}/api/epg/current-programs/`, { + method: 'POST', + body: { channel_ids: channelIds }, + }); return response; } catch (e) { @@ -1318,9 +1313,15 @@ export default class API { errorNotification('Failed to retrieve timezones', e); // Return fallback data instead of throwing return { - timezones: ['UTC', 'US/Eastern', 'US/Central', 'US/Mountain', 'US/Pacific'], + timezones: [ + 'UTC', + 'US/Eastern', + 'US/Central', + 'US/Mountain', + 'US/Pacific', + ], grouped: {}, - count: 5 + count: 5, }; } } @@ -1444,16 +1445,22 @@ export default class API { static async refreshAccountInfo(profileId) { try { - const response = await request(`${host}/api/m3u/refresh-account-info/${profileId}/`, { - method: 'POST', - }); + const response = await request( + `${host}/api/m3u/refresh-account-info/${profileId}/`, + { + method: 'POST', + } + ); return response; } catch (e) { // If it's a structured error response, return it instead of throwing if (e.body && typeof e.body === 'object') { return e.body; } - errorNotification(`Failed to refresh account info for profile ${profileId}`, e); + errorNotification( + `Failed to refresh account info for profile ${profileId}`, + e + ); throw e; } } @@ -1580,7 +1587,11 @@ export default class API { }); // Wait for the task to complete using token for auth - const result = await API.waitForBackupTask(response.task_id, onProgress, response.task_token); + const result = await API.waitForBackupTask( + response.task_id, + onProgress, + response.task_token + ); return result; } catch (e) { errorNotification('Failed to create backup', e); @@ -1593,13 +1604,10 @@ export default class API { const formData = new FormData(); formData.append('file', file); - const response = await request( - `${host}/api/backups/upload/`, - { - method: 'POST', - body: formData, - } - ); + const response = await request(`${host}/api/backups/upload/`, { + method: 'POST', + body: formData, + }); return response; } catch (e) { errorNotification('Failed to upload backup', e); @@ -1622,7 +1630,9 @@ export default class API { static async getDownloadToken(filename) { // Get a download token from the server try { - const response = await request(`${host}/api/backups/${encodeURIComponent(filename)}/download-token/`); + const response = await request( + `${host}/api/backups/${encodeURIComponent(filename)}/download-token/` + ); return response.token; } catch (e) { throw e; @@ -1666,7 +1676,11 @@ export default class API { // Wait for the task to complete using token for auth // Token-based auth allows status polling even after DB restore invalidates user sessions - const result = await API.waitForBackupTask(response.task_id, onProgress, response.task_token); + const result = await API.waitForBackupTask( + response.task_id, + onProgress, + response.task_token + ); return result; } catch (e) { errorNotification('Failed to restore backup', e); @@ -1741,17 +1755,27 @@ export default class API { return response; } catch (e) { // Show only the concise error message for plugin import - const msg = (e?.body && (e.body.error || e.body.detail)) || e?.message || 'Failed to import plugin'; - notifications.show({ title: 'Import failed', message: msg, color: 'red' }); + const msg = + (e?.body && (e.body.error || e.body.detail)) || + e?.message || + 'Failed to import plugin'; + notifications.show({ + title: 'Import failed', + message: msg, + color: 'red', + }); throw e; } } static async deletePlugin(key) { try { - const response = await request(`${host}/api/plugins/plugins/${key}/delete/`, { - method: 'DELETE', - }); + const response = await request( + `${host}/api/plugins/plugins/${key}/delete/`, + { + method: 'DELETE', + } + ); return response; } catch (e) { errorNotification('Failed to delete plugin', e); @@ -1776,10 +1800,13 @@ export default class API { static async runPluginAction(key, action, params = {}) { try { - const response = await request(`${host}/api/plugins/plugins/${key}/run/`, { - method: 'POST', - body: { action, params }, - }); + const response = await request( + `${host}/api/plugins/plugins/${key}/run/`, + { + method: 'POST', + body: { action, params }, + } + ); return response; } catch (e) { errorNotification('Failed to run plugin action', e); @@ -1788,10 +1815,13 @@ export default class API { static async setPluginEnabled(key, enabled) { try { - const response = await request(`${host}/api/plugins/plugins/${key}/enabled/`, { - method: 'POST', - body: { enabled }, - }); + const response = await request( + `${host}/api/plugins/plugins/${key}/enabled/`, + { + method: 'POST', + body: { enabled }, + } + ); return response; } catch (e) { errorNotification('Failed to update plugin enabled state', e); @@ -1973,7 +2003,7 @@ export default class API { if (!logoIds || logoIds.length === 0) return []; const params = new URLSearchParams(); - logoIds.forEach(id => params.append('ids', id)); + logoIds.forEach((id) => params.append('ids', id)); // Disable pagination for ID-based queries to get all matching logos params.append('no_pagination', 'true'); @@ -2440,10 +2470,13 @@ export default class API { static async updateRecurringRule(ruleId, payload) { try { - const response = await request(`${host}/api/channels/recurring-rules/${ruleId}/`, { - method: 'PATCH', - body: payload, - }); + const response = await request( + `${host}/api/channels/recurring-rules/${ruleId}/`, + { + method: 'PATCH', + body: payload, + } + ); return response; } catch (e) { errorNotification(`Failed to update recurring rule ${ruleId}`, e); @@ -2462,9 +2495,13 @@ export default class API { static async deleteRecording(id) { try { - await request(`${host}/api/channels/recordings/${id}/`, { method: 'DELETE' }); + await request(`${host}/api/channels/recordings/${id}/`, { + method: 'DELETE', + }); // Optimistically remove locally for instant UI update - try { useChannelsStore.getState().removeRecording(id); } catch {} + try { + useChannelsStore.getState().removeRecording(id); + } catch {} } catch (e) { errorNotification(`Failed to delete recording ${id}`, e); } @@ -2472,9 +2509,12 @@ export default class API { static async runComskip(recordingId) { try { - const resp = await request(`${host}/api/channels/recordings/${recordingId}/comskip/`, { - method: 'POST', - }); + const resp = await request( + `${host}/api/channels/recordings/${recordingId}/comskip/`, + { + method: 'POST', + } + ); // Refresh recordings list to reflect comskip status when done later // This endpoint just queues the task; the websocket/refresh will update eventually return resp; @@ -2512,7 +2552,9 @@ export default class API { static async deleteSeriesRule(tvgId) { try { const encodedTvgId = encodeURIComponent(tvgId); - await request(`${host}/api/channels/series-rules/${encodedTvgId}/`, { method: 'DELETE' }); + await request(`${host}/api/channels/series-rules/${encodedTvgId}/`, { + method: 'DELETE', + }); notifications.show({ title: 'Series rule removed' }); } catch (e) { errorNotification('Failed to remove series rule', e); @@ -2522,9 +2564,12 @@ export default class API { static async deleteAllUpcomingRecordings() { try { - const resp = await request(`${host}/api/channels/recordings/bulk-delete-upcoming/`, { - method: 'POST', - }); + const resp = await request( + `${host}/api/channels/recordings/bulk-delete-upcoming/`, + { + method: 'POST', + } + ); notifications.show({ title: `Removed ${resp.removed || 0} upcoming` }); useChannelsStore.getState().fetchRecordings(); return resp; @@ -2545,12 +2590,19 @@ export default class API { } } - static async bulkRemoveSeriesRecordings({ tvg_id, title = null, scope = 'title' }) { + static async bulkRemoveSeriesRecordings({ + tvg_id, + title = null, + scope = 'title', + }) { try { - const resp = await request(`${host}/api/channels/series-rules/bulk-remove/`, { - method: 'POST', - body: { tvg_id, title, scope }, - }); + const resp = await request( + `${host}/api/channels/series-rules/bulk-remove/`, + { + method: 'POST', + body: { tvg_id, title, scope }, + } + ); notifications.show({ title: `Removed ${resp.removed || 0} scheduled` }); return resp; } catch (e) { @@ -2712,13 +2764,10 @@ export default class API { try { // Use POST for large ID lists to avoid URL length limitations if (ids.length > 50) { - const response = await request( - `${host}/api/channels/streams/by-ids/`, - { - method: 'POST', - body: { ids }, - } - ); + const response = await request(`${host}/api/channels/streams/by-ids/`, { + method: 'POST', + body: { ids }, + }); return response; } else { // Use GET for small ID lists for backward compatibility @@ -2744,8 +2793,9 @@ export default class API { return response; } catch (e) { // Don't show error notification for "Invalid page" errors as they're handled gracefully - const isInvalidPage = e.body?.detail?.includes('Invalid page') || - e.message?.includes('Invalid page'); + const isInvalidPage = + e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); if (!isInvalidPage) { errorNotification('Failed to retrieve movies', e); @@ -2762,8 +2812,9 @@ export default class API { return response; } catch (e) { // Don't show error notification for "Invalid page" errors as they're handled gracefully - const isInvalidPage = e.body?.detail?.includes('Invalid page') || - e.message?.includes('Invalid page'); + const isInvalidPage = + e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); if (!isInvalidPage) { errorNotification('Failed to retrieve series', e); @@ -2774,7 +2825,10 @@ export default class API { static async getAllContent(params = new URLSearchParams()) { try { - console.log('Calling getAllContent with URL:', `${host}/api/vod/all/?${params.toString()}`); + console.log( + 'Calling getAllContent with URL:', + `${host}/api/vod/all/?${params.toString()}` + ); const response = await request( `${host}/api/vod/all/?${params.toString()}` ); @@ -2787,8 +2841,9 @@ export default class API { console.error('Error message:', e.message); // Don't show error notification for "Invalid page" errors as they're handled gracefully - const isInvalidPage = e.body?.detail?.includes('Invalid page') || - e.message?.includes('Invalid page'); + const isInvalidPage = + e.body?.detail?.includes('Invalid page') || + e.message?.includes('Invalid page'); if (!isInvalidPage) { errorNotification('Failed to retrieve content', e); @@ -2911,9 +2966,8 @@ export default class API { ); // Update the store with fetched notifications - const { default: useNotificationsStore } = await import( - './store/notifications' - ); + const { default: useNotificationsStore } = + await import('./store/notifications'); useNotificationsStore.getState().setNotifications(response.notifications); return response; @@ -2928,9 +2982,8 @@ export default class API { const response = await request(`${host}/api/core/notifications/count/`); // Update the store with the count - const { default: useNotificationsStore } = await import( - './store/notifications' - ); + const { default: useNotificationsStore } = + await import('./store/notifications'); useNotificationsStore.getState().setUnreadCount(response.unread_count); return response; @@ -2962,10 +3015,11 @@ export default class API { ); // Update the store - const { default: useNotificationsStore } = await import( - './store/notifications' - ); - useNotificationsStore.getState().dismissNotification(response.notification_key); + const { default: useNotificationsStore } = + await import('./store/notifications'); + useNotificationsStore + .getState() + .dismissNotification(response.notification_key); return response; } catch (e) { @@ -2984,9 +3038,8 @@ export default class API { ); // Update the store - const { default: useNotificationsStore } = await import( - './store/notifications' - ); + const { default: useNotificationsStore } = + await import('./store/notifications'); useNotificationsStore.getState().dismissAllNotifications(); return response; @@ -2994,4 +3047,124 @@ export default class API { errorNotification('Failed to dismiss all notifications', e); } } + + static async getConnectIntegrations() { + try { + return await request(`${host}/api/connect/integrations/`); + } catch (e) { + errorNotification('Failed to fetch connect integrations', e); + } + } + + static async createConnectIntegration(values) { + try { + const response = await request(`${host}/api/connect/integrations/`, { + method: 'POST', + body: values, + }); + + useConnectStore.getState().addIntegration(response); + + return response; + } catch (e) { + errorNotification('Failed to create integration', e); + } + } + + static async updateConnectIntegration(id, values) { + try { + const response = await request( + `${host}/api/connect/integrations/${id}/`, + { + method: 'PUT', + body: values, + } + ); + + if (response.id) { + useConnectStore.getState().updateIntegration(response); + } + + return response; + } catch (e) { + errorNotification('Failed to update integration', e); + } + } + + static async deleteConnectIntegration(id) { + try { + await request(`${host}/api/connect/integrations/${id}/`, { + method: 'DELETE', + }); + + useConnectStore.getState().removeIntegration(id); + + return true; + } catch (e) { + errorNotification('Failed to delete integration', e); + throw e; + } + } + + static async createConnectSubscription(values) { + try { + await request(`${host}/api/connect/subscriptions/`, { + method: 'POST', + body: values, + }); + + return true; + } catch (e) { + errorNotification('Failed to create subscription', e); + } + } + + static async listConnectSubscriptions(integrationId) { + try { + return await request( + `${host}/api/connect/integrations/${integrationId}/subscriptions/` + ); + } catch (e) { + errorNotification('Failed to fetch subscriptions', e); + } + } + + static async setConnectSubscriptions(integrationId, subscriptions) { + // subscriptions: [{ event, enabled, payload_template }] + console.log(subscriptions); + try { + const response = await request( + `${host}/api/connect/integrations/${integrationId}/subscriptions/set/`, + { + method: 'PUT', + body: subscriptions, + } + ); + + useConnectStore + .getState() + .updateIntegrationSubscriptions(integrationId, response); + + return true; + } catch (e) { + errorNotification('Failed to set subscriptions', e); + throw e; + } + } + + static async getConnectLogs(params = {}) { + try { + const search = new URLSearchParams(); + if (params.page) search.set('page', params.page); + if (params.page_size) search.set('page_size', params.page_size); + if (params.type) search.set('type', params.type); + if (params.integration) search.set('integration', params.integration); + + return await request( + `${host}/api/connect/logs/${search.toString() ? `?${search.toString()}` : ''}` + ); + } catch (e) { + errorNotification('Failed to fetch connect logs', e); + } + } } diff --git a/frontend/src/components/Sidebar.jsx b/frontend/src/components/Sidebar.jsx index 6c997cfe..4598427d 100644 --- a/frontend/src/components/Sidebar.jsx +++ b/frontend/src/components/Sidebar.jsx @@ -15,6 +15,11 @@ import { LogOut, User, FileImage, + Webhook, + Logs, + ChevronDown, + ChevronRight, + MonitorCog, } from 'lucide-react'; import { Avatar, @@ -27,6 +32,7 @@ import { TextInput, ActionIcon, Menu, + ScrollArea, } from '@mantine/core'; import { notifications } from '@mantine/notifications'; import logo from '../images/logo.png'; @@ -70,6 +76,74 @@ const NavLink = ({ item, isActive, collapsed }) => { ); }; +function NavGroup({ label, icon, paths, location, collapsed }) { + const [open, setOpen] = useState(() => + location.pathname.startsWith('/connect') + ); + + const parentActive = paths + .map((path) => path.path) + .includes(location.pathname); + + return ( + + setOpen((o) => !o)} + className={`navlink ${parentActive ? 'navlink-parent-active' : ''} ${open ? 'navlink-collapsed' : ''}`} + style={{ width: '100%' }} + > + {icon} + {!collapsed && ( + + + {label} + + + + {open ? : } + + + )} + + + {open && ( + + + {paths.map((child) => { + const active = location.pathname === child.path; + return ( + + + + ); + })} + + + )} + + ); +} + const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { const location = useLocation(); @@ -111,19 +185,41 @@ const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { { label: 'Stats', icon: , path: '/stats' }, { label: 'Plugins', icon: , path: '/plugins' }, { - label: 'Users', - icon: , - path: '/users', - }, - { - label: 'Logo Manager', - icon: , - path: '/logos', + label: 'Connect', + icon: , + paths: [ + { + label: 'Connections', + icon: , + path: '/connect', + }, + { + label: 'Logs', + icon: , + path: '/connect/logs', + }, + ], }, { label: 'Settings', icon: , - path: '/settings', + paths: [ + { + label: 'Users', + icon: , + path: '/users', + }, + { + label: 'Logo Manager', + icon: , + path: '/logos', + }, + { + label: 'System', + icon: , + path: '/settings', + }, + ], }, ] : [ @@ -205,20 +301,44 @@ const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { {/* Navigation Links */} - - {navItems.map((item) => { - const isActive = location.pathname === item.path; + + + {navItems.map((item) => { + if (item.paths) { + return ( + + ); + } - return ( - - ); - })} - + const isActive = location.pathname === item.path; + + return ( + + ); + })} + + {/* Profile Section */} ({ + value, + label, + }) +); + +const ConnectionForm = ({ connection = null, isOpen, onClose }) => { + const [submitting, setSubmitting] = useState(false); + const [selectedEvents, setSelectedEvents] = useState([]); + + // One-time form + const form = useForm({ + mode: 'controlled', + initialValues: { + name: connection?.name || '', + type: connection?.type || 'webhook', + url: connection?.config?.url || '', + script_path: connection?.config?.path || '', + enabled: connection?.enabled ?? true, + }, + validate: { + name: isNotEmpty('Provide a name'), + type: isNotEmpty('Select a type'), + url: (value, values) => { + if (values.type === 'webhook' && !value.trim()) { + return 'Provide a webhook URL'; + } + return null; + }, + script_path: (value, values) => { + if (values.type === 'script' && !value.trim()) { + return 'Provide a script path'; + } + return null; + }, + }, + }); + + useEffect(() => { + if (connection) { + const values = { + name: connection.name, + type: connection.type, + url: connection.config?.url, + script_path: connection.config?.path, + enabled: connection.enabled, + }; + form.setValues(values); + setSelectedEvents( + connection.subscriptions.reduce((acc, sub) => { + if (sub.enabled) acc.push(sub.event); + return acc; + }, []) + ); + } else { + form.reset(); + } + }, [connection]); + + const handleClose = () => { + onClose?.(); + }; + + const onSubmit = async (values) => { + console.log(values); + try { + setSubmitting(true); + const config = + values.type === 'webhook' + ? { url: values.url } + : { path: values.script_path }; + + if (connection) { + await API.updateConnectIntegration(connection.id, { + name: values.name, + type: values.type, + config, + enabled: values.enabled, + }); + } else { + connection = await API.createConnectIntegration({ + name: values.name, + type: values.type, + config, + enabled: values.enabled, + }); + } + + await API.setConnectSubscriptions( + connection.id, + Object.keys(SUBSCRIPTION_EVENTS).map((event) => ({ + event, + enabled: selectedEvents.includes(event), + })) + ); + handleClose(); + } catch (error) { + console.error('Failed to create connection', error); + } finally { + setSubmitting(false); + } + }; + + const toggleEvent = (event) => { + setSelectedEvents((prev) => + prev.includes(event) ? prev.filter((e) => e !== event) : [...prev, event] + ); + }; + + if (!isOpen) return null; + + return ( + + +
+ + + setFilters((prev) => ({ ...prev, type: value })) + } + style={{ width: 150 }} + /> + Integration + handleScheduleChange('frequency', value)} + data={[ + { value: 'daily', label: 'Daily' }, + { value: 'weekly', label: 'Weekly' }, + ]} + disabled={!schedule.enabled} + /> + {schedule.frequency === 'weekly' && ( + { + const minute = displayTime + ? displayTime.split(':')[1] + : '00'; + handleTimeChange12h(`${value}:${minute}`, null); + }} + data={Array.from({ length: 12 }, (_, i) => ({ + value: String(i + 1), + label: String(i + 1), + }))} + disabled={!schedule.enabled} + searchable + /> + handleTimeChange12h(null, value)} + data={[ + { value: 'AM', label: 'AM' }, + { value: 'PM', label: 'PM' }, + ]} + disabled={!schedule.enabled} + /> + + ) : ( + <> + { + const hour = schedule.time + ? schedule.time.split(':')[0] + : '00'; + handleTimeChange24h(`${hour}:${value}`); + }} + data={Array.from({ length: 60 }, (_, i) => ({ + value: String(i).padStart(2, '0'), + label: String(i).padStart(2, '0'), + }))} + disabled={!schedule.enabled} + searchable + /> + + )} + + + {scheduleLoading ? ( ) : ( <> - {advancedMode ? ( - <> - - - handleScheduleChange( - 'cron_expression', - e.currentTarget.value - ) - } - placeholder="0 3 * * *" - description="Format: minute hour day month weekday (e.g., '0 3 * * *' = 3:00 AM daily)" - disabled={!schedule.enabled} - error={cronError} - /> - - Examples:
• 0 3 * * * - Every day at 3:00 - AM -
• 0 2 * * 0 - Every Sunday at 2:00 AM -
• 0 */6 * * * - Every 6 hours -
• 30 14 1 * * - 1st of every month at - 2:30 PM - - - - - handleScheduleChange('retention_count', value || 0) - } - min={0} - disabled={!schedule.enabled} - /> - - - - ) : ( - - - - handleScheduleChange('day_of_week', parseInt(value, 10)) - } - data={DAYS_OF_WEEK} - disabled={!schedule.enabled} - /> - )} - {is12Hour ? ( - <> - { - const hour = displayTime - ? displayTime.split(':')[0] - : '12'; - handleTimeChange12h(`${hour}:${value}`, null); - }} - data={Array.from({ length: 60 }, (_, i) => ({ - value: String(i).padStart(2, '0'), - label: String(i).padStart(2, '0'), - }))} - disabled={!schedule.enabled} - searchable - /> - { - const minute = schedule.time - ? schedule.time.split(':')[1] - : '00'; - handleTimeChange24h(`${value}:${minute}`); - }} - data={Array.from({ length: 24 }, (_, i) => ({ - value: String(i).padStart(2, '0'), - label: String(i).padStart(2, '0'), - }))} - disabled={!schedule.enabled} - searchable - /> - } + /> + + {frequency !== 'hourly' && ( + } + /> + )} + + } + /> + + {frequency === 'weekly' && ( + { + setGroupStates( + groupStates.map((state) => { + if ( + state.channel_group === group.channel_group + ) { + return { + ...state, + custom_properties: { + ...state.custom_properties, + channel_numbering_mode: value || 'fixed', + }, + }; + } + return state; + }) + ); + }} + data={[ + { + value: 'fixed', + label: 'Fixed Start Number', + }, + { + value: 'provider', + label: 'Use Provider Number', + }, + { + value: 'next_available', + label: 'Next Available', + }, + ]} + size="xs" + /> + + + {(!group.custom_properties?.channel_numbering_mode || + group.custom_properties?.channel_numbering_mode === + 'fixed') && ( + + updateChannelStart(group.channel_group, value) + } + min={1} + step={1} + size="xs" + precision={0} + /> + )} + + {group.custom_properties?.channel_numbering_mode === + 'provider' && ( + { + setGroupStates( + groupStates.map((state) => { + if ( + state.channel_group === group.channel_group + ) { + return { + ...state, + custom_properties: { + ...state.custom_properties, + channel_numbering_fallback: value || 1, + }, + }; + } + return state; + }) + ); + }} + min={1} + step={1} + size="xs" + precision={0} + /> + )} {/* Auto Channel Sync Options Multi-Select */} Date: Fri, 13 Feb 2026 15:07:24 -0600 Subject: [PATCH 039/503] Bug Fix: Fixes a bug with auto channel sync where channel numbers could be duplicated if multiple groups overlapped. --- apps/m3u/tasks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 80880671..0a1d4fc4 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -1648,6 +1648,13 @@ def sync_auto_channels(account_id, scan_start_time=None): channels_updated = 0 channels_deleted = 0 + # Get all channel numbers that are already in use by other channels (not auto-created by this account) + used_numbers = set( + Channel.objects.exclude( + auto_created=True, auto_created_by=account + ).values_list("channel_number", flat=True) + ) + for group_relation in auto_sync_groups: channel_group = group_relation.channel_group start_number = group_relation.auto_sync_channel_start or 1.0 @@ -1841,13 +1848,6 @@ def sync_auto_channels(account_id, scan_start_time=None): channels_to_renumber = [] temp_channel_number = start_number - # Get all channel numbers that are already in use by other channels (not auto-created by this account) - used_numbers = set( - Channel.objects.exclude( - auto_created=True, auto_created_by=account - ).values_list("channel_number", flat=True) - ) - for stream in current_streams: if stream.id in existing_channel_map: channel = existing_channel_map[stream.id] From 5ed8b7a954dff84d7fd9983eab52e7782254a8ba Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 13 Feb 2026 15:39:27 -0600 Subject: [PATCH 040/503] changelog: Update changelog for auto channel sync changes. --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c768e1e..3ef85239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Channel numbering modes for auto channel sync: Added three channel numbering modes when auto-syncing channels from M3U groups: + - **Fixed Start Number** (default): Start at a specified number and increment sequentially + - **Use Provider Number**: Use channel numbers from the M3U source (tvg-chno), with configurable fallback if provider number is missing + - **Next Available**: Auto-assign starting from 1, skipping all used channel numbers + Each mode includes its own configuration options accessible via the "Channel Numbering Mode" dropdown in auto sync settings. (Closes #956, #433) + ### Fixed +- Auto channel sync duplicate channel numbers across groups: Fixed issue where multiple auto-sync groups starting at the same number would create duplicate channel numbers. The used channel number tracking now persists across all groups in a single sync operation, ensuring each assigned channel number is globally unique. - Modular mode PostgreSQL/Redis connection checks: Replaced raw Python socket checks with native tools (`pg_isready` for PostgreSQL and `socket.create_connection` for Redis) in modular deployment mode to prevent indefinite hangs in Docker environments with non-standard networking or DNS configurations. Now properly supports IPv4 and IPv6 configurations. (Fixes #952) - Thanks [@CodeBormen](https://github.com/CodeBormen) ## [0.19.0] - 2026-02-10 From 4842bb87fa793578109c2f0ad59a8c7810b65cc6 Mon Sep 17 00:00:00 2001 From: None Date: Fri, 13 Feb 2026 19:37:06 -0600 Subject: [PATCH 041/503] Fix: VOD proxy connection counter leak on client disconnect Three fixes: - TOCTOU: Replace GET-check-INCR with atomic INCR-first-then-check pattern in both connection managers to prevent concurrent requests exceeding max_streams - Immediate DECR: Decrement profile counter directly in stream generator exit paths (normal completion, client disconnect, error) instead of deferring to daemon thread cleanup which may never execute - Rollback: Decrement profile counter on create_connection() failure so the reserved slot is released Fixes #962 --- apps/proxy/vod_proxy/connection_manager.py | 49 ++++++++++-- .../multi_worker_connection_manager.py | 77 ++++++++++++++++--- 2 files changed, 111 insertions(+), 15 deletions(-) diff --git a/apps/proxy/vod_proxy/connection_manager.py b/apps/proxy/vod_proxy/connection_manager.py index aafc75cd..c4c2130f 100644 --- a/apps/proxy/vod_proxy/connection_manager.py +++ b/apps/proxy/vod_proxy/connection_manager.py @@ -459,13 +459,12 @@ class VODConnectionManager: return False try: - # Check profile connection limits using standardized key - if not self._check_profile_limits(m3u_profile): + # Atomically check and reserve a profile connection slot (INCR-first) + if not self._check_and_reserve_profile_slot(m3u_profile): logger.warning(f"Profile {m3u_profile.name} connection limit exceeded") return False connection_key = self._get_connection_key(content_type, content_uuid, client_id) - profile_connections_key = self._get_profile_connections_key(m3u_profile.id) content_connections_key = self._get_content_connections_key(content_type, content_uuid) # Check if connection already exists to prevent duplicate counting @@ -473,6 +472,9 @@ class VODConnectionManager: logger.info(f"Connection already exists for {client_id} - {content_type} {content_name}") # Update activity but don't increment profile counter self.redis_client.hset(connection_key, "last_activity", str(time.time())) + # Roll back the reservation — connection already counted + if m3u_profile.max_streams > 0: + self.redis_client.decr(self._get_profile_connections_key(m3u_profile.id)) return True # Connection data @@ -499,8 +501,7 @@ class VODConnectionManager: pipe.hset(connection_key, mapping=connection_data) pipe.expire(connection_key, self.connection_ttl) - # Increment profile connections using standardized method - pipe.incr(profile_connections_key) + # Profile counter already incremented atomically above — no pipe.incr needed # Add to content connections set pipe.sadd(content_connections_key, client_id) @@ -513,6 +514,9 @@ class VODConnectionManager: return True except Exception as e: + # Roll back the profile reservation on failure + if m3u_profile.max_streams > 0: + self.redis_client.decr(self._get_profile_connections_key(m3u_profile.id)) logger.error(f"Error creating VOD connection: {e}") return False @@ -531,6 +535,41 @@ class VODConnectionManager: logger.error(f"Error checking profile limits: {e}") return False + def _check_and_reserve_profile_slot(self, m3u_profile: M3UAccountProfile) -> bool: + """ + Atomically check and reserve a connection slot for the given profile. + + Uses an INCR-first-then-check pattern to eliminate the TOCTOU race + condition where separate GET > check > INCR operations could allow + concurrent requests to both pass the capacity check. + + For profiles with max_streams=0 (unlimited), no reservation is needed. + + Returns: + bool: True if slot was reserved (or unlimited), False if at capacity + """ + if m3u_profile.max_streams == 0: # Unlimited + return True + + try: + profile_connections_key = self._get_profile_connections_key(m3u_profile.id) + + # Atomically increment first — single Redis command eliminates race window + new_count = self.redis_client.incr(profile_connections_key) + + if new_count <= m3u_profile.max_streams: + logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: {new_count}/{m3u_profile.max_streams}") + return True + + # Over capacity — roll back the increment + self.redis_client.decr(profile_connections_key) + logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: {new_count - 1}/{m3u_profile.max_streams}") + return False + + except Exception as e: + logger.error(f"Error reserving profile slot: {e}") + return False + def update_connection_activity(self, content_type: str, content_uuid: str, client_id: str, bytes_sent: int = 0, position_seconds: int = 0) -> bool: diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index 1534f761..c8da7cc1 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -674,6 +674,41 @@ class MultiWorkerVODConnectionManager: logger.error(f"Error checking profile limits: {e}") return False + def _check_and_reserve_profile_slot(self, m3u_profile) -> bool: + """ + Atomically check and reserve a connection slot for the given profile. + + Uses an INCR-first-then-check pattern to eliminate the TOCTOU race + condition where separate GET > check > INCR operations could allow + concurrent requests to both pass the capacity check. + + For profiles with max_streams=0 (unlimited), no reservation is needed. + + Returns: + bool: True if slot was reserved (or unlimited), False if at capacity + """ + if m3u_profile.max_streams == 0: # Unlimited + return True + + try: + profile_connections_key = self._get_profile_connections_key(m3u_profile.id) + + # Atomically increment first — single Redis command eliminates race window + new_count = self.redis_client.incr(profile_connections_key) + + if new_count <= m3u_profile.max_streams: + logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} slot reserved: {new_count}/{m3u_profile.max_streams}") + return True + + # Over capacity — roll back the increment + self.redis_client.decr(profile_connections_key) + logger.info(f"[PROFILE-RESERVE] Profile {m3u_profile.id} at capacity: {new_count - 1}/{m3u_profile.max_streams}") + return False + + except Exception as e: + logger.error(f"Error reserving profile slot: {e}") + return False + def _increment_profile_connections(self, m3u_profile): """Increment profile connection count""" try: @@ -756,10 +791,11 @@ class MultiWorkerVODConnectionManager: if not existing_state: logger.info(f"[{client_id}] Worker {self.worker_id} - Creating new Redis-backed connection") - # Check profile limits before creating new connection - if not self._check_profile_limits(m3u_profile): + # Atomically check and reserve a profile connection slot (INCR-first) + if not self._check_and_reserve_profile_slot(m3u_profile): logger.warning(f"[{client_id}] Profile {m3u_profile.name} connection limit exceeded") return HttpResponse("Connection limit exceeded for profile", status=429) + profile_connections_incremented = True # Apply timeshift parameters modified_stream_url = self._apply_timeshift_parameters(stream_url, utc_start, utc_end, offset) @@ -802,12 +838,11 @@ class MultiWorkerVODConnectionManager: worker_id=self.worker_id ): logger.error(f"[{client_id}] Worker {self.worker_id} - Failed to create Redis connection") + # Roll back the profile slot reservation since connection failed + self._decrement_profile_connections(m3u_profile.id) + profile_connections_incremented = False return HttpResponse("Failed to create connection", status=500) - # Increment profile connections after successful connection creation - self._increment_profile_connections(m3u_profile) - profile_connections_incremented = True - logger.info(f"[{client_id}] Worker {self.worker_id} - Created consolidated connection with session metadata") else: logger.info(f"[{client_id}] Worker {self.worker_id} - Using existing Redis-backed connection") @@ -898,11 +933,18 @@ class MultiWorkerVODConnectionManager: # Schedule smart cleanup if no active streams after normal completion if not redis_connection.has_active_streams(): + # Decrement profile counter immediately — don't defer to daemon thread + state = redis_connection._get_connection_state() + if state and state.m3u_profile_id: + self._decrement_profile_connections(state.m3u_profile_id) + logger.info(f"[{client_id}] Profile counter decremented for profile {state.m3u_profile_id} on normal completion") + def delayed_cleanup(): time.sleep(1) # Wait 1 second # Smart cleanup: check active streams and ownership logger.info(f"[{client_id}] Worker {self.worker_id} - Checking for smart cleanup after normal completion") - redis_connection.cleanup(connection_manager=self, current_worker_id=self.worker_id) + # No connection_manager — profile already decremented above + redis_connection.cleanup(current_worker_id=self.worker_id) import threading cleanup_thread = threading.Thread(target=delayed_cleanup) @@ -917,11 +959,18 @@ class MultiWorkerVODConnectionManager: # Schedule smart cleanup if no active streams if not redis_connection.has_active_streams(): + # Decrement profile counter immediately — don't defer to daemon thread + state = redis_connection._get_connection_state() + if state and state.m3u_profile_id: + self._decrement_profile_connections(state.m3u_profile_id) + logger.info(f"[{client_id}] Profile counter decremented for profile {state.m3u_profile_id} on client disconnect") + def delayed_cleanup(): time.sleep(1) # Wait 1 second # Smart cleanup: check active streams and ownership logger.info(f"[{client_id}] Worker {self.worker_id} - Checking for smart cleanup after client disconnect") - redis_connection.cleanup(connection_manager=self, current_worker_id=self.worker_id) + # No connection_manager — profile already decremented above + redis_connection.cleanup(current_worker_id=self.worker_id) import threading cleanup_thread = threading.Thread(target=delayed_cleanup) @@ -933,8 +982,16 @@ class MultiWorkerVODConnectionManager: if not decremented: redis_connection.decrement_active_streams() decremented = True - # Smart cleanup on error - immediate cleanup since we're in error state - redis_connection.cleanup(connection_manager=self, current_worker_id=self.worker_id) + + # Decrement profile counter immediately if no other active streams + if not redis_connection.has_active_streams(): + state = redis_connection._get_connection_state() + if state and state.m3u_profile_id: + self._decrement_profile_connections(state.m3u_profile_id) + logger.info(f"[{client_id}] Profile counter decremented for profile {state.m3u_profile_id} on stream error") + # Smart cleanup on error - immediate cleanup since we're in error state + # No connection_manager — profile already decremented above + redis_connection.cleanup(current_worker_id=self.worker_id) yield b"Error: Stream interrupted" finally: From 45d0f180fe34dc05216253b66bf20be823a4b072 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 15 Feb 2026 07:36:08 -0500 Subject: [PATCH 042/503] added endpoint for testing connections with a dummy payload --- apps/connect/api_views.py | 67 +++++++++++++++++++++++++++++++++++-- apps/connect/serializers.py | 22 ++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/apps/connect/api_views.py b/apps/connect/api_views.py index 1402de04..de3903c0 100644 --- a/apps/connect/api_views.py +++ b/apps/connect/api_views.py @@ -3,6 +3,7 @@ from rest_framework.pagination import PageNumberPagination from django_filters.rest_framework import DjangoFilterBackend from rest_framework.response import Response from rest_framework.decorators import action +from django.utils import timezone from .models import Integration, EventSubscription, DeliveryLog from .serializers import ( IntegrationSerializer, @@ -12,7 +13,10 @@ from .serializers import ( from apps.accounts.permissions import ( Authenticated, permission_classes_by_action, + IsAdmin, ) +from .handlers.webhook import WebhookHandler +from .handlers.script import ScriptHandler class IntegrationViewSet(viewsets.ModelViewSet): @@ -21,9 +25,11 @@ class IntegrationViewSet(viewsets.ModelViewSet): def get_permissions(self): try: - return [perm() for perm in permission_classes_by_action[self.action]] + perms = permission_classes_by_action[self.action] except KeyError: - return [Authenticated()] + # Respect view/action-specific permission_classes if provided; fallback to Authenticated + perms = getattr(self, "permission_classes", [Authenticated]) + return [perm() for perm in perms] @action(detail=True, methods=["get"], url_path="subscriptions") def list_subscriptions(self, request, pk=None): @@ -99,6 +105,63 @@ class IntegrationViewSet(viewsets.ModelViewSet): serializer = EventSubscriptionSerializer(updated, many=True) return Response(serializer.data, status=status.HTTP_200_OK) + @action(detail=True, methods=["post"], url_path="test", permission_classes=[IsAdmin]) + def test(self, request, pk=None): + """ + Execute a saved integration (connect) with a dummy payload to verify configuration. + """ + try: + integration = Integration.objects.get(pk=pk) + except Integration.DoesNotExist: + return Response({"detail": "Integration not found"}, status=status.HTTP_404_NOT_FOUND) + + # Build a dummy payload similar to system events + now = timezone.now().isoformat() + dummy_payload = { + "event": "test", + "timestamp": now, + "channel_name": "Test Channel", + "stream_name": "Test Stream", + "stream_url": "http://example.com/stream.m3u8", + "channel_url": "http://example.com/stream.m3u8", + "provider_name": "Test Provider", + "profile_used": "Default", + "test": True, + } + + # Choose handler based on saved type + if integration.type == "webhook": + handler = WebhookHandler(integration, None, dummy_payload) + elif integration.type == "script": + handler = ScriptHandler(integration, None, dummy_payload) + else: + return Response( + {"success": False, "error": f"Unsupported integration type: {integration.type}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + result = handler.execute() + return Response( + { + "success": bool(result.get("success")), + "type": integration.type, + "request_payload": dummy_payload, + "result": result, + }, + status=status.HTTP_200_OK, + ) + except Exception as e: + return Response( + { + "success": False, + "type": integration.type, + "request_payload": dummy_payload, + "error": str(e), + }, + status=status.HTTP_502_BAD_GATEWAY, + ) + class EventSubscriptionViewSet(viewsets.ModelViewSet): queryset = EventSubscription.objects.all() diff --git a/apps/connect/serializers.py b/apps/connect/serializers.py index 859295f3..832fc14b 100644 --- a/apps/connect/serializers.py +++ b/apps/connect/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers from .models import Integration, EventSubscription, DeliveryLog +import os class EventSubscriptionSerializer(serializers.ModelSerializer): @@ -29,6 +30,27 @@ class IntegrationSerializer(serializers.ModelSerializer): "subscriptions", ] + def validate(self, attrs): + type = attrs.get("type") if "type" in attrs else getattr(self.instance, "type", None) + config = attrs.get("config") if "config" in attrs else getattr(self.instance, "config", {}) + + if type == "script": + path = (config or {}).get("path") + if not path or not isinstance(path, str): + raise serializers.ValidationError({"config": "Script config must include a 'path' string"}) + + real_path = os.path.abspath(os.path.realpath(path)) + if not os.path.exists(real_path): + raise serializers.ValidationError({"config": f"Script path does not exist: {path}"}) + elif type == "webhook": + url = (config or {}).get("url") + if not url or not isinstance(url, str): + raise serializers.ValidationError({"config": "Webhook config must include a 'url' string"}) + else: + raise serializers.ValidationError({"type": "Unsupported integration type"}) + + return attrs + class DeliveryLogSerializer(serializers.ModelSerializer): subscription = EventSubscriptionSerializer(read_only=True) From 055d2604ca609e035d63b84283bf9a660bcb3e03 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sun, 15 Feb 2026 07:36:29 -0500 Subject: [PATCH 043/503] better form validation --- frontend/src/components/forms/Connection.jsx | 40 +++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/forms/Connection.jsx b/frontend/src/components/forms/Connection.jsx index d55ee522..e8077d43 100644 --- a/frontend/src/components/forms/Connection.jsx +++ b/frontend/src/components/forms/Connection.jsx @@ -25,6 +25,7 @@ const EVENT_OPTIONS = Object.entries(SUBSCRIPTION_EVENTS).map( const ConnectionForm = ({ connection = null, isOpen, onClose }) => { const [submitting, setSubmitting] = useState(false); const [selectedEvents, setSelectedEvents] = useState([]); + const [apiError, setApiError] = useState(''); // One-time form const form = useForm({ @@ -77,6 +78,7 @@ const ConnectionForm = ({ connection = null, isOpen, onClose }) => { }, [connection]); const handleClose = () => { + setApiError(''); onClose?.(); }; @@ -84,6 +86,7 @@ const ConnectionForm = ({ connection = null, isOpen, onClose }) => { console.log(values); try { setSubmitting(true); + setApiError(''); const config = values.type === 'webhook' ? { url: values.url } @@ -114,7 +117,37 @@ const ConnectionForm = ({ connection = null, isOpen, onClose }) => { ); handleClose(); } catch (error) { - console.error('Failed to create connection', error); + console.error('Failed to create/update connection', error); + // Try to map server-side validation errors to form fields + const body = error?.body; + + if (body && typeof body === 'object') { + const fieldErrors = {}; + if (body.name) { + fieldErrors.name = body.name; + } + if (body.type) { + fieldErrors.type = body.type; + } + if (body.config) { + if (values.type === 'webhook') { + fieldErrors.url = msg; + } else { + fieldErrors.script_path = msg; + } + } + + const nonField = body.non_field_errors || body.detail; + if (Object.keys(fieldErrors).length > 0) { + form.setErrors(fieldErrors); + } + if (nonField) setApiError(nonField); + if (!nonField && Object.keys(fieldErrors).length === 0) { + setApiError(body); + } + } else { + setApiError(error?.message || 'Unknown error'); + } } finally { setSubmitting(false); } @@ -132,6 +165,11 @@ const ConnectionForm = ({ connection = null, isOpen, onClose }) => { + {apiError ? ( + + {apiError} + + ) : null} Date: Sun, 15 Feb 2026 07:37:00 -0500 Subject: [PATCH 044/503] hardening of script handling with configurable variables - making script execution more secure --- apps/connect/handlers/script.py | 69 +++++++++++++++++++++++++++++---- dispatcharr/settings.py | 15 +++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/apps/connect/handlers/script.py b/apps/connect/handlers/script.py index 3fa8215b..b6aef79c 100644 --- a/apps/connect/handlers/script.py +++ b/apps/connect/handlers/script.py @@ -1,28 +1,81 @@ # connect/handlers/script.py import os +import stat import subprocess +from django.conf import settings from .base import IntegrationHandler + +def _is_path_allowed(real_path: str) -> bool: + # Ensure path is within one of the allowed directories + for base in getattr(settings, "CONNECT_ALLOWED_SCRIPT_DIRS", []): + base_abs = os.path.abspath(base) + os.sep + if real_path.startswith(base_abs): + return True + return False + + class ScriptHandler(IntegrationHandler): def execute(self): - script_path = self.integration.config.get("path") + raw_path = self.integration.config.get("path") + if not raw_path: + raise ValueError("Missing 'path' in integration config") - # Build environment variables from payload (prefixed for clarity) - env = os.environ.copy() + # Resolve and validate path + real_path = os.path.abspath(os.path.realpath(raw_path)) + + if not os.path.exists(real_path): + raise FileNotFoundError(f"Script not found: {real_path}") + + if not _is_path_allowed(real_path): + raise PermissionError( + f"Script path '{real_path}' not within allowed directories: " + f"{getattr(settings, 'CONNECT_ALLOWED_SCRIPT_DIRS', [])}" + ) + + if getattr(settings, "CONNECT_SCRIPT_REQUIRE_EXECUTABLE", True): + if not os.access(real_path, os.X_OK): + raise PermissionError(f"Script is not executable: {real_path}") + + if getattr(settings, "CONNECT_SCRIPT_DISALLOW_WORLD_WRITABLE", True): + st = os.stat(real_path) + if st.st_mode & stat.S_IWOTH: + raise PermissionError( + f"Refusing to execute world-writable script: {real_path}" + ) + + # Build a sanitized minimal environment; avoid inheriting secrets + env = { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } for key, value in (self.payload or {}).items(): - # Convert keys to upper snake case and prefix env_key = f"DISPATCHARR_{str(key).upper()}" - env[env_key] = str(value) if value is not None else "" + env[env_key] = "" if value is None else str(value) + + # Run with a timeout to prevent hanging scripts + timeout = getattr(settings, "CONNECT_SCRIPT_TIMEOUT", 10) + max_out = getattr(settings, "CONNECT_SCRIPT_MAX_OUTPUT", 65536) result = subprocess.run( - [script_path], + [real_path], capture_output=True, text=True, env=env, + timeout=timeout, + cwd=os.path.dirname(real_path) or None, ) + + # Truncate outputs to avoid excessive memory/logging + stdout = result.stdout or "" + stderr = result.stderr or "" + if len(stdout) > max_out: + stdout = stdout[:max_out] + "... [truncated]" + if len(stderr) > max_out: + stderr = stderr[:max_out] + "... [truncated]" + return { "exit_code": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr, + "stdout": stdout, + "stderr": stderr, "success": result.returncode == 0, } diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 8fba3143..1fbf566b 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -412,3 +412,18 @@ LOGGING = { "level": LOG_LEVEL, # Use user-configured level instead of hardcoded 'INFO' }, } + +# Connect script execution safety settings +# Allowed base directories for custom scripts; real paths must be inside +_allowed_dirs_env = os.environ.get("DISPATCHARR_ALLOWED_SCRIPT_DIRS", "/data/plugins") +CONNECT_ALLOWED_SCRIPT_DIRS = [p for p in _allowed_dirs_env.split(":") if p] + +# Max execution time (seconds) for scripts +CONNECT_SCRIPT_TIMEOUT = int(os.environ.get("DISPATCHARR_SCRIPT_TIMEOUT", "10")) + +# Truncate stdout/stderr to this many characters to avoid large outputs +CONNECT_SCRIPT_MAX_OUTPUT = int(os.environ.get("DISPATCHARR_SCRIPT_MAX_OUTPUT", "65536")) + +# Require executable bit and disallow world-writable files +CONNECT_SCRIPT_REQUIRE_EXECUTABLE = True +CONNECT_SCRIPT_DISALLOW_WORLD_WRITABLE = True From 968551c1a6e712b6142ac4f865ab53831aa393f8 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 16 Feb 2026 13:45:57 -0600 Subject: [PATCH 045/503] Enhancement: M3U generation for Xtream Codes API endpoints now use proper XC-style stream URLs (`/live/username/password/channel_id`) instead of UUID-based stream endpoints, ensuring full compatibility with XC clients. (Fixes #839) --- CHANGELOG.md | 4 ++++ apps/output/views.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6718253a..688c11bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Next Available**: Auto-assign starting from 1, skipping all used channel numbers Each mode includes its own configuration options accessible via the "Channel Numbering Mode" dropdown in auto sync settings. (Closes #956, #433) +### Changed + +- XC API M3U stream URLs: M3U generation for Xtream Codes API endpoints now use proper XC-style stream URLs (`/live/username/password/channel_id`) instead of UUID-based stream endpoints, ensuring full compatibility with XC clients. (Fixes #839) + ### Fixed - Auto channel sync duplicate channel numbers across groups: Fixed issue where multiple auto-sync groups starting at the same number would create duplicate channel numbers. The used channel number tracking now persists across all groups in a single sync operation, ensuring each assigned channel number is globally unique. diff --git a/apps/output/views.py b/apps/output/views.py index 1d53237a..21377489 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -183,8 +183,9 @@ def generate_m3u(request, profile_name=None, user=None): # Check if this is an XC API request (has username/password in GET params and user is authenticated) xc_username = request.GET.get('username') xc_password = request.GET.get('password') + is_xc_request = user is not None and xc_username and xc_password - if user is not None and xc_username and xc_password: + if is_xc_request: # This is an XC API request - use XC-style EPG URL base_url = build_absolute_uri_with_port(request, '') epg_url = f"{base_url}/xmltv.php?username={xc_username}&password={xc_password}" @@ -254,8 +255,12 @@ def generate_m3u(request, profile_name=None, user=None): f'tvg-chno="{formatted_channel_number}" {tvc_guide_stationid}group-title="{group_title}",{channel.name}\n' ) - # Determine the stream URL based on the direct parameter - if use_direct_urls: + # Determine the stream URL based on request type + if is_xc_request: + # XC API request - use XC-style stream URL format + base_url = build_absolute_uri_with_port(request, '') + stream_url = f"{base_url}/live/{xc_username}/{xc_password}/{channel.id}" + elif use_direct_urls: # Try to get the first stream's direct URL first_stream = channel.streams.order_by('channelstream__order').first() if first_stream and first_stream.url: From 50918b45fb2830694ef30de71038bb74854690a2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 16 Feb 2026 15:34:09 -0600 Subject: [PATCH 046/503] changelog: Update changelog for webhooks. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 688c11bd..320cf90b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Event-driven webhooks and script execution (Connect): Added new Connect feature that enables event-driven execution of custom scripts and webhooks in response to system events. Supports multiple event types including channel lifecycle (start, stop, reconnect, error, failover), stream operations (switch), recording events (start, end), data refreshes (EPG, M3U), and client activity (connect, disconnect). Event data is available as environment variables in scripts (prefixed with `DISPATCHARR_`), POST payloads for webhooks, and plugin execution payloads. Plugins can now subscribe to events by specifying an `events` array in their action definitions. Includes connection testing endpoint with dummy payloads for validation. (Closes #203) - Cron scheduling support for M3U and EPG refreshes: Added interactive cron expression builder with preset buttons and custom field editors, plus info popover with common cron examples. Refactored backup scheduling to use shared ScheduleInput component for consistency across all scheduling interfaces. (Closes #165) - Channel numbering modes for auto channel sync: Added three channel numbering modes when auto-syncing channels from M3U groups: - **Fixed Start Number** (default): Start at a specified number and increment sequentially From 74cc9bd376d1ed1ffea570ce3c1971cd7fde8471 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 16 Feb 2026 16:10:51 -0600 Subject: [PATCH 047/503] changelog: Update changelog for pr 946 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 320cf90b..26665f72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- XC profile refresh credential extraction with sub-paths: Fixed credential extraction in `get_transformed_credentials()` to use negative indices anchored to the known tail structure instead of hardcoded indices that broke when server URLs contained sub-paths (e.g., `http://server.com/portal/a/`). This ensures XC accounts with sub-paths in their server URLs work correctly for profile refreshes. (Fixes #945) - Thanks [@CodeBormen](https://github.com/CodeBormen) +- XC EPG URL construction for accounts with sub-paths or trailing slashes: Fixed EPG URL construction in M3U forms to normalize server URL to origin before appending `xmltv.php` endpoint, preventing double slashes and incorrect path placement when server URLs include sub-paths or trailing slashes. (Fixes #800) - Thanks [@CodeBormen](https://github.com/CodeBormen) - Auto channel sync duplicate channel numbers across groups: Fixed issue where multiple auto-sync groups starting at the same number would create duplicate channel numbers. The used channel number tracking now persists across all groups in a single sync operation, ensuring each assigned channel number is globally unique. - Modular mode PostgreSQL/Redis connection checks: Replaced raw Python socket checks with native tools (`pg_isready` for PostgreSQL and `socket.create_connection` for Redis) in modular deployment mode to prevent indefinite hangs in Docker environments with non-standard networking or DNS configurations. Now properly supports IPv4 and IPv6 configurations. (Fixes #952) - Thanks [@CodeBormen](https://github.com/CodeBormen) From b1376386d5cd8862668aea4c61b27c55e954884f Mon Sep 17 00:00:00 2001 From: Patrick McDonagh Date: Tue, 17 Feb 2026 09:50:50 -0600 Subject: [PATCH 048/503] docs: add legacy NumPy support instructions in docker-compose --- docker/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 519b288a..bb4397d4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -117,6 +117,11 @@ services: # Process Priority Configuration (Optional) #- CELERY_NICE_LEVEL=5 # Celery/EPG/Background tasks (default: 5, low priority; Range: -20 to 19) + # Legacy CPU Support (Optional) + # Uncomment to enable legacy NumPy build for older CPUs (circa 2009) + # that lack support for newer baseline CPU features: + #- USE_LEGACY_NUMPY=true + # Django Configuration - DJANGO_SETTINGS_MODULE=dispatcharr.settings - PYTHONUNBUFFERED=1 From 967704d7a00379e04c6326b87909e9d9dc3351eb Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 17 Feb 2026 10:59:19 -0600 Subject: [PATCH 049/503] changelog: Update changelog for modular celery container numpy detection pr. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26665f72..f7dadeef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Use Provider Number**: Use channel numbers from the M3U source (tvg-chno), with configurable fallback if provider number is missing - **Next Available**: Auto-assign starting from 1, skipping all used channel numbers Each mode includes its own configuration options accessible via the "Channel Numbering Mode" dropdown in auto sync settings. (Closes #956, #433) +- Legacy NumPy for modular Docker: Added entrypoint detection and automatic installation for the Celery container (use `USE_LEGACY_NUMPY`) to support older CPUs.- Thanks [@patrickjmcd](https://github.com/patrickjmcd) ### Changed From 47853bdb5bcf0132642c967dc4f7ae576f614a33 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 17 Feb 2026 12:00:24 -0600 Subject: [PATCH 050/503] Enhancement: Added optional subtitle template field to custom dummy EPG configuration. Users can now define subtitle patterns using extracted regex groups and time/date placeholders (e.g., `{starttime} - {endtime}`). (Closes #942) --- CHANGELOG.md | 3 +- apps/output/views.py | 31 +++++++++++++++ frontend/src/components/forms/DummyEPG.jsx | 44 ++++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7dadeef..da99fb5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Custom Dummy EPG subtitle template support: Added optional subtitle template field to custom dummy EPG configuration. Users can now define subtitle patterns using extracted regex groups and time/date placeholders (e.g., `{starttime} - {endtime}`). (Closes #942) - Event-driven webhooks and script execution (Connect): Added new Connect feature that enables event-driven execution of custom scripts and webhooks in response to system events. Supports multiple event types including channel lifecycle (start, stop, reconnect, error, failover), stream operations (switch), recording events (start, end), data refreshes (EPG, M3U), and client activity (connect, disconnect). Event data is available as environment variables in scripts (prefixed with `DISPATCHARR_`), POST payloads for webhooks, and plugin execution payloads. Plugins can now subscribe to events by specifying an `events` array in their action definitions. Includes connection testing endpoint with dummy payloads for validation. (Closes #203) - Cron scheduling support for M3U and EPG refreshes: Added interactive cron expression builder with preset buttons and custom field editors, plus info popover with common cron examples. Refactored backup scheduling to use shared ScheduleInput component for consistency across all scheduling interfaces. (Closes #165) - Channel numbering modes for auto channel sync: Added three channel numbering modes when auto-syncing channels from M3U groups: @@ -16,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Use Provider Number**: Use channel numbers from the M3U source (tvg-chno), with configurable fallback if provider number is missing - **Next Available**: Auto-assign starting from 1, skipping all used channel numbers Each mode includes its own configuration options accessible via the "Channel Numbering Mode" dropdown in auto sync settings. (Closes #956, #433) -- Legacy NumPy for modular Docker: Added entrypoint detection and automatic installation for the Celery container (use `USE_LEGACY_NUMPY`) to support older CPUs.- Thanks [@patrickjmcd](https://github.com/patrickjmcd) +- Legacy NumPy for modular Docker: Added entrypoint detection and automatic installation for the Celery container (use `USE_LEGACY_NUMPY`) to support older CPUs. - Thanks [@patrickjmcd](https://github.com/patrickjmcd) ### Changed diff --git a/apps/output/views.py b/apps/output/views.py index 21377489..21670e5a 100644 --- a/apps/output/views.py +++ b/apps/output/views.py @@ -511,6 +511,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust output_timezone_value = custom_properties.get('output_timezone', '') # Optional: display times in different timezone program_duration = custom_properties.get('program_duration', 180) # Minutes title_template = custom_properties.get('title_template', '') + subtitle_template = custom_properties.get('subtitle_template', '') description_template = custom_properties.get('description_template', '') # Templates for upcoming/ended programs @@ -916,6 +917,11 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust title_parts.append(all_groups['title']) main_event_title = ' - '.join(title_parts) if title_parts else channel_name + if subtitle_template: + main_event_subtitle = format_template(subtitle_template, all_groups) + else: + main_event_subtitle = None + if description_template: main_event_description = format_template(description_template, all_groups) else: @@ -966,6 +972,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust "start_time": program_start_utc, "end_time": program_end_utc, "title": upcoming_title, + "sub_title": None, # No subtitle for filler programs "description": upcoming_description, "custom_properties": program_custom_properties, "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation @@ -1005,6 +1012,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust "start_time": event_start_utc, "end_time": event_end_utc, "title": main_event_title, + "sub_title": main_event_subtitle, "description": main_event_description, "custom_properties": main_event_custom_properties, "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation @@ -1049,6 +1057,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust "start_time": program_start_utc, "end_time": program_end_utc, "title": ended_title, + "sub_title": None, # No subtitle for filler programs "description": ended_description, "custom_properties": program_custom_properties, "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation @@ -1109,6 +1118,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust "start_time": program_start_utc, "end_time": program_end_utc, "title": program_title, + "sub_title": None, # No subtitle for filler programs "description": program_description, "custom_properties": program_custom_properties, "channel_logo_url": channel_logo_url, @@ -1136,6 +1146,11 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust title_parts.append(all_groups['title']) title = ' - '.join(title_parts) if title_parts else channel_name + if subtitle_template: + subtitle = format_template(subtitle_template, all_groups) + else: + subtitle = None + if description_template: description = format_template(description_template, all_groups) else: @@ -1172,6 +1187,7 @@ def generate_custom_dummy_programs(channel_id, channel_name, now, num_days, cust "start_time": program_start_utc, "end_time": program_end_utc, "title": title, + "sub_title": subtitle, "description": description, "custom_properties": program_custom_properties, "channel_logo_url": channel_logo_url, # Pass channel logo for EPG generation @@ -1211,6 +1227,11 @@ def generate_dummy_epg( f' ' ) xml_lines.append(f" {html.escape(program['title'])}") + + # Add subtitle if available + if program.get('sub_title'): + xml_lines.append(f" {html.escape(program['sub_title'])}") + xml_lines.append(f" {html.escape(program['description'])}") # Add custom_properties if present @@ -1530,6 +1551,11 @@ def generate_epg(request, profile_name=None, user=None): # Create program entry with escaped channel name yield f' \n' yield f" {html.escape(program['title'])}\n" + + # Add subtitle if available + if program.get('sub_title'): + yield f" {html.escape(program['sub_title'])}\n" + yield f" {html.escape(program['description'])}\n" # Add custom_properties if present @@ -1579,6 +1605,11 @@ def generate_epg(request, profile_name=None, user=None): yield f' \n' yield f" {html.escape(program['title'])}\n" + + # Add subtitle if available + if program.get('sub_title'): + yield f" {html.escape(program['sub_title'])}\n" + yield f" {html.escape(program['description'])}\n" # Add custom_properties if present diff --git a/frontend/src/components/forms/DummyEPG.jsx b/frontend/src/components/forms/DummyEPG.jsx index 9f9346da..06731ac9 100644 --- a/frontend/src/components/forms/DummyEPG.jsx +++ b/frontend/src/components/forms/DummyEPG.jsx @@ -43,6 +43,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { const [datePattern, setDatePattern] = useState(''); const [sampleTitle, setSampleTitle] = useState(''); const [titleTemplate, setTitleTemplate] = useState(''); + const [subtitleTemplate, setSubtitleTemplate] = useState(''); const [descriptionTemplate, setDescriptionTemplate] = useState(''); const [upcomingTitleTemplate, setUpcomingTitleTemplate] = useState(''); const [upcomingDescriptionTemplate, setUpcomingDescriptionTemplate] = @@ -71,6 +72,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { program_duration: 180, sample_title: '', title_template: '', + subtitle_template: '', description_template: '', upcoming_title_template: '', upcoming_description_template: '', @@ -125,6 +127,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { dateGroups: {}, calculatedPlaceholders: {}, formattedTitle: '', + formattedSubtitle: '', formattedDescription: '', formattedUpcomingTitle: '', formattedUpcomingDescription: '', @@ -459,6 +462,14 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { ); } + // Format subtitle template + if (subtitleTemplate && (result.titleMatch || result.timeMatch)) { + result.formattedSubtitle = subtitleTemplate.replace( + /\{(\w+)\}/g, + (match, key) => allGroups[key] || match + ); + } + // Format description template if (descriptionTemplate && (result.titleMatch || result.timeMatch)) { result.formattedDescription = descriptionTemplate.replace( @@ -533,6 +544,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { datePattern, sampleTitle, titleTemplate, + subtitleTemplate, descriptionTemplate, upcomingTitleTemplate, upcomingDescriptionTemplate, @@ -565,6 +577,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { program_duration: custom.program_duration || 180, sample_title: custom.sample_title || '', title_template: custom.title_template || '', + subtitle_template: custom.subtitle_template || '', description_template: custom.description_template || '', upcoming_title_template: custom.upcoming_title_template || '', upcoming_description_template: @@ -591,6 +604,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { setDatePattern(custom.date_pattern || ''); setSampleTitle(custom.sample_title || ''); setTitleTemplate(custom.title_template || ''); + setSubtitleTemplate(custom.subtitle_template || ''); setDescriptionTemplate(custom.description_template || ''); setUpcomingTitleTemplate(custom.upcoming_title_template || ''); setUpcomingDescriptionTemplate( @@ -611,6 +625,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { setDatePattern(''); setSampleTitle(''); setTitleTemplate(''); + setSubtitleTemplate(''); setDescriptionTemplate(''); setUpcomingTitleTemplate(''); setUpcomingDescriptionTemplate(''); @@ -682,6 +697,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { program_duration: custom.program_duration || 180, sample_title: custom.sample_title || '', title_template: custom.title_template || '', + subtitle_template: custom.subtitle_template || '', description_template: custom.description_template || '', upcoming_title_template: custom.upcoming_title_template || '', upcoming_description_template: @@ -708,6 +724,7 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { setDatePattern(custom.date_pattern || ''); setSampleTitle(custom.sample_title || ''); setTitleTemplate(custom.title_template || ''); + setSubtitleTemplate(custom.subtitle_template || ''); setDescriptionTemplate(custom.description_template || ''); setUpcomingTitleTemplate(custom.upcoming_title_template || ''); setUpcomingDescriptionTemplate(custom.upcoming_description_template || ''); @@ -904,6 +921,20 @@ const DummyEPGForm = ({ epg, isOpen, onClose }) => { }} /> + { + const value = e.target.value; + setSubtitleTemplate(value); + form.setFieldValue('custom_properties.subtitle_template', value); + }} + /> +