From 50e9075bb50f99bff878266ebf8648da5bb72f51 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 25 Oct 2025 08:15:39 -0400 Subject: [PATCH 01/67] initial run of a binary and encoded redis client - no more encoding / decoding data into redis, huge PITA (still some outstanding spots I need to patch) --- apps/channels/api_views.py | 6 +- apps/channels/tasks.py | 4 +- apps/proxy/tasks.py | 2 +- apps/proxy/ts_proxy/channel_status.py | 208 ++++++------------ apps/proxy/ts_proxy/server.py | 75 +++---- .../ts_proxy/services/channel_service.py | 20 +- apps/proxy/ts_proxy/stream_manager.py | 14 +- apps/proxy/ts_proxy/url_utils.py | 8 +- .../multi_worker_connection_manager.py | 14 +- apps/proxy/vod_proxy/views.py | 16 +- core/redis_pubsub.py | 4 - core/tasks.py | 2 +- core/utils.py | 192 ++++++++-------- dispatcharr/persistent_lock.py | 2 +- 14 files changed, 253 insertions(+), 314 deletions(-) diff --git a/apps/channels/api_views.py b/apps/channels/api_views.py index 862de7f9..aa25ca85 100644 --- a/apps/channels/api_views.py +++ b/apps/channels/api_views.py @@ -1849,14 +1849,12 @@ class RecordingViewSet(viewsets.ModelViewSet): client_set_key = RedisKeys.clients(channel_uuid) client_ids = r.smembers(client_set_key) or [] stopped = 0 - for raw_id in client_ids: + for cid in client_ids: try: - cid = raw_id.decode("utf-8") if isinstance(raw_id, (bytes, bytearray)) else str(raw_id) meta_key = RedisKeys.client_metadata(channel_uuid, cid) ua = r.hget(meta_key, "user_agent") - ua_s = ua.decode("utf-8") if isinstance(ua, (bytes, bytearray)) else (ua or "") # Identify DVR recording client by its user agent - if ua_s and "Dispatcharr-DVR" in ua_s: + if ua and "Dispatcharr-DVR" in ua: try: ChannelService.stop_client(channel_uuid, cid) stopped += 1 diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 3943cf16..28da31b3 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1875,14 +1875,14 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str): md = r.hgetall(metadata_key) if md: def _gv(bkey): - return md.get(bkey.encode('utf-8')) + return md.get(bkey) def _d(bkey, cast=str): v = _gv(bkey) try: if v is None: return None - s = v.decode('utf-8') + s = v return cast(s) if cast is not str else s except Exception: return None diff --git a/apps/proxy/tasks.py b/apps/proxy/tasks.py index 68843712..d42e4d9a 100644 --- a/apps/proxy/tasks.py +++ b/apps/proxy/tasks.py @@ -31,7 +31,7 @@ def fetch_channel_stats(): while True: cursor, keys = redis_client.scan(cursor, match=channel_pattern) for key in keys: - channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8')) + channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key) if channel_id_match: ch_id = channel_id_match.group(1) channel_info = ChannelStatus.get_basic_channel_info(ch_id) diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 8f1d0649..015d5ca2 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -38,19 +38,19 @@ class ChannelStatus: info = { 'channel_id': channel_id, - 'state': metadata.get(ChannelMetadataField.STATE.encode('utf-8'), b'unknown').decode('utf-8'), - 'url': metadata.get(ChannelMetadataField.URL.encode('utf-8'), b'').decode('utf-8'), - 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8'), b'').decode('utf-8'), - 'started_at': metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0').decode('utf-8'), - 'owner': metadata.get(ChannelMetadataField.OWNER.encode('utf-8'), b'unknown').decode('utf-8'), - 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, + 'state': metadata.get(ChannelMetadataField.STATE, 'unknown'), + 'url': metadata.get(ChannelMetadataField.URL, ''), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ''), + 'started_at': metadata.get(ChannelMetadataField.INIT_TIME, '0'), + 'owner': metadata.get(ChannelMetadataField.OWNER, 'unknown'), + 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, } # Add stream ID and name information - stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8')) + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: - stream_id = int(stream_id_bytes.decode('utf-8')) + stream_id = int(stream_id_bytes) info['stream_id'] = stream_id # Look up stream name from database @@ -65,10 +65,10 @@ class ChannelStatus: logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}") # Add M3U profile information - m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8')) + m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE) if m3u_profile_id_bytes: try: - m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8')) + m3u_profile_id = int(m3u_profile_id_bytes) info['m3u_profile_id'] = m3u_profile_id # Look up M3U profile name from database @@ -83,22 +83,22 @@ class ChannelStatus: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}") # Add timing information - state_changed_field = ChannelMetadataField.STATE_CHANGED_AT.encode('utf-8') + state_changed_field = ChannelMetadataField.STATE_CHANGED_AT if state_changed_field in metadata: - state_changed_at = float(metadata[state_changed_field].decode('utf-8')) + state_changed_at = float(metadata[state_changed_field]) info['state_changed_at'] = state_changed_at info['state_duration'] = time.time() - state_changed_at - init_time_field = ChannelMetadataField.INIT_TIME.encode('utf-8') + init_time_field = ChannelMetadataField.INIT_TIME if init_time_field in metadata: - created_at = float(metadata[init_time_field].decode('utf-8')) + created_at = float(metadata[init_time_field]) info['started_at'] = created_at info['uptime'] = time.time() - created_at # Add data throughput information - total_bytes_field = ChannelMetadataField.TOTAL_BYTES.encode('utf-8') + total_bytes_field = ChannelMetadataField.TOTAL_BYTES if total_bytes_field in metadata: - total_bytes = int(metadata[total_bytes_field].decode('utf-8')) + total_bytes = int(metadata[total_bytes_field]) info['total_bytes'] = total_bytes # Format total bytes in human-readable form @@ -128,40 +128,40 @@ class ChannelStatus: clients = [] for client_id in client_ids: - client_id_str = client_id.decode('utf-8') + client_id_str = client_id client_key = RedisKeys.client_metadata(channel_id, client_id_str) client_data = proxy_server.redis_client.hgetall(client_key) if client_data: client_info = { 'client_id': client_id_str, - 'user_agent': client_data.get(b'user_agent', b'unknown').decode('utf-8'), - 'worker_id': client_data.get(b'worker_id', b'unknown').decode('utf-8'), + 'user_agent': client_data.get('user_agent', 'unknown'), + 'worker_id': client_data.get('worker_id', 'unknown'), } - if b'connected_at' in client_data: - connected_at = float(client_data[b'connected_at'].decode('utf-8')) + if 'connected_at' in client_data: + connected_at = float(client_data['connected_at']) client_info['connected_at'] = connected_at client_info['connection_duration'] = time.time() - connected_at - if b'last_active' in client_data: - last_active = float(client_data[b'last_active'].decode('utf-8')) + if 'last_active' in client_data: + last_active = float(client_data['last_active']) client_info['last_active'] = last_active client_info['last_active_ago'] = time.time() - last_active # Add transfer rate statistics - if b'bytes_sent' in client_data: - client_info['bytes_sent'] = int(client_data[b'bytes_sent'].decode('utf-8')) + if 'bytes_sent' in client_data: + client_info['bytes_sent'] = int(client_data['bytes_sent']) # Add average transfer rate - if b'avg_rate_KBps' in client_data: - client_info['avg_rate_KBps'] = float(client_data[b'avg_rate_KBps'].decode('utf-8')) - elif b'transfer_rate_KBps' in client_data: # For backward compatibility - client_info['avg_rate_KBps'] = float(client_data[b'transfer_rate_KBps'].decode('utf-8')) + if 'avg_rate_KBps' in client_data: + client_info['avg_rate_KBps'] = float(client_data['avg_rate_KBps']) + elif 'transfer_rate_KBps' in client_data: # For backward compatibility + client_info['avg_rate_KBps'] = float(client_data['transfer_rate_KBps']) # Add current transfer rate - if b'current_rate_KBps' in client_data: - client_info['current_rate_KBps'] = float(client_data[b'current_rate_KBps'].decode('utf-8')) + if 'current_rate_KBps' in client_data: + client_info['current_rate_KBps'] = float(client_data['current_rate_KBps']) clients.append(client_info) @@ -235,7 +235,7 @@ class ChannelStatus: while True: cursor, keys = proxy_server.redis_client.scan(cursor, match=buffer_key_pattern, count=100) if keys: - all_buffer_keys.extend([k.decode('utf-8') for k in keys]) + all_buffer_keys.extend([k for k in keys]) if cursor == 0 or len(all_buffer_keys) >= 20: # Limit to 20 keys break @@ -265,61 +265,22 @@ class ChannelStatus: } # Add FFmpeg stream information - video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8')) - if video_codec: - info['video_codec'] = video_codec.decode('utf-8') - - resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8')) - if resolution: - info['resolution'] = resolution.decode('utf-8') - - source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8')) - if source_fps: - info['source_fps'] = float(source_fps.decode('utf-8')) - - pixel_format = metadata.get(ChannelMetadataField.PIXEL_FORMAT.encode('utf-8')) - if pixel_format: - info['pixel_format'] = pixel_format.decode('utf-8') - - source_bitrate = metadata.get(ChannelMetadataField.SOURCE_BITRATE.encode('utf-8')) - if source_bitrate: - info['source_bitrate'] = float(source_bitrate.decode('utf-8')) - - audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8')) - if audio_codec: - info['audio_codec'] = audio_codec.decode('utf-8') - - sample_rate = metadata.get(ChannelMetadataField.SAMPLE_RATE.encode('utf-8')) - if sample_rate: - info['sample_rate'] = int(sample_rate.decode('utf-8')) - - audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8')) - if audio_channels: - info['audio_channels'] = audio_channels.decode('utf-8') - - audio_bitrate = metadata.get(ChannelMetadataField.AUDIO_BITRATE.encode('utf-8')) - if audio_bitrate: - info['audio_bitrate'] = float(audio_bitrate.decode('utf-8')) + info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) + info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) + info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) + info['pixel_format'] = metadata.get(ChannelMetadataField.PIXEL_FORMAT) + info['source_bitrate'] = metadata.get(ChannelMetadataField.SOURCE_BITRATE) + info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) + info['sample_rate'] = metadata.get(ChannelMetadataField.SAMPLE_RATE) + info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + info['audio_bitrate'] = metadata.get(ChannelMetadataField.AUDIO_BITRATE) # Add FFmpeg performance stats - ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8')) - if ffmpeg_speed: - info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8')) - - ffmpeg_fps = metadata.get(ChannelMetadataField.FFMPEG_FPS.encode('utf-8')) - if ffmpeg_fps: - info['ffmpeg_fps'] = float(ffmpeg_fps.decode('utf-8')) - - actual_fps = metadata.get(ChannelMetadataField.ACTUAL_FPS.encode('utf-8')) - if actual_fps: - info['actual_fps'] = float(actual_fps.decode('utf-8')) - - ffmpeg_bitrate = metadata.get(ChannelMetadataField.FFMPEG_BITRATE.encode('utf-8')) - if ffmpeg_bitrate: - info['ffmpeg_bitrate'] = float(ffmpeg_bitrate.decode('utf-8')) - stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8')) - if stream_type: - info['stream_type'] = stream_type.decode('utf-8') + info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + info['ffmpeg_fps'] = metadata.get(ChannelMetadataField.FFMPEG_FPS) + info['actual_fps'] = metadata.get(ChannelMetadataField.ACTUAL_FPS) + info['ffmpeg_bitrate'] = metadata.get(ChannelMetadataField.FFMPEG_BITRATE) + info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) return info @@ -364,33 +325,27 @@ class ChannelStatus: client_count = proxy_server.redis_client.scard(client_set_key) or 0 # Calculate uptime - init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME.encode('utf-8'), b'0') - created_at = float(init_time_bytes.decode('utf-8')) + init_time_bytes = metadata.get(ChannelMetadataField.INIT_TIME, '0') + created_at = float(init_time_bytes) uptime = time.time() - created_at if created_at > 0 else 0 - # Safely decode bytes or use defaults - def safe_decode(bytes_value, default="unknown"): - if bytes_value is None: - return default - return bytes_value.decode('utf-8') - # Simplified info info = { 'channel_id': channel_id, - 'state': safe_decode(metadata.get(ChannelMetadataField.STATE.encode('utf-8'))), - 'url': safe_decode(metadata.get(ChannelMetadataField.URL.encode('utf-8')), ""), - 'stream_profile': safe_decode(metadata.get(ChannelMetadataField.STREAM_PROFILE.encode('utf-8')), ""), - 'owner': safe_decode(metadata.get(ChannelMetadataField.OWNER.encode('utf-8'))), - 'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0, + 'state': metadata.get(ChannelMetadataField.STATE), + 'url': metadata.get(ChannelMetadataField.URL, ""), + 'stream_profile': metadata.get(ChannelMetadataField.STREAM_PROFILE, ""), + 'owner': metadata.get(ChannelMetadataField.OWNER), + 'buffer_index': int(buffer_index_value) if buffer_index_value else 0, 'client_count': client_count, 'uptime': uptime } # Add stream ID and name information - stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID.encode('utf-8')) + stream_id_bytes = metadata.get(ChannelMetadataField.STREAM_ID) if stream_id_bytes: try: - stream_id = int(stream_id_bytes.decode('utf-8')) + stream_id = int(stream_id_bytes) info['stream_id'] = stream_id # Look up stream name from database @@ -405,9 +360,9 @@ class ChannelStatus: logger.warning(f"Invalid stream_id format in Redis: {stream_id_bytes}") # Add data throughput information to basic info - total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES.encode('utf-8')) + total_bytes_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.TOTAL_BYTES) if total_bytes_bytes: - total_bytes = int(total_bytes_bytes.decode('utf-8')) + total_bytes = int(total_bytes_bytes) info['total_bytes'] = total_bytes # Calculate and add bitrate @@ -434,26 +389,25 @@ class ChannelStatus: if client_ids: # Get up to 10 clients for the basic view for client_id in list(client_ids)[:10]: - client_id_str = client_id.decode('utf-8') - client_key = RedisKeys.client_metadata(channel_id, client_id_str) + client_key = RedisKeys.client_metadata(channel_id, client_id) # Efficient way - just retrieve the essentials client_info = { - 'client_id': client_id_str, + 'client_id': client_id, } # Safely get user_agent and ip_address user_agent_bytes = proxy_server.redis_client.hget(client_key, 'user_agent') - client_info['user_agent'] = safe_decode(user_agent_bytes) + client_info['user_agent'] = user_agent_bytes ip_address_bytes = proxy_server.redis_client.hget(client_key, 'ip_address') if ip_address_bytes: - client_info['ip_address'] = safe_decode(ip_address_bytes) + client_info['ip_address'] = ip_address_bytes # Just get connected_at for client age connected_at_bytes = proxy_server.redis_client.hget(client_key, 'connected_at') if connected_at_bytes: - connected_at = float(connected_at_bytes.decode('utf-8')) + connected_at = float(connected_at_bytes) client_info['connected_since'] = time.time() - connected_at clients.append(client_info) @@ -462,10 +416,10 @@ class ChannelStatus: info['clients'] = clients # Add M3U profile information - m3u_profile_id_bytes = metadata.get(ChannelMetadataField.M3U_PROFILE.encode('utf-8')) - if m3u_profile_id_bytes: + m3u_profile_id = metadata.get(ChannelMetadataField.M3U_PROFILE) + if m3u_profile_id: try: - m3u_profile_id = int(m3u_profile_id_bytes.decode('utf-8')) + m3u_profile_id = int(m3u_profile_id) info['m3u_profile_id'] = m3u_profile_id # Look up M3U profile name from database @@ -477,32 +431,16 @@ class ChannelStatus: except (ImportError, DatabaseError) as e: logger.warning(f"Failed to get M3U profile name for ID {m3u_profile_id}: {e}") except ValueError: - logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id_bytes}") + logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") # Add stream info to basic info as well - video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC.encode('utf-8')) - if video_codec: - info['video_codec'] = video_codec.decode('utf-8') - - resolution = metadata.get(ChannelMetadataField.RESOLUTION.encode('utf-8')) - if resolution: - info['resolution'] = resolution.decode('utf-8') - - source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS.encode('utf-8')) - if source_fps: - info['source_fps'] = float(source_fps.decode('utf-8')) - ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED.encode('utf-8')) - if ffmpeg_speed: - info['ffmpeg_speed'] = float(ffmpeg_speed.decode('utf-8')) - audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC.encode('utf-8')) - if audio_codec: - info['audio_codec'] = audio_codec.decode('utf-8') - audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS.encode('utf-8')) - if audio_channels: - info['audio_channels'] = audio_channels.decode('utf-8') - stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE.encode('utf-8')) - if stream_type: - info['stream_type'] = stream_type.decode('utf-8') + info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) + info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) + info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) + info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) + info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) return info except Exception as e: diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index cca827a9..7a59f530 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -157,7 +157,8 @@ class ProxyServer: socket_timeout=60, socket_connect_timeout=10, socket_keepalive=True, - health_check_interval=30 + health_check_interval=30, + decode_responses=True ) logger.info("Created fallback Redis PubSub client for event listener") @@ -178,8 +179,8 @@ class ProxyServer: continue try: - channel = message["channel"].decode("utf-8") - data = json.loads(message["data"].decode("utf-8")) + channel = message["channel"] + data = json.loads(message["data"]) event_type = data.get("event") channel_id = data.get("channel_id") @@ -374,7 +375,7 @@ class ProxyServer: try: lock_key = RedisKeys.channel_owner(channel_id) return self._execute_redis_command( - lambda: self.redis_client.get(lock_key).decode('utf-8') if self.redis_client.get(lock_key) else None + lambda: self.redis_client.get(lock_key) if self.redis_client.get(lock_key) else None ) except Exception as e: logger.error(f"Error getting channel owner: {e}") @@ -415,7 +416,7 @@ class ProxyServer: current_owner = self._execute_redis_command( lambda: self.redis_client.get(lock_key) ) - if current_owner and current_owner.decode('utf-8') == self.worker_id: + if current_owner and current_owner == self.worker_id: # Refresh TTL self._execute_redis_command( lambda: self.redis_client.expire(lock_key, ttl) @@ -440,7 +441,7 @@ class ProxyServer: # Only delete if we're the current owner to prevent race conditions current = self.redis_client.get(lock_key) - if current and current.decode('utf-8') == self.worker_id: + if current and current == self.worker_id: self.redis_client.delete(lock_key) logger.info(f"Released ownership of channel {channel_id}") @@ -462,7 +463,7 @@ class ProxyServer: current = self.redis_client.get(lock_key) # Only extend if we're still the owner - if current and current.decode('utf-8') == self.worker_id: + if current and current == self.worker_id: self.redis_client.expire(lock_key, ttl) return True return False @@ -478,15 +479,15 @@ class ProxyServer: metadata_key = RedisKeys.channel_metadata(channel_id) if self.redis_client.exists(metadata_key): metadata = self.redis_client.hgetall(metadata_key) - if b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if 'state' in metadata: + state = metadata['state'] active_states = [ChannelState.INITIALIZING, ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS, ChannelState.ACTIVE, ChannelState.BUFFERING] if state in active_states: logger.info(f"Channel {channel_id} already being initialized with state {state}") # Create buffer and client manager only if we don't have them if channel_id not in self.stream_buffers: - self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=self.redis_client) + self.stream_buffers[channel_id] = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer()) if channel_id not in self.client_managers: self.client_managers[channel_id] = ClientManager( channel_id, @@ -497,7 +498,7 @@ class ProxyServer: # Create buffer and client manager instances (or reuse if they exist) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer if channel_id not in self.client_managers: @@ -536,18 +537,18 @@ class ProxyServer: # If no url was passed, try to get from Redis if not url and existing_metadata: - url_bytes = existing_metadata.get(b'url') + url_bytes = existing_metadata.get('url') if url_bytes: - channel_url = url_bytes.decode('utf-8') + channel_url = url_bytes - ua_bytes = existing_metadata.get(b'user_agent') + ua_bytes = existing_metadata.get('user_agent') if ua_bytes: - channel_user_agent = ua_bytes.decode('utf-8') + channel_user_agent = ua_bytes # Get stream ID from metadata if not provided - if not channel_stream_id and b'stream_id' in existing_metadata: + if not channel_stream_id and 'stream_id' in existing_metadata: try: - channel_stream_id = int(existing_metadata[b'stream_id'].decode('utf-8')) + channel_stream_id = int(existing_metadata['stream_id']) logger.debug(f"Found stream_id {channel_stream_id} in metadata for channel {channel_id}") except (ValueError, TypeError) as e: logger.debug(f"Could not parse stream_id from metadata: {e}") @@ -562,7 +563,7 @@ class ProxyServer: # Create buffer but not stream manager (only if not already exists) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer # Create client manager with channel_id and redis_client (only if not already exists) @@ -585,7 +586,7 @@ class ProxyServer: # Create buffer but not stream manager (only if not already exists) if channel_id not in self.stream_buffers: - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) self.stream_buffers[channel_id] = buffer # Create client manager with channel_id and redis_client (only if not already exists) @@ -624,12 +625,12 @@ class ProxyServer: # Verify the stream_id was set correctly in Redis stream_id_value = self.redis_client.hget(metadata_key, "stream_id") if stream_id_value: - logger.info(f"Verified stream_id {stream_id_value.decode('utf-8')} is set in Redis for channel {channel_id}") + logger.info(f"Verified stream_id {stream_id_value} is set in Redis for channel {channel_id}") else: logger.warning(f"Failed to set stream_id in Redis for channel {channel_id}") # Create stream buffer - buffer = StreamBuffer(channel_id=channel_id, redis_client=self.redis_client) + buffer = StreamBuffer(channel_id=channel_id, redis_client=RedisClient.get_buffer()) logger.debug(f"Created StreamBuffer for channel {channel_id}") self.stream_buffers[channel_id] = buffer @@ -700,8 +701,8 @@ class ProxyServer: metadata = self.redis_client.hgetall(metadata_key) # Get channel state and owner - state = metadata.get(b'state', b'unknown').decode('utf-8') - owner = metadata.get(b'owner', b'').decode('utf-8') + state = metadata.get('state', 'unknown') + owner = metadata.get('owner', '') # States that indicate the channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, @@ -726,8 +727,8 @@ class ProxyServer: return True else: # Unknown or initializing state, check how long it's been in this state - if b'state_changed_at' in metadata: - state_changed_at = float(metadata[b'state_changed_at'].decode('utf-8')) + if 'state_changed_at' in metadata: + state_changed_at = float(metadata['state_changed_at']) state_age = time.time() - state_changed_at # If in initializing state for too long, consider it stale @@ -762,8 +763,8 @@ class ProxyServer: # If we have metadata, log details for debugging if metadata: - state = metadata.get(b'state', b'unknown').decode('utf-8') - owner = metadata.get(b'owner', b'unknown').decode('utf-8') + state = metadata.get('state', 'unknown') + owner = metadata.get('owner', 'unknown') logger.info(f"Zombie channel details - state: {state}, owner: {owner}") # Clean up Redis keys @@ -931,8 +932,8 @@ class ProxyServer: if self.redis_client: metadata_key = RedisKeys.channel_metadata(channel_id) metadata = self.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - channel_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + channel_state = metadata['state'] # Check if channel has any clients left total_clients = 0 @@ -948,9 +949,9 @@ class ProxyServer: if channel_state in [ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS]: # Get connection ready time from metadata connection_ready_time = None - if metadata and b'connection_ready_time' in metadata: + if metadata and 'connection_ready_time' in metadata: try: - connection_ready_time = float(metadata[b'connection_ready_time'].decode('utf-8')) + connection_ready_time = float(metadata['connection_ready_time']) except (ValueError, TypeError): pass @@ -981,8 +982,8 @@ class ProxyServer: # Grace period expired but we have clients - mark channel as active logger.info(f"Grace period expired with {total_clients} clients - marking channel {channel_id} as active") old_state = "unknown" - if metadata and b'state' in metadata: - old_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + old_state = metadata['state'] if self.update_channel_state(channel_id, ChannelState.ACTIVE, { "grace_period_ended_at": str(time.time()), "clients_at_activation": str(total_clients) @@ -998,7 +999,7 @@ class ProxyServer: disconnect_value = self.redis_client.get(disconnect_key) if disconnect_value: try: - disconnect_time = float(disconnect_value.decode('utf-8')) + disconnect_time = float(disconnect_value) except (ValueError, TypeError) as e: logger.error(f"Invalid disconnect time for channel {channel_id}: {e}") @@ -1076,7 +1077,7 @@ class ProxyServer: for key in channel_keys: try: - channel_id = key.decode('utf-8').split(':')[2] + channel_id = key.split(':')[2] # Skip channels we already have locally if channel_id in self.stream_buffers: @@ -1170,8 +1171,8 @@ class ProxyServer: # Get current state for logging current_state = None metadata = self.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - current_state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + current_state = metadata['state'] # Only update if state is actually changing if current_state == new_state: diff --git a/apps/proxy/ts_proxy/services/channel_service.py b/apps/proxy/ts_proxy/services/channel_service.py index 551e2d27..6e7d3878 100644 --- a/apps/proxy/ts_proxy/services/channel_service.py +++ b/apps/proxy/ts_proxy/services/channel_service.py @@ -59,7 +59,7 @@ class ChannelService: # Verify the stream_id was set stream_id_value = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) if stream_id_value: - logger.debug(f"Verified stream_id {stream_id_value.decode('utf-8')} is now set in Redis") + logger.debug(f"Verified stream_id {stream_id_value} is now set in Redis") else: logger.error(f"Failed to set stream_id {stream_id} in Redis before initialization") @@ -129,7 +129,7 @@ class ChannelService: try: # This is inefficient but used for diagnostics - in production would use more targeted checks redis_keys = proxy_server.redis_client.keys(f"ts_proxy:*:{channel_id}*") - redis_keys = [k.decode('utf-8') for k in redis_keys] if redis_keys else [] + redis_keys = [k for k in redis_keys] if redis_keys else [] except Exception as e: logger.error(f"Error checking Redis keys: {e}") @@ -234,8 +234,8 @@ class ChannelService: metadata_key = RedisKeys.channel_metadata(channel_id) try: metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] channel_info = {"state": state} # Immediately mark as stopping in metadata so clients detect it faster @@ -375,8 +375,8 @@ class ChannelService: metadata = proxy_server.redis_client.hgetall(metadata_key) # Extract state and owner - state = metadata.get(ChannelMetadataField.STATE.encode(), b'unknown').decode('utf-8') - owner = metadata.get(ChannelMetadataField.OWNER.encode(), b'unknown').decode('utf-8') + state = metadata.get(ChannelMetadataField.STATE.encode(), 'unknown') + owner = metadata.get(ChannelMetadataField.OWNER.encode(), 'unknown') # Valid states indicate channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING] @@ -402,7 +402,7 @@ class ChannelService: } if last_data: - last_data_time = float(last_data.decode('utf-8')) + last_data_time = float(last_data) data_age = time.time() - last_data_time details["last_data_age"] = data_age @@ -598,7 +598,7 @@ class ChannelService: def _update_stream_stats_in_db(stream_id, **stats): """Update stream stats in database""" from django.db import connection - + try: from apps.channels.models import Stream from django.utils import timezone @@ -624,7 +624,7 @@ class ChannelService: except Exception as e: logger.error(f"Error updating stream stats in database for stream {stream_id}: {e}") return False - + finally: # Always close database connection after update try: @@ -645,7 +645,7 @@ class ChannelService: metadata_key = RedisKeys.channel_metadata(channel_id) # First check if the key exists and what type it is - key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8') + key_type = proxy_server.redis_client.type(metadata_key) logger.debug(f"Redis key {metadata_key} is of type: {key_type}") # Build metadata update dict diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index 99ae8027..73118768 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -89,7 +89,7 @@ class StreamManager: # Try to get stream_id specifically stream_id_bytes = buffer.redis_client.hget(metadata_key, "stream_id") if stream_id_bytes: - self.current_stream_id = int(stream_id_bytes.decode('utf-8')) + self.current_stream_id = int(stream_id_bytes) self.tried_stream_ids.add(self.current_stream_id) logger.info(f"Loaded stream ID {self.current_stream_id} from Redis for channel {buffer.channel_id}") else: @@ -362,7 +362,7 @@ class StreamManager: current_owner = self.buffer.redis_client.get(owner_key) # Use the worker_id that was passed in during initialization - if current_owner and self.worker_id and current_owner.decode('utf-8') == self.worker_id: + if current_owner and self.worker_id and current_owner == self.worker_id: # Determine the appropriate error message based on retry failures if self.tried_stream_ids and len(self.tried_stream_ids) > 0: error_message = f"All {len(self.tried_stream_ids)} stream options failed" @@ -948,10 +948,10 @@ class StreamManager: logger.debug(f"Updated m3u profile for channel {self.channel_id} to use profile from stream {stream_id}") else: logger.warning(f"Failed to update stream profile for channel {self.channel_id}") - + except Exception as e: logger.error(f"Error updating stream profile for channel {self.channel_id}: {e}") - + finally: # Always close database connection after profile update try: @@ -1348,9 +1348,9 @@ class StreamManager: current_state = None try: metadata = redis_client.hgetall(metadata_key) - state_field = ChannelMetadataField.STATE.encode('utf-8') + state_field = ChannelMetadataField.STATE if metadata and state_field in metadata: - current_state = metadata[state_field].decode('utf-8') + current_state = metadata[state_field] except Exception as e: logger.error(f"Error checking current state: {e}") @@ -1555,4 +1555,4 @@ class StreamManager: """Safely reset the URL switching state if it gets stuck""" self.url_switching = False self.url_switch_start_time = 0 - logger.info(f"Reset URL switching state for channel {self.channel_id}") \ No newline at end of file + logger.info(f"Reset URL switching state for channel {self.channel_id}") diff --git a/apps/proxy/ts_proxy/url_utils.py b/apps/proxy/ts_proxy/url_utils.py index db53cc74..0bfde312 100644 --- a/apps/proxy/ts_proxy/url_utils.py +++ b/apps/proxy/ts_proxy/url_utils.py @@ -215,9 +215,9 @@ def get_stream_info_for_switch(channel_id: str, target_stream_id: Optional[int] existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id.decode('utf-8') + existing_stream_id = existing_stream_id existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") - if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id: + if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True logger.debug(f"Channel {channel.id} already using profile {profile.id}") @@ -353,9 +353,9 @@ def get_alternate_streams(channel_id: str, current_stream_id: Optional[int] = No existing_stream_id = redis_client.get(f"channel_stream:{channel.id}") if existing_stream_id: # Decode bytes to string/int for proper Redis key lookup - existing_stream_id = existing_stream_id.decode('utf-8') + existing_stream_id = existing_stream_id existing_profile_id = redis_client.get(f"stream_profile:{existing_stream_id}") - if existing_profile_id and int(existing_profile_id.decode('utf-8')) == profile.id: + if existing_profile_id and int(existing_profile_id) == profile.id: channel_using_profile = True logger.debug(f"Channel {channel.id} already using profile {profile.id}") diff --git a/apps/proxy/vod_proxy/multi_worker_connection_manager.py b/apps/proxy/vod_proxy/multi_worker_connection_manager.py index fefc8739..e1fe4c4b 100644 --- a/apps/proxy/vod_proxy/multi_worker_connection_manager.py +++ b/apps/proxy/vod_proxy/multi_worker_connection_manager.py @@ -219,7 +219,7 @@ class RedisBackedVODConnection: # Convert bytes keys/values to strings if needed if isinstance(list(data.keys())[0], bytes): - data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()} + data = {k: v for k, v in data.items()} return SerializableConnectionState.from_dict(data) except Exception as e: @@ -1115,14 +1115,14 @@ class MultiWorkerVODConnectionManager: # Convert bytes to strings if needed if isinstance(list(data.keys())[0], bytes): - data = {k.decode('utf-8'): v.decode('utf-8') for k, v in data.items()} + data = {k: v for k, v in data.items()} last_activity = float(data.get('last_activity', 0)) active_streams = int(data.get('active_streams', 0)) # Clean up if stale and no active streams if (current_time - last_activity > max_age_seconds) and active_streams == 0: - session_id = key.decode('utf-8').replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') logger.info(f"Cleaning up stale connection: {session_id}") # Clean up connection and related keys @@ -1219,7 +1219,7 @@ class MultiWorkerVODConnectionManager: if connection_data: # Convert bytes to strings if needed if isinstance(list(connection_data.keys())[0], bytes): - connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()} + connection_data = {k: v for k, v in connection_data.items()} profile_id = connection_data.get('m3u_profile_id') if profile_id: @@ -1279,7 +1279,7 @@ class MultiWorkerVODConnectionManager: # Convert bytes keys/values to strings if needed if isinstance(list(connection_data.keys())[0], bytes): - connection_data = {k.decode('utf-8'): v.decode('utf-8') for k, v in connection_data.items()} + connection_data = {k: v for k, v in connection_data.items()} # Check if content matches (using consolidated data) stored_content_type = connection_data.get('content_obj_type', '') @@ -1289,7 +1289,7 @@ class MultiWorkerVODConnectionManager: continue # Extract session ID - session_id = key.decode('utf-8').replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') # Check if Redis-backed connection exists and has no active streams redis_connection = RedisBackedVODConnection(session_id, self.redis_client) @@ -1367,4 +1367,4 @@ class MultiWorkerVODConnectionManager: return redis_connection.get_session_metadata() except Exception as e: logger.error(f"Error getting session info for {session_id}: {e}") - return None \ No newline at end of file + return None diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 00ed8a10..12e1d071 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -550,14 +550,7 @@ class VODStreamView(View): connection_data = redis_client.hgetall(persistent_connection_key) if connection_data: - # Decode Redis hash data - decoded_data = {} - for k, v in connection_data.items(): - k_str = k.decode('utf-8') if isinstance(k, bytes) else k - v_str = v.decode('utf-8') if isinstance(v, bytes) else v - decoded_data[k_str] = v_str - - existing_profile_id = decoded_data.get('m3u_profile_id') + existing_profile_id = connection_data.get('m3u_profile_id') if existing_profile_id: try: existing_profile = M3UAccountProfile.objects.get( @@ -770,19 +763,16 @@ class VODStatsView(View): for key in keys: try: - key_str = key.decode('utf-8') if isinstance(key, bytes) else key connection_data = redis_client.hgetall(key) if connection_data: # Extract session ID from key - session_id = key_str.replace('vod_persistent_connection:', '') + session_id = key.replace('vod_persistent_connection:', '') # Decode Redis hash data combined_data = {} for k, v in connection_data.items(): - k_str = k.decode('utf-8') if isinstance(k, bytes) else k - v_str = v.decode('utf-8') if isinstance(v, bytes) else v - combined_data[k_str] = v_str + combined_data[k] = v # Get content info from the connection data (using correct field names) content_type = combined_data.get('content_obj_type', 'unknown') diff --git a/core/redis_pubsub.py b/core/redis_pubsub.py index 5d0032b0..b1f48b1f 100644 --- a/core/redis_pubsub.py +++ b/core/redis_pubsub.py @@ -201,10 +201,6 @@ class RedisPubSubManager: channel = message.get('channel') if channel: - # Decode binary channel name if needed - if isinstance(channel, bytes): - channel = channel.decode('utf-8') - # Find and call the appropriate handler handler = self.message_handlers.get(channel) if handler: diff --git a/core/tasks.py b/core/tasks.py index f757613b..257d8660 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -404,7 +404,7 @@ def fetch_channel_stats(): while True: cursor, keys = redis_client.scan(cursor, match=channel_pattern) for key in keys: - channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key.decode('utf-8')) + channel_id_match = re.search(r"ts_proxy:channel:(.*):metadata", key) if channel_id_match: ch_id = channel_id_match.group(1) channel_info = ChannelStatus.get_basic_channel_info(ch_id) diff --git a/core/utils.py b/core/utils.py index 36ac5fef..9cc4c26f 100644 --- a/core/utils.py +++ b/core/utils.py @@ -42,101 +42,116 @@ def natural_sort_key(text): return [convert(c) for c in re.split('([0-9]+)', text)] class RedisClient: + _initialized = False _client = None + _buffer = None _pubsub_client = None + @classmethod + def _init_client(cls, decode_responses=True, max_retries=5, retry_interval=1): + retry_count = 0 + while retry_count < max_retries: + try: + # Get connection parameters from settings or environment + redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) + redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) + redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) + + # Use standardized settings + socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) + socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) + health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) + socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) + retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + + # Create Redis client with better defaults + client = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + socket_timeout=socket_timeout, + socket_connect_timeout=socket_connect_timeout, + socket_keepalive=socket_keepalive, + health_check_interval=health_check_interval, + retry_on_timeout=retry_on_timeout, + decode_responses=decode_responses + ) + + # Validate connection with ping + client.ping() + if cls._initialized is False: + client.flushdb() + cls._initialized = True + + # Disable persistence on first connection - improves performance + # Only try to disable if not in a read-only environment + try: + client.config_set('save', '') # Disable RDB snapshots + client.config_set('appendonly', 'no') # Disable AOF logging + + # Set optimal memory settings with environment variable support + # Get max memory from environment or use a larger default (512MB instead of 256MB) + #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') + #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') + + # Apply memory settings + #client.config_set('maxmemory-policy', eviction_policy) + #client.config_set('maxmemory', max_memory) + + #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") + + # Disable protected mode when in debug mode + if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': + client.config_set('protected-mode', 'no') # Disable protected mode in debug + logger.warning("Redis protected mode disabled for debug environment") + + logger.trace("Redis persistence disabled for better performance") + except redis.exceptions.ResponseError as e: + # Improve error handling for Redis configuration errors + if "OOM" in str(e): + logger.error(f"Redis OOM during configuration: {e}") + # Try to increase maxmemory as an emergency measure + try: + client.config_set('maxmemory', '768mb') + logger.warning("Applied emergency Redis memory increase to 768MB") + except: + pass + else: + logger.error(f"Redis configuration error: {e}") + + logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") + break + + except (ConnectionError, TimeoutError) as e: + retry_count += 1 + if retry_count >= max_retries: + logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") + return None + else: + # Use exponential backoff for retries + wait_time = retry_interval * (2 ** (retry_count - 1)) + logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") + time.sleep(wait_time) + + except Exception as e: + logger.error(f"Unexpected error connecting to Redis: {e}") + return None + + return client + @classmethod def get_client(cls, max_retries=5, retry_interval=1): if cls._client is None: - retry_count = 0 - while retry_count < max_retries: - try: - # Get connection parameters from settings or environment - redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) - redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) - redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) - - # Use standardized settings - socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) - socket_connect_timeout = getattr(settings, 'REDIS_SOCKET_CONNECT_TIMEOUT', 5) - health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) - socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) - retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) - - # Create Redis client with better defaults - client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - socket_timeout=socket_timeout, - socket_connect_timeout=socket_connect_timeout, - socket_keepalive=socket_keepalive, - health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout - ) - - # Validate connection with ping - client.ping() - client.flushdb() - - # Disable persistence on first connection - improves performance - # Only try to disable if not in a read-only environment - try: - client.config_set('save', '') # Disable RDB snapshots - client.config_set('appendonly', 'no') # Disable AOF logging - - # Set optimal memory settings with environment variable support - # Get max memory from environment or use a larger default (512MB instead of 256MB) - #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') - #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') - - # Apply memory settings - #client.config_set('maxmemory-policy', eviction_policy) - #client.config_set('maxmemory', max_memory) - - #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") - - # Disable protected mode when in debug mode - if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': - client.config_set('protected-mode', 'no') # Disable protected mode in debug - logger.warning("Redis protected mode disabled for debug environment") - - logger.trace("Redis persistence disabled for better performance") - except redis.exceptions.ResponseError as e: - # Improve error handling for Redis configuration errors - if "OOM" in str(e): - logger.error(f"Redis OOM during configuration: {e}") - # Try to increase maxmemory as an emergency measure - try: - client.config_set('maxmemory', '768mb') - logger.warning("Applied emergency Redis memory increase to 768MB") - except: - pass - else: - logger.error(f"Redis configuration error: {e}") - - logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") - - cls._client = client - break - - except (ConnectionError, TimeoutError) as e: - retry_count += 1 - if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") - return None - else: - # Use exponential backoff for retries - wait_time = retry_interval * (2 ** (retry_count - 1)) - logger.warning(f"Redis connection failed. Retrying in {wait_time}s... ({retry_count}/{max_retries})") - time.sleep(wait_time) - - except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}") - return None - + cls._client = cls._init_client(decode_responses=True, max_retries=max_retries, retry_interval=retry_interval) return cls._client + @classmethod + def get_buffer(cls, max_retries=5, retry_interval=1): + """Get Redis client optimized for binary data (no decoding)""" + if cls._buffer is None: + cls._buffer = cls._init_client(decode_responses=False, max_retries=max_retries, retry_interval=retry_interval) + return cls._buffer + @classmethod def get_pubsub_client(cls, max_retries=5, retry_interval=1): """Get Redis client optimized for PubSub operations""" @@ -165,7 +180,8 @@ class RedisClient: socket_connect_timeout=socket_connect_timeout, socket_keepalive=socket_keepalive, health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout + retry_on_timeout=retry_on_timeout, + decode_responses=True ) # Validate connection with ping diff --git a/dispatcharr/persistent_lock.py b/dispatcharr/persistent_lock.py index 360c9b5d..fe62be43 100644 --- a/dispatcharr/persistent_lock.py +++ b/dispatcharr/persistent_lock.py @@ -48,7 +48,7 @@ class PersistentLock: Returns True if the expiration was successfully extended. """ current_value = self.redis_client.get(self.lock_key) - if current_value and current_value.decode("utf-8") == self.lock_token: + if current_value and current_value == self.lock_token: self.redis_client.expire(self.lock_key, self.lock_timeout) self.has_lock = False return True From 16183329623e163ec01145b721e2197571c12097 Mon Sep 17 00:00:00 2001 From: dekzter Date: Wed, 12 Nov 2025 17:52:11 -0500 Subject: [PATCH 02/67] byte to string fix --- apps/proxy/vod_proxy/connection_manager.py | 55 ++++++++++------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/apps/proxy/vod_proxy/connection_manager.py b/apps/proxy/vod_proxy/connection_manager.py index dea5759b..00ca77aa 100644 --- a/apps/proxy/vod_proxy/connection_manager.py +++ b/apps/proxy/vod_proxy/connection_manager.py @@ -329,15 +329,15 @@ class VODConnectionManager: continue # Extract session info - stored_content_type = session_data.get(b'content_type', b'').decode('utf-8') - stored_content_uuid = session_data.get(b'content_uuid', b'').decode('utf-8') + stored_content_type = session_data.get('content_type', '') + stored_content_uuid = session_data.get('content_uuid', '') # Check if content matches if stored_content_type != content_type or stored_content_uuid != content_uuid: continue # Extract session ID from key - session_id = key.decode('utf-8').replace('vod_session:', '') + session_id = key.replace('vod_session:', '') # Check if session has an active persistent connection persistent_conn = self._persistent_connections.get(session_id) @@ -351,13 +351,13 @@ class VODConnectionManager: continue # Get stored client info for comparison - stored_client_ip = session_data.get(b'client_ip', b'').decode('utf-8') - stored_user_agent = session_data.get(b'user_agent', b'').decode('utf-8') + stored_client_ip = session_data.get('client_ip', '') + stored_user_agent = session_data.get('user_agent', '') # Check timeshift parameters match - stored_utc_start = session_data.get(b'utc_start', b'').decode('utf-8') - stored_utc_end = session_data.get(b'utc_end', b'').decode('utf-8') - stored_offset = session_data.get(b'offset', b'').decode('utf-8') + stored_utc_start = session_data.get('utc_start', '') + stored_utc_end = session_data.get('utc_end', '') + stored_offset = session_data.get('offset', '') current_utc_start = utc_start or "" current_utc_end = utc_end or "" @@ -394,7 +394,7 @@ class VODConnectionManager: 'session_id': session_id, 'score': score, 'reasons': match_reasons, - 'last_activity': float(session_data.get(b'last_activity', b'0').decode('utf-8')) + 'last_activity': float(session_data.get('last_activity', '0')) }) except Exception as e: @@ -536,7 +536,7 @@ class VODConnectionManager: # Get current bytes and add to it current_bytes = self.redis_client.hget(connection_key, "bytes_sent") if current_bytes: - total_bytes = int(current_bytes.decode('utf-8')) + bytes_sent + total_bytes = int(current_bytes) + bytes_sent else: total_bytes = bytes_sent update_data["bytes_sent"] = str(total_bytes) @@ -571,7 +571,7 @@ class VODConnectionManager: profile_id = None if b"m3u_profile_id" in connection_data: try: - profile_id = int(connection_data[b"m3u_profile_id"].decode('utf-8')) + profile_id = int(connection_data["m3u_profile_id"]) except ValueError: pass @@ -617,16 +617,14 @@ class VODConnectionManager: # Convert bytes to strings and parse numbers info = {} for key, value in connection_data.items(): - key_str = key.decode('utf-8') - value_str = value.decode('utf-8') # Parse numeric fields - if key_str in ['connected_at', 'last_activity']: - info[key_str] = float(value_str) - elif key_str in ['bytes_sent', 'position_seconds', 'm3u_profile_id']: - info[key_str] = int(value_str) + if key in ['connected_at', 'last_activity']: + info[key] = float(value) + elif key in ['bytes_sent', 'position_seconds', 'm3u_profile_id']: + info[key] = int(valuvaluee_str) else: - info[key_str] = value_str + info[key] = value return info @@ -676,14 +674,13 @@ class VODConnectionManager: for key in keys: try: - key_str = key.decode('utf-8') last_activity = self.redis_client.hget(key, "last_activity") if last_activity: - last_activity_time = float(last_activity.decode('utf-8')) + last_activity_time = float(last_activity) if current_time - last_activity_time > max_age_seconds: # Extract info for cleanup - parts = key_str.split(':') + parts = key.split(':') if len(parts) >= 5: content_type = parts[2] content_uuid = parts[3] @@ -1342,9 +1339,9 @@ class VODConnectionManager: session_data = self.redis_client.hgetall(session_key) if session_data: # Get session details for connection cleanup - content_type = session_data.get(b'content_type', b'').decode('utf-8') - content_uuid = session_data.get(b'content_uuid', b'').decode('utf-8') - profile_id = session_data.get(b'profile_id') + content_type = session_data.get('content_type', '') + content_uuid = session_data.get('content_uuid', '') + profile_id = session_data.get('profile_id') # Generate client_id from session_id (matches what's used during streaming) client_id = session_id @@ -1355,12 +1352,12 @@ class VODConnectionManager: self.remove_connection(content_type, content_uuid, client_id) # Remove from profile connections if counted (additional safety check) - if session_data.get(b'connection_counted') == b'True' and profile_id: - profile_key = self._get_profile_connections_key(int(profile_id.decode('utf-8'))) + if session_data.get('connection_counted') == 'True' and profile_id: + profile_key = self._get_profile_connections_key(int(profile_id)) current_count = int(self.redis_client.get(profile_key) or 0) if current_count > 0: self.redis_client.decr(profile_key) - logger.info(f"[{session_id}] Decremented profile {profile_id.decode('utf-8')} connections") + logger.info(f"[{session_id}] Decremented profile {profile_id} connections") # Remove session tracking key self.redis_client.delete(session_key) @@ -1380,7 +1377,7 @@ class VODConnectionManager: if session_related_keys: # Filter out keys we already deleted - remaining_keys = [k for k in session_related_keys if k.decode('utf-8') != session_key] + remaining_keys = [k for k in session_related_keys if k != session_key] if remaining_keys: self.redis_client.delete(*remaining_keys) logger.info(f"[{session_id}] Cleaned up {len(remaining_keys)} additional session-related keys") @@ -1410,7 +1407,7 @@ class VODConnectionManager: if self.redis_client: session_data = self.redis_client.hgetall(session_key) if session_data: - created_at = float(session_data.get(b'created_at', b'0').decode('utf-8')) + created_at = float(session_data.get('created_at', '0')) if current_time - created_at > max_age_seconds: logger.info(f"[{session_id}] Session older than {max_age_seconds}s") stale_sessions.append(session_id) From cd35e671e56a29e57e256351010147a4c8145c88 Mon Sep 17 00:00:00 2001 From: dekzter Date: Wed, 12 Nov 2025 17:52:48 -0500 Subject: [PATCH 03/67] byte to string fix --- apps/proxy/ts_proxy/stream_generator.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/proxy/ts_proxy/stream_generator.py b/apps/proxy/ts_proxy/stream_generator.py index 368691b8..25122d74 100644 --- a/apps/proxy/ts_proxy/stream_generator.py +++ b/apps/proxy/ts_proxy/stream_generator.py @@ -108,13 +108,13 @@ class StreamGenerator: metadata_key = RedisKeys.channel_metadata(self.channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] if state in ['waiting_for_clients', 'active']: logger.info(f"[{self.client_id}] Channel {self.channel_id} now ready (state={state})") return True elif state in ['error', 'stopped', 'stopping']: # Added 'stopping' to error states - error_message = metadata.get(b'error_message', b'Unknown error').decode('utf-8') + error_message = metadata.get('error_message', 'Unknown error') logger.error(f"[{self.client_id}] Channel {self.channel_id} in error state: {state}, message: {error_message}") # Send error packet before giving up yield create_ts_packet('error', f"Error: {error_message}") @@ -122,9 +122,9 @@ class StreamGenerator: else: # Improved logging to track initialization progress init_time = "unknown" - if b'init_time' in metadata: + if 'init_time' in metadata: try: - init_time_float = float(metadata[b'init_time'].decode('utf-8')) + init_time_float = float(metadata['init_time']) init_duration = time.time() - init_time_float init_time = f"{init_duration:.1f}s ago" except: @@ -267,8 +267,8 @@ class StreamGenerator: # Also check channel state in metadata metadata_key = RedisKeys.channel_metadata(self.channel_id) metadata = proxy_server.redis_client.hgetall(metadata_key) - if metadata and b'state' in metadata: - state = metadata[b'state'].decode('utf-8') + if metadata and 'state' in metadata: + state = metadata['state'] if state in ['error', 'stopped', 'stopping']: logger.info(f"[{self.client_id}] Channel in {state} state, terminating stream") return False @@ -397,8 +397,6 @@ class StreamGenerator: if metadata: stream_id_bytes = proxy_server.redis_client.hget(metadata_key, ChannelMetadataField.STREAM_ID) if stream_id_bytes: - stream_id = int(stream_id_bytes.decode('utf-8')) - # Check if we're the last client if self.channel_id in proxy_server.client_managers: client_count = proxy_server.client_managers[self.channel_id].get_total_client_count() @@ -459,4 +457,4 @@ def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, Returns a function that can be passed to StreamingHttpResponse. """ generator = StreamGenerator(channel_id, client_id, client_ip, client_user_agent, channel_initializing) - return generator.generate \ No newline at end of file + return generator.generate From 2ca2a51e2e63dcdc8c8ab7942472caaf2b9f4a0d Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 6 Dec 2025 09:36:50 -0500 Subject: [PATCH 04/67] reverted fix so only updated values are modified --- apps/proxy/ts_proxy/channel_status.py | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 015d5ca2..4ac8f63f 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -434,13 +434,33 @@ class ChannelStatus: logger.warning(f"Invalid m3u_profile_id format in Redis: {m3u_profile_id}") # Add stream info to basic info as well - info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) - info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) - info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) - info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) - info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) - info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) - info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) + video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC) + if video_codec: + info['video_codec'] = video_codec + + resolution = metadata.get(ChannelMetadataField.RESOLUTION) + if resolution: + info['resolution'] = resolution + + source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS) + if source_fps: + info['source_fps'] = float(source_fps) + + ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + if ffmpeg_speed: + info['ffmpeg_speed'] = float(ffmpeg_speed) + + audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC) + if audio_codec: + info['audio_codec'] = audio_codec + + audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + if audio_channels: + info['audio_channels'] = audio_channels + + stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE) + if stream_type: + info['stream_type'] = stream_type return info except Exception as e: From dba70b3c585e0f2dce6cdcf157045992e617b236 Mon Sep 17 00:00:00 2001 From: dekzter Date: Tue, 27 Jan 2026 15:23:07 -0500 Subject: [PATCH 05/67] outstanding commits, may revisit later --- .../migrations/0004_user_stream_priority.py | 18 +++ apps/channels/models.py | 153 +++++++++++++++++- apps/channels/tasks.py | 5 +- apps/proxy/ts_proxy/client_manager.py | 16 +- apps/proxy/ts_proxy/server.py | 6 - apps/proxy/ts_proxy/utils.py | 4 +- apps/proxy/ts_proxy/views.py | 26 +-- docker/init/02-postgres.sh | 2 - docker/init/03-init-dispatcharr.sh | 2 +- docker/init/99-init-dev.sh | 50 +++--- docker/uwsgi.dev.ini | 4 +- frontend/prettier.config.js | 4 +- frontend/src/components/forms/User.jsx | 40 +++-- frontend/src/pages/Users.jsx | 35 ---- 14 files changed, 242 insertions(+), 123 deletions(-) create mode 100644 apps/accounts/migrations/0004_user_stream_priority.py diff --git a/apps/accounts/migrations/0004_user_stream_priority.py b/apps/accounts/migrations/0004_user_stream_priority.py new file mode 100644 index 00000000..35c8bbae --- /dev/null +++ b/apps/accounts/migrations/0004_user_stream_priority.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.4 on 2025-10-21 19:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0003_alter_user_custom_properties'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='stream_priority', + field=models.IntegerField(default=0, help_text='Priority level for streaming tasks. Lower values indicate higher priority.'), + ), + ] diff --git a/apps/channels/models.py b/apps/channels/models.py index 3dfb392b..7282d025 100644 --- a/apps/channels/models.py +++ b/apps/channels/models.py @@ -166,7 +166,7 @@ class Stream(models.Model): return stream_profile - def get_stream(self): + def get_stream(self, requester=None): """ Finds an available stream for the requested channel and returns the selected stream and profile. """ @@ -349,7 +349,145 @@ class Channel(models.Model): return stream_profile - def get_stream(self): + def _pick_channel_to_preempt( + self, + profile_id, + requester_level, + redis_client, + exclude_channel_ids=None, + cooldown_seconds=30, + ): + """ + Pick the lowest-impact channel to terminate on the given profile. + Returns: Optional[int] channel_id to preempt + """ + exclude_channel_ids = set(exclude_channel_ids or []) + candidates = [] + + # 1) Try to get active channel IDs for this profile from an index set if available + ch_set_key = f"ts_proxy:profile:{profile_id}:channels" + try: + ch_ids = { (int(x) if not isinstance(x, int) else x) for x in (redis_client.smembers(ch_set_key) or set()) } + except Exception: + ch_ids = set() + + logger.debug("Candidate channels for preemption:") + logger.debug(ch_ids) + + # 2) Fallback: scan metadata keys and filter by m3u_profile == profile_id + if not ch_ids: + cursor = 0 + pattern = "ts_proxy:channel:*:metadata" + while True: + cursor, keys = redis_client.scan(cursor=cursor, match=pattern, count=500) + if keys: + # Prefer HGET m3u_profile if metadata is a hash + pipe = redis_client.pipeline() + for k in keys: + pipe.hget(k, "m3u_profile") + prof_vals = pipe.execute() + for k, prof_val in zip(keys, prof_vals): + try: + pid = int(prof_val) if prof_val is not None else None + except Exception: + pid = None + + if pid == profile_id: + parts = k.split(":") # ts_proxy:channel:{id}:metadata + if len(parts) >= 4: + try: + ch_ids.add(int(parts[2])) + except Exception: + pass + if cursor == 0: + break + + logger.debug("Candidate channels for preemption:") + logger.debug(ch_ids) + + if not ch_ids: + return None + + # 3) Score candidates + for ch_id in ch_ids: + if ch_id in exclude_channel_ids: + continue + + # Skip if recently preempted + last_preempt_key = f"ts_proxy:channel:{ch_id}:last_preempt" + try: + last_preempt = float(redis_client.get(last_preempt_key) or 0.0) + except Exception: + last_preempt = 0.0 + if last_preempt and (time.time() - last_preempt) < cooldown_seconds: + continue + + # Clients and their levels + clients_key = f"ts_proxy:channel:{ch_id}:clients" + member_ids = list(redis_client.smembers(clients_key) or []) + viewer_count = len(member_ids) + max_viewer_level = 0 + if viewer_count: + pipe = redis_client.pipeline() + for cid in member_ids: + pipe.hget(f"ts_proxy:channel:{ch_id}:clients:{cid}", "user_level") + levels_raw = pipe.execute() + levels = [] + for lv in levels_raw: + try: + levels.append(int(lv or 0)) + except Exception: + levels.append(0) + max_viewer_level = max(levels or [0]) + + # Only preempt if requester strictly outranks this channel's viewers + if requester_level <= max_viewer_level: + continue + + # Metadata (protected/recording/started_at_ts) + meta_key = f"ts_proxy:channel:{ch_id}:metadata" + try: + protected, recording, started_at_ts = redis_client.hmget( + meta_key, "protected", "recording", "started_at_ts" + ) + except Exception: + protected = recording = started_at_ts = None + + protected = str(protected or "0") in ("1", "true", "True") + recording = str(recording or "0") in ("1", "true", "True") + if protected or recording: + continue + + try: + started_at_ts = float(started_at_ts) if started_at_ts is not None else None + except Exception: + started_at_ts = None + if started_at_ts is None: + started_at_ts = time.time() # treat unknown as newest + + # Score: lower is safer to terminate + has_viewers = 1 if viewer_count > 0 else 0 + score = (has_viewers, max_viewer_level, viewer_count, started_at_ts) + candidates.append((score, ch_id)) + + logger.debug("Candidate channels after scoring:") + logger.debug(candidates) + + if not candidates: + return None + + candidates.sort(key=lambda x: x[0]) + victim_id = candidates[0][1] + + # Mark preempt timestamp to avoid thrashing + try: + redis_client.set(f"ts_proxy:channel:{victim_id}:last_preempt", str(time.time()), ex=3600) + except Exception: + pass + + return victim_id + + def get_stream(self, requester=None): """ Finds an available stream for the requested channel and returns the selected stream and profile. @@ -438,6 +576,17 @@ class Channel(models.Model): None, ) # Return newly assigned stream and matched profile else: + # At capacity: try to preempt a lower-impact channel on this profile + victim_channel_id = self._pick_channel_to_preempt( + profile_id=profile.id, + requester_level=requester.user_level if requester else 100, + redis_client=redis_client, + exclude_channel_ids=None, + ) + if victim_channel_id: + logger.info(f"Preempting channel {victim_channel_id} for new stream on profile {profile.id}") + # return self.id, profile.id, victim_channel_id + # This profile is at max connections has_streams_but_maxed_out = True logger.debug( diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index ffb2b97a..b63577d2 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -1900,11 +1900,8 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str): metadata_key = RedisKeys.channel_metadata(str(channel.uuid)) md = r.hgetall(metadata_key) if md: - def _gv(bkey): - return md.get(bkey) - def _d(bkey, cast=str): - v = _gv(bkey) + v = md.get(key) try: if v is None: return None diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index bffecdde..80be26f1 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -1,10 +1,8 @@ """Client connection management for TS streams""" import threading -import logging import time import json -import gevent from typing import Set, Optional from apps.proxy.config import TSConfig as Config from redis.exceptions import ConnectionError, TimeoutError @@ -123,7 +121,7 @@ class ClientManager: # Check for stale activity using last_active field last_active = self.redis_client.hget(client_key, "last_active") if last_active: - last_active_time = float(last_active.decode('utf-8')) + last_active_time = float(last_active) ghost_timeout = self.heartbeat_interval * getattr(Config, 'GHOST_CLIENT_MULTIPLIER', 5.0) if current_time - last_active_time > ghost_timeout: @@ -224,7 +222,7 @@ class ClientManager: except Exception as e: logger.error(f"Error notifying owner of client activity: {e}") - def add_client(self, client_id, client_ip, user_agent=None): + def add_client(self, client_id, client_ip, user_agent=None, user=None): """Add a client with duplicate prevention""" if client_id in self._registered_clients: logger.debug(f"Client {client_id} already registered, skipping") @@ -242,7 +240,8 @@ class ClientManager: "ip_address": client_ip, "connected_at": current_time, "last_active": current_time, - "worker_id": self.worker_id or "unknown" + "worker_id": self.worker_id or "unknown", + "user_level": user.user_level if user is not None else 100, # default to a high value since no user means the non-user specific M3U/HDHR } try: @@ -303,8 +302,6 @@ class ClientManager: def remove_client(self, client_id): """Remove a client from this channel and Redis""" - client_ip = None - with self.lock: if client_id in self.clients: self.clients.remove(client_id) @@ -317,11 +314,6 @@ class ClientManager: if self.redis_client: # Get client IP before removing the data client_key = f"ts_proxy:channel:{self.channel_id}:clients:{client_id}" - client_data = self.redis_client.hgetall(client_key) - if client_data and b'ip_address' in client_data: - client_ip = client_data[b'ip_address'].decode('utf-8') - elif client_data and 'ip_address' in client_data: - client_ip = client_data['ip_address'] # Remove from channel's client set self.redis_client.srem(self.client_set_key, client_id) diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index 9773e487..7d4a9f89 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -1121,12 +1121,6 @@ class ProxyServer: else: # Grace period expired with clients - mark channel as active logger.info(f"Grace period expired with {total_clients} clients - marking channel {channel_id} as active") -<<<<<<< HEAD - old_state = "unknown" - if metadata and 'state' in metadata: - old_state = metadata['state'] -======= ->>>>>>> origin/dev if self.update_channel_state(channel_id, ChannelState.ACTIVE, { "grace_period_ended_at": str(time.time()), "clients_at_activation": str(total_clients) diff --git a/apps/proxy/ts_proxy/utils.py b/apps/proxy/ts_proxy/utils.py index 20a6e140..ff9e1780 100644 --- a/apps/proxy/ts_proxy/utils.py +++ b/apps/proxy/ts_proxy/utils.py @@ -83,7 +83,7 @@ def create_ts_packet(packet_type='null', message=None): # Add message to payload if provided if message: - msg_bytes = message.encode('utf-8') + msg_bytes = message packet[4:4+min(len(msg_bytes), 180)] = msg_bytes[:180] return bytes(packet) @@ -113,4 +113,4 @@ def get_logger(component_name=None): # Default if detection fails logger_name = "ts_proxy" - return logging.getLogger(logger_name) \ No newline at end of file + return logging.getLogger(logger_name) diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index 91f254a7..0f9c1228 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -45,7 +45,7 @@ logger = get_logger() @api_view(["GET"]) -def stream_ts(request, channel_id): +def stream_ts(request, channel_id, user=None): if not network_access_allowed(request, "STREAMS"): return JsonResponse({"error": "Forbidden"}, status=403) @@ -80,9 +80,9 @@ def stream_ts(request, channel_id): metadata_key = RedisKeys.channel_metadata(channel_id) if proxy_server.redis_client.exists(metadata_key): metadata = proxy_server.redis_client.hgetall(metadata_key) - state_field = ChannelMetadataField.STATE.encode("utf-8") + state_field = ChannelMetadataField.STATE if state_field in metadata: - channel_state = metadata[state_field].decode("utf-8") + channel_state = metadata[state_field] # Active/running states - channel is operational, don't reinitialize if channel_state in [ @@ -120,7 +120,7 @@ def stream_ts(request, channel_id): else: owner_field = ChannelMetadataField.OWNER.encode("utf-8") if owner_field in metadata: - owner = metadata[owner_field].decode("utf-8") + owner = metadata[owner_field] owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat" if proxy_server.redis_client.exists(owner_heartbeat_key): # Owner is still active with unknown state - don't reinitialize @@ -373,7 +373,7 @@ def stream_ts(request, channel_id): metadata_key, ChannelMetadataField.STATE ) if state_bytes: - current_state = state_bytes.decode("utf-8") + current_state = state_bytes logger.debug( f"[{client_id}] Current state of channel {channel_id}: {current_state}" ) @@ -449,12 +449,12 @@ def stream_ts(request, channel_id): ) if url_bytes: - url = url_bytes.decode("utf-8") + url = url_bytes if ua_bytes: - stream_user_agent = ua_bytes.decode("utf-8") + stream_user_agent = ua_bytes # Extract transcode setting from Redis if profile_bytes: - profile_str = profile_bytes.decode("utf-8") + profile_str = profile_bytes use_transcode = ( profile_str == PROXY_PROFILE_NAME or profile_str == "None" ) @@ -490,7 +490,7 @@ def stream_ts(request, channel_id): # Register client buffer = proxy_server.stream_buffers[channel_id] client_manager = proxy_server.client_managers[channel_id] - client_manager.add_client(client_id, client_ip, client_user_agent) + client_manager.add_client(client_id, client_ip, client_user_agent, user) logger.info(f"[{client_id}] Client registered with channel {channel_id}") # Create a stream generator for this client @@ -553,7 +553,7 @@ def stream_xc(request, username, password, channel_id): channel = get_object_or_404(Channel, id=channel_id) # @TODO: we've got the file 'type' via extension, support this when we support multiple outputs - return stream_ts(request._request, str(channel.uuid)) + return stream_ts(request._request, str(channel.uuid), user) @csrf_exempt @@ -681,7 +681,7 @@ def channel_status(request, channel_id=None): ) for key in keys: channel_id_match = re.search( - r"ts_proxy:channel:(.*):metadata", key.decode("utf-8") + r"ts_proxy:channel:(.*):metadata", key ) if channel_id_match: ch_id = channel_id_match.group(1) @@ -802,7 +802,7 @@ def next_stream(request, channel_id): metadata_key, ChannelMetadataField.STREAM_ID ) if stream_id_bytes: - current_stream_id = int(stream_id_bytes.decode("utf-8")) + current_stream_id = int(stream_id_bytes) logger.info( f"Found current stream ID {current_stream_id} in Redis for channel {channel_id}" ) @@ -812,7 +812,7 @@ def next_stream(request, channel_id): metadata_key, ChannelMetadataField.M3U_PROFILE ) if profile_id_bytes: - profile_id = int(profile_id_bytes.decode("utf-8")) + profile_id = int(profile_id_bytes) logger.info( f"Found M3U profile ID {profile_id} in Redis for channel {channel_id}" ) diff --git a/docker/init/02-postgres.sh b/docker/init/02-postgres.sh index e36dd744..233a5fb8 100644 --- a/docker/init/02-postgres.sh +++ b/docker/init/02-postgres.sh @@ -162,5 +162,3 @@ ensure_utf8_encoding() { echo "Database $POSTGRES_DB converted to UTF8 and permissions set." fi } - - diff --git a/docker/init/03-init-dispatcharr.sh b/docker/init/03-init-dispatcharr.sh index 5fbef23d..bfe081e2 100644 --- a/docker/init/03-init-dispatcharr.sh +++ b/docker/init/03-init-dispatcharr.sh @@ -64,4 +64,4 @@ if [ "$(id -u)" = "0" ]; then fi chmod +x /data -fi \ No newline at end of file +fi diff --git a/docker/init/99-init-dev.sh b/docker/init/99-init-dev.sh index 861c8307..fc8ef18a 100644 --- a/docker/init/99-init-dev.sh +++ b/docker/init/99-init-dev.sh @@ -1,25 +1,33 @@ #!/bin/bash -echo "🚀 Development Mode - Setting up Frontend..." +if [ ! -e "/tmp/init" ]; then + echo "🚀 Development Mode - Setting up Frontend..." -# Install Node.js -if ! command -v node 2>&1 >/dev/null -then - echo "=== setting up nodejs ===" - curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh - bash /tmp/nodesource_setup.sh - apt-get update - apt-get install -y --no-install-recommends \ - nodejs -fi - -# Install frontend dependencies -cd /app/frontend && npm install -# Install pip dependencies -cd /app && pip install -r requirements.txt - -# Install debugpy for remote debugging -if [ "$DISPATCHARR_DEBUG" = "true" ]; then - echo "=== setting up debugpy ===" - pip install debugpy + # Install Node.js + if ! command -v node 2>&1 >/dev/null + then + echo "=== setting up nodejs ===" + curl -sL https://deb.nodesource.com/setup_23.x -o /tmp/nodesource_setup.sh + bash /tmp/nodesource_setup.sh + apt-get update + apt-get install -y --no-install-recommends \ + nodejs + fi + + # Install frontend dependencies + cd /app/frontend && npm install + # Install pip dependencies + cd /app && pip install -r requirements.txt + + # Install debugpy for remote debugging + if [ "$DISPATCHARR_DEBUG" = "true" ]; then + echo "=== setting up debugpy ===" + pip install debugpy + fi + + if [[ "$DISPATCHARR_ENV" = "dev" ]]; then + touch /tmp/init + fi +else + echo "Development mode initialization already done. Skipping dev setup." fi diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index e476e216..e6d0806d 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -8,7 +8,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first -attach-daemon = redis-server +attach-daemon = redis-server --protected-mode no ; Then start other services with configurable nice level (default: 5 for low priority) ; Users can override via CELERY_NICE_LEVEL environment variable in docker-compose attach-daemon = nice -n $(CELERY_NICE_LEVEL) celery -A dispatcharr worker --autoscale=6,1 @@ -59,4 +59,4 @@ logformat-strftime = true log-date = %%Y-%%m-%%d %%H:%%M:%%S,000 # Use formatted time with environment variable for log level log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms -log-buffering = 1024 # Add buffer size limit for logging \ No newline at end of file +log-buffering = 1024 # Add buffer size limit for logging diff --git a/frontend/prettier.config.js b/frontend/prettier.config.js index d2b86878..e143dfef 100644 --- a/frontend/prettier.config.js +++ b/frontend/prettier.config.js @@ -3,8 +3,8 @@ export default { semi: true, // Add semicolons at the end of statements singleQuote: true, // Use single quotes instead of double tabWidth: 2, // Set the indentation width - trailingComma: "es5", // Add trailing commas where valid in ES5 + trailingComma: 'es5', // Add trailing commas where valid in ES5 printWidth: 80, // Wrap lines at 80 characters bracketSpacing: true, // Add spaces inside object braces - arrowParens: "always", // Always include parentheses around arrow function parameters + arrowParens: 'always', // Always include parentheses around arrow function parameters }; diff --git a/frontend/src/components/forms/User.jsx b/frontend/src/components/forms/User.jsx index 619b156f..6ec8705e 100644 --- a/frontend/src/components/forms/User.jsx +++ b/frontend/src/components/forms/User.jsx @@ -12,8 +12,9 @@ import { Stack, MultiSelect, ActionIcon, + NumberInput, } from '@mantine/core'; -import { RotateCcwKey, X } from 'lucide-react'; +import { RotateCcwKey } from 'lucide-react'; import { useForm } from '@mantine/form'; import useChannelsStore from '../../store/channels'; import { USER_LEVELS, USER_LEVEL_LABELS } from '../../constants'; @@ -209,26 +210,23 @@ const User = ({ user = null, isOpen, onClose }) => { key={form.key('last_name')} /> - - - - - } - /> - + + + + } + /> {showPermissions && ( { const authUser = useAuthStore((s) => s.user); - const [selectedUser, setSelectedUser] = useState(null); - const [userModalOpen, setUserModalOpen] = useState(false); - const [confirmDeleteOpen, setConfirmDeleteOpen] = useState(false); - const [deleteTarget, setDeleteTarget] = useState(null); - const [userToDelete, setUserToDelete] = useState(null); - if (!authUser.id) { return <>; } - const closeUserModal = () => { - setSelectedUser(null); - setUserModalOpen(false); - }; - const editUser = (user) => { - setSelectedUser(user); - setUserModalOpen(true); - }; - - const deleteUser = (id) => { - // Get user details for the confirmation dialog - const user = users.find((u) => u.id === id); - setUserToDelete(user); - setDeleteTarget(id); - - // Skip warning if it's been suppressed - if (isWarningSuppressed('delete-user')) { - return executeDeleteUser(id); - } - - setConfirmDeleteOpen(true); - }; - - const executeDeleteUser = async (id) => { - await API.deleteUser(id); - setConfirmDeleteOpen(false); - }; - return ( From 1dd368278ca733b9976b4a01a02df438c6d10eeb Mon Sep 17 00:00:00 2001 From: dekzter Date: Fri, 13 Mar 2026 09:57:22 -0400 Subject: [PATCH 06/67] reverted value checking for consistency, added in user id to client info --- .../migrations/0004_user_stream_priority.py | 18 ----- apps/proxy/ts_proxy/channel_status.py | 70 +++++++++++++++---- apps/proxy/ts_proxy/client_manager.py | 3 +- apps/proxy/ts_proxy/server.py | 7 +- 4 files changed, 59 insertions(+), 39 deletions(-) delete mode 100644 apps/accounts/migrations/0004_user_stream_priority.py diff --git a/apps/accounts/migrations/0004_user_stream_priority.py b/apps/accounts/migrations/0004_user_stream_priority.py deleted file mode 100644 index 35c8bbae..00000000 --- a/apps/accounts/migrations/0004_user_stream_priority.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.2.4 on 2025-10-21 19:32 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('accounts', '0003_alter_user_custom_properties'), - ] - - operations = [ - migrations.AddField( - model_name='user', - name='stream_priority', - field=models.IntegerField(default=0, help_text='Priority level for streaming tasks. Lower values indicate higher priority.'), - ), - ] diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index 4ac8f63f..ffb6be2c 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -265,22 +265,64 @@ class ChannelStatus: } # Add FFmpeg stream information - info['video_codec'] = metadata.get(ChannelMetadataField.VIDEO_CODEC) - info['resolution'] = metadata.get(ChannelMetadataField.RESOLUTION) - info['source_fps'] = metadata.get(ChannelMetadataField.SOURCE_FPS) - info['pixel_format'] = metadata.get(ChannelMetadataField.PIXEL_FORMAT) - info['source_bitrate'] = metadata.get(ChannelMetadataField.SOURCE_BITRATE) - info['audio_codec'] = metadata.get(ChannelMetadataField.AUDIO_CODEC) - info['sample_rate'] = metadata.get(ChannelMetadataField.SAMPLE_RATE) - info['audio_channels'] = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) - info['audio_bitrate'] = metadata.get(ChannelMetadataField.AUDIO_BITRATE) + video_codec = metadata.get(ChannelMetadataField.VIDEO_CODEC) + if video_codec: + info['video_codec'] = video_codec + + resolution = metadata.get(ChannelMetadataField.RESOLUTION) + if resolution: + info['resolution'] = resolution + + source_fps = metadata.get(ChannelMetadataField.SOURCE_FPS) + if source_fps: + info['source_fps'] = source_fps + + pixel_format = metadata.get(ChannelMetadataField.PIXEL_FORMAT) + if pixel_format: + info['pixel_format'] = pixel_format + + source_bitrate = metadata.get(ChannelMetadataField.SOURCE_BITRATE) + if source_bitrate: + info['source_bitrate'] = source_bitrate + + audio_codec = metadata.get(ChannelMetadataField.AUDIO_CODEC) + if audio_codec: + info['audio_codec'] = audio_codec + + sample_rate = metadata.get(ChannelMetadataField.SAMPLE_RATE) + if sample_rate: + info['sample_rate'] = sample_rate + + audio_channels = metadata.get(ChannelMetadataField.AUDIO_CHANNELS) + if audio_channels: + info['audio_channels'] = audio_channels + + audio_bitrate = metadata.get(ChannelMetadataField.AUDIO_BITRATE) + if audio_bitrate: + info['audio_bitrate'] = audio_bitrate + # Add FFmpeg performance stats - info['ffmpeg_speed'] = metadata.get(ChannelMetadataField.FFMPEG_SPEED) - info['ffmpeg_fps'] = metadata.get(ChannelMetadataField.FFMPEG_FPS) - info['actual_fps'] = metadata.get(ChannelMetadataField.ACTUAL_FPS) - info['ffmpeg_bitrate'] = metadata.get(ChannelMetadataField.FFMPEG_BITRATE) - info['stream_type'] = metadata.get(ChannelMetadataField.STREAM_TYPE) + ffmpeg_speed = metadata.get(ChannelMetadataField.FFMPEG_SPEED) + if ffmpeg_speed: + info['ffmpeg_speed'] = ffmpeg_speed + + ffmpeg_fps = metadata.get(ChannelMetadataField.FFMPEG_FPS) + if ffmpeg_fps: + info['ffmpeg_fps'] = ffmpeg_fps + + actual_fps = metadata.get(ChannelMetadataField.ACTUAL_FPS) + if actual_fps: + info['actual_fps'] = actual_fps + + ffmpeg_bitrate = metadata.get(ChannelMetadataField.FFMPEG_BITRATE) + if ffmpeg_bitrate: + info['ffmpeg_bitrate'] = ffmpeg_bitrate + + stream_type = metadata.get(ChannelMetadataField.STREAM_TYPE) + if stream_type: + info['stream_type'] = stream_type + return info diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index 394ad5ea..c0e5ab0b 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -247,7 +247,8 @@ class ClientManager: "connected_at": current_time, "last_active": current_time, "worker_id": self.worker_id or "unknown", - "user_level": user.user_level if user is not None else 100, # default to a high value since no user means the non-user specific M3U/HDHR + "user_id": str(user.id) if user is not None else "unknown", + # "user_level": user.user_level if user is not None else 100, # default to a high value since no user means the non-user specific M3U/HDHR } try: diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index 1999b491..4c51768a 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -459,10 +459,6 @@ class ProxyServer: lock_key = RedisKeys.channel_owner(channel_id) current = self.redis_client.get(lock_key) -<<<<<<< HEAD - # Only extend if we're still the owner - if current and current == self.worker_id: -======= if current is None: # Key expired — re-acquire if we have the stream_manager if channel_id in self.stream_managers: @@ -476,8 +472,7 @@ class ProxyServer: return False return False - if current.decode('utf-8') == self.worker_id: ->>>>>>> origin/dev + if current == self.worker_id: self.redis_client.expire(lock_key, ttl) return True From 1f97143bf892905579e655526f55c6ad16ac4380 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Sun, 15 Mar 2026 00:59:58 -0500 Subject: [PATCH 07/67] fix: resolve HTML named entities in XMLTV files before parsing lxml 6.0.2 with recover=True silently drops HTML named entities (e.g. é, î) that are not valid XML, causing channel names and program data to lose characters during EPG ingestion. - Add preprocessing step in fetch_xmltv() that converts HTML named entities to Unicode while preserving XML-predefined entities - Detect encoding from XML declaration, fall back to UTF-8 for unknown codecs, skip files that fail to decode cleanly - Guard against entities resolving to XML-special characters and html.unescape partial matching edge cases - Add 28 unit tests covering entity resolution, encoding detection, and file-level processing --- apps/epg/tasks.py | 89 +++++++++++ apps/epg/tests/test_entity_resolution.py | 195 +++++++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 apps/epg/tests/test_entity_resolution.py diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index f4c85256..5d567e6e 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -1,8 +1,11 @@ # apps/epg/tasks.py +import codecs import logging import gzip +import html as html_module import os +import re import uuid import requests import time # Add import for tracking download progress @@ -28,6 +31,85 @@ from core.utils import acquire_task_lock, release_task_lock, TaskLockRenewer, se logger = logging.getLogger(__name__) +# Regex and helpers for resolving HTML named entities in XMLTV files. +# lxml only recognises the 5 XML-predefined entities; HTML entities like +# é are silently dropped in recovery mode. This substitution +# converts them to Unicode before the file reaches the parser. +_XML_ENTITIES = frozenset({'amp', 'lt', 'gt', 'quot', 'apos'}) +_XML_SPECIAL_CHARS = frozenset('&<>\'"') +_NAMED_ENTITY_RE = re.compile(r'&([a-zA-Z][a-zA-Z0-9]*);') + + +def _replace_html_entity(match): + """Replace a single HTML named entity, preserving XML-special ones. + + Skips lowercase XML entities (fast path) and any entity whose resolved + output contains XML-special characters that would break parsing. This + also guards against html.unescape partially matching sub-patterns + (e.g. &ersand; being split into & + ersand;). + """ + original = match.group(0) + name = match.group(1) + if name in _XML_ENTITIES: + return original + resolved = html_module.unescape(original) + if resolved == original: + return original + # If the resolved output contains any XML-special character, replacing + # it would produce invalid XML (bare &, <, >, etc.) + if _XML_SPECIAL_CHARS.intersection(resolved): + return original + return resolved + + +def _detect_xml_encoding(header): + """Detect encoding from raw XML header bytes, defaulting to UTF-8. + + Falls back to UTF-8 if the declared encoding is not recognized by Python. + """ + m = re.match(rb'<\?xml[^>]*encoding=["\']([^"\']+)["\']', header) + if m: + declared = m.group(1).decode('ascii') + try: + codecs.lookup(declared) + return declared + except LookupError: + logger.warning(f"Unknown encoding '{declared}', falling back to UTF-8") + return 'utf-8' + return 'utf-8' + + +def _resolve_html_entities(file_path): + """Replace HTML named entities in an XMLTV file with Unicode characters. + + Processes line-by-line to keep memory usage low, writes to a temp file, + then atomically replaces the original. Honors the encoding declared + in the XML declaration. If the file cannot be decoded cleanly it is + left untouched for lxml to handle with its own recovery mode. + """ + # Read the first 200 bytes to detect encoding from the XML declaration + with open(file_path, 'rb') as f: + header = f.read(200) + encoding = _detect_xml_encoding(header) + + temp_path = file_path + '.entity_tmp' + success = False + try: + with open(file_path, 'r', encoding=encoding) as src, \ + open(temp_path, 'w', encoding=encoding) as dst: + for line in src: + dst.write(_NAMED_ENTITY_RE.sub(_replace_html_entity, line)) + os.replace(temp_path, file_path) + success = True + logger.debug(f"Resolved HTML entities in {file_path} (encoding: {encoding})") + except (UnicodeDecodeError, UnicodeEncodeError): + # Leave the file untouched for lxml to handle with its own + # encoding detection and recovery mode. + logger.debug(f"Skipping entity resolution for {file_path}: encoding error with {encoding}") + finally: + if not success and os.path.exists(temp_path): + os.unlink(temp_path) + def validate_icon_url_fast(icon_url, max_length=None): """ @@ -280,6 +362,9 @@ def fetch_xmltv(source): # Send a download complete notification send_epg_update(source.id, "downloading", 100, status="success") + # Resolve HTML named entities so lxml doesn't drop them during parsing + _resolve_html_entities(source.extracted_file_path or source.file_path) + # Return True to indicate successful fetch, processing will continue with parse_channels_only return True @@ -526,6 +611,10 @@ def fetch_xmltv(source): source.save(update_fields=['status']) logger.info(f"Cached EPG file saved to {source.file_path}") + + # Resolve HTML named entities so lxml doesn't drop them during parsing + _resolve_html_entities(source.extracted_file_path or source.file_path) + return True except requests.exceptions.HTTPError as e: diff --git a/apps/epg/tests/test_entity_resolution.py b/apps/epg/tests/test_entity_resolution.py new file mode 100644 index 00000000..353d9b51 --- /dev/null +++ b/apps/epg/tests/test_entity_resolution.py @@ -0,0 +1,195 @@ +import os +import tempfile + +from django.test import TestCase + +from apps.epg.tasks import ( + _NAMED_ENTITY_RE, + _detect_xml_encoding, + _replace_html_entity, + _resolve_html_entities, +) + + +class ReplaceHtmlEntityTests(TestCase): + """Tests for the regex callback that resolves individual HTML entities.""" + + def _sub(self, text): + return _NAMED_ENTITY_RE.sub(_replace_html_entity, text) + + def test_french_accented(self): + self.assertEqual(self._sub("Chaîne Télé"), "Chaîne Télé") + + def test_german_umlauts(self): + self.assertEqual(self._sub("München Übersicht ß"), "München Übersicht ß") + + def test_spanish(self): + self.assertEqual(self._sub("España ¿Qué?"), "España ¿Qué?") + + def test_portuguese(self): + self.assertEqual(self._sub("Comunicação"), "Comunicação") + + def test_scandinavian(self): + self.assertEqual(self._sub("Norsk ø å æ"), "Norsk ø å æ") + + def test_greek_letters(self): + self.assertEqual(self._sub("αβγ"), "αβγ") + + def test_currency_and_symbols(self): + self.assertEqual(self._sub("© € £ ¥"), "© € £ ¥") + + def test_preserves_xml_amp(self): + self.assertEqual(self._sub("A & B"), "A & B") + + def test_preserves_xml_lt_gt(self): + self.assertEqual(self._sub("<tag>"), "<tag>") + + def test_preserves_xml_quot_apos(self): + self.assertEqual(self._sub(""hello'"), ""hello'") + + def test_preserves_uppercase_xml_entities(self): + """&, <, >, " resolve to XML-special chars; must not be replaced.""" + self.assertEqual(self._sub("&"), "&") + self.assertEqual(self._sub("<"), "<") + self.assertEqual(self._sub(">"), ">") + self.assertEqual(self._sub("""), """) + + def test_partial_entity_match_preserved(self): + """html.unescape can partially match & inside &ersand; — must not corrupt.""" + self.assertEqual(self._sub("&ersand;"), "&ersand;") + + def test_mixed_html_and_xml_entities(self): + self.assertEqual( + self._sub("Résumé & Co <test>"), + "Résumé & Co <test>", + ) + + def test_plain_ascii_unchanged(self): + self.assertEqual(self._sub("Plain ASCII text"), "Plain ASCII text") + + def test_direct_utf8_unchanged(self): + self.assertEqual(self._sub("日本語テレビ"), "日本語テレビ") + + def test_unknown_entity_preserved(self): + self.assertEqual(self._sub("&zzfakeentity;"), "&zzfakeentity;") + + +class ResolveHtmlEntitiesFileTests(TestCase): + """Tests for the file-level preprocessing function.""" + + def _make_file(self, content): + fd, path = tempfile.mkstemp(suffix=".xml") + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(content) + self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None) + return path + + def test_resolves_entities_in_file(self): + path = self._make_file( + '\nTélé' + ) + _resolve_html_entities(path) + with open(path, "r", encoding="utf-8") as f: + content = f.read() + self.assertIn("Télé", content) + self.assertNotIn("é", content) + + def test_preserves_xml_entities_in_file(self): + path = self._make_file("A & B <C>") + _resolve_html_entities(path) + with open(path, "r", encoding="utf-8") as f: + content = f.read() + self.assertIn("&", content) + self.assertIn("<", content) + self.assertIn(">", content) + + def test_no_temp_file_left_on_success(self): + path = self._make_file("test") + _resolve_html_entities(path) + self.assertFalse(os.path.exists(path + ".entity_tmp")) + + def test_plain_file_unchanged(self): + original = '\nPlain' + path = self._make_file(original) + _resolve_html_entities(path) + with open(path, "r", encoding="utf-8") as f: + content = f.read() + self.assertEqual(content, original) + + def test_utf8_content_preserved(self): + original = "日本語テレビ" + path = self._make_file(original) + _resolve_html_entities(path) + with open(path, "r", encoding="utf-8") as f: + content = f.read() + self.assertIn("日本語テレビ", content) + + def test_iso_8859_1_encoding(self): + """Files declaring ISO-8859-1 should be read in that encoding.""" + xml = '\nChaîne' + fd, path = tempfile.mkstemp(suffix=".xml") + with os.fdopen(fd, "wb") as f: + f.write(xml.encode("iso-8859-1")) + self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None) + + _resolve_html_entities(path) + with open(path, "r", encoding="iso-8859-1") as f: + content = f.read() + self.assertIn("Cha\u00eene", content) + self.assertNotIn("î", content) + + def test_detect_encoding_utf8_default(self): + """Headers without an encoding declaration default to UTF-8.""" + self.assertEqual(_detect_xml_encoding(b''), "utf-8") + + def test_detect_encoding_iso_8859_1(self): + """Encoding is read from the XML declaration.""" + self.assertEqual( + _detect_xml_encoding(b''), + "ISO-8859-1", + ) + + def test_detect_encoding_single_quotes(self): + """Encoding detection works with single-quoted attributes.""" + self.assertEqual( + _detect_xml_encoding(b""), + "windows-1252", + ) + + def test_detect_encoding_unknown_falls_back(self): + """Unrecognized encoding falls back to UTF-8.""" + self.assertEqual( + _detect_xml_encoding(b''), + "utf-8", + ) + + def test_iso_8859_1_with_entities_roundtrip(self): + """ISO-8859-1 file with entities: resolved without corrupting existing accented chars.""" + # Mix of direct ISO-8859-1 chars and HTML entities + xml_str = '\nD\xe9j\xe0 émission' + fd, path = tempfile.mkstemp(suffix=".xml") + with os.fdopen(fd, "wb") as f: + f.write(xml_str.encode("iso-8859-1")) + self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None) + + _resolve_html_entities(path) + with open(path, "r", encoding="iso-8859-1") as f: + content = f.read() + self.assertIn("D\xe9j\xe0", content, "Existing accented chars should be preserved") + self.assertIn("\xe9mission", content, "Entity should be resolved") + self.assertNotIn("é", content) + + def test_mismatched_encoding_leaves_file_untouched(self): + """File declaring UTF-8 but containing Latin-1 bytes is left alone.""" + # \xe9 is valid ISO-8859-1 but invalid as a standalone UTF-8 byte + raw = b'\n\xe9' + fd, path = tempfile.mkstemp(suffix=".xml") + with os.fdopen(fd, "wb") as f: + f.write(raw) + self.addCleanup(lambda: os.unlink(path) if os.path.exists(path) else None) + + original_bytes = raw # save for comparison + _resolve_html_entities(path) + with open(path, "rb") as f: + result_bytes = f.read() + self.assertEqual(result_bytes, original_bytes, "File should be untouched on decode error") From 01a214851de1fffd76e829a8aff884d6e15dc5c3 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Sun, 15 Mar 2026 01:11:29 -0500 Subject: [PATCH 08/67] fix: catch all exceptions in entity resolution to avoid breaking EPG refresh Previously only UnicodeDecodeError/UnicodeEncodeError were caught. Unexpected errors like disk full or permission denied would propagate and crash the EPG refresh, which would have succeeded before the preprocessing step existed. Now all exceptions are caught and logged, leaving the original file untouched for lxml to handle as before. --- apps/epg/tasks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 5d567e6e..df1e0836 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -84,8 +84,10 @@ def _resolve_html_entities(file_path): Processes line-by-line to keep memory usage low, writes to a temp file, then atomically replaces the original. Honors the encoding declared - in the XML declaration. If the file cannot be decoded cleanly it is - left untouched for lxml to handle with its own recovery mode. + in the XML declaration. + + This function is strictly additive — on any error the original file is + left untouched so lxml can handle it as it always did. """ # Read the first 200 bytes to detect encoding from the XML declaration with open(file_path, 'rb') as f: @@ -102,10 +104,12 @@ def _resolve_html_entities(file_path): os.replace(temp_path, file_path) success = True logger.debug(f"Resolved HTML entities in {file_path} (encoding: {encoding})") - except (UnicodeDecodeError, UnicodeEncodeError): - # Leave the file untouched for lxml to handle with its own - # encoding detection and recovery mode. - logger.debug(f"Skipping entity resolution for {file_path}: encoding error with {encoding}") + except Exception as e: + # On any error, leave the original file untouched for lxml to + # handle with its own encoding detection and recovery mode. + # This ensures the preprocessing step never breaks a previously + # working EPG refresh. + logger.debug(f"Skipping entity resolution for {file_path}: {e}") finally: if not success and os.path.exists(temp_path): os.unlink(temp_path) From 99a328ca171e864f1b0b9d301b82cbcf77716a08 Mon Sep 17 00:00:00 2001 From: dekzter Date: Thu, 19 Mar 2026 12:45:01 -0400 Subject: [PATCH 09/67] fixed bug when filtering after category was selected --- apps/vod/api_views.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/vod/api_views.py b/apps/vod/api_views.py index 3bd984e6..8b778c81 100644 --- a/apps/vod/api_views.py +++ b/apps/vod/api_views.py @@ -578,13 +578,15 @@ class UnifiedContentViewSet(viewsets.ReadOnlyModelViewSet): "series.id IN (SELECT DISTINCT series_id FROM vod_m3useriesrelation msr JOIN m3u_m3uaccount ma ON msr.m3u_account_id = ma.id WHERE ma.is_active = true)" ] - params = [] + movie_params = [] + series_params = [] if search: where_conditions[0] += " AND LOWER(movies.name) LIKE %s" where_conditions[1] += " AND LOWER(series.name) LIKE %s" search_param = f"%{search.lower()}%" - params.extend([search_param, search_param]) + movie_params.append(search_param) + series_params.append(search_param) if category: if '|' in category: @@ -592,15 +594,20 @@ class UnifiedContentViewSet(viewsets.ReadOnlyModelViewSet): if cat_type == 'movie': where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)" where_conditions[1] = "1=0" # Exclude series - params.append(cat_name) + movie_params.append(cat_name) + series_params = [] # no params needed for "1=0" elif cat_type == 'series': where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)" where_conditions[0] = "1=0" # Exclude movies - params.append(cat_name) + series_params.append(cat_name) + movie_params = [] # no params needed for "1=0" else: where_conditions[0] += " AND movies.id IN (SELECT movie_id FROM vod_m3umovierelation mmr JOIN vod_vodcategory c ON mmr.category_id = c.id WHERE c.name = %s)" where_conditions[1] += " AND series.id IN (SELECT series_id FROM vod_m3useriesrelation msr JOIN vod_vodcategory c ON msr.category_id = c.id WHERE c.name = %s)" - params.extend([category, category]) + movie_params.append(category) + series_params.append(category) + + params = movie_params + series_params # Use UNION ALL with ORDER BY and LIMIT/OFFSET for true unified pagination # This is much more efficient than Python sorting From ba0036445c102a41dbd92add1bb1eeac6ed79549 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 20 Mar 2026 10:23:52 -0500 Subject: [PATCH 10/67] Enhancement: Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) --- CHANGELOG.md | 4 ++ .../src/components/forms/LiveGroupFilter.jsx | 37 ++++++++++++++----- .../components/forms/VODCategoryFilter.jsx | 37 ++++++++++++++----- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71b8b8c1..e01acf2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) + ## [0.21.1] - 2026-03-18 ### Fixed diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index b06024e4..09d073f5 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -19,6 +19,7 @@ import { Popover, ScrollArea, Center, + SegmentedControl, } from '@mantine/core'; import { Info } from 'lucide-react'; import useChannelsStore from '../../store/channels'; @@ -54,6 +55,7 @@ const LiveGroupFilter = ({ const streamProfiles = useStreamProfilesStore((s) => s.profiles); const fetchStreamProfiles = useStreamProfilesStore((s) => s.fetchProfiles); const [groupFilter, setGroupFilter] = useState(''); + const [statusFilter, setStatusFilter] = useState('all'); const [epgSources, setEpgSources] = useState([]); // Logo selection functionality @@ -177,13 +179,22 @@ const LiveGroupFilter = ({ setCurrentEditingGroupId(null); }; + const isVisible = (group) => { + const matchesText = group.name + .toLowerCase() + .includes(groupFilter.toLowerCase()); + const matchesStatus = + statusFilter === 'all' || + (statusFilter === 'enabled' && group.enabled) || + (statusFilter === 'disabled' && !group.enabled); + return matchesText && matchesStatus; + }; + const selectAll = () => { setGroupStates( groupStates.map((state) => ({ ...state, - enabled: state.name.toLowerCase().includes(groupFilter.toLowerCase()) - ? true - : state.enabled, + enabled: isVisible(state) ? true : state.enabled, })) ); }; @@ -192,9 +203,7 @@ const LiveGroupFilter = ({ setGroupStates( groupStates.map((state) => ({ ...state, - enabled: state.name.toLowerCase().includes(groupFilter.toLowerCase()) - ? false - : state.enabled, + enabled: isVisible(state) ? false : state.enabled, })) ); }; @@ -220,7 +229,7 @@ const LiveGroupFilter = ({ description="When disabled, new groups from the M3U source will be created but disabled by default. You can enable them manually later." /> - + + @@ -245,9 +264,7 @@ const LiveGroupFilter = ({ verticalSpacing="xs" > {groupStates - .filter((group) => - group.name.toLowerCase().includes(groupFilter.toLowerCase()) - ) + .filter((group) => isVisible(group)) .sort((a, b) => a.name.localeCompare(b.name)) .map((group) => ( { const categories = useVODStore((s) => s.categories); const [filter, setFilter] = useState(''); + const [statusFilter, setStatusFilter] = useState('all'); useEffect(() => { if (Object.keys(categories).length === 0) { @@ -64,13 +66,22 @@ const VODCategoryFilter = ({ ); }; + const isVisible = (category) => { + const matchesText = category.name + .toLowerCase() + .includes(filter.toLowerCase()); + const matchesStatus = + statusFilter === 'all' || + (statusFilter === 'enabled' && category.enabled) || + (statusFilter === 'disabled' && !category.enabled); + return matchesText && matchesStatus; + }; + const selectAll = () => { setCategoryStates( categoryStates.map((state) => ({ ...state, - enabled: state.name.toLowerCase().includes(filter.toLowerCase()) - ? true - : state.enabled, + enabled: isVisible(state) ? true : state.enabled, })) ); }; @@ -79,9 +90,7 @@ const VODCategoryFilter = ({ setCategoryStates( categoryStates.map((state) => ({ ...state, - enabled: state.name.toLowerCase().includes(filter.toLowerCase()) - ? false - : state.enabled, + enabled: isVisible(state) ? false : state.enabled, })) ); }; @@ -98,7 +107,7 @@ const VODCategoryFilter = ({ description="When disabled, new categories from the provider will be created but disabled by default. You can enable them manually later." /> - + + @@ -121,9 +140,7 @@ const VODCategoryFilter = ({ verticalSpacing="xs" > {categoryStates - .filter((category) => { - return category.name.toLowerCase().includes(filter.toLowerCase()); - }) + .filter((category) => isVisible(category)) .sort((a, b) => a.name.localeCompare(b.name)) .map((category) => ( Date: Fri, 20 Mar 2026 11:37:51 -0500 Subject: [PATCH 11/67] Refactor: Removed unused imports from `M3UGroupFilter`, `LiveGroupFilter`, and `VODCategoryFilter` (`Yup`, `M3UProfiles`, several unused Mantine components, dead `OptionWithTooltip` component, duplicate lucide-react imports, and `Divider` in `VODCategoryFilter`). No behaviour changes. --- CHANGELOG.md | 4 +++ .../src/components/forms/LiveGroupFilter.jsx | 3 +- .../src/components/forms/M3UGroupFilter.jsx | 34 +------------------ .../components/forms/VODCategoryFilter.jsx | 1 - 4 files changed, 6 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01acf2a..a2300474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) +### Changed + +- Frontend cleanup: removed unused imports from `M3UGroupFilter`, `LiveGroupFilter`, and `VODCategoryFilter` (`Yup`, `M3UProfiles`, several unused Mantine components, dead `OptionWithTooltip` component, duplicate lucide-react imports, and `Divider` in `VODCategoryFilter`). No behaviour changes. + ## [0.21.1] - 2026-03-18 ### Fixed diff --git a/frontend/src/components/forms/LiveGroupFilter.jsx b/frontend/src/components/forms/LiveGroupFilter.jsx index 09d073f5..4cfdaca6 100644 --- a/frontend/src/components/forms/LiveGroupFilter.jsx +++ b/frontend/src/components/forms/LiveGroupFilter.jsx @@ -21,10 +21,9 @@ import { Center, SegmentedControl, } from '@mantine/core'; -import { Info } from 'lucide-react'; +import { Info, CircleCheck, CircleX } from 'lucide-react'; import useChannelsStore from '../../store/channels'; import useStreamProfilesStore from '../../store/streamProfiles'; -import { CircleCheck, CircleX } from 'lucide-react'; import { useChannelLogoSelection } from '../../hooks/useSmartLogos'; import { FixedSizeList as List } from 'react-window'; import LazyLogo from '../LazyLogo'; diff --git a/frontend/src/components/forms/M3UGroupFilter.jsx b/frontend/src/components/forms/M3UGroupFilter.jsx index 0a7dc224..20713d4e 100644 --- a/frontend/src/components/forms/M3UGroupFilter.jsx +++ b/frontend/src/components/forms/M3UGroupFilter.jsx @@ -1,52 +1,20 @@ // Modal.js -import React, { useState, useEffect, forwardRef } from 'react'; -import * as Yup from 'yup'; +import React, { useState, useEffect } from 'react'; import API from '../../api'; -import M3UProfiles from './M3UProfiles'; import { LoadingOverlay, - TextInput, Button, - Checkbox, Modal, Flex, - NativeSelect, - FileInput, - Select, - Space, - Chip, Stack, - Group, - Center, - SimpleGrid, - Text, - NumberInput, - Divider, - Alert, - Box, - MultiSelect, - Tooltip, Tabs, } from '@mantine/core'; -import { Info } from 'lucide-react'; import useChannelsStore from '../../store/channels'; import useVODStore from '../../store/useVODStore'; -import { CircleCheck, CircleX } from 'lucide-react'; import { notifications } from '@mantine/notifications'; import LiveGroupFilter from './LiveGroupFilter'; import VODCategoryFilter from './VODCategoryFilter'; -// Custom item component for MultiSelect with tooltip -const OptionWithTooltip = forwardRef( - ({ label, description, ...others }, ref) => ( - -
- {label} -
-
- ) -); - const M3UGroupFilter = ({ playlist = null, isOpen, onClose }) => { const channelGroups = useChannelsStore((s) => s.channelGroups); const fetchCategories = useVODStore((s) => s.fetchCategories); diff --git a/frontend/src/components/forms/VODCategoryFilter.jsx b/frontend/src/components/forms/VODCategoryFilter.jsx index 73932249..646aeca1 100644 --- a/frontend/src/components/forms/VODCategoryFilter.jsx +++ b/frontend/src/components/forms/VODCategoryFilter.jsx @@ -8,7 +8,6 @@ import { Group, SimpleGrid, Text, - Divider, Box, Checkbox, SegmentedControl, From 36a8e92d941e29ce7cbcd5de715d792873a2dae9 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:48:06 -0500 Subject: [PATCH 12/67] fix(dvr): prevent duplicate recordings after EPG refresh - Replace ProgramData.id-based dedup with stable (tvg_id, start_time, end_time) key from Recording.custom_properties.program. ProgramData IDs change on every EPG refresh; the old dedup always passed after refresh, creating duplicates. - Fix secondary timeslot guard to compare original program times (stored in custom_properties) instead of offset-adjusted Recording times. With DVR pre/post offsets configured, the old guard never matched. - Add acquire_task_lock/release_task_lock to serialize concurrent evaluate_series_rules calls. Multiple EPG source refreshes firing evaluate_series_rules.delay() simultaneously raced to create duplicate recordings. --- apps/channels/tasks.py | 75 +- apps/channels/tests/test_series_rule_dedup.py | 718 ++++++++++++++++++ 2 files changed, 776 insertions(+), 17 deletions(-) create mode 100644 apps/channels/tests/test_series_rule_dedup.py diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index b5969cd1..449a3f4a 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -19,6 +19,7 @@ from rapidfuzz import fuzz from apps.channels.models import Channel from apps.epg.models import EPGData from core.models import CoreSettings +from core.utils import acquire_task_lock, release_task_lock from django.db import OperationalError, close_old_connections from channels.layers import get_channel_layer @@ -1070,12 +1071,39 @@ def match_single_channel_epg(channel_id): def evaluate_series_rules_impl(tvg_id: str | None = None): """Synchronous implementation of series rule evaluation; returns details for debugging.""" + result = {"scheduled": 0, "details": []} + + # Serialize all invocations to prevent concurrent evaluations from + # racing to create duplicate recordings (e.g. multiple EPG sources + # refreshing simultaneously each firing evaluate_series_rules.delay()). + # If Redis is unavailable, proceed without lock — the primary and + # secondary dedup guards still prevent duplicates. + lock_acquired = False + try: + lock_acquired = acquire_task_lock('evaluate_series_rules', 'all') + if not lock_acquired: + result["details"].append({"status": "skipped", "reason": "concurrent evaluation in progress"}) + return result + except (ConnectionError, OSError): + logger.warning("Could not acquire series rule evaluation lock (Redis unavailable), proceeding without lock") + + try: + return _evaluate_series_rules_locked(tvg_id, result) + finally: + if lock_acquired: + try: + release_task_lock('evaluate_series_rules', 'all') + except (ConnectionError, OSError): + logger.warning("Could not release series rule evaluation lock") + + +def _evaluate_series_rules_locked(tvg_id, result): + """Inner implementation of series rule evaluation, called under lock.""" from django.utils import timezone from apps.channels.models import Recording, Channel from apps.epg.models import EPGData, ProgramData rules = CoreSettings.get_dvr_series_rules() - result = {"scheduled": 0, "details": []} if not isinstance(rules, list) or not rules: return result @@ -1089,14 +1117,23 @@ def evaluate_series_rules_impl(tvg_id: str | None = None): now = timezone.now() horizon = now + timedelta(days=7) - # Preload existing recordings' program ids to avoid duplicates - existing_program_ids = set() - for rec in Recording.objects.all().only("custom_properties"): + # Preload existing recordings keyed by stable program attributes that + # survive EPG refreshes (tvg_id + original start/end times stored in + # custom_properties). ProgramData.id changes on every EPG refresh so + # it cannot be used for deduplication. Only load future recordings + # to bound the set size — past recordings cannot collide with newly + # scheduled future programs. + existing_program_keys = set() + for cp in Recording.objects.filter( + end_time__gte=now, + ).values_list("custom_properties", flat=True): try: - pid = rec.custom_properties.get("program", {}).get("id") if rec.custom_properties else None - if pid is not None: - # Normalize to string for consistent comparisons - existing_program_ids.add(str(pid)) + prog_data = (cp or {}).get("program", {}) + tvg_id_val = prog_data.get("tvg_id") + st = prog_data.get("start_time") + et = prog_data.get("end_time") + if tvg_id_val and st and et: + existing_program_keys.add((str(tvg_id_val), str(st), str(et))) except Exception: continue @@ -1191,17 +1228,21 @@ def evaluate_series_rules_impl(tvg_id: str | None = None): created_here = 0 for prog in unique_programs: try: - # Skip if already scheduled by program id - if str(prog.id) in existing_program_ids: + # Skip if a recording already exists for this exact airing + # (keyed by tvg_id + original program times, which are stable + # across EPG refreshes unlike ProgramData.id). + prog_key = (str(prog.tvg_id), prog.start_time.isoformat(), prog.end_time.isoformat()) + if prog_key in existing_program_keys: continue - # Extra guard: skip if a recording exists for the same channel + timeslot + # Extra guard: DB query using the same stable attributes + # stored in custom_properties (unadjusted program times, + # not offset-adjusted Recording.start_time/end_time). try: - from django.db.models import Q if Recording.objects.filter( - channel=channel, - start_time=prog.start_time, - end_time=prog.end_time, - ).filter(Q(custom_properties__program__id=prog.id) | Q(custom_properties__program__title=prog.title)).exists(): + custom_properties__program__tvg_id=prog.tvg_id, + custom_properties__program__start_time=prog.start_time.isoformat(), + custom_properties__program__end_time=prog.end_time.isoformat(), + ).exists(): continue except Exception: continue # already scheduled/recorded @@ -1245,7 +1286,7 @@ def evaluate_series_rules_impl(tvg_id: str | None = None): } }, ) - existing_program_ids.add(str(prog.id)) + existing_program_keys.add(prog_key) created_here += 1 try: prefetch_recording_artwork.apply_async(args=[rec.id], countdown=1) diff --git a/apps/channels/tests/test_series_rule_dedup.py b/apps/channels/tests/test_series_rule_dedup.py new file mode 100644 index 00000000..33aff695 --- /dev/null +++ b/apps/channels/tests/test_series_rule_dedup.py @@ -0,0 +1,718 @@ +"""Tests for series rule evaluation deduplication. + +Unit tests verify the dedup logic in evaluate_series_rules_impl. +Integration tests exercise the full path: EPG refresh → series rule +evaluation → Recording creation → post_save signal chain. +""" +from datetime import timedelta +from unittest.mock import patch, MagicMock + +from django.test import TestCase +from django.utils import timezone + +from apps.channels.models import Channel, Recording +from apps.epg.models import EPGSource, EPGData, ProgramData +from core.models import CoreSettings + + +def _set_series_rules(rules): + """Helper to store series rules in CoreSettings.""" + CoreSettings.set_dvr_series_rules(rules) + + +def _set_dvr_offsets(pre_min=0, post_min=0): + """Helper to store DVR pre/post offsets.""" + CoreSettings._update_group("dvr_settings", "DVR Settings", { + "pre_offset_minutes": pre_min, + "post_offset_minutes": post_min, + }) + + +class SeriesRuleDedupBaseTestCase(TestCase): + """Shared setup for series rule dedup tests.""" + + def setUp(self): + self.now = timezone.now() + self.epg_source = EPGSource.objects.create( + name="Test EPG", source_type="xmltv" + ) + self.epg = EPGData.objects.create( + tvg_id="test.channel.1", + name="Test Channel EPG", + epg_source=self.epg_source, + ) + self.channel = Channel.objects.create( + channel_number=1, name="Test Channel", epg_data=self.epg + ) + + _set_series_rules([{ + "tvg_id": "test.channel.1", + "mode": "all", + "title": "Test Show", + }]) + _set_dvr_offsets(pre_min=0, post_min=0) + + def _create_program(self, hours_from_now=1, title="Test Show", + sub_title="Episode 1", tvg_id="test.channel.1"): + """Create a ProgramData at the given offset.""" + start = self.now + timedelta(hours=hours_from_now) + end = start + timedelta(hours=1) + return ProgramData.objects.create( + epg=self.epg, + tvg_id=tvg_id, + start_time=start, + end_time=end, + title=title, + sub_title=sub_title, + ) + + def _simulate_epg_refresh(self, programs_data): + """Delete all ProgramData and recreate with new IDs (simulates EPG refresh).""" + ProgramData.objects.filter(epg=self.epg).delete() + new_programs = [] + for data in programs_data: + prog = ProgramData.objects.create(epg=self.epg, **data) + new_programs.append(prog) + return new_programs + + def _program_data_for_refresh(self, prog): + """Build the dict needed by _simulate_epg_refresh from a ProgramData.""" + return { + "tvg_id": prog.tvg_id, + "start_time": prog.start_time, + "end_time": prog.end_time, + "title": prog.title, + "sub_title": prog.sub_title, + } + + +# --------------------------------------------------------------------------- +# Unit tests: dedup logic in evaluate_series_rules_impl +# --------------------------------------------------------------------------- + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class ProgramIdStabilityTests(SeriesRuleDedupBaseTestCase): + """Verify dedup works after EPG refresh changes ProgramData IDs.""" + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_no_duplicate_after_epg_refresh(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Same program should not be recorded twice after EPG refresh.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2) + old_id = prog.id + result1 = evaluate_series_rules_impl() + self.assertEqual(result1["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 1) + + new_programs = self._simulate_epg_refresh( + [self._program_data_for_refresh(prog)] + ) + self.assertNotEqual(old_id, new_programs[0].id) + + result2 = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + self.assertEqual(result2["scheduled"], 0) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_no_duplicate_with_offsets_after_refresh(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Dedup works when DVR offsets shift Recording times away from program times.""" + from apps.channels.tasks import evaluate_series_rules_impl + + _set_dvr_offsets(pre_min=5, post_min=5) + prog = self._create_program(hours_from_now=2) + result1 = evaluate_series_rules_impl() + self.assertEqual(result1["scheduled"], 1) + + rec = Recording.objects.first() + self.assertEqual(rec.start_time, prog.start_time - timedelta(minutes=5)) + self.assertEqual(rec.end_time, prog.end_time + timedelta(minutes=5)) + + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + result2 = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_different_episodes_still_recorded(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Different episodes on the same channel should each get a recording.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2, sub_title="Episode 1") + self._create_program(hours_from_now=4, sub_title="Episode 2") + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 2) + self.assertEqual(Recording.objects.count(), 2) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_new_episode_after_refresh_is_recorded(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """A genuinely new episode appearing after EPG refresh should be recorded.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2, sub_title="Episode 1") + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + self._simulate_epg_refresh([ + self._program_data_for_refresh(prog), + { + "tvg_id": "test.channel.1", + "start_time": prog.end_time, + "end_time": prog.end_time + timedelta(hours=1), + "title": "Test Show", + "sub_title": "Episode 2", + }, + ]) + + result2 = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 2) + self.assertEqual(result2["scheduled"], 1) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_multiple_epg_refreshes_no_duplicates(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Multiple consecutive EPG refreshes should not accumulate duplicates.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + for _ in range(5): + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + evaluate_series_rules_impl() + + self.assertEqual(Recording.objects.count(), 1) + + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class ConcurrencyGuardTests(SeriesRuleDedupBaseTestCase): + """Verify the task lock prevents concurrent evaluation.""" + + def test_lock_acquired_and_released(self, mock_schedule, mock_artwork): + """evaluate_series_rules_impl acquires and releases the task lock.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2) + + with patch("apps.channels.tasks.acquire_task_lock", return_value=True) as mock_lock, \ + patch("apps.channels.tasks.release_task_lock") as mock_release: + evaluate_series_rules_impl() + mock_lock.assert_called_once_with('evaluate_series_rules', 'all') + mock_release.assert_called_once_with('evaluate_series_rules', 'all') + + def test_skips_when_lock_held(self, mock_schedule, mock_artwork): + """Returns early with skip reason when lock is already held.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2) + + with patch("apps.channels.tasks.acquire_task_lock", return_value=False): + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 0) + self.assertTrue( + any(d.get("reason") == "concurrent evaluation in progress" + for d in result["details"]), + ) + self.assertEqual(Recording.objects.count(), 0) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_lock_released_on_exception(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Lock is released even if the inner implementation raises.""" + from apps.channels.tasks import evaluate_series_rules_impl + + with patch("apps.channels.tasks._evaluate_series_rules_locked", + side_effect=RuntimeError("test error")): + with self.assertRaises(RuntimeError): + evaluate_series_rules_impl() + mock_release.assert_called_once_with('evaluate_series_rules', 'all') + + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class SecondaryGuardTests(SeriesRuleDedupBaseTestCase): + """Verify the secondary DB guard uses stable program attributes.""" + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_secondary_guard_catches_duplicate_with_offsets(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Secondary guard works with stale program IDs and DVR offsets.""" + from apps.channels.tasks import evaluate_series_rules_impl + + _set_dvr_offsets(pre_min=10, post_min=10) + prog = self._create_program(hours_from_now=2) + + # Pre-existing recording with a stale program ID (from previous EPG refresh) + Recording.objects.create( + channel=self.channel, + start_time=prog.start_time - timedelta(minutes=10), + end_time=prog.end_time + timedelta(minutes=10), + custom_properties={ + "program": { + "id": 99999, + "tvg_id": prog.tvg_id, + "title": prog.title, + "start_time": prog.start_time.isoformat(), + "end_time": prog.end_time.isoformat(), + } + }, + ) + + result = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + self.assertEqual(result["scheduled"], 0) + + +# --------------------------------------------------------------------------- +# Integration tests: full path from EPG refresh through recording creation +# --------------------------------------------------------------------------- + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class IntegrationEPGRefreshTests(SeriesRuleDedupBaseTestCase): + """End-to-end tests simulating the EPG refresh → evaluate → record flow. + + These exercise the full signal chain: evaluate_series_rules_impl creates + a Recording, the post_save signal fires schedule_recording_task, and + subsequent evaluations (after EPG refresh) must not create duplicates. + """ + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_single_episode_no_duplicates(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Simulate: create rule → evaluate → EPG refresh → re-evaluate. + + The full recording lifecycle must result in exactly 1 recording. + """ + from apps.channels.tasks import evaluate_series_rules_impl + + # Initial EPG data + prog = self._create_program(hours_from_now=2, sub_title="Pilot") + + # First evaluation creates the recording + result1 = evaluate_series_rules_impl() + self.assertEqual(result1["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 1) + + # Verify the recording was created with correct program metadata + rec = Recording.objects.first() + self.assertEqual(rec.custom_properties["program"]["tvg_id"], "test.channel.1") + self.assertEqual(rec.custom_properties["program"]["title"], "Test Show") + self.assertEqual( + rec.custom_properties["program"]["start_time"], + prog.start_time.isoformat() + ) + + # Verify the post_save signal scheduled a task + mock_schedule.assert_called() + initial_schedule_count = mock_schedule.call_count + + # Simulate EPG refresh (programs get new DB IDs) + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + + # Re-evaluate after refresh (this is what EPG refresh triggers) + result2 = evaluate_series_rules_impl() + self.assertEqual(result2["scheduled"], 0) + self.assertEqual(Recording.objects.count(), 1) + + # No additional task scheduling should have occurred + self.assertEqual(mock_schedule.call_count, initial_schedule_count) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_with_offsets_no_duplicates(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Full flow with DVR offsets: recording times differ from program times.""" + from apps.channels.tasks import evaluate_series_rules_impl + + _set_dvr_offsets(pre_min=5, post_min=10) + prog = self._create_program(hours_from_now=3, sub_title="Episode 1") + + result1 = evaluate_series_rules_impl() + self.assertEqual(result1["scheduled"], 1) + + rec = Recording.objects.first() + # Verify offset-adjusted recording times + self.assertEqual(rec.start_time, prog.start_time - timedelta(minutes=5)) + self.assertEqual(rec.end_time, prog.end_time + timedelta(minutes=10)) + # Verify original (unadjusted) program times in custom_properties + self.assertEqual( + rec.custom_properties["program"]["start_time"], + prog.start_time.isoformat() + ) + self.assertEqual( + rec.custom_properties["program"]["end_time"], + prog.end_time.isoformat() + ) + + # EPG refresh + re-evaluate + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + result2 = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + self.assertEqual(result2["scheduled"], 0) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_multiple_episodes_across_refreshes(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """New episodes appear across multiple EPG refreshes; each recorded once.""" + from apps.channels.tasks import evaluate_series_rules_impl + + ep1 = self._create_program(hours_from_now=2, sub_title="Episode 1") + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + # EPG refresh adds episode 2 alongside episode 1 + ep1_data = self._program_data_for_refresh(ep1) + ep2_start = ep1.end_time + ep2_data = { + "tvg_id": "test.channel.1", + "start_time": ep2_start, + "end_time": ep2_start + timedelta(hours=1), + "title": "Test Show", + "sub_title": "Episode 2", + } + self._simulate_epg_refresh([ep1_data, ep2_data]) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 2) + + # Another EPG refresh adds episode 3 + ep3_start = ep2_start + timedelta(hours=1) + ep3_data = { + "tvg_id": "test.channel.1", + "start_time": ep3_start, + "end_time": ep3_start + timedelta(hours=1), + "title": "Test Show", + "sub_title": "Episode 3", + } + self._simulate_epg_refresh([ep1_data, ep2_data, ep3_data]) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 3) + + # Final EPG refresh with no new episodes — count must stay at 3 + self._simulate_epg_refresh([ep1_data, ep2_data, ep3_data]) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 3) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_multiple_series_rules(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Multiple series rules on different channels, each evaluated correctly.""" + from apps.channels.tasks import evaluate_series_rules_impl + + # Second channel with its own EPG + epg2 = EPGData.objects.create( + tvg_id="test.channel.2", + name="Channel 2 EPG", + epg_source=self.epg_source, + ) + channel2 = Channel.objects.create( + channel_number=2, name="Test Channel 2", epg_data=epg2 + ) + + _set_series_rules([ + {"tvg_id": "test.channel.1", "mode": "all", "title": "Show A"}, + {"tvg_id": "test.channel.2", "mode": "all", "title": "Show B"}, + ]) + + # Programs on both channels + start1 = self.now + timedelta(hours=2) + prog1 = ProgramData.objects.create( + epg=self.epg, tvg_id="test.channel.1", + start_time=start1, end_time=start1 + timedelta(hours=1), + title="Show A", sub_title="Episode 1", + ) + start2 = self.now + timedelta(hours=3) + prog2 = ProgramData.objects.create( + epg=epg2, tvg_id="test.channel.2", + start_time=start2, end_time=start2 + timedelta(hours=1), + title="Show B", sub_title="Episode 1", + ) + + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 2) + self.assertEqual(Recording.objects.filter(channel=self.channel).count(), 1) + self.assertEqual(Recording.objects.filter(channel=channel2).count(), 1) + + # EPG refresh for both channels + ProgramData.objects.filter(epg=self.epg).delete() + ProgramData.objects.filter(epg=epg2).delete() + ProgramData.objects.create( + epg=self.epg, tvg_id="test.channel.1", + start_time=start1, end_time=start1 + timedelta(hours=1), + title="Show A", sub_title="Episode 1", + ) + ProgramData.objects.create( + epg=epg2, tvg_id="test.channel.2", + start_time=start2, end_time=start2 + timedelta(hours=1), + title="Show B", sub_title="Episode 1", + ) + + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 2, + "No duplicates across multiple series rules after EPG refresh") + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_rapid_epg_refreshes_simulate_user_report( + self, mock_release, mock_lock, mock_schedule, mock_artwork + ): + """Reproduce the user-reported scenario: series rule + multiple EPG refreshes + causing count to balloon from 6 to 25 and 5 simultaneous recordings. + + Simulates 6 episodes with 5 EPG refreshes (each assigning new ProgramData IDs). + """ + from apps.channels.tasks import evaluate_series_rules_impl + + # Create 6 episodes (the user had "next of 6") + episodes = [] + for i in range(6): + start = self.now + timedelta(hours=2 + i * 2) + episodes.append({ + "tvg_id": "test.channel.1", + "start_time": start, + "end_time": start + timedelta(hours=1), + "title": "Test Show", + "sub_title": f"Episode {i + 1}", + }) + + # Create initial ProgramData + for ep in episodes: + ProgramData.objects.create(epg=self.epg, **ep) + + # First evaluation: should create exactly 6 recordings + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 6) + + # Simulate 5 EPG refreshes (the user saw count balloon to 25) + for refresh_num in range(5): + self._simulate_epg_refresh(episodes) + result = evaluate_series_rules_impl() + self.assertEqual( + Recording.objects.count(), 6, + f"After EPG refresh #{refresh_num + 1}, expected 6 recordings " + f"but got {Recording.objects.count()}" + ) + self.assertEqual(result["scheduled"], 0) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_recording_survives_program_removal_and_readd( + self, mock_release, mock_lock, mock_schedule, mock_artwork + ): + """Program temporarily disappears from EPG then reappears — no duplicate.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2, sub_title="Episode 1") + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + # EPG refresh removes the program entirely + self._simulate_epg_refresh([]) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1, + "Existing recording preserved when program disappears from EPG") + + # EPG refresh adds the program back (new ID) + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1, + "No duplicate when program reappears with new ID") + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_celery_task_wrapper_calls_impl(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """The @shared_task evaluate_series_rules delegates to _impl correctly.""" + from apps.channels.tasks import evaluate_series_rules + + self._create_program(hours_from_now=2) + result = evaluate_series_rules() + self.assertEqual(result["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 1) + + # Call again (simulating a second EPG refresh trigger) + result2 = evaluate_series_rules() + self.assertEqual(result2["scheduled"], 0) + self.assertEqual(Recording.objects.count(), 1) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_tvg_id_scoped_evaluation(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Scoped evaluation (tvg_id parameter) still prevents duplicates.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2) + result1 = evaluate_series_rules_impl(tvg_id="test.channel.1") + self.assertEqual(result1["scheduled"], 1) + + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + result2 = evaluate_series_rules_impl(tvg_id="test.channel.1") + self.assertEqual(result2["scheduled"], 0) + self.assertEqual(Recording.objects.count(), 1) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_full_flow_offset_change_between_refreshes(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Changing DVR offsets between EPG refreshes doesn't create duplicates. + + Even though Recording.start_time/end_time change when offsets change, + the dedup key uses the original program times from custom_properties. + """ + from apps.channels.tasks import evaluate_series_rules_impl + + _set_dvr_offsets(pre_min=5, post_min=5) + prog = self._create_program(hours_from_now=2) + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + rec = Recording.objects.first() + original_start = rec.start_time + original_end = rec.end_time + + # Change offsets + _set_dvr_offsets(pre_min=10, post_min=15) + + # EPG refresh + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + result = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1, + "Changing offsets between refreshes should not create duplicates") + self.assertEqual(result["scheduled"], 0) + + +# --------------------------------------------------------------------------- +# Edge case tests: Redis unavailability, non-series recordings, robustness +# --------------------------------------------------------------------------- + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class RedisUnavailabilityTests(SeriesRuleDedupBaseTestCase): + """Verify evaluation works when Redis is unavailable (lock cannot be acquired).""" + + def test_proceeds_when_redis_down(self, mock_schedule, mock_artwork): + """Evaluation succeeds (with dedup guards) when Redis raises on lock acquire.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2) + + with patch("apps.channels.tasks.acquire_task_lock", + side_effect=ConnectionError("Redis unavailable")): + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 1) + + def test_dedup_still_works_without_lock(self, mock_schedule, mock_artwork): + """Dedup guards prevent duplicates even when the lock is unavailable.""" + from apps.channels.tasks import evaluate_series_rules_impl + + prog = self._create_program(hours_from_now=2) + + # First call: Redis down, proceeds without lock + with patch("apps.channels.tasks.acquire_task_lock", + side_effect=ConnectionError("Redis unavailable")): + evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1) + + # EPG refresh + self._simulate_epg_refresh([self._program_data_for_refresh(prog)]) + + # Second call: Redis still down + with patch("apps.channels.tasks.acquire_task_lock", + side_effect=ConnectionError("Redis unavailable")): + result = evaluate_series_rules_impl() + self.assertEqual(Recording.objects.count(), 1, + "Dedup guards prevent duplicates even without lock") + self.assertEqual(result["scheduled"], 0) + + def test_lock_not_released_when_not_acquired(self, mock_schedule, mock_artwork): + """release_task_lock is not called if acquire raised an exception.""" + from apps.channels.tasks import evaluate_series_rules_impl + + self._create_program(hours_from_now=2) + + with patch("apps.channels.tasks.acquire_task_lock", + side_effect=ConnectionError("Redis unavailable")), \ + patch("apps.channels.tasks.release_task_lock") as mock_release: + evaluate_series_rules_impl() + mock_release.assert_not_called() + + +@patch("apps.channels.tasks.prefetch_recording_artwork") +@patch("apps.channels.signals.schedule_recording_task", return_value="mock-task-id") +class NonSeriesRecordingTests(SeriesRuleDedupBaseTestCase): + """Verify non-series recordings don't interfere with series rule dedup.""" + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_manual_recording_without_program_data_ignored(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Recordings without custom_properties.program are skipped by dedup key builder.""" + from apps.channels.tasks import evaluate_series_rules_impl + + # Manual recording with no program metadata + Recording.objects.create( + channel=self.channel, + start_time=self.now + timedelta(hours=2), + end_time=self.now + timedelta(hours=3), + custom_properties={}, + ) + + prog = self._create_program(hours_from_now=2) + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 2) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_recurring_rule_recording_does_not_interfere(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Recordings from recurring rules (custom_properties.rule) don't block series rules.""" + from apps.channels.tasks import evaluate_series_rules_impl + + Recording.objects.create( + channel=self.channel, + start_time=self.now + timedelta(hours=2), + end_time=self.now + timedelta(hours=3), + custom_properties={"rule": {"id": 1, "name": "Daily News"}}, + ) + + prog = self._create_program(hours_from_now=2) + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 1) + self.assertEqual(Recording.objects.count(), 2) + + @patch("apps.channels.tasks.acquire_task_lock", return_value=True) + @patch("apps.channels.tasks.release_task_lock") + def test_recording_with_null_custom_properties_ignored(self, mock_release, mock_lock, + mock_schedule, mock_artwork): + """Recordings with None custom_properties don't crash the dedup key builder.""" + from apps.channels.tasks import evaluate_series_rules_impl + + Recording.objects.create( + channel=self.channel, + start_time=self.now + timedelta(hours=2), + end_time=self.now + timedelta(hours=3), + custom_properties=None, + ) + + prog = self._create_program(hours_from_now=2) + result = evaluate_series_rules_impl() + self.assertEqual(result["scheduled"], 1) From 7cd4abb6946a9c1a306918b6ee326279bc87f0a2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 21 Mar 2026 14:07:30 -0500 Subject: [PATCH 13/67] Enhancement: Network Access settings: leaving a field blank no longer shows a validation error. The default CIDR range for that field is saved automatically and a "Defaults Restored" warning is displayed listing which fields were reset. (Closes #726) --- CHANGELOG.md | 3 ++ .../forms/settings/NetworkAccessForm.jsx | 43 +++++++++++++++++-- .../forms/settings/NetworkAccessFormUtils.js | 4 ++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2300474..bbc76777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Frontend cleanup: removed unused imports from `M3UGroupFilter`, `LiveGroupFilter`, and `VODCategoryFilter` (`Yup`, `M3UProfiles`, several unused Mantine components, dead `OptionWithTooltip` component, duplicate lucide-react imports, and `Divider` in `VODCategoryFilter`). No behaviour changes. +- Network Access settings: leaving a field blank no longer shows a validation error. The default CIDR range for that field is saved automatically and a "Defaults Restored" warning is displayed listing which fields were reset. (Closes #726) + +### Fixed ## [0.21.1] - 2026-03-18 diff --git a/frontend/src/components/forms/settings/NetworkAccessForm.jsx b/frontend/src/components/forms/settings/NetworkAccessForm.jsx index 04ab60b0..ca165ce6 100644 --- a/frontend/src/components/forms/settings/NetworkAccessForm.jsx +++ b/frontend/src/components/forms/settings/NetworkAccessForm.jsx @@ -1,6 +1,6 @@ import { NETWORK_ACCESS_OPTIONS } from '../../../constants.js'; import useSettingsStore from '../../../store/settings.jsx'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { useForm } from '@mantine/form'; import { checkSetting, @@ -19,12 +19,14 @@ const NetworkAccessForm = React.memo(({ active }) => { const [networkAccessError, setNetworkAccessError] = useState(null); const [saved, setSaved] = useState(false); + const [restoredDefaults, setRestoredDefaults] = useState([]); const [networkAccessConfirmOpen, setNetworkAccessConfirmOpen] = useState(false); const [saving, setSaving] = useState(false); const [netNetworkAccessConfirmCIDRs, setNetNetworkAccessConfirmCIDRs] = useState([]); const [clientIpAddress, setClientIpAddress] = useState(null); + const pendingSaveValuesRef = useRef(null); const networkAccessForm = useForm({ mode: 'controlled', @@ -33,7 +35,10 @@ const NetworkAccessForm = React.memo(({ active }) => { }); useEffect(() => { - if (!active) setSaved(false); + if (!active) { + setSaved(false); + setRestoredDefaults([]); + } }, [active]); useEffect(() => { @@ -58,9 +63,31 @@ const NetworkAccessForm = React.memo(({ active }) => { const onNetworkAccessSubmit = async () => { setSaved(false); setNetworkAccessError(null); + setRestoredDefaults([]); + + // Check for blank fields and substitute defaults before saving + const currentValues = networkAccessForm.getValues(); + const defaults = getNetworkAccessDefaults(); + const restoredLabels = []; + const submitValues = { ...currentValues }; + + Object.keys(currentValues).forEach((key) => { + if (!currentValues[key] || currentValues[key].trim() === '') { + submitValues[key] = defaults[key]; + restoredLabels.push(NETWORK_ACCESS_OPTIONS[key]?.label || key); + } + }); + + if (restoredLabels.length > 0) { + networkAccessForm.setValues(submitValues); + setRestoredDefaults(restoredLabels); + } + + pendingSaveValuesRef.current = submitValues; + const check = await checkSetting({ ...settings['network_access'], - value: networkAccessForm.getValues(), // Send as object + value: submitValues, }); if (check.error && check.message) { @@ -84,10 +111,12 @@ const NetworkAccessForm = React.memo(({ active }) => { const saveNetworkAccess = async () => { setSaved(false); setSaving(true); + const values = + pendingSaveValuesRef.current || networkAccessForm.getValues(); try { await updateSetting({ ...settings['network_access'], - value: networkAccessForm.getValues(), // Send as object + value: values, }); setSaved(true); } catch (e) { @@ -113,6 +142,12 @@ const NetworkAccessForm = React.memo(({ active }) => { title="Saved Successfully" > )} + {restoredDefaults.length > 0 && ( + + The following fields were empty and have been restored to their + defaults: {restoredDefaults.join(', ')} + + )} {networkAccessError && ( { export const getNetworkAccessFormValidation = () => { return Object.keys(NETWORK_ACCESS_OPTIONS).reduce((acc, key) => { acc[key] = (value) => { + if (!value || value.trim() === '') { + return null; // Empty values will be replaced with defaults on submit + } + if ( value .split(',') From 61d713fcda5ac6ffd90286c8512d64ff83e222c6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 21 Mar 2026 15:14:58 -0500 Subject: [PATCH 14/67] test: update validation for empty CIDR range to allow defaults on submit --- .../settings/__tests__/NetworkAccessForm.test.jsx | 15 ++++++++------- .../__tests__/NetworkAccessFormUtils.test.js | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/forms/settings/__tests__/NetworkAccessForm.test.jsx b/frontend/src/components/forms/settings/__tests__/NetworkAccessForm.test.jsx index 15e5e430..d3008354 100644 --- a/frontend/src/components/forms/settings/__tests__/NetworkAccessForm.test.jsx +++ b/frontend/src/components/forms/settings/__tests__/NetworkAccessForm.test.jsx @@ -313,10 +313,10 @@ describe('NetworkAccessForm', () => { fireEvent.click(screen.getByTestId('confirm-ok')); await waitFor(() => { - expect(screen.getByTestId('alert')).toBeInTheDocument(); - expect(screen.getByTestId('alert-title')).toHaveTextContent( - 'Saved Successfully' - ); + const alertTitles = screen.getAllByTestId('alert-title'); + expect( + alertTitles.some((el) => el.textContent === 'Saved Successfully') + ).toBe(true); }); }); @@ -399,9 +399,10 @@ describe('NetworkAccessForm', () => { fireEvent.click(screen.getByTestId('confirm-ok')); await waitFor(() => { - expect(screen.getByTestId('alert-title')).toHaveTextContent( - 'Saved Successfully' - ); + const alertTitles = screen.getAllByTestId('alert-title'); + expect( + alertTitles.some((el) => el.textContent === 'Saved Successfully') + ).toBe(true); }); }); }); diff --git a/frontend/src/utils/forms/settings/__tests__/NetworkAccessFormUtils.test.js b/frontend/src/utils/forms/settings/__tests__/NetworkAccessFormUtils.test.js index 11438d00..2cc885ec 100644 --- a/frontend/src/utils/forms/settings/__tests__/NetworkAccessFormUtils.test.js +++ b/frontend/src/utils/forms/settings/__tests__/NetworkAccessFormUtils.test.js @@ -131,7 +131,8 @@ describe('NetworkAccessFormUtils', () => { NetworkAccessFormUtils.getNetworkAccessFormValidation(); const validator = validation['network-access-admin']; - expect(validator('')).toBe('Invalid CIDR range'); + // Empty values are allowed — defaults are substituted on submit + expect(validator('')).toBe(null); }); it('should return empty object when NETWORK_ACCESS_OPTIONS is empty', () => { From b30a24e2fb384383466a033a4ebb809ac9399a66 Mon Sep 17 00:00:00 2001 From: None <190783071+CodeBormen@users.noreply.github.com> Date: Sat, 21 Mar 2026 17:24:09 -0500 Subject: [PATCH 15/67] feat: add TLS connection support for Redis and PostgreSQL (#950) - Add 10 env vars for Redis/PostgreSQL TLS (REDIS_SSL, REDIS_SSL_VERIFY, REDIS_SSL_CA_CERT, REDIS_SSL_CERT, REDIS_SSL_KEY, POSTGRES_SSL, POSTGRES_SSL_MODE, POSTGRES_SSL_CA_CERT, POSTGRES_SSL_CERT, POSTGRES_SSL_KEY) - Propagate SSL params to all Redis connection points: RedisClient, stream view, PubSub, client manager, Celery broker/result backend, Django Channels, and pre-Django startup scripts - Add Celery SSL config via URL query string params (required by Kombu's internal URL parsing) - Add PostgreSQL sslmode and certificate options to DATABASES config - Add startup validation: cert file existence, URL scheme conflict detection, TLS status logging - Add TLS error hints to Redis connection failure messages - Add Connection Security panel in System Settings (modular mode only) showing encryption, verification, and mTLS status - Add PG client key permission fix in web and celery entrypoints for Docker Desktop compatibility (0777 to 0600) - Add TLS env var passthrough in entrypoint for su - login shells - Document TLS env vars in modular docker-compose.yml - Change env_mode from dev/prod to actual DISPATCHARR_ENV value for modular mode detection --- apps/proxy/ts_proxy/client_manager.py | 3 +- apps/proxy/ts_proxy/server.py | 4 +- core/api_views.py | 17 +- core/utils.py | 25 ++- core/views.py | 4 +- dispatcharr/persistent_lock.py | 24 ++- dispatcharr/settings.py | 176 ++++++++++++++++-- docker/docker-compose.yml | 36 +++- docker/entrypoint.celery.sh | 17 ++ docker/entrypoint.sh | 30 +++ .../settings/ConnectionSecurityPanel.jsx | 174 +++++++++++++++++ .../forms/settings/SystemSettingsForm.jsx | 13 +- .../__tests__/SystemSettingsForm.test.jsx | 7 + .../src/store/__tests__/settings.test.jsx | 6 +- frontend/src/store/settings.jsx | 4 +- scripts/wait_for_redis.py | 48 ++++- 16 files changed, 540 insertions(+), 48 deletions(-) create mode 100644 frontend/src/components/forms/settings/ConnectionSecurityPanel.jsx diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index ea2aa5b0..d62551c1 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -58,7 +58,8 @@ class ClientManager: from django.conf import settings redis_url = getattr(settings, 'REDIS_URL', 'redis://localhost:6379/0') - redis_client = redis.Redis.from_url(redis_url, decode_responses=True) + ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) + redis_client = redis.Redis.from_url(redis_url, decode_responses=True, **ssl_params) all_channels = [] cursor = 0 diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index b72b350a..50729991 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -166,6 +166,7 @@ class ProxyServer: redis_password = os.environ.get("REDIS_PASSWORD", getattr(settings, 'REDIS_PASSWORD', '')) redis_user = os.environ.get("REDIS_USER", getattr(settings, 'REDIS_USER', '')) + ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) pubsub_client = redis.Redis( host=redis_host, port=redis_port, @@ -175,7 +176,8 @@ class ProxyServer: socket_timeout=60, socket_connect_timeout=10, socket_keepalive=True, - health_check_interval=30 + health_check_interval=30, + **ssl_params ) logger.info("Created fallback Redis PubSub client for event listener") diff --git a/core/api_views.py b/core/api_views.py index dd15886c..a81ddf54 100644 --- a/core/api_views.py +++ b/core/api_views.py @@ -3,6 +3,7 @@ import json import ipaddress import logging +from django.conf import settings as django_settings from django.db import models from rest_framework import viewsets, status from rest_framework.response import Response @@ -301,7 +302,9 @@ def environment(request): country_code = None country_name = None - # 4) Get environment mode from system environment variable + # 4) Get environment mode and TLS status from settings + postgres_ssl = getattr(django_settings, "POSTGRES_SSL", False) + return Response( { "authenticated": True, @@ -309,7 +312,17 @@ def environment(request): "local_ip": local_ip, "country_code": country_code, "country_name": country_name, - "env_mode": "dev" if os.getenv("DISPATCHARR_ENV") == "dev" else "prod", + "env_mode": os.getenv("DISPATCHARR_ENV", "aio"), + "redis_tls": { + "enabled": getattr(django_settings, "REDIS_SSL", False), + "verify": getattr(django_settings, "REDIS_SSL_VERIFY", True), + "mtls": bool(getattr(django_settings, "REDIS_SSL_CERT", "") and getattr(django_settings, "REDIS_SSL_KEY", "")), + }, + "postgres_tls": { + "enabled": postgres_ssl, + "ssl_mode": getattr(django_settings, "POSTGRES_SSL_MODE", "verify-full") if postgres_ssl else None, + "mtls": bool(getattr(django_settings, "POSTGRES_SSL_CERT", "") and getattr(django_settings, "POSTGRES_SSL_KEY", "")), + }, } ) diff --git a/core/utils.py b/core/utils.py index ef5aa702..20714724 100644 --- a/core/utils.py +++ b/core/utils.py @@ -13,6 +13,8 @@ from django.core.validators import URLValidator from django.core.exceptions import ValidationError import gc +_REDIS_TLS_HINT = " (TLS is enabled — verify certificate paths and that Redis is configured for TLS)" + logger = logging.getLogger(__name__) # Import the command detector @@ -65,6 +67,9 @@ class RedisClient: socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + # TLS params from settings (empty dict when TLS is disabled) + ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) + # Create Redis client with better defaults client = redis.Redis( host=redis_host, @@ -76,7 +81,8 @@ class RedisClient: socket_connect_timeout=socket_connect_timeout, socket_keepalive=socket_keepalive, health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout + retry_on_timeout=retry_on_timeout, + **ssl_params ) # Validate connection with ping @@ -125,8 +131,9 @@ class RedisClient: except (ConnectionError, TimeoutError) as e: retry_count += 1 + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}") + logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}{_tls_hint}") return None else: # Use exponential backoff for retries @@ -135,7 +142,8 @@ class RedisClient: time.sleep(wait_time) except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}") + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" + logger.error(f"Unexpected error connecting to Redis: {e}{_tls_hint}") return None return cls._client @@ -161,6 +169,8 @@ class RedisClient: health_check_interval = getattr(settings, 'REDIS_HEALTH_CHECK_INTERVAL', 30) retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) + # Create Redis client with PubSub-optimized settings - no timeout client = redis.Redis( host=redis_host, @@ -172,7 +182,8 @@ class RedisClient: socket_connect_timeout=socket_connect_timeout, socket_keepalive=socket_keepalive, health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout + retry_on_timeout=retry_on_timeout, + **ssl_params ) # Validate connection with ping @@ -185,8 +196,9 @@ class RedisClient: except (ConnectionError, TimeoutError) as e: retry_count += 1 + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}") + logger.error(f"Failed to connect to Redis for PubSub after {max_retries} attempts: {e}{_tls_hint}") return None else: # Use exponential backoff for retries @@ -195,7 +207,8 @@ class RedisClient: time.sleep(wait_time) except Exception as e: - logger.error(f"Unexpected error connecting to Redis for PubSub: {e}") + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" + logger.error(f"Unexpected error connecting to Redis for PubSub: {e}{_tls_hint}") return None return cls._pubsub_client diff --git a/core/views.py b/core/views.py index 082cccc3..b3e6dfb2 100644 --- a/core/views.py +++ b/core/views.py @@ -42,12 +42,14 @@ def stream_view(request, channel_uuid): redis_db = int(getattr(settings, "REDIS_DB", "0")) redis_password = getattr(settings, "REDIS_PASSWORD", "") redis_user = getattr(settings, "REDIS_USER", "") + ssl_params = getattr(settings, "REDIS_SSL_PARAMS", {}) redis_client = redis.Redis( host=redis_host, port=redis_port, db=redis_db, password=redis_password if redis_password else None, - username=redis_user if redis_user else None + username=redis_user if redis_user else None, + **ssl_params ) # Retrieve the channel by the provided stream_id. diff --git a/dispatcharr/persistent_lock.py b/dispatcharr/persistent_lock.py index 2ed72bf4..64e9e6ba 100644 --- a/dispatcharr/persistent_lock.py +++ b/dispatcharr/persistent_lock.py @@ -74,18 +74,40 @@ class PersistentLock: # Example usage (for testing purposes only): if __name__ == "__main__": import os + import sys # Connect to Redis using environment variables; adjust connection parameters as needed. redis_host = os.environ.get("REDIS_HOST", "localhost") redis_port = int(os.environ.get("REDIS_PORT", 6379)) redis_db = int(os.environ.get("REDIS_DB", 0)) redis_password = os.environ.get("REDIS_PASSWORD", "") redis_user = os.environ.get("REDIS_USER", "") + ssl_kwargs = {} + if os.environ.get("REDIS_SSL", "false").lower() == "true": + import ssl as _ssl + ssl_kwargs["ssl"] = True + ssl_kwargs["ssl_cert_reqs"] = ( + _ssl.CERT_REQUIRED if os.environ.get("REDIS_SSL_VERIFY", "true").lower() == "true" + else _ssl.CERT_NONE + ) + for env_var, key in [ + ("REDIS_SSL_CA_CERT", "ssl_ca_certs"), + ("REDIS_SSL_CERT", "ssl_certfile"), + ("REDIS_SSL_KEY", "ssl_keyfile"), + ]: + path = os.environ.get(env_var, "") + if path: + if not os.path.isfile(path): + print(f"Redis TLS: {env_var}={path!r} — file not found.") + sys.exit(1) + ssl_kwargs[key] = path + client = redis.Redis( host=redis_host, port=redis_port, db=redis_db, password=redis_password if redis_password else None, - username=redis_user if redis_user else None + username=redis_user if redis_user else None, + **ssl_kwargs ) lock = PersistentLock(client, "lock:example_account", lock_timeout=120) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index b0d387b2..ec61eb4d 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -1,7 +1,23 @@ import os +import ssl from pathlib import Path from datetime import timedelta from urllib.parse import quote_plus +from django.core.exceptions import ImproperlyConfigured + + +def _validate_tls_cert_paths(paths, service_name): + """Validate that configured TLS certificate file paths exist on disk. + + Raises ImproperlyConfigured with a clear message identifying the + service and missing file so operators can fix their environment. + """ + for env_var, file_path in paths: + if file_path and not Path(file_path).is_file(): + raise ImproperlyConfigured( + f"{service_name} TLS: {env_var}={file_path!r} — file not found. " + f"Check that the certificate file exists and the volume is mounted correctly." + ) BASE_DIR = Path(__file__).resolve().parent.parent @@ -12,6 +28,37 @@ REDIS_DB = os.environ.get("REDIS_DB", "0") REDIS_USER = os.environ.get("REDIS_USER", "") REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "") +# Redis TLS configuration +REDIS_SSL = os.environ.get("REDIS_SSL", "false").lower() == "true" +REDIS_SSL_VERIFY = os.environ.get("REDIS_SSL_VERIFY", "true").lower() == "true" +REDIS_SSL_CA_CERT = os.environ.get("REDIS_SSL_CA_CERT", "") +REDIS_SSL_CERT = os.environ.get("REDIS_SSL_CERT", "") +REDIS_SSL_KEY = os.environ.get("REDIS_SSL_KEY", "") + +# Reusable dict of SSL kwargs for redis.Redis() constructors +REDIS_SSL_PARAMS = {} +if REDIS_SSL: + _validate_tls_cert_paths([ + ("REDIS_SSL_CA_CERT", REDIS_SSL_CA_CERT), + ("REDIS_SSL_CERT", REDIS_SSL_CERT), + ("REDIS_SSL_KEY", REDIS_SSL_KEY), + ], "Redis") + + REDIS_SSL_PARAMS["ssl"] = True + REDIS_SSL_PARAMS["ssl_cert_reqs"] = ssl.CERT_REQUIRED if REDIS_SSL_VERIFY else ssl.CERT_NONE + if REDIS_SSL_CA_CERT: + REDIS_SSL_PARAMS["ssl_ca_certs"] = REDIS_SSL_CA_CERT + if REDIS_SSL_CERT: + REDIS_SSL_PARAMS["ssl_certfile"] = REDIS_SSL_CERT + if REDIS_SSL_KEY: + REDIS_SSL_PARAMS["ssl_keyfile"] = REDIS_SSL_KEY + + _mtls = "enabled" if REDIS_SSL_CERT and REDIS_SSL_KEY else "disabled" + _verify = "on" if REDIS_SSL_VERIFY else "off" + print(f"Redis TLS: enabled (verify={_verify}, mTLS={_mtls})") +else: + print("Redis TLS: disabled") + # Set DEBUG to True for development, False for production if os.environ.get("DISPATCHARR_DEBUG", "False").lower() == "true": DEBUG = True @@ -120,20 +167,46 @@ TEMPLATES = [ WSGI_APPLICATION = "dispatcharr.wsgi.application" ASGI_APPLICATION = "dispatcharr.asgi.application" +_redis_scheme = "rediss" if REDIS_SSL else "redis" + +# URL-encoded auth string shared by CHANNEL_LAYERS and Celery broker URLs +if REDIS_PASSWORD: + _encoded_password = quote_plus(REDIS_PASSWORD) + if REDIS_USER: + _redis_auth = f"{quote_plus(REDIS_USER)}:{_encoded_password}@" + else: + _redis_auth = f":{_encoded_password}@" +else: + _redis_auth = "" + +_channels_redis_url = f"{_redis_scheme}://{_redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" +# channels_redis accepts either a URL string or a dict with "address" + kwargs. +# When TLS is enabled, pass SSL params alongside the URL so the connection pool +# uses the correct CA cert and verification settings. +if REDIS_SSL: + # Filter out "ssl" key — the rediss:// scheme already enables SSL. + # Passing ssl=True as a kwarg to aioredis from_url causes an error. + _channels_ssl = {k: v for k, v in REDIS_SSL_PARAMS.items() if k != "ssl"} + _channels_host = {"address": _channels_redis_url, **_channels_ssl} +else: + _channels_host = _channels_redis_url + CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": ["redis://{redis_auth}{host}:{port}/{db}".format( - redis_auth=f"{quote_plus(REDIS_USER)}:{quote_plus(REDIS_PASSWORD)}@" if REDIS_PASSWORD and REDIS_USER else f":{quote_plus(REDIS_PASSWORD)}@" if REDIS_PASSWORD else "", - host=REDIS_HOST, - port=REDIS_PORT, - db=REDIS_DB - )], # URL format supports authentication + "hosts": [_channels_host], }, }, } +# PostgreSQL TLS configuration (defined before DATABASES for module-level access) +POSTGRES_SSL = os.environ.get("POSTGRES_SSL", "false").lower() == "true" +POSTGRES_SSL_MODE = os.environ.get("POSTGRES_SSL_MODE", "verify-full") +POSTGRES_SSL_CA_CERT = os.environ.get("POSTGRES_SSL_CA_CERT", "") +POSTGRES_SSL_CERT = os.environ.get("POSTGRES_SSL_CERT", "") +POSTGRES_SSL_KEY = os.environ.get("POSTGRES_SSL_KEY", "") + if os.getenv("DB_ENGINE", None) == "sqlite": DATABASES = { "default": { @@ -154,6 +227,28 @@ else: } } + if POSTGRES_SSL: + _validate_tls_cert_paths([ + ("POSTGRES_SSL_CA_CERT", POSTGRES_SSL_CA_CERT), + ("POSTGRES_SSL_CERT", POSTGRES_SSL_CERT), + ("POSTGRES_SSL_KEY", POSTGRES_SSL_KEY), + ], "PostgreSQL") + + DATABASES["default"]["OPTIONS"] = { + "sslmode": POSTGRES_SSL_MODE, + } + if POSTGRES_SSL_CA_CERT: + DATABASES["default"]["OPTIONS"]["sslrootcert"] = POSTGRES_SSL_CA_CERT + if POSTGRES_SSL_CERT: + DATABASES["default"]["OPTIONS"]["sslcert"] = POSTGRES_SSL_CERT + if POSTGRES_SSL_KEY: + DATABASES["default"]["OPTIONS"]["sslkey"] = POSTGRES_SSL_KEY + + _mtls = "enabled" if POSTGRES_SSL_CERT and POSTGRES_SSL_KEY else "disabled" + print(f"PostgreSQL TLS: enabled (sslmode={POSTGRES_SSL_MODE}, mTLS={_mtls})") + else: + print("PostgreSQL TLS: disabled") + AUTH_PASSWORD_VALIDATORS = [ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", @@ -208,22 +303,52 @@ STATICFILES_DIRS = [ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" AUTH_USER_MODEL = "accounts.User" -# Build default Redis URL from components for Celery with optional authentication -# Build auth string conditionally with URL encoding for special characters -if REDIS_PASSWORD: - encoded_password = quote_plus(REDIS_PASSWORD) - if REDIS_USER: - encoded_user = quote_plus(REDIS_USER) - redis_auth = f"{encoded_user}:{encoded_password}@" - else: - redis_auth = f":{encoded_password}@" +_default_redis_url = f"{_redis_scheme}://{_redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" +# Celery/Kombu require SSL parameters in the URL query string because +# internal URL parsing can overwrite the CELERY_BROKER_USE_SSL dict. +if REDIS_SSL: + _celery_ssl_params = [ + f"ssl_cert_reqs={'CERT_REQUIRED' if REDIS_SSL_VERIFY else 'CERT_NONE'}", + ] + if REDIS_SSL_CA_CERT: + _celery_ssl_params.append(f"ssl_ca_certs={REDIS_SSL_CA_CERT}") + if REDIS_SSL_CERT: + _celery_ssl_params.append(f"ssl_certfile={REDIS_SSL_CERT}") + if REDIS_SSL_KEY: + _celery_ssl_params.append(f"ssl_keyfile={REDIS_SSL_KEY}") + _default_celery_url = f"{_default_redis_url}?{'&'.join(_celery_ssl_params)}" else: - redis_auth = "" - -_default_redis_url = f"redis://{redis_auth}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" -CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", _default_redis_url) + _default_celery_url = _default_redis_url +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", _default_celery_url) CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", CELERY_BROKER_URL) +# Validate that URL overrides don't conflict with TLS settings +for _url_var, _url_val in [ + ("CELERY_BROKER_URL", CELERY_BROKER_URL), + ("CELERY_RESULT_BACKEND", CELERY_RESULT_BACKEND), +]: + _is_override = os.environ.get(_url_var) is not None + if not _is_override: + continue + _url_is_ssl = _url_val.startswith("rediss://") + if REDIS_SSL and not _url_is_ssl: + raise ImproperlyConfigured( + f"REDIS_SSL is enabled but {_url_var} uses redis:// (plaintext). " + f"Change the URL scheme to rediss:// or remove the {_url_var} override." + ) + if not REDIS_SSL and _url_is_ssl: + raise ImproperlyConfigured( + f"{_url_var} uses rediss:// (TLS) but REDIS_SSL is not enabled. " + f"Set REDIS_SSL=true and configure the TLS certificate settings." + ) + +# Celery TLS configuration — required in addition to the rediss:// URL scheme. +# Uses the same cert params as REDIS_SSL_PARAMS, minus the "ssl" key that +# redis-py needs but Celery/Kombu does not. +if REDIS_SSL: + CELERY_BROKER_USE_SSL = {k: v for k, v in REDIS_SSL_PARAMS.items() if k != "ssl"} + CELERY_RESULT_BACKEND_USE_SSL = CELERY_BROKER_USE_SSL + # Configure Redis key prefix CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = { "global_keyprefix": "celery-tasks:", # Set the Redis key prefix for Celery @@ -299,8 +424,19 @@ SIMPLE_JWT = { "BLACKLIST_AFTER_ROTATION": True, # Optional: Whether to blacklist refresh tokens } -# Redis connection settings +# Redis connection settings — _default_redis_url uses rediss:// when REDIS_SSL is enabled REDIS_URL = os.environ.get("REDIS_URL", _default_redis_url) +if os.environ.get("REDIS_URL") is not None: + if REDIS_SSL and not REDIS_URL.startswith("rediss://"): + raise ImproperlyConfigured( + "REDIS_SSL is enabled but REDIS_URL uses redis:// (plaintext). " + "Change the URL scheme to rediss:// or remove the REDIS_URL override." + ) + if not REDIS_SSL and REDIS_URL.startswith("rediss://"): + raise ImproperlyConfigured( + "REDIS_URL uses rediss:// (TLS) but REDIS_SSL is not enabled. " + "Set REDIS_SSL=true and configure the TLS certificate settings." + ) REDIS_SOCKET_TIMEOUT = 60 # Socket timeout in seconds REDIS_SOCKET_CONNECT_TIMEOUT = 5 # Connection timeout in seconds REDIS_HEALTH_CHECK_INTERVAL = 15 # Health check every 15 seconds diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d674460a..547ac249 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,6 +14,7 @@ services: - 9191:9191 volumes: - ./data:/data + #- ./certs:/certs:ro # TLS certificates (optional) depends_on: db: condition: service_healthy @@ -33,15 +34,26 @@ services: - POSTGRES_DB=dispatcharr - POSTGRES_USER=dispatch - POSTGRES_PASSWORD=secret + # PostgreSQL TLS (optional) — mount certs via the volume above + #- POSTGRES_SSL=true # required to enable TLS + #- POSTGRES_SSL_MODE=verify-full # optional: verify-full (default) | verify-ca | require + #- POSTGRES_SSL_CA_CERT=/certs/postgres/ca.crt # optional: CA cert to verify the server + #- POSTGRES_SSL_CERT=/certs/postgres/client.crt # optional: client cert (only if server requires client auth) + #- POSTGRES_SSL_KEY=/certs/postgres/client.key # optional: client key (only if server requires client auth) # Redis Connection - REDIS_HOST=redis - REDIS_PORT=6379 - # Redis Authentication (Optional) # Uncomment and set if your Redis requires authentication: #- REDIS_PASSWORD=your_strong_redis_password #- REDIS_USER=your_redis_username # For Redis 6+ ACL - see Redis service below + # Redis TLS (optional) — mount certs via the volume above + #- REDIS_SSL=true # required to enable TLS + #- REDIS_SSL_VERIFY=true # optional: set false for self-signed certs without a CA + #- REDIS_SSL_CA_CERT=/certs/redis/ca.crt # optional: CA cert to verify the server + #- REDIS_SSL_CERT=/certs/redis/client.crt # optional: client cert (only if server requires client auth) + #- REDIS_SSL_KEY=/certs/redis/client.key # optional: client key (only if server requires client auth) # Logging - DISPATCHARR_LOG_LEVEL=info @@ -95,6 +107,7 @@ services: condition: service_started volumes: - ./data:/data + #- ./certs:/certs:ro # TLS certificates (optional) extra_hosts: - "host.docker.internal:host-gateway" entrypoint: ["/app/docker/entrypoint.celery.sh"] @@ -108,21 +121,30 @@ services: # Must match the web service port for DVR recording and internal API calls - DISPATCHARR_PORT=9191 - # PostgreSQL Connection + # PostgreSQL — must match web service settings - POSTGRES_HOST=db - POSTGRES_PORT=5432 - POSTGRES_DB=dispatcharr - POSTGRES_USER=dispatch - POSTGRES_PASSWORD=secret + # PostgreSQL TLS — must match web service + #- POSTGRES_SSL=true + #- POSTGRES_SSL_MODE=verify-full + #- POSTGRES_SSL_CA_CERT=/certs/postgres/ca.crt + #- POSTGRES_SSL_CERT=/certs/postgres/client.crt + #- POSTGRES_SSL_KEY=/certs/postgres/client.key - # Redis Connection + # Redis — must match web service settings - REDIS_HOST=redis - REDIS_PORT=6379 - - # Redis Authentication (Optional) - # Uncomment and set if your Redis requires authentication: #- REDIS_PASSWORD=your_strong_redis_password - #- REDIS_USER=your_redis_username # For Redis 6+ ACL - see Redis service below + #- REDIS_USER=your_redis_username + # Redis TLS — must match web service + #- REDIS_SSL=true + #- REDIS_SSL_VERIFY=true + #- REDIS_SSL_CA_CERT=/certs/redis/ca.crt + #- REDIS_SSL_CERT=/certs/redis/client.crt + #- REDIS_SSL_KEY=/certs/redis/client.key # Logging - DISPATCHARR_LOG_LEVEL=info diff --git a/docker/entrypoint.celery.sh b/docker/entrypoint.celery.sh index 7a69f430..d7a43255 100644 --- a/docker/entrypoint.celery.sh +++ b/docker/entrypoint.celery.sh @@ -36,6 +36,23 @@ if [ "$USE_LEGACY_NUMPY" = "true" ]; then fi fi +# Fix TLS client key permissions for PostgreSQL (same issue as web entrypoint). +# libpq requires 0600 but Docker Desktop mounts files as 0777. +if [ -n "${POSTGRES_SSL_KEY:-}" ] && [ -f "$POSTGRES_SSL_KEY" ]; then + _key_perms=$(stat -c '%a' "$POSTGRES_SSL_KEY" 2>/dev/null) + if [ "$_key_perms" != "600" ] && [ "$_key_perms" != "640" ]; then + _fixed_key="/data/.pg-client-celery.key" + cp "$POSTGRES_SSL_KEY" "$_fixed_key" + chmod 600 "$_fixed_key" + # Match ownership to the user running Celery if running as root + if [ "$(id -u)" = "0" ] && [ -n "${PUID:-}" ]; then + chown "${PUID}:${PGID:-$PUID}" "$_fixed_key" + fi + export POSTGRES_SSL_KEY="$_fixed_key" + echo "Fixed PostgreSQL client key permissions (${_key_perms} → 600)" + fi +fi + # Wait for migrations to complete # Uses 'migrate --check' which exits 0 only when all migrations are applied, # and exits 1 on unapplied migrations OR connection errors (safe either way) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0de0afd5..7365a300 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -112,6 +112,14 @@ variables=( CELERY_NICE_LEVEL UWSGI_NICE_LEVEL DJANGO_SECRET_KEY ) +# TLS variables are optional — only propagate when set to avoid noisy warnings +for _tls_var in POSTGRES_SSL POSTGRES_SSL_MODE POSTGRES_SSL_CA_CERT POSTGRES_SSL_CERT POSTGRES_SSL_KEY \ + REDIS_SSL REDIS_SSL_VERIFY REDIS_SSL_CA_CERT REDIS_SSL_CERT REDIS_SSL_KEY; do + if [ -n "${!_tls_var+x}" ]; then + variables+=("$_tls_var") + fi +done + # Truncate files before rewriting > /etc/profile.d/dispatcharr.sh @@ -229,6 +237,28 @@ if [ "$USE_LEGACY_NUMPY" = "true" ]; then fi fi +# Fix TLS client key permissions for PostgreSQL. +# libpq requires the client key to be 0600 or stricter. Volume-mounted files +# from Windows/macOS hosts often have 0755 permissions, so copy the key to a +# writable location with correct permissions if needed. +if [ -n "${POSTGRES_SSL_KEY:-}" ] && [ -f "$POSTGRES_SSL_KEY" ]; then + _key_perms=$(stat -c '%a' "$POSTGRES_SSL_KEY" 2>/dev/null) + if [ "$_key_perms" != "600" ] && [ "$_key_perms" != "640" ]; then + _fixed_key="/data/.pg-client.key" + cp "$POSTGRES_SSL_KEY" "$_fixed_key" + chmod 600 "$_fixed_key" + if [ "$(id -u)" = "0" ] && [ -n "${PUID:-}" ]; then + chown "${PUID}:${PGID}" "$_fixed_key" + fi + export POSTGRES_SSL_KEY="$_fixed_key" + # Update /etc/environment so login shells see the fixed path + sed -i "/^POSTGRES_SSL_KEY=/d" /etc/environment + echo "POSTGRES_SSL_KEY='$_fixed_key'" >> /etc/environment + sed -i "s|export POSTGRES_SSL_KEY=.*|export POSTGRES_SSL_KEY='$_fixed_key'|" /etc/profile.d/dispatcharr.sh + echo "Fixed PostgreSQL client key permissions (${_key_perms} → 600)" + fi +fi + # Run Django commands as non-root user to prevent permission issues su - "$POSTGRES_USER" -c "cd /app && python manage.py migrate --noinput" su - "$POSTGRES_USER" -c "cd /app && python manage.py collectstatic --noinput" diff --git a/frontend/src/components/forms/settings/ConnectionSecurityPanel.jsx b/frontend/src/components/forms/settings/ConnectionSecurityPanel.jsx new file mode 100644 index 00000000..f8c9614a --- /dev/null +++ b/frontend/src/components/forms/settings/ConnectionSecurityPanel.jsx @@ -0,0 +1,174 @@ +import React from 'react'; +import { + Badge, + Group, + Paper, + SimpleGrid, + Stack, + Text, + Tooltip, +} from '@mantine/core'; +import useSettingsStore from '../../../store/settings.jsx'; + +const TlsOption = ({ label, description, tooltip, badgeText, badgeColor }) => ( +
+ + + {label} + + + + {badgeText} + + + + + {description} + +
+); + +const TlsServiceCard = ({ serviceName, children }) => ( + + + {serviceName} + {children} + + +); + +const RedisStatus = ({ tls }) => { + const enabled = tls?.enabled ?? false; + const verify = tls?.verify ?? true; + const mtls = tls?.mtls ?? false; + + let verifyColor = 'gray'; + let verifyTooltip = 'Verification is not active — TLS is disabled'; + if (enabled) { + verifyColor = verify ? 'green' : 'yellow'; + verifyTooltip = verify + ? "The server's identity is verified using a trusted certificate" + : 'The connection is encrypted but the server identity is not verified'; + } + + let mtlsColor = 'gray'; + let mtlsTooltip = 'Mutual authentication is not active'; + if (enabled && mtls) { + mtlsColor = 'green'; + mtlsTooltip = 'Both client and server verify each other with certificates'; + } + + return ( + + + + + + ); +}; + +const PostgresStatus = ({ tls }) => { + const enabled = tls?.enabled ?? false; + const sslMode = tls?.ssl_mode; + const mtls = tls?.mtls ?? false; + + let modeColor = 'gray'; + let modeTooltip = 'Verification mode is not active — TLS is disabled'; + let modeBadge = 'Off'; + if (enabled && sslMode) { + modeBadge = sslMode; + if (sslMode === 'verify-full') { + modeColor = 'green'; + modeTooltip = 'Server certificate and hostname are both verified'; + } else if (sslMode === 'verify-ca') { + modeColor = 'yellow'; + modeTooltip = + 'Server certificate is verified, but hostname is not checked'; + } else { + modeColor = 'yellow'; + modeTooltip = 'Connection is encrypted but the server is not verified'; + } + } + + let mtlsColor = 'gray'; + let mtlsTooltip = 'Mutual authentication is not active'; + if (enabled && mtls) { + mtlsColor = 'green'; + mtlsTooltip = 'Both client and server verify each other with certificates'; + } + + return ( + + + + + + ); +}; + +const ConnectionSecurityPanel = React.memo(() => { + const environment = useSettingsStore((s) => s.environment); + const isModular = environment.env_mode === 'modular'; + + return ( + + + {isModular + ? 'Encrypt connections to Redis and PostgreSQL using environment variables in the docker compose file.' + : 'Connection encryption is only available in modular deployment mode with external services.'} + + {isModular && ( + + + + + )} + + ); +}); + +export default ConnectionSecurityPanel; diff --git a/frontend/src/components/forms/settings/SystemSettingsForm.jsx b/frontend/src/components/forms/settings/SystemSettingsForm.jsx index a8c8d45b..132830ff 100644 --- a/frontend/src/components/forms/settings/SystemSettingsForm.jsx +++ b/frontend/src/components/forms/settings/SystemSettingsForm.jsx @@ -5,7 +5,16 @@ import { parseSettings, saveChangedSettings, } from '../../../utils/pages/SettingsUtils.js'; -import { Alert, Button, Flex, NumberInput, Stack, Text } from '@mantine/core'; +import { + Alert, + Button, + Divider, + Flex, + NumberInput, + Stack, + Text, +} from '@mantine/core'; +import ConnectionSecurityPanel from './ConnectionSecurityPanel.jsx'; import { useForm } from '@mantine/form'; import { getSystemSettingsFormInitialValues } from '../../../utils/forms/settings/SystemSettingsFormUtils.js'; @@ -68,6 +77,8 @@ const SystemSettingsForm = React.memo(({ active }) => { max={1000} step={10} /> + + + + + + + ); +}); + +export default UserLimitsForm; diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 0c19eea1..e8638cc8 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -62,6 +62,33 @@ export const PROXY_SETTINGS_OPTIONS = { }, }; +export const USER_LIMITS_OPTIONS = { + terminate_on_limit_exceeded: { + label: 'Terminate on Limit Exceeded', + description: + 'Terminate a stream (based on below criteria) when the user exceeds the allowed limits', + default: true, + }, + prioritize_single_client_channels: { + label: 'Prioritize Single Client Channels', + description: + 'Prioritize terminating channels only a single client belonging to the user', + default: true, + }, + ignore_same_channel_connections: { + label: 'Ignore Same-Channel Connections', + description: + 'Multiple user connections to the same channel count as 1 connection toward user limits', + default: false, + }, + terminate_oldest: { + label: 'Terminate Oldest', + description: + 'Prioritize terminating the oldest stream when limits are exceeded. Setting to false prioritizes the newest stream.', + default: true, + }, +}; + export const M3U_FILTER_TYPES = [ { label: 'Group', diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index 8bef5b47..71bbc68c 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -23,6 +23,7 @@ const BackupManager = React.lazy( import useAuthStore from '../store/auth'; import { USER_LEVELS } from '../constants'; import UiSettingsForm from '../components/forms/settings/UiSettingsForm.jsx'; +import UserLimitsForm from '../components/forms/settings/UserLimitsForm.jsx'; import ErrorBoundary from '../components/ErrorBoundary.jsx'; const NetworkAccessForm = React.lazy( () => import('../components/forms/settings/NetworkAccessForm.jsx') @@ -200,6 +201,19 @@ const SettingsPage = () => { + + + User Limts + + + }> + + + + + )} From af54366d76cafec9a02fd0b0397823a4f8f81468 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 10:15:24 -0500 Subject: [PATCH 28/67] changelog: Update changelog for XML parsing fixes. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5dace89..58a41e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) +- TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) — Thanks [@CodeBormen](https://github.com/CodeBormen) - Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) ### Changed @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- HTML named entities in XMLTV EPG files are now resolved to Unicode characters before lxml parsing. Some EPG providers (particularly French and other European sources) use HTML named entities like `é`, `î`, `ü` in channel names, program titles, and metadata. These are not valid XML entities — lxml 6.0.2 with `recover=True` silently drops them, causing characters to go missing (e.g., "Chaîne Télé" becomes "Chane Tl"). A preprocessing step in `fetch_xmltv()` now resolves HTML named entities while preserving the 5 XML-predefined entities, detecting encoding from the XML declaration (falling back to UTF-8), and processing line-by-line to a temp file before atomically replacing the original. (Closes #1095) — Thanks [@CodeBormen](https://github.com/CodeBormen) - Duplicate recordings created when EPG sources refresh and re-evaluate series rules — Thanks [@CodeBormen](https://github.com/CodeBormen): - **Program ID instability**: `parse_programs_for_source()` deletes and recreates all `ProgramData` rows with new auto-increment IDs on every EPG refresh. The dedup set used these IDs, so it never matched after a refresh. Deduplication now uses a stable `(tvg_id, start_time, end_time)` composite key sourced from `Recording.custom_properties.program`. - **Secondary guard using wrong times**: The DB guard compared unadjusted program times against offset-adjusted `Recording.start_time`/`end_time`, so it never matched when any DVR pre/post offset was configured. It now queries `custom_properties__program__start_time/end_time` (the original, unadjusted program times stored at recording creation). From b671a72707867d2340932ad060075a3bb74ac637 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 18:41:42 -0500 Subject: [PATCH 29/67] fix: resolve decode_responses migration bugs and user-limit regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a wave of bugs introduced by the user-priority branch's migration from manual .decode('utf-8') calls to decode_responses=True on the metadata Redis client: core/utils.py - Rewrite _init_client: fix SyntaxError at line 138, missing redis_password/redis_user params, cls._client corruption when decode_responses=False was requested, and dead retry backoff logic apps/proxy/utils.py - Fix get_user_limit_settings() → get_user_limits_settings() (2 sites) - Fix VOD scan key vod_persistent_connections:* → vod_persistent_connection:* - Fix undefined channel_id in VOD loop (use content_uuid from Redis hash) - Remove leftover print(active_connections) debug statement - Refactor manual cursor SCAN while-loops to scan_iter() apps/channels/tasks.py - Fix closure bug in _d(): md.get(key) → md.get(bkey) (was reading the outer loop variable instead of the local bytes key) apps/proxy/ts_proxy/server.py - Replace stale b'init_time', b'total_bytes', b'state' byte-string key lookups and .decode() calls with plain string equivalents apps/proxy/ts_proxy/services/channel_service.py - Fix ChannelMetadataField.STATE.encode() / .OWNER.encode() → .STATE / .OWNER (pre-existing, broke under decoded client) apps/proxy/ts_proxy/views.py - Fix ChannelMetadataField.OWNER.encode("utf-8") → .OWNER apps/proxy/ts_proxy/client_manager.py - Fix cid.decode('utf-8') in remove_ghost_clients(): smembers() already returns str with decode_responses=True --- apps/channels/tasks.py | 2 +- apps/proxy/ts_proxy/client_manager.py | 3 +- apps/proxy/ts_proxy/server.py | 10 +- .../ts_proxy/services/channel_service.py | 8 +- apps/proxy/ts_proxy/views.py | 2 +- apps/proxy/utils.py | 108 ++++++++--------- core/utils.py | 109 ++++++------------ debug_redis_check.py | 21 ++++ 8 files changed, 114 insertions(+), 149 deletions(-) create mode 100644 debug_redis_check.py diff --git a/apps/channels/tasks.py b/apps/channels/tasks.py index 52956d53..59122d9f 100755 --- a/apps/channels/tasks.py +++ b/apps/channels/tasks.py @@ -2494,7 +2494,7 @@ def run_recording(recording_id, channel_id, start_time_str, end_time_str): md = r.hgetall(metadata_key) if md: def _d(bkey, cast=str): - v = md.get(key) + v = md.get(bkey) try: if v is None: return None diff --git a/apps/proxy/ts_proxy/client_manager.py b/apps/proxy/ts_proxy/client_manager.py index e2cfd460..e1c92256 100644 --- a/apps/proxy/ts_proxy/client_manager.py +++ b/apps/proxy/ts_proxy/client_manager.py @@ -428,8 +428,7 @@ class ClientManager: client_id_list = list(client_ids) pipe = redis_client.pipeline() for cid in client_id_list: - cid_str = cid.decode('utf-8') - pipe.exists(RedisKeys.client_metadata(channel_id, cid_str)) + pipe.exists(RedisKeys.client_metadata(channel_id, cid)) results = pipe.execute() stale_ids = [ diff --git a/apps/proxy/ts_proxy/server.py b/apps/proxy/ts_proxy/server.py index b3449693..5316ec58 100644 --- a/apps/proxy/ts_proxy/server.py +++ b/apps/proxy/ts_proxy/server.py @@ -940,14 +940,14 @@ class ProxyServer: metadata = self.redis_client.hgetall(metadata_key) if metadata: # Calculate runtime from init_time - if b'init_time' in metadata: + if 'init_time' in metadata: try: init_time = float(metadata['init_time']) runtime = round(time.time() - init_time, 2) except Exception: pass # Get total bytes transferred - if b'total_bytes' in metadata: + if 'total_bytes' in metadata: try: total_bytes = int(metadata['total_bytes']) except Exception: @@ -1113,9 +1113,9 @@ class ProxyServer: # Also get init time as a fallback init_time = None - if metadata and b'init_time' in metadata: + if metadata and 'init_time' in metadata: try: - init_time = float(metadata[b'init_time']) + init_time = float(metadata['init_time']) except (ValueError, TypeError): pass @@ -1400,7 +1400,7 @@ class ProxyServer: real_count = max(0, client_count - len(stale_ids)) if real_count <= 0: # No real clients remain — safe to clean up. - state = metadata.get(b'state', b'unknown').decode('utf-8') if b'state' in metadata else 'unknown' + state = metadata.get('state', 'unknown') logger.warning( f"Orphaned channel {channel_id} (state: {state}, " f"owner: {owner}) had {client_count} ghost client(s) " diff --git a/apps/proxy/ts_proxy/services/channel_service.py b/apps/proxy/ts_proxy/services/channel_service.py index 0a0945d6..d0478e8f 100644 --- a/apps/proxy/ts_proxy/services/channel_service.py +++ b/apps/proxy/ts_proxy/services/channel_service.py @@ -382,8 +382,8 @@ class ChannelService: metadata = proxy_server.redis_client.hgetall(metadata_key) # Extract state and owner - state = metadata.get(ChannelMetadataField.STATE.encode(), 'unknown') - owner = metadata.get(ChannelMetadataField.OWNER.encode(), 'unknown') + state = metadata.get(ChannelMetadataField.STATE, 'unknown') + owner = metadata.get(ChannelMetadataField.OWNER, 'unknown') # Valid states indicate channel is running properly valid_states = [ChannelState.ACTIVE, ChannelState.WAITING_FOR_CLIENTS, ChannelState.CONNECTING] @@ -432,13 +432,13 @@ class ChannelService: try: # Use factory to parse the line based on stream type parsed_data = LogParserFactory.parse(stream_type, stream_info_line) - + if not parsed_data: return # Update Redis and database with parsed data ChannelService._update_stream_info_in_redis( - channel_id, + channel_id, parsed_data.get('video_codec'), parsed_data.get('resolution'), parsed_data.get('width'), diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index fabc1053..d94a03cf 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -127,7 +127,7 @@ def stream_ts(request, channel_id, user=None): ) # Unknown/empty state - check if owner is alive else: - owner_field = ChannelMetadataField.OWNER.encode("utf-8") + owner_field = ChannelMetadataField.OWNER if owner_field in metadata: owner = metadata[owner_field] owner_heartbeat_key = f"ts_proxy:worker:{owner}:heartbeat" diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 0ea1bd27..1c71b5cd 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -11,7 +11,7 @@ def attempt_stream_termination(user_id, requesting_client_id, active_connections try: logger.info("[stream limits]" f"[{requesting_client_id}] User {user_id} has {len(active_connections)} active connections, checking termination candidates") - user_limit_settings = CoreSettings.get_user_limit_settings() + user_limit_settings = CoreSettings.get_user_limits_settings() terminate_oldest = user_limit_settings.get("terminate_oldest", True) prioritize_single = user_limit_settings.get("prioritize_single_client_channels", True) ignore_same_channel = user_limit_settings.get("ignore_same_channel_connections", False) @@ -79,74 +79,58 @@ def get_user_active_connections(user_id): connections = [] try: - cursor = 0 - # Grab live streams - while True: - cursor, keys = redis_client.scan(cursor=cursor, match="ts_proxy:channel:*:clients:*", count=1000) + for key in redis_client.scan_iter(match="ts_proxy:channel:*:clients:*", count=1000): + parts = key.split(':') + if len(parts) >= 5: + channel_id = parts[2] + client_id = parts[4] - for key in keys: - parts = key.split(':') - if len(parts) >= 5: - channel_id = parts[2] - client_id = parts[4] + client_user_id = redis_client.hget(key, 'user_id') + connected_at = redis_client.hget(key, 'connected_at') - client_user_id = redis_client.hget(key, 'user_id') - connected_at = redis_client.hget(key, 'connected_at') + logger.info(f"[stream limits] user_id = {user_id}") + logger.info(f"[stream limits] channel_id = {channel_id}") + logger.info(f"[stream limits] client_id = {client_id}") - logger.info(f"[stream limits] user_id = {user_id}") - logger.info(f"[stream limits] channel_id = {channel_id}") - logger.info(f"[stream limits] client_id = {client_id}") - - if client_user_id and int(client_user_id) == user_id: - try: - logger.info(f"[stream limits] Found LIVE connection for user {user_id} on channel {channel_id} with client ID {client_id}") - connected_at = float(connected_at) if connected_at else 0 - connections.append({ - 'media_id': channel_id, - 'client_id': client_id, - 'connected_at': connected_at, - 'type': 'live', - }) - except (ValueError, TypeError): - pass - - if cursor == 0: - break - - - cursor = 0 + if client_user_id and int(client_user_id) == user_id: + try: + logger.info(f"[stream limits] Found LIVE connection for user {user_id} on channel {channel_id} with client ID {client_id}") + connected_at = float(connected_at) if connected_at else 0 + connections.append({ + 'media_id': channel_id, + 'client_id': client_id, + 'connected_at': connected_at, + 'type': 'live', + }) + except (ValueError, TypeError): + pass # Grab VOD - while True: - cursor, keys = redis_client.scan(cursor=cursor, match="vod_persistent_connections:*", count=1000) + for key in redis_client.scan_iter(match="vod_persistent_connection:*", count=1000): + parts = key.split(':') + if len(parts) >= 2: + client_id = parts[1] - for key in keys: - parts = key.split(':') - if len(parts) >= 2: - client_id = parts[1] + client_user_id = redis_client.hget(key, 'user_id') + connected_at = redis_client.hget(key, 'created_at') + content_uuid = redis_client.hget(key, 'content_uuid') - client_user_id = redis_client.hget(key, 'user_id') - connected_at = redis_client.hget(key, 'created_at') + logger.info(f"[stream limits] user_id = {user_id}") + logger.info(f"[stream limits] client_id = {client_id}") - logger.info(f"[stream limits] user_id = {user_id}") - logger.info(f"[stream limits] client_id = {client_id}") - - if client_user_id and int(client_user_id) == user_id: - try: - logger.info(f"[stream limits] Found VOD connection for user {user_id} on channel {channel_id} with client ID {client_id}") - connected_at = float(connected_at) if connected_at else 0 - connections.append({ - 'media_id': channel_id, - 'client_id': client_id, - 'connected_at': connected_at, - 'type': 'vod', - }) - except (ValueError, TypeError): - pass - - if cursor == 0: - break + if client_user_id and int(client_user_id) == user_id: + try: + logger.info(f"[stream limits] Found VOD connection for user {user_id} on content {content_uuid} with client ID {client_id}") + connected_at = float(connected_at) if connected_at else 0 + connections.append({ + 'media_id': content_uuid or client_id, + 'client_id': client_id, + 'connected_at': connected_at, + 'type': 'vod', + }) + except (ValueError, TypeError): + pass return connections @@ -159,14 +143,12 @@ def check_user_stream_limits(user, client_id): # Check user stream limits if user and user.stream_limit > 0: logger.info("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) is requesting a stream (stream_limit: {user.stream_limit})") - user_limit_settings = CoreSettings.get_user_limit_settings() + user_limit_settings = CoreSettings.get_user_limits_settings() active_connections = get_user_active_connections(user.id) unique_channel_count = set([conn['media_id'] for conn in active_connections]) user_stream_count = len(unique_channel_count) if user_limit_settings.get("ignore_same_channel_connections", False) else len(active_connections) - print(active_connections) - logger.info(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if user_limit_settings.get('ignore_same_channel_connections', False) else 'total connections'})") if user.stream_limit > 0 and user_stream_count >= user.stream_limit: diff --git a/core/utils.py b/core/utils.py index 4d65aa91..583075c3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -57,6 +57,8 @@ class RedisClient: redis_host = os.environ.get("REDIS_HOST", getattr(settings, 'REDIS_HOST', 'localhost')) redis_port = int(os.environ.get("REDIS_PORT", getattr(settings, 'REDIS_PORT', 6379))) redis_db = int(os.environ.get("REDIS_DB", getattr(settings, 'REDIS_DB', 0))) + redis_password = os.environ.get("REDIS_PASSWORD", getattr(settings, 'REDIS_PASSWORD', '')) + redis_user = os.environ.get("REDIS_USER", getattr(settings, 'REDIS_USER', '')) # Use standardized settings socket_timeout = getattr(settings, 'REDIS_SOCKET_TIMEOUT', 5) @@ -65,17 +67,23 @@ class RedisClient: socket_keepalive = getattr(settings, 'REDIS_SOCKET_KEEPALIVE', True) retry_on_timeout = getattr(settings, 'REDIS_RETRY_ON_TIMEOUT', True) + # TLS params from settings (empty dict when TLS is disabled) + ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) + # Create Redis client with better defaults client = redis.Redis( host=redis_host, port=redis_port, db=redis_db, + password=redis_password if redis_password else None, + username=redis_user if redis_user else None, socket_timeout=socket_timeout, socket_connect_timeout=socket_connect_timeout, socket_keepalive=socket_keepalive, health_check_interval=health_check_interval, retry_on_timeout=retry_on_timeout, - decode_responses=decode_responses + decode_responses=decode_responses, + **ssl_params ) # Validate connection with ping @@ -87,84 +95,34 @@ class RedisClient: client.config_set('save', '') # Disable RDB snapshots client.config_set('appendonly', 'no') # Disable AOF logging - # Set optimal memory settings with environment variable support - # Get max memory from environment or use a larger default (512MB instead of 256MB) - #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') - #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') - - # TLS params from settings (empty dict when TLS is disabled) - ssl_params = getattr(settings, 'REDIS_SSL_PARAMS', {}) - - # Create Redis client with better defaults - client = redis.Redis( - host=redis_host, - port=redis_port, - db=redis_db, - password=redis_password if redis_password else None, - username=redis_user if redis_user else None, - socket_timeout=socket_timeout, - socket_connect_timeout=socket_connect_timeout, - socket_keepalive=socket_keepalive, - health_check_interval=health_check_interval, - retry_on_timeout=retry_on_timeout, - decode_responses=decode_responses, - **ssl_params - ) - - #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") - # Disable protected mode when in debug mode if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': client.config_set('protected-mode', 'no') # Disable protected mode in debug logger.warning("Redis protected mode disabled for debug environment") - # Set optimal memory settings with environment variable support - # Get max memory from environment or use a larger default (512MB instead of 256MB) - #max_memory = os.environ.get('REDIS_MAX_MEMORY', '512mb') - #eviction_policy = os.environ.get('REDIS_EVICTION_POLICY', 'allkeys-lru') - - # Apply memory settings - #client.config_set('maxmemory-policy', eviction_policy) - #client.config_set('maxmemory', max_memory) - - #logger.info(f"Redis configured with maxmemory={max_memory}, policy={eviction_policy}") - - # Disable protected mode when in debug mode - if os.environ.get('DISPATCHARR_DEBUG', '').lower() == 'true': - client.config_set('protected-mode', 'no') # Disable protected mode in debug - logger.warning("Redis protected mode disabled for debug environment") - - logger.trace("Redis persistence disabled for better performance") - except redis.exceptions.ResponseError as e: - # Improve error handling for Redis configuration errors - if "OOM" in str(e): - logger.error(f"Redis OOM during configuration: {e}") - # Try to increase maxmemory as an emergency measure - try: - client.config_set('maxmemory', '768mb') - logger.warning("Applied emergency Redis memory increase to 768MB") - except: - pass - else: - logger.error(f"Redis configuration error: {e}") - - logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") - - cls._client = client - break - - except (ConnectionError, TimeoutError) as e: - retry_count += 1 - _tls_hint = _REDIS_TLS_HINT if ssl_params else "" - if retry_count >= max_retries: - logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}{_tls_hint}") - return None + logger.trace("Redis persistence disabled for better performance") + except redis.exceptions.ResponseError as e: + # Improve error handling for Redis configuration errors + if "OOM" in str(e): + logger.error(f"Redis OOM during configuration: {e}") + # Try to increase maxmemory as an emergency measure + try: + client.config_set('maxmemory', '768mb') + logger.warning("Applied emergency Redis memory increase to 768MB") + except: + pass else: logger.error(f"Redis configuration error: {e}") - except Exception as e: - _tls_hint = _REDIS_TLS_HINT if ssl_params else "" - logger.error(f"Unexpected error connecting to Redis: {e}{_tls_hint}") + logger.info(f"Connected to Redis at {redis_host}:{redis_port}/{redis_db}") + + return client + + except (ConnectionError, TimeoutError) as e: + retry_count += 1 + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" + if retry_count >= max_retries: + logger.error(f"Failed to connect to Redis after {max_retries} attempts: {e}{_tls_hint}") return None else: # Use exponential backoff for retries @@ -173,10 +131,15 @@ class RedisClient: time.sleep(wait_time) except Exception as e: - logger.error(f"Unexpected error connecting to Redis: {e}") + _tls_hint = "" + try: + _tls_hint = _REDIS_TLS_HINT if ssl_params else "" + except NameError: + pass + logger.error(f"Unexpected error connecting to Redis: {e}{_tls_hint}") return None - return client + return None @classmethod def get_client(cls, max_retries=5, retry_interval=1): diff --git a/debug_redis_check.py b/debug_redis_check.py new file mode 100644 index 00000000..9deab6d4 --- /dev/null +++ b/debug_redis_check.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +import os, sys +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') +import django +django.setup() + +from core.utils import RedisClient + +r = RedisClient.get_client() +print(f"Client: {r}") +print(f"decode_responses: {r.connection_pool.connection_kwargs.get('decode_responses')}") + +keys = list(r.scan_iter(match='ts_proxy:channel:*:metadata', count=100)) +print(f"Found {len(keys)} metadata keys") +for k in keys[:3]: + print(f" Key: {k!r}") + data = r.hgetall(k) + field_names = list(data.keys())[:8] + print(f" Fields ({len(data)} total): {field_names}") + state = data.get('state', 'MISSING') + print(f" state={state!r}") From 58befaa52bc3f55f5180c6b4ce4f7c07575c1cc1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 19:14:46 -0500 Subject: [PATCH 30/67] fix: correct indentation in check_user_stream_limits function --- apps/proxy/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 1c71b5cd..fd4e88f2 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -164,4 +164,4 @@ def check_user_stream_limits(user, client_id): if not attempt_stream_termination(user.id, client_id, active_connections): return False - return True + return True From aa6fb033d30c39fac6137b8afa9e3d51e8a04bec Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 20:02:54 -0500 Subject: [PATCH 31/67] fix: update user stream limit checks to include media_id and rename user_limits_settings key --- apps/proxy/ts_proxy/views.py | 3 +-- apps/proxy/utils.py | 22 ++++++++++++++++------ apps/proxy/vod_proxy/views.py | 2 +- core/models.py | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/apps/proxy/ts_proxy/views.py b/apps/proxy/ts_proxy/views.py index d94a03cf..64f73664 100644 --- a/apps/proxy/ts_proxy/views.py +++ b/apps/proxy/ts_proxy/views.py @@ -73,7 +73,7 @@ def stream_ts(request, channel_id, user=None): break if user: - if not check_user_stream_limits(user, client_id): + if not check_user_stream_limits(user, client_id, media_id=channel_id): return JsonResponse( {"error": f"Stream limit exceeded ({user.stream_limit} concurrent streams allowed)"}, status=429 @@ -565,7 +565,6 @@ def stream_xc(request, username, password, channel_id): if custom_properties["xc_password"] != password: return Response({"error": "Invalid credentials"}, status=401) - print(f"Fetchin channel with ID: {channel_id}") if user.user_level < 10: user_profile_count = user.channel_profiles.count() diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index fd4e88f2..21097da6 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -139,26 +139,36 @@ def get_user_active_connections(user_id): return [] -def check_user_stream_limits(user, client_id): +def check_user_stream_limits(user, client_id, media_id=None): # Check user stream limits if user and user.stream_limit > 0: logger.info("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) is requesting a stream (stream_limit: {user.stream_limit})") user_limit_settings = CoreSettings.get_user_limits_settings() + ignore_same_channel = user_limit_settings.get("ignore_same_channel_connections", False) active_connections = get_user_active_connections(user.id) unique_channel_count = set([conn['media_id'] for conn in active_connections]) - user_stream_count = len(unique_channel_count) if user_limit_settings.get("ignore_same_channel_connections", False) else len(active_connections) + user_stream_count = len(unique_channel_count) if ignore_same_channel else len(active_connections) - logger.info(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if user_limit_settings.get('ignore_same_channel_connections', False) else 'total connections'})") + logger.info(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if ignore_same_channel else 'total connections'})") - if user.stream_limit > 0 and user_stream_count >= user.stream_limit: + # If ignore_same_channel is enabled and this request is for a live channel the user + # is already watching, allow it through without counting against the limit. + # VOD is excluded: connections aren't shared so multiple VOD connections to the + # same content would mean multiple upstream connections. + live_channel_ids = {str(conn['media_id']) for conn in active_connections if conn['type'] == 'live'} + if ignore_same_channel and media_id and str(media_id) in live_channel_ids: + logger.info(f"[stream limits][{client_id}] Same-channel reconnect for {media_id} allowed (ignore_same_channel=True)") + return True + + if user_stream_count >= user.stream_limit: if user_limit_settings.get("terminate_on_limit_exceeded", True) == False: return False - if len(active_connections) >= user.stream_limit: + if user_stream_count >= user.stream_limit: logger.warning("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) has reached stream limit " - f"({len(active_connections)}/{user.stream_limit} channels), attempting to free up slot" + f"({user_stream_count}/{user.stream_limit} streams), attempting to free up slot" ) if not attempt_stream_termination(user.id, client_id, active_connections): diff --git a/apps/proxy/vod_proxy/views.py b/apps/proxy/vod_proxy/views.py index 14e1b631..d7250bcf 100644 --- a/apps/proxy/vod_proxy/views.py +++ b/apps/proxy/vod_proxy/views.py @@ -411,7 +411,7 @@ def stream_vod(request, content_type, content_id, session_id=None, profile_id=No ) if user: - if not check_user_stream_limits(user, session_id): + if not check_user_stream_limits(user, session_id, media_id=content_id): return JsonResponse( {"error": f"Stream limit exceeded ({user.stream_limit} concurrent streams allowed)"}, status=429 diff --git a/core/models.py b/core/models.py index 83b58495..a1526e73 100644 --- a/core/models.py +++ b/core/models.py @@ -156,7 +156,7 @@ PROXY_SETTINGS_KEY = "proxy_settings" NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" -USER_LIMITS_SETTINGS_KEY = "user_limits_settings" +USER_LIMITS_SETTINGS_KEY = "user_limits" class CoreSettings(models.Model): From 46d406352443c974ae3e47f787aed525ab3121a7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 20:10:03 -0500 Subject: [PATCH 32/67] fix: change logging level from info to debug in user connection checks --- apps/proxy/utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 21097da6..4dbb2551 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -89,13 +89,13 @@ def get_user_active_connections(user_id): client_user_id = redis_client.hget(key, 'user_id') connected_at = redis_client.hget(key, 'connected_at') - logger.info(f"[stream limits] user_id = {user_id}") - logger.info(f"[stream limits] channel_id = {channel_id}") - logger.info(f"[stream limits] client_id = {client_id}") + logger.debug(f"[stream limits] user_id = {user_id}") + logger.debug(f"[stream limits] channel_id = {channel_id}") + logger.debug(f"[stream limits] client_id = {client_id}") if client_user_id and int(client_user_id) == user_id: try: - logger.info(f"[stream limits] Found LIVE connection for user {user_id} on channel {channel_id} with client ID {client_id}") + logger.debug(f"[stream limits] Found LIVE connection for user {user_id} on channel {channel_id} with client ID {client_id}") connected_at = float(connected_at) if connected_at else 0 connections.append({ 'media_id': channel_id, @@ -116,12 +116,12 @@ def get_user_active_connections(user_id): connected_at = redis_client.hget(key, 'created_at') content_uuid = redis_client.hget(key, 'content_uuid') - logger.info(f"[stream limits] user_id = {user_id}") - logger.info(f"[stream limits] client_id = {client_id}") + logger.debug(f"[stream limits] user_id = {user_id}") + logger.debug(f"[stream limits] client_id = {client_id}") if client_user_id and int(client_user_id) == user_id: try: - logger.info(f"[stream limits] Found VOD connection for user {user_id} on content {content_uuid} with client ID {client_id}") + logger.debug(f"[stream limits] Found VOD connection for user {user_id} on content {content_uuid} with client ID {client_id}") connected_at = float(connected_at) if connected_at else 0 connections.append({ 'media_id': content_uuid or client_id, @@ -142,7 +142,7 @@ def get_user_active_connections(user_id): def check_user_stream_limits(user, client_id, media_id=None): # Check user stream limits if user and user.stream_limit > 0: - logger.info("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) is requesting a stream (stream_limit: {user.stream_limit})") + logger.debug("[stream limits]" f"[{client_id}] User {user.username} (ID: {user.id}) is requesting a stream (stream_limit: {user.stream_limit})") user_limit_settings = CoreSettings.get_user_limits_settings() ignore_same_channel = user_limit_settings.get("ignore_same_channel_connections", False) @@ -150,7 +150,7 @@ def check_user_stream_limits(user, client_id, media_id=None): unique_channel_count = set([conn['media_id'] for conn in active_connections]) user_stream_count = len(unique_channel_count) if ignore_same_channel else len(active_connections) - logger.info(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if ignore_same_channel else 'total connections'})") + logger.debug(f"[stream limits]" f"[{client_id}] User {user.username} currently has {len(active_connections)} active connections across {len(unique_channel_count)} unique channels (counting method: {'unique channels' if ignore_same_channel else 'total connections'})") # If ignore_same_channel is enabled and this request is for a live channel the user # is already watching, allow it through without counting against the limit. @@ -158,7 +158,7 @@ def check_user_stream_limits(user, client_id, media_id=None): # same content would mean multiple upstream connections. live_channel_ids = {str(conn['media_id']) for conn in active_connections if conn['type'] == 'live'} if ignore_same_channel and media_id and str(media_id) in live_channel_ids: - logger.info(f"[stream limits][{client_id}] Same-channel reconnect for {media_id} allowed (ignore_same_channel=True)") + logger.debug(f"[stream limits][{client_id}] Same-channel reconnect for {media_id} allowed (ignore_same_channel=True)") return True if user_stream_count >= user.stream_limit: From 7e6041286a42e72fafe4cf88d46353b5907621a7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 20:16:16 -0500 Subject: [PATCH 33/67] fix: update user limit settings key in migration and frontend form for consistency with other settings keys. --- core/migrations/022_default_user_limit_settings.py | 4 ++-- core/models.py | 2 +- frontend/src/components/forms/settings/UserLimitsForm.jsx | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/migrations/022_default_user_limit_settings.py b/core/migrations/022_default_user_limit_settings.py index af758a11..4c2b3c33 100644 --- a/core/migrations/022_default_user_limit_settings.py +++ b/core/migrations/022_default_user_limit_settings.py @@ -7,8 +7,8 @@ from django.utils.text import slugify def preload_user_limit_settings(apps, schema_editor): CoreSettings = apps.get_model("core", "CoreSettings") CoreSettings.objects.create( - key="user_limits", - name="User Limits", + key="user_limit_settings", + name="User Limit Settings", value={}, ) diff --git a/core/models.py b/core/models.py index a1526e73..5e86e98f 100644 --- a/core/models.py +++ b/core/models.py @@ -156,7 +156,7 @@ PROXY_SETTINGS_KEY = "proxy_settings" NETWORK_ACCESS_KEY = "network_access" SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" -USER_LIMITS_SETTINGS_KEY = "user_limits" +USER_LIMITS_SETTINGS_KEY = "user_limit_settings" class CoreSettings(models.Model): diff --git a/frontend/src/components/forms/settings/UserLimitsForm.jsx b/frontend/src/components/forms/settings/UserLimitsForm.jsx index 3481a7db..b6a9ed5e 100644 --- a/frontend/src/components/forms/settings/UserLimitsForm.jsx +++ b/frontend/src/components/forms/settings/UserLimitsForm.jsx @@ -37,10 +37,10 @@ const UserLimitsForm = React.memo(({ active }) => { useEffect(() => { if (settings) { - if (settings['user_limits']?.value) { + if (settings['user_limit_settings']?.value) { userLimitSettingsForm.setValues({ ...USER_LIMIT_DEFAULTS, - ...settings['user_limits'].value, + ...settings['user_limit_settings'].value, }); } } @@ -55,7 +55,7 @@ const UserLimitsForm = React.memo(({ active }) => { try { const result = await updateSetting({ - ...settings['user_limits'], + ...settings['user_limit_settings'], value: userLimitSettingsForm.getValues(), }); if (result) { From 7964b2c2cb07e94292bf272fb301c09bf541a52b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 20:33:03 -0500 Subject: [PATCH 34/67] fix: enhance stream termination logic to handle multiple connections on the same channel --- apps/proxy/utils.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/apps/proxy/utils.py b/apps/proxy/utils.py index 4dbb2551..6503d1df 100644 --- a/apps/proxy/utils.py +++ b/apps/proxy/utils.py @@ -47,27 +47,35 @@ def attempt_stream_termination(user_id, requesting_client_id, active_connections f"on media {target['media_id']} (connected_at={target['connected_at']})" ) - if target['type'] == 'live': - result = ChannelService.stop_client(target['media_id'], target['client_id']) - if result.get("status") == "error": - return False - else: - connection_manager = MultiWorkerVODConnectionManager.get_instance() - redis_client = connection_manager.redis_client + # When counting by unique channel, freeing one connection from a multi-connection + # channel doesn't free a slot — terminate all connections to that channel so the + # unique-channel count actually drops by one. + targets = ( + [c for c in active_connections if c['media_id'] == target['media_id']] + if ignore_same_channel + else [target] + ) - if not redis_client: - return False + for t in targets: + if t['type'] == 'live': + result = ChannelService.stop_client(t['media_id'], t['client_id']) + if result.get("status") == "error": + logger.warning(f"[stream limits][{requesting_client_id}] Failed to stop client {t['client_id']} on channel {t['media_id']}") + else: + connection_manager = MultiWorkerVODConnectionManager.get_instance() + redis_client = connection_manager.redis_client - # Check if connection exists - connection_key = f"vod_persistent_connection:{target['client_id']}" - connection_data = redis_client.hgetall(connection_key) - if not connection_data: - logger.warning(f"VOD connection not found: {target['client_id']}") - return False + if not redis_client: + return False - # Set a stop signal key that the worker will check - stop_key = get_vod_client_stop_key(target['client_id']) - redis_client.setex(stop_key, 60, "true") # 60 second TTL + connection_key = f"vod_persistent_connection:{t['client_id']}" + connection_data = redis_client.hgetall(connection_key) + if not connection_data: + logger.warning(f"VOD connection not found: {t['client_id']}") + continue + + stop_key = get_vod_client_stop_key(t['client_id']) + redis_client.setex(stop_key, 60, "true") # 60 second TTL return True except Exception as e: From 8e9a6285a4692f5e8372afdd232cb4aff52ccd7b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 26 Mar 2026 21:16:18 -0500 Subject: [PATCH 35/67] Enhancement: Donate button added to the sidebar footer. A heart icon links to the project's Open Collective page, visible in both expanded and collapsed states. Hovering shows a "Support Dispatcharr" tooltip. The version string is also now clickable to copy it to the clipboard. --- CHANGELOG.md | 1 + frontend/src/components/Sidebar.jsx | 71 +++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58a41e3d..36657d3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Donate button added to the sidebar footer. A heart icon links to the project's Open Collective page, visible in both expanded and collapsed states. Hovering shows a "Support Dispatcharr" tooltip. The version string is also now clickable to copy it to the clipboard. - TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) — Thanks [@CodeBormen](https://github.com/CodeBormen) - Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) diff --git a/frontend/src/components/Sidebar.jsx b/frontend/src/components/Sidebar.jsx index 73ee13c3..f5332d3a 100644 --- a/frontend/src/components/Sidebar.jsx +++ b/frontend/src/components/Sidebar.jsx @@ -1,12 +1,7 @@ import React, { useRef, useState, useMemo } from 'react'; import { Link, useLocation } from 'react-router-dom'; import { copyToClipboard } from '../utils'; -import { - Copy, - LogOut, - ChevronDown, - ChevronRight, -} from 'lucide-react'; +import { Copy, LogOut, ChevronDown, ChevronRight, Heart } from 'lucide-react'; import { getOrderedNavItems } from '../config/navigation'; import { Avatar, @@ -19,6 +14,7 @@ import { ActionIcon, AppShellNavbar, ScrollArea, + Tooltip, } from '@mantine/core'; import logo from '../images/logo.png'; import useChannelsStore from '../store/channels'; @@ -29,6 +25,21 @@ import { USER_LEVELS } from '../constants'; import UserForm from './forms/User'; import NotificationCenter from './NotificationCenter'; +const DonateButton = ({ tooltipPosition = 'top' }) => ( + + + + + +); + const NavLink = ({ item, isActive, collapsed }) => { const IconComponent = item.icon; return ( @@ -328,24 +339,56 @@ const Sidebar = ({ collapsed, toggleDrawer, drawerWidth, miniDrawerWidth }) => { {!collapsed && ( - - v{appVersion?.version || '0.0.0'} - {appVersion?.timestamp ? `-${appVersion.timestamp}` : ''} - - {isAuthenticated && } + + + copyToClipboard( + `v${appVersion?.version || '0.0.0'}${appVersion?.timestamp ? `-${appVersion.timestamp}` : ''}`, + { + successTitle: 'Copied', + successMessage: 'Version copied to clipboard', + } + ) + } + > + v{appVersion?.version || '0.0.0'} + {appVersion?.timestamp ? `-${appVersion.timestamp}` : ''} + + + + + {isAuthenticated && } + )} - {collapsed && isAuthenticated && ( + {collapsed && ( - + {isAuthenticated && } + )} From 7d8619f644d0c7a7d626f9ce9aacb6863a00af8f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 09:30:01 -0500 Subject: [PATCH 36/67] Security: Update npm dependencies to resolve frontend vulernabilities. --- CHANGELOG.md | 8 +++++++ frontend/package-lock.json | 44 +++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58a41e3d..35b283b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- Updated frontend npm dependencies to resolve 4 vulnerabilities (2 high, 2 moderate): + - Updated `brace-expansion` to 5.0.5, resolving **moderate** zero-step sequence causing process hang and memory exhaustion ([GHSA-f886-m6hf-6m8v](https://github.com/advisories/GHSA-f886-m6hf-6m8v)) + - Updated `flatted` to 3.4.2, resolving **high** Prototype Pollution via `parse()` in NodeJS flatted ([GHSA-rf6f-7fwh-wjgh](https://github.com/advisories/GHSA-rf6f-7fwh-wjgh)) + - Updated `picomatch` to 4.0.4, resolving **high** method injection in POSIX character classes causing incorrect glob matching ([GHSA-3v7f-55p6-f55p](https://github.com/advisories/GHSA-3v7f-55p6-f55p)) and a ReDoS vulnerability via extglob quantifiers ([GHSA-c2c7-rcm5-vvqj](https://github.com/advisories/GHSA-c2c7-rcm5-vvqj)) + - Updated `yaml` to 1.10.3, resolving **moderate** stack overflow via deeply nested YAML collections ([GHSA-48c2-rrv3-qjmp](https://github.com/advisories/GHSA-48c2-rrv3-qjmp)) + ### Added - TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) — Thanks [@CodeBormen](https://github.com/CodeBormen) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9a0adea6..6880b99a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -2704,16 +2704,16 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", - "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" } }, "node_modules/cac": { @@ -2850,9 +2850,9 @@ } }, "node_modules/cosmiconfig/node_modules/yaml": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", - "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", + "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", "license": "ISC", "engines": { "node": ">= 6" @@ -3554,9 +3554,9 @@ } }, "node_modules/flatted": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz", - "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -4362,9 +4362,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -5794,6 +5794,24 @@ "dev": true, "license": "MIT" }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "dev": true, + "license": "ISC", + "optional": true, + "peer": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", From b20c42f25a2efdf4a5f5afd3f5e1bea93ae8ccd1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 11:03:55 -0500 Subject: [PATCH 37/67] chore: update dependencies for celery, requests, torch, and sentence-transformers --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fecc4e68..0f2d7afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,9 @@ dynamic = ["version"] dependencies = [ "Django==6.0.3", "psycopg2-binary==2.9.11", - "celery[redis]==5.6.2", + "celery[redis]==5.6.3", "djangorestframework==3.16.1", - "requests==2.32.5", + "requests==2.33.0", "psutil==7.2.2", "pillow", "drf-spectacular>=0.29.0", @@ -27,8 +27,8 @@ dependencies = [ "regex", "tzlocal", "pytz", - "torch==2.10.0+cpu", - "sentence-transformers==5.2.3", + "torch==2.11.0+cpu", + "sentence-transformers==5.3.0", "channels", "channels-redis==4.3.0", "django-filter", From 2253e18ef8767f2045253422fa6f4658af881089 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 11:04:37 -0500 Subject: [PATCH 38/67] changelog: Update changelog for dependency upgrades --- CHANGELOG.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b283b5..9d7cfc23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Updated frontend npm dependencies to resolve 4 vulnerabilities (2 high, 2 moderate): - - Updated `brace-expansion` to 5.0.5, resolving **moderate** zero-step sequence causing process hang and memory exhaustion ([GHSA-f886-m6hf-6m8v](https://github.com/advisories/GHSA-f886-m6hf-6m8v)) - - Updated `flatted` to 3.4.2, resolving **high** Prototype Pollution via `parse()` in NodeJS flatted ([GHSA-rf6f-7fwh-wjgh](https://github.com/advisories/GHSA-rf6f-7fwh-wjgh)) - - Updated `picomatch` to 4.0.4, resolving **high** method injection in POSIX character classes causing incorrect glob matching ([GHSA-3v7f-55p6-f55p](https://github.com/advisories/GHSA-3v7f-55p6-f55p)) and a ReDoS vulnerability via extglob quantifiers ([GHSA-c2c7-rcm5-vvqj](https://github.com/advisories/GHSA-c2c7-rcm5-vvqj)) - - Updated `yaml` to 1.10.3, resolving **moderate** stack overflow via deeply nested YAML collections ([GHSA-48c2-rrv3-qjmp](https://github.com/advisories/GHSA-48c2-rrv3-qjmp)) +- Updated `requests` 2.32.5 → 2.33.0, resolving the following CVE: + - **CVE-2026-25645** (moderate): Insecure temp file reuse in `extract_zipped_paths()` utility function. +- Updated frontend npm dependencies to resolve 4 audit vulnerabilities (2 moderate, 2 high): + - Updated `brace-expansion` 5.0.2 → 5.0.5, resolving **moderate** zero-step sequence causing process hang and memory exhaustion ([GHSA-f886-m6hf-6m8v](https://github.com/advisories/GHSA-f886-m6hf-6m8v)) + - Updated `flatted` 3.4.1 → 3.4.2, resolving **high** Prototype Pollution via `parse()` in NodeJS flatted ([GHSA-rf6f-7fwh-wjgh](https://github.com/advisories/GHSA-rf6f-7fwh-wjgh)) + - Updated `picomatch` 4.0.3 → 4.0.4, resolving **high** method injection in POSIX character classes causing incorrect glob matching ([GHSA-3v7f-55p6-f55p](https://github.com/advisories/GHSA-3v7f-55p6-f55p)) and a ReDoS vulnerability via extglob quantifiers ([GHSA-c2c7-rcm5-vvqj](https://github.com/advisories/GHSA-c2c7-rcm5-vvqj)) + - Updated `yaml` 1.10.2 → 1.10.3, resolving **moderate** stack overflow via deeply nested YAML collections ([GHSA-48c2-rrv3-qjmp](https://github.com/advisories/GHSA-48c2-rrv3-qjmp)) ### Added @@ -22,6 +24,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Dependency updates: + - `requests` 2.32.5 → 2.33.0 (security patch; see Security section) + - `celery` 5.6.2 → 5.6.3 + - `torch` 2.10.0+cpu → 2.11.0+cpu + - `sentence-transformers` 5.2.3 → 5.3.0 + - `yt-dlp` 2026.3.13 → 2026.3.17 - M3U table **Max Streams** column now reflects the combined limit across all active profiles. When a playlist has multiple active profiles, the column displays their summed total (or ∞ if any profile is unlimited) and a hover tooltip lists each profile's individual limit by name. (Closes #816) - Toggling an M3U profile's active state now immediately updates the playlist store (including the `playlists` array), so the **Max Streams** total in the M3U table reflects the change without a page reload. - M3U account form: **Max Streams** field changed from a plain text input to a number input with increment/decrement controls, consistent with other integer fields. From 9c9de12b19408a7e9923594211a84fee516ed58a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 11:49:28 -0500 Subject: [PATCH 39/67] chore: remove unnecessary python-is-python3, python3-pip and streamlink from Dockerfile dependencies --- docker/DispatcharrBase | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index d2e8ceaa..3040e189 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -15,9 +15,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ && apt-get update \ && apt-get install --no-install-recommends -y \ python3.13 python3.13-dev python3.13-venv libpython3.13 \ - python-is-python3 python3-pip \ libpcre3 libpcre3-dev libpq-dev procps pciutils \ - nginx streamlink comskip \ + nginx comskip \ vlc-bin vlc-plugin-base \ build-essential gcc g++ gfortran libopenblas-dev libopenblas0 ninja-build From 36ce5a8c7184f9831a573815d95f279f5cbcddd8 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 11:54:00 -0500 Subject: [PATCH 40/67] changelog: Update changelog for base image changes. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d7cfc23..f25d8572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `torch` 2.10.0+cpu → 2.11.0+cpu - `sentence-transformers` 5.2.3 → 5.3.0 - `yt-dlp` 2026.3.13 → 2026.3.17 +- Docker base image cleanup: removed `python-is-python3`, `python3-pip`, and `streamlink` from the apt package list in `DispatcharrBase`. `python3-pip` and `streamlink` were pulling outdated system Python packages (e.g. `requests 2.31.0`, `cryptography 41.0.7`, `lxml 5.2.1`) into the system Python's site-packages despite the app running entirely in the uv-managed venv at `/dispatcharrpy`. `streamlink` is already installed in the venv via `pyproject.toml`. `python-is-python3` is unnecessary as `PATH` resolves bare `python` to the venv binary. - M3U table **Max Streams** column now reflects the combined limit across all active profiles. When a playlist has multiple active profiles, the column displays their summed total (or ∞ if any profile is unlimited) and a hover tooltip lists each profile's individual limit by name. (Closes #816) - Toggling an M3U profile's active state now immediately updates the playlist store (including the `playlists` array), so the **Max Streams** total in the M3U table reflects the change without a page reload. - M3U account form: **Max Streams** field changed from a plain text input to a number input with increment/decrement controls, consistent with other integer fields. From 5f45d3e9adcd7404c5bf90a189d7ae5afb129e03 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 13:55:16 -0500 Subject: [PATCH 41/67] changelog: User stream limits --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58a41e3d..6806f29a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- User stream limits: administrators can now set a maximum number of concurrent streams per user account. When a user reaches their limit, the system can automatically terminate an existing stream to free a slot based on configurable rules. Limit enforcement applies to both live channels and VOD. (Closes #544) + - Each user account has a new **Stream Limit** field (0 = unlimited) configurable from the user edit form in Settings → Users. + - Global enforcement behaviour is configurable in Settings → User Limits: + - **Terminate on Limit Exceeded**: automatically stop an existing stream when the user's limit is reached (vs. rejecting the new connection). + - **Terminate Oldest**: prefer terminating the oldest stream when freeing a slot; disable to prefer the newest. + - **Prioritize Single-Client Channels**: prefer terminating streams on channels that only this user is watching. + - **Ignore Same-Channel Connections**: count multiple connections to the same live channel as one stream toward the limit. Same-channel reconnects are always allowed through. When this is enabled and a channel must be freed, all connections to the chosen channel are terminated together so that the unique-channel count actually decreases. VOD is explicitly excluded from this bypass since VOD connections are not shared upstream. - TLS and mutual TLS (mTLS) support for Redis and PostgreSQL connections in modular deployments. Supports encrypted connections, server certificate verification (Redis: on/off; PostgreSQL: verify-full, verify-ca, require), CA certificate configuration, and client certificate authentication. Configured via environment variables in the docker compose file. Includes startup validation for certificate paths and TLS/URL scheme conflicts, and a read-only Connection Security panel in System Settings. (Closes #950) — Thanks [@CodeBormen](https://github.com/CodeBormen) - Status filter for M3U group and VOD category filter modals: A new **All / Enabled / Disabled** segmented control is now shown alongside the text search input in the Live, VOD - Movies, and VOD - Series tabs of the M3U Group Filter modal. The status filter works in combination with the text search and also scopes the "Select Visible" / "Deselect Visible" buttons so they only act on the currently visible subset. (Closes #312) From 110c8623ed2c28247d6ace39b6ef5208ed959d56 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 13:59:50 -0500 Subject: [PATCH 42/67] delete temp script used for debugging. --- debug_redis_check.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 debug_redis_check.py diff --git a/debug_redis_check.py b/debug_redis_check.py deleted file mode 100644 index 9deab6d4..00000000 --- a/debug_redis_check.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -import os, sys -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') -import django -django.setup() - -from core.utils import RedisClient - -r = RedisClient.get_client() -print(f"Client: {r}") -print(f"decode_responses: {r.connection_pool.connection_kwargs.get('decode_responses')}") - -keys = list(r.scan_iter(match='ts_proxy:channel:*:metadata', count=100)) -print(f"Found {len(keys)} metadata keys") -for k in keys[:3]: - print(f" Key: {k!r}") - data = r.hgetall(k) - field_names = list(data.keys())[:8] - print(f" Fields ({len(data)} total): {field_names}") - state = data.get('state', 'MISSING') - print(f" state={state!r}") From 0827908b538b28edb9456ae490e0913077d6c65d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 14:12:21 -0500 Subject: [PATCH 43/67] fix: lazy load UserLimitsForm and correct spelling in user limits section --- frontend/src/pages/Settings.jsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Settings.jsx b/frontend/src/pages/Settings.jsx index 71bbc68c..570e9f4b 100644 --- a/frontend/src/pages/Settings.jsx +++ b/frontend/src/pages/Settings.jsx @@ -23,8 +23,10 @@ const BackupManager = React.lazy( import useAuthStore from '../store/auth'; import { USER_LEVELS } from '../constants'; import UiSettingsForm from '../components/forms/settings/UiSettingsForm.jsx'; -import UserLimitsForm from '../components/forms/settings/UserLimitsForm.jsx'; import ErrorBoundary from '../components/ErrorBoundary.jsx'; +const UserLimitsForm = React.lazy( + () => import('../components/forms/settings/UserLimitsForm.jsx') +); const NetworkAccessForm = React.lazy( () => import('../components/forms/settings/NetworkAccessForm.jsx') ); @@ -203,7 +205,7 @@ const SettingsPage = () => { - User Limts + User Limits }> From 3064be8194fd831a056762826b354cdf8e141737 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 14:15:20 -0500 Subject: [PATCH 44/67] tests: Fix settings page test. --- frontend/src/pages/__tests__/Settings.test.jsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/pages/__tests__/Settings.test.jsx b/frontend/src/pages/__tests__/Settings.test.jsx index fc0bc323..7a01b95a 100644 --- a/frontend/src/pages/__tests__/Settings.test.jsx +++ b/frontend/src/pages/__tests__/Settings.test.jsx @@ -78,6 +78,13 @@ vi.mock('../../components/forms/settings/NavOrderForm', () => ({ ), })); +vi.mock('../../components/forms/settings/UserLimitsForm', () => ({ + default: ({ active }) => ( +
+ UserLimitsForm {active ? 'active' : 'inactive'} +
+ ), +})); vi.mock('../../components/ErrorBoundary', () => ({ default: ({ children }) =>
{children}
, })); From 3ee925ca71f714d3fff3afb09a77a79704a71727 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 15:14:04 -0500 Subject: [PATCH 45/67] tests: add Heart icon and Tooltip mock to Sidebar tests --- frontend/src/components/__tests__/Sidebar.test.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/__tests__/Sidebar.test.jsx b/frontend/src/components/__tests__/Sidebar.test.jsx index cc5e8be2..e92b2f39 100644 --- a/frontend/src/components/__tests__/Sidebar.test.jsx +++ b/frontend/src/components/__tests__/Sidebar.test.jsx @@ -56,6 +56,7 @@ vi.mock('lucide-react', () => ({ ChevronRight: () =>
, MonitorCog: () =>
, Blocks: () =>
, + Heart: () =>
, })); // Mock UserForm component @@ -114,6 +115,7 @@ vi.mock('@mantine/core', async () => { ), ScrollArea: ({ children }) =>
{children}
, + Tooltip: ({ children }) => <>{children}, }; }); From 49f6871ba8d078859bc44ef4b51c7ee97e3ddcea Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 27 Mar 2026 16:36:34 -0500 Subject: [PATCH 46/67] changelog: Update formatting a bit. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7df81dc0..c2d5a865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,10 +50,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - HTML named entities in XMLTV EPG files are now resolved to Unicode characters before lxml parsing. Some EPG providers (particularly French and other European sources) use HTML named entities like `é`, `î`, `ü` in channel names, program titles, and metadata. These are not valid XML entities — lxml 6.0.2 with `recover=True` silently drops them, causing characters to go missing (e.g., "Chaîne Télé" becomes "Chane Tl"). A preprocessing step in `fetch_xmltv()` now resolves HTML named entities while preserving the 5 XML-predefined entities, detecting encoding from the XML declaration (falling back to UTF-8), and processing line-by-line to a temp file before atomically replacing the original. (Closes #1095) — Thanks [@CodeBormen](https://github.com/CodeBormen) -- Duplicate recordings created when EPG sources refresh and re-evaluate series rules — Thanks [@CodeBormen](https://github.com/CodeBormen): +- Duplicate recordings created when EPG sources refresh and re-evaluate series rules (Fixes #940) — Thanks [@CodeBormen](https://github.com/CodeBormen): - **Program ID instability**: `parse_programs_for_source()` deletes and recreates all `ProgramData` rows with new auto-increment IDs on every EPG refresh. The dedup set used these IDs, so it never matched after a refresh. Deduplication now uses a stable `(tvg_id, start_time, end_time)` composite key sourced from `Recording.custom_properties.program`. - **Secondary guard using wrong times**: The DB guard compared unadjusted program times against offset-adjusted `Recording.start_time`/`end_time`, so it never matched when any DVR pre/post offset was configured. It now queries `custom_properties__program__start_time/end_time` (the original, unadjusted program times stored at recording creation). - - **No concurrency guard**: Each EPG source refresh fired `evaluate_series_rules.delay()` independently. Concurrent tasks loaded the dedup set before others committed, allowing races. Evaluation is now serialized with `acquire_task_lock` (reusing the existing EPG task pattern). Gracefully degrades if Redis is unavailable — the primary and secondary dedup guards still protect. (Fixes #940) + - **No concurrency guard**: Each EPG source refresh fired `evaluate_series_rules.delay()` independently. Concurrent tasks loaded the dedup set before others committed, allowing races. Evaluation is now serialized with `acquire_task_lock` (reusing the existing EPG task pattern). Gracefully degrades if Redis is unavailable — the primary and secondary dedup guards still protect. ## [0.21.1] - 2026-03-18 From 5f4e0661478e0b372c1dc8cd3f71c0ad75417e27 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 28 Mar 2026 13:33:29 -0500 Subject: [PATCH 47/67] Enhancements: - Connection cards on the Stats page now show the **username** of the connected user in a new User column (between IP Address and Connected). The username is resolved from the user store using the `user_id` stored in Redis client metadata; unauthenticated connections display "Anonymous". (Closes #766) - `ip_address` and `user_id` were not included in the client info returned by `get_detailed_channel_info()` despite being available in the Redis hash. Both fields are now extracted and returned. (Closes #586) --- CHANGELOG.md | 2 + apps/proxy/ts_proxy/channel_status.py | 6 ++ .../components/cards/StreamConnectionCard.jsx | 55 ++++++++++++++++--- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d5a865..450a33ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Connection cards on the Stats page now show the **username** of the connected user in a new User column (between IP Address and Connected). The username is resolved from the user store using the `user_id` stored in Redis client metadata; unauthenticated connections display "Anonymous". (Closes #766) +- `ip_address` and `user_id` were not included in the client info returned by `get_detailed_channel_info()` despite being available in the Redis hash. Both fields are now extracted and returned. (Closes #586) - Donate button added to the sidebar footer. A heart icon links to the project's Open Collective page, visible in both expanded and collapsed states. Hovering shows a "Support Dispatcharr" tooltip. The version string is also now clickable to copy it to the clipboard. - User stream limits: administrators can now set a maximum number of concurrent streams per user account. When a user reaches their limit, the system can automatically terminate an existing stream to free a slot based on configurable rules. Limit enforcement applies to both live channels and VOD. (Closes #544) - Each user account has a new **Stream Limit** field (0 = unlimited) configurable from the user edit form in Settings → Users. diff --git a/apps/proxy/ts_proxy/channel_status.py b/apps/proxy/ts_proxy/channel_status.py index b7c77024..c4a54cdc 100644 --- a/apps/proxy/ts_proxy/channel_status.py +++ b/apps/proxy/ts_proxy/channel_status.py @@ -143,6 +143,8 @@ class ChannelStatus: 'client_id': client_id_str, 'user_agent': client_data.get('user_agent', 'unknown'), 'worker_id': client_data.get('worker_id', 'unknown'), + 'ip_address': client_data.get('ip_address', 'unknown'), + 'user_id': client_data.get('user_id', '0'), } if 'connected_at' in client_data: @@ -473,6 +475,10 @@ class ChannelStatus: connected_at = float(connected_at_bytes) client_info['connected_since'] = time.time() - connected_at + user_id_bytes = proxy_server.redis_client.hget(client_key, 'user_id') + if user_id_bytes: + client_info['user_id'] = user_id_bytes + clients.append(client_info) # Add clients to info diff --git a/frontend/src/components/cards/StreamConnectionCard.jsx b/frontend/src/components/cards/StreamConnectionCard.jsx index 56d4d174..86dbc240 100644 --- a/frontend/src/components/cards/StreamConnectionCard.jsx +++ b/frontend/src/components/cards/StreamConnectionCard.jsx @@ -2,6 +2,7 @@ import { useLocation } from 'react-router-dom'; import React, { useEffect, useMemo, useState } from 'react'; import usePlaylistsStore from '../../store/playlists.jsx'; import useSettingsStore from '../../store/settings.jsx'; +import useUsersStore from '../../store/users.jsx'; import { ActionIcon, Badge, @@ -123,6 +124,8 @@ const StreamConnectionCard = ({ // Get M3U account data from the playlists store const m3uAccounts = usePlaylistsStore((s) => s.playlists); + // Get users for resolving user_id → username on client rows + const users = useUsersStore((s) => s.users); // Get settings for speed threshold and environment mode const settings = useSettingsStore((s) => s.settings); const env_mode = @@ -138,6 +141,15 @@ const StreamConnectionCard = ({ return getM3uAccountsMap(m3uAccounts); }, [m3uAccounts]); + // Create a map of user IDs to usernames for quick lookup + const usersMap = useMemo(() => { + const map = {}; + users.forEach((u) => { + map[String(u.id)] = u.username; + }); + return map; + }, [users]); + // Update M3U profile information when channel data changes useEffect(() => { // If the channel data includes M3U profile information, update our state @@ -314,7 +326,8 @@ const StreamConnectionCard = ({ { header: 'IP Address', accessorKey: 'ip_address', - size: 150, + grow: true, + minSize: 85, cell: ({ cell }) => ( @@ -323,10 +336,29 @@ const StreamConnectionCard = ({ ), }, + { + id: 'user', + header: 'User', + grow: true, + minSize: 60, + accessorFn: (row) => { + const uid = row.user_id ? String(row.user_id) : null; + if (!uid || uid === '0') return 'Anonymous'; + return usersMap[uid] || `User ${uid}`; + }, + cell: ({ cell }) => ( + + {cell.getValue()} + + ), + }, // Updated Connected column with tooltip { id: 'connected', header: 'Connected', + grow: 1.5, + minSize: 70, + maxSize: 150, accessorFn: connectedAccessor(fullDateTimeFormat), cell: ({ cell }) => ( - {cell.getValue()} + + {cell.getValue()} + ), }, @@ -344,6 +378,8 @@ const StreamConnectionCard = ({ { id: 'duration', header: 'Duration', + size: 82, + minSize: 60, accessorFn: durationAccessor(), cell: ({ cell, row }) => { const exactDuration = @@ -356,7 +392,9 @@ const StreamConnectionCard = ({ : 'Unknown duration' } > - {cell.getValue()} + + {cell.getValue()} + ); }, @@ -364,10 +402,11 @@ const StreamConnectionCard = ({ { id: 'actions', header: 'Actions', - size: 100, + size: 60, + minSize: 40, }, ], - [fullDateTimeFormat] + [fullDateTimeFormat, usersMap] ); const channelClientsTable = useTable({ @@ -383,6 +422,7 @@ const StreamConnectionCard = ({ }), headerCellRenderFns: { ip_address: renderHeaderCell, + user: renderHeaderCell, connected: renderHeaderCell, duration: renderHeaderCell, actions: renderHeaderCell, @@ -610,8 +650,9 @@ const StreamConnectionCard = ({ {currentProgram && isProgramDescExpanded && currentProgram.start_time && - currentProgram.end_time && - } + currentProgram.end_time && ( + + )} {/* Add stream selection dropdown and preview button */} {availableStreams.length > 0 && ( From 6f516c0074e535fcee250ab0b88a8ed0c836d4d9 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 28 Mar 2026 13:36:47 -0500 Subject: [PATCH 48/67] Enhancement: `CustomTable` column layout now supports flexible (`grow`) columns alongside fixed-width ones: - Column definitions accept a `grow` property (boolean or number) to opt into flex layout. A numeric value sets the flex-grow weight, allowing relative sizing between grow columns (e.g. `grow: 2` gives a column twice the share of spare space as `grow: 1`). - `maxSize` is now respected on grow columns, capping how wide they expand via `maxWidth`. - The wrapper's `minWidth` calculation now uses `minSize` (not TanStack's 150px default) for grow columns, preventing the table from overflowing its container when columns would otherwise be sized larger than available space. --- CHANGELOG.md | 4 ++++ .../src/components/tables/CustomTable/CustomTable.jsx | 11 ++++++++--- .../components/tables/CustomTable/CustomTableBody.jsx | 8 ++++---- .../tables/CustomTable/CustomTableHeader.jsx | 5 ++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 450a33ae..c15fc6c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `CustomTable` column layout now supports flexible (`grow`) columns alongside fixed-width ones: + - Column definitions accept a `grow` property (boolean or number) to opt into flex layout. A numeric value sets the flex-grow weight, allowing relative sizing between grow columns (e.g. `grow: 2` gives a column twice the share of spare space as `grow: 1`). + - `maxSize` is now respected on grow columns, capping how wide they expand via `maxWidth`. + - The wrapper's `minWidth` calculation now uses `minSize` (not TanStack's 150px default) for grow columns, preventing the table from overflowing its container when columns would otherwise be sized larger than available space. - Dependency updates: - `requests` 2.32.5 → 2.33.0 (security patch; see Security section) - `celery` 5.6.2 → 5.6.3 diff --git a/frontend/src/components/tables/CustomTable/CustomTable.jsx b/frontend/src/components/tables/CustomTable/CustomTable.jsx index e14209d2..f1f081dd 100644 --- a/frontend/src/components/tables/CustomTable/CustomTable.jsx +++ b/frontend/src/components/tables/CustomTable/CustomTable.jsx @@ -6,17 +6,22 @@ import CustomTableBody from './CustomTableBody'; const CustomTable = ({ table }) => { const tableSize = table?.tableSize ?? 'default'; - // Get column sizing state for dependency tracking + // columnSizing is read here so the memo below re-runs when columns are resized. const columnSizing = table.getState().columnSizing; - // Calculate minimum table width reactively based on column sizes + // Calculate minimum table width reactively based on column sizes. + // Grow columns contribute only their minSize (not TanStack's default 150px) + // so the wrapper doesn't force the table wider than its container. const minTableWidth = useMemo(() => { + void columnSizing; // reactive trigger: recalculate when column sizes change const headerGroups = table.getHeaderGroups(); if (!headerGroups || headerGroups.length === 0) return 0; const width = headerGroups[0]?.headers.reduce((total, header) => { - return total + header.getSize(); + const colDef = header.column.columnDef; + const size = colDef.grow ? colDef.minSize || 0 : header.getSize(); + return total + size; }, 0) || 0; return width; diff --git a/frontend/src/components/tables/CustomTable/CustomTableBody.jsx b/frontend/src/components/tables/CustomTable/CustomTableBody.jsx index d579a41a..c8178eeb 100644 --- a/frontend/src/components/tables/CustomTable/CustomTableBody.jsx +++ b/frontend/src/components/tables/CustomTable/CustomTableBody.jsx @@ -126,9 +126,6 @@ const CustomTableBody = ({ }} > {row.getVisibleCells().map((cell) => { - const hasFixedSize = cell.column.columnDef.size; - const isFlexible = !hasFixedSize; - return ( Date: Sat, 28 Mar 2026 14:21:37 -0500 Subject: [PATCH 49/67] Bug Fix: preserve original file mtime during HTML entity resolution to prevent infinite refresh loop --- apps/epg/tasks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index df1e0836..9683c85a 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -94,6 +94,15 @@ def _resolve_html_entities(file_path): header = f.read(200) encoding = _detect_xml_encoding(header) + # Capture the original mtime before modifying the file so that os.replace() + # does not update the file's mtime. The file watcher in core/tasks.py uses + # mtime to detect changes; if we let mtime advance it would trigger an + # infinite refresh loop for local file-based EPG sources. + try: + original_stat = os.stat(file_path) + except OSError: + original_stat = None + temp_path = file_path + '.entity_tmp' success = False try: @@ -102,6 +111,10 @@ def _resolve_html_entities(file_path): for line in src: dst.write(_NAMED_ENTITY_RE.sub(_replace_html_entity, line)) os.replace(temp_path, file_path) + # Restore the original mtime (and atime) so the file watcher does not + # mistake the rewrite for an external update. + if original_stat is not None: + os.utime(file_path, (original_stat.st_atime, original_stat.st_mtime)) success = True logger.debug(f"Resolved HTML entities in {file_path} (encoding: {encoding})") except Exception as e: From 7ece48bdf2fd95b8dc9c0208c7f5f76245cd770b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 28 Mar 2026 15:12:24 -0500 Subject: [PATCH 50/67] tests: Update frontend tests --- .../__tests__/M3URefreshNotification.test.jsx | 1 + .../__tests__/NotificationCenter.test.jsx | 119 ++++- .../components/__tests__/SeriesModal.test.jsx | 172 +++++-- .../cards/__tests__/RecordingCard.test.jsx | 280 ++++++++--- .../cards/__tests__/SeriesCard.test.jsx | 60 ++- .../__tests__/StreamConnectionCard.test.jsx | 150 ++++-- .../cards/__tests__/VODCard.test.jsx | 58 ++- .../__tests__/VodConnectionCard.test.jsx | 446 ++++++++++++++---- 8 files changed, 1006 insertions(+), 280 deletions(-) diff --git a/frontend/src/components/__tests__/M3URefreshNotification.test.jsx b/frontend/src/components/__tests__/M3URefreshNotification.test.jsx index 2c5a5572..43375fa7 100644 --- a/frontend/src/components/__tests__/M3URefreshNotification.test.jsx +++ b/frontend/src/components/__tests__/M3URefreshNotification.test.jsx @@ -57,6 +57,7 @@ vi.mock('@mantine/core', async () => { // Mock lucide-react icons vi.mock('lucide-react', () => ({ + ListOrdered: () =>
, CircleCheck: () =>
, })); diff --git a/frontend/src/components/__tests__/NotificationCenter.test.jsx b/frontend/src/components/__tests__/NotificationCenter.test.jsx index 9dfb13f5..2231c951 100644 --- a/frontend/src/components/__tests__/NotificationCenter.test.jsx +++ b/frontend/src/components/__tests__/NotificationCenter.test.jsx @@ -33,8 +33,12 @@ vi.mock('@mantine/core', async () => { {children}
), - PopoverTarget: ({ children }) =>
{children}
, - PopoverDropdown: ({ children }) =>
{children}
, + PopoverTarget: ({ children }) => ( +
{children}
+ ), + PopoverDropdown: ({ children }) => ( +
{children}
+ ), Indicator: ({ children, label, disabled, processing }) => (
{
), ActionIcon: ({ children, onClick, 'aria-label': ariaLabel, ...props }) => ( - ), - ScrollAreaAutosize: ({ children }) =>
{children}
, - Badge: ({ children, ...props }) => {children}, - Card: ({ children, ...props }) =>
{children}
, - ThemeIcon: ({ children, ...props }) =>
{children}
, - Group: ({ children, ...props }) =>
{children}
, - Stack: ({ children, ...props }) =>
{children}
, - Box: ({ children, ...props }) =>
{children}
, - Text: ({ children, ...props }) => {children}, + ScrollAreaAutosize: ({ children }) => ( +
{children}
+ ), + Badge: ({ children, ...props }) => ( + + {children} + + ), + Card: ({ children, ...props }) => ( +
+ {children} +
+ ), + ThemeIcon: ({ children, ...props }) => ( +
+ {children} +
+ ), + Group: ({ children, ...props }) => ( +
+ {children} +
+ ), + Stack: ({ children, ...props }) => ( +
+ {children} +
+ ), + Box: ({ children, ...props }) => ( +
+ {children} +
+ ), + Text: ({ children, ...props }) => ( + + {children} + + ), Button: ({ children, onClick, ...props }) => ( +
{children}
); }, - Box: ({ children, ...props }) =>
{children}
, + Box: ({ children, ...props }) => ( +
+ {children} +
+ ), Button: ({ children, onClick, disabled, ...props }) => ( - ), - Flex: ({ children, ...props }) =>
{children}
, - Group: ({ children, ...props }) =>
{children}
, + Flex: ({ children, ...props }) => ( +
+ {children} +
+ ), + Group: ({ children, ...props }) => ( +
+ {children} +
+ ), Image: ({ src, alt, ...props }) => ( {alt} ), - Text: ({ children, ...props }) =>
{children}
, - Title: ({ children, order, ...props }) => ( -
{children}
+ Text: ({ children, ...props }) => ( +
+ {children} +
), - Select: ({ value, onChange, data, label, placeholder, disabled, ...props }) => ( + Title: ({ children, order, ...props }) => ( +
+ {children} +
+ ), + Select: ({ + value, + onChange, + data, + label, + placeholder, + disabled, + ...props + }) => (
), - Badge: ({ children, ...props }) => {children}, + Badge: ({ children, ...props }) => ( + + {children} + + ), Loader: (props) =>
, - Stack: ({ children, ...props }) =>
{children}
, + Stack: ({ children, ...props }) => ( +
+ {children} +
+ ), ActionIcon: ({ children, onClick, disabled, ...props }) => ( - ), Tabs: ({ children, value, onChange, ...props }) => (
-
{ - const tab = e.target.closest('[data-tab-value]'); - if (tab) onChange?.(tab.dataset.tabValue); - }}> +
{ + const tab = e.target.closest('[data-tab-value]'); + if (tab) onChange?.(tab.dataset.tabValue); + }} + > {children}
), TabsList: ({ children }) =>
{children}
, TabsTab: ({ children, value }) => ( - + ), TabsPanel: ({ children, value }) => ( -
{children}
+
+ {children} +
+ ), + Table: ({ children, ...props }) => ( + + {children} +
+ ), + TableThead: ({ children }) => ( + {children} + ), + TableTbody: ({ children }) => ( + {children} ), - Table: ({ children, ...props }) => {children}
, - TableThead: ({ children }) => {children}, - TableTbody: ({ children }) => {children}, TableTr: ({ children, onClick, ...props }) => ( - {children} + + {children} + + ), + TableTh: ({ children, ...props }) => ( + + {children} + + ), + TableTd: ({ children, ...props }) => ( + + {children} + ), - TableTh: ({ children, ...props }) => {children}, - TableTd: ({ children, ...props }) => {children}, Divider: (props) =>
, }; }); @@ -168,7 +239,7 @@ describe('SeriesModal', () => { m3u_account: { name: 'Provider 2' }, stream_name: 'Test Series 720p', quality_info: null, - } + }, ]; beforeEach(() => { @@ -187,9 +258,15 @@ describe('SeriesModal', () => { environment: { env_mode: 'prod' }, }; - useVODStore.mockImplementation((selector) => selector ? selector(mockVODStore) : mockVODStore); - useVideoStore.mockImplementation((selector) => selector ? selector(mockVideoStore) : mockVideoStore); - useSettingsStore.mockImplementation((selector) => selector ? selector(mockSettingsStore) : mockSettingsStore); + useVODStore.mockImplementation((selector) => + selector ? selector(mockVODStore) : mockVODStore + ); + useVideoStore.mockImplementation((selector) => + selector ? selector(mockVideoStore) : mockVideoStore + ); + useSettingsStore.mockImplementation((selector) => + selector ? selector(mockSettingsStore) : mockSettingsStore + ); copyToClipboard.mockResolvedValue(undefined); }); @@ -325,23 +402,37 @@ describe('SeriesModal', () => { it('should display IMDB link when imdb_id exists', async () => { render( - + ); await waitFor(() => { const link = screen.getByText(/IMDB/i).closest('a'); - expect(link).toHaveAttribute('href', 'https://www.imdb.com/title/tt1234567'); + expect(link).toHaveAttribute( + 'href', + 'https://www.imdb.com/title/tt1234567' + ); }); }); it('should display TMDB link when tmdb_id exists', async () => { render( - + ); await waitFor(() => { const link = screen.getByText(/TMDB/i).closest('a'); - expect(link).toHaveAttribute('href', 'https://www.themoviedb.org/tv/12345'); + expect(link).toHaveAttribute( + 'href', + 'https://www.themoviedb.org/tv/12345' + ); }); }); }); @@ -446,7 +537,12 @@ describe('SeriesModal', () => { }); it('should sort episodes by episode number', async () => { - const episode2 = { ...mockEpisode, id: 2, episode_number: 2, name: 'Second Episode' }; + const episode2 = { + ...mockEpisode, + id: 2, + episode_number: 2, + name: 'Second Episode', + }; mockVODStore.fetchSeriesInfo.mockResolvedValue({ ...mockDetailedSeries, episodesList: [episode2, mockEpisode], @@ -592,7 +688,12 @@ describe('SeriesModal', () => { describe('Season Tabs', () => { it('should create tabs for each season', async () => { - const season2Episode = { ...mockEpisode, id: 2, season_number: 2, episode_num: 1 }; + const season2Episode = { + ...mockEpisode, + id: 2, + season_number: 2, + episode_num: 1, + }; mockVODStore.fetchSeriesInfo.mockResolvedValue({ ...mockDetailedSeries, episodesList: [mockEpisode, season2Episode], @@ -670,7 +771,6 @@ describe('SeriesModal', () => { ); - await waitFor(() => { expect(screen.getByText(/Provider 1 - 1080p/)).toBeInTheDocument(); }); @@ -797,7 +897,7 @@ describe('SeriesModal', () => { ); await waitFor(() => { - expect(screen.getByText('Test Account')).toBeInTheDocument() + expect(screen.getByText('Test Account')).toBeInTheDocument(); }); }); }); diff --git a/frontend/src/components/cards/__tests__/RecordingCard.test.jsx b/frontend/src/components/cards/__tests__/RecordingCard.test.jsx index 9566f2b5..24e8dc46 100644 --- a/frontend/src/components/cards/__tests__/RecordingCard.test.jsx +++ b/frontend/src/components/cards/__tests__/RecordingCard.test.jsx @@ -109,13 +109,12 @@ vi.mock('@mantine/core', async () => ({ {children} ), - Tooltip: ({ children, label }) => ( -
{children}
- ), + Tooltip: ({ children, label }) =>
{children}
, })); // ── lucide-react ─────────────────────────────────────────────────────────────── vi.mock('lucide-react', () => ({ + ListOrdered: () => , AlertTriangle: () => , Plus: () => , Square: () => , @@ -140,8 +139,13 @@ vi.mock('../../../images/logo.png', () => ({ default: 'default-logo.png' })); import useChannelsStore from '../../../store/channels.jsx'; import useSettingsStore from '../../../store/settings.jsx'; import useVideoStore from '../../../store/useVideoStore.jsx'; -import { useDateTimeFormat, useTimeHelpers, format, isAfter, isBefore } - from '../../../utils/dateTimeUtils.js'; +import { + useDateTimeFormat, + useTimeHelpers, + format, + isAfter, + isBefore, +} from '../../../utils/dateTimeUtils.js'; import { notifications } from '@mantine/notifications'; import * as RecordingCardUtils from '../../../utils/cards/RecordingCardUtils.js'; import dayjs from 'dayjs'; @@ -187,7 +191,11 @@ const makeChannel = () => ({ }); /** Wire up all store/utility mocks with sensible defaults */ -const setupMocks = ({ now = NOW, recording = makeRecording(), channel = makeChannel() } = {}) => { +const setupMocks = ({ + now = NOW, + recording = makeRecording(), + channel = makeChannel(), +} = {}) => { const nowMoment = makeMoment(now); const startMoment = makeMoment(recording.start_time); const endMoment = makeMoment(recording.end_time); @@ -226,9 +234,13 @@ const setupMocks = ({ now = NOW, recording = makeRecording(), channel = makeChan vi.mocked(RecordingCardUtils.getPosterUrl).mockReturnValue('/poster.jpg'); vi.mocked(RecordingCardUtils.getChannelLogoUrl).mockReturnValue('/logo.png'); - vi.mocked(RecordingCardUtils.getRecordingUrl).mockReturnValue('/recordings/test.ts'); + vi.mocked(RecordingCardUtils.getRecordingUrl).mockReturnValue( + '/recordings/test.ts' + ); vi.mocked(RecordingCardUtils.getSeasonLabel).mockReturnValue(''); - vi.mocked(RecordingCardUtils.getSeriesInfo).mockReturnValue({ seriesId: 's1' }); + vi.mocked(RecordingCardUtils.getSeriesInfo).mockReturnValue({ + seriesId: 's1', + }); vi.mocked(RecordingCardUtils.getShowVideoUrl).mockReturnValue('/live/ch-1'); return { mockShowVideo, mockFetchRecordings }; @@ -237,10 +249,18 @@ const setupMocks = ({ now = NOW, recording = makeRecording(), channel = makeChan describe('RecordingCard', () => { beforeEach(() => { vi.clearAllMocks(); - vi.mocked(RecordingCardUtils.stopRecordingById).mockResolvedValue(undefined); - vi.mocked(RecordingCardUtils.deleteRecordingById).mockResolvedValue(undefined); - vi.mocked(RecordingCardUtils.deleteSeriesAndRule).mockResolvedValue(undefined); - vi.mocked(RecordingCardUtils.extendRecordingById).mockResolvedValue(undefined); + vi.mocked(RecordingCardUtils.stopRecordingById).mockResolvedValue( + undefined + ); + vi.mocked(RecordingCardUtils.deleteRecordingById).mockResolvedValue( + undefined + ); + vi.mocked(RecordingCardUtils.deleteSeriesAndRule).mockResolvedValue( + undefined + ); + vi.mocked(RecordingCardUtils.extendRecordingById).mockResolvedValue( + undefined + ); vi.mocked(RecordingCardUtils.runComSkip).mockResolvedValue(undefined); vi.mocked(RecordingCardUtils.removeRecording).mockReturnValue(undefined); }); @@ -250,17 +270,23 @@ describe('RecordingCard', () => { describe('rendering', () => { it('renders the recording title', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('Test Show')).toBeInTheDocument(); }); it('renders "Custom Recording" when no program title', () => { setupMocks({ - recording: makeRecording({ custom_properties: { status: 'completed', program: {} } }), + recording: makeRecording({ + custom_properties: { status: 'completed', program: {} }, + }), }); render( ); @@ -269,7 +295,9 @@ describe('RecordingCard', () => { it('renders channel info', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('501 • HBO')).toBeInTheDocument(); }); @@ -281,27 +309,35 @@ describe('RecordingCard', () => { it('shows description via RecordingSynopsis for non-series completed recording', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByTestId('recording-synopsis')).toBeInTheDocument(); expect(screen.getByText('A test description')).toBeInTheDocument(); }); it('shows sub_title when present', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('Pilot')).toBeInTheDocument(); }); it('shows season/episode label when getSeasonLabel returns a value', () => { setupMocks(); vi.mocked(RecordingCardUtils.getSeasonLabel).mockReturnValue('S01E02'); - render(); + render( + + ); expect(screen.getByText('S01E02')).toBeInTheDocument(); }); it('renders the poster image', () => { setupMocks(); - render(); + render( + + ); const img = screen.getByAltText('Test Show'); expect(img).toHaveAttribute('src', '/poster.jpg'); }); @@ -312,7 +348,9 @@ describe('RecordingCard', () => { describe('status badge', () => { it('shows "Completed" badge for a completed recording', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('Completed')).toBeInTheDocument(); }); @@ -320,7 +358,10 @@ describe('RecordingCard', () => { const recording = makeRecording({ start_time: PAST, end_time: FUTURE, - custom_properties: { status: 'recording', program: { title: 'Live Show' } }, + custom_properties: { + status: 'recording', + program: { title: 'Live Show' }, + }, }); setupMocks({ recording }); render(); @@ -331,7 +372,10 @@ describe('RecordingCard', () => { const recording = makeRecording({ start_time: FUTURE, end_time: FUTURE, - custom_properties: { status: 'scheduled', program: { title: 'Future Show' } }, + custom_properties: { + status: 'scheduled', + program: { title: 'Future Show' }, + }, }); setupMocks({ recording }); render(); @@ -373,8 +417,7 @@ describe('RecordingCard', () => { // ── Series group ─────────────────────────────────────────────────────────── describe('series group', () => { - const makeSeriesRecording = () => - makeRecording({ _group_count: 3 }); + const makeSeriesRecording = () => makeRecording({ _group_count: 3 }); it('shows "Series" badge when _group_count > 1', () => { const recording = makeSeriesRecording(); @@ -394,7 +437,9 @@ describe('RecordingCard', () => { const recording = makeSeriesRecording(); setupMocks({ recording }); render(); - expect(screen.queryByTestId('recording-synopsis')).not.toBeInTheDocument(); + expect( + screen.queryByTestId('recording-synopsis') + ).not.toBeInTheDocument(); }); }); @@ -500,7 +545,9 @@ describe('RecordingCard', () => { it('does not show "Watch Live" for a completed recording', () => { setupMocks(); - render(); + render( + + ); expect(screen.queryByText('Watch Live')).not.toBeInTheDocument(); }); @@ -523,7 +570,9 @@ describe('RecordingCard', () => { it('shows "Watch" button for a completed recording', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('Watch')).toBeInTheDocument(); }); @@ -531,7 +580,10 @@ describe('RecordingCard', () => { const recording = makeRecording({ start_time: FUTURE, end_time: FUTURE, - custom_properties: { status: 'scheduled', program: { title: 'Future' } }, + custom_properties: { + status: 'scheduled', + program: { title: 'Future' }, + }, }); setupMocks({ recording }); render(); @@ -540,7 +592,9 @@ describe('RecordingCard', () => { it('calls showVideo with vod params when Watch is clicked', () => { const { mockShowVideo } = setupMocks(); - render(); + render( + + ); fireEvent.click(screen.getByText('Watch')); expect(mockShowVideo).toHaveBeenCalledWith( '/recordings/test.ts', @@ -552,7 +606,9 @@ describe('RecordingCard', () => { it('does not call showVideo when Watch is clicked but no file url', () => { const { mockShowVideo } = setupMocks(); vi.mocked(RecordingCardUtils.getRecordingUrl).mockReturnValue(null); - render(); + render( + + ); fireEvent.click(screen.getByText('Watch')); expect(mockShowVideo).not.toHaveBeenCalled(); }); @@ -575,7 +631,9 @@ describe('RecordingCard', () => { describe('"Remove commercials" button', () => { it('shows "Remove commercials" for a completed recording without comskip', () => { setupMocks(); - render(); + render( + + ); expect(screen.getByText('Remove commercials')).toBeInTheDocument(); }); @@ -597,7 +655,10 @@ describe('RecordingCard', () => { const recording = makeRecording({ start_time: FUTURE, end_time: FUTURE, - custom_properties: { status: 'scheduled', program: { title: 'Future' } }, + custom_properties: { + status: 'scheduled', + program: { title: 'Future' }, + }, }); setupMocks({ recording }); render(); @@ -606,20 +667,31 @@ describe('RecordingCard', () => { it('calls runComSkip and shows notification on success', async () => { setupMocks(); - render(); + render( + + ); fireEvent.click(screen.getByText('Remove commercials')); await waitFor(() => { - expect(RecordingCardUtils.runComSkip).toHaveBeenCalledWith(makeRecording()); + expect(RecordingCardUtils.runComSkip).toHaveBeenCalledWith( + makeRecording() + ); expect(notifications.show).toHaveBeenCalledWith( - expect.objectContaining({ title: 'Removing commercials', color: 'blue.5' }) + expect.objectContaining({ + title: 'Removing commercials', + color: 'blue.5', + }) ); }); }); it('does not show notification when runComSkip throws', async () => { - vi.mocked(RecordingCardUtils.runComSkip).mockRejectedValue(new Error('fail')); + vi.mocked(RecordingCardUtils.runComSkip).mockRejectedValue( + new Error('fail') + ); setupMocks(); - render(); + render( + + ); fireEvent.click(screen.getByText('Remove commercials')); await waitFor(() => { expect(notifications.show).not.toHaveBeenCalled(); @@ -634,7 +706,11 @@ describe('RecordingCard', () => { makeRecording({ start_time: PAST, end_time: FUTURE, - custom_properties: { status: 'recording', program: { title: 'Live Show' }, file_url: '/f.ts' }, + custom_properties: { + status: 'recording', + program: { title: 'Live Show' }, + file_url: '/f.ts', + }, }); it('shows extend menu for in-progress recording', () => { @@ -650,9 +726,15 @@ describe('RecordingCard', () => { render(); fireEvent.click(screen.getByText('+15 minutes')); await waitFor(() => { - expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith('rec-1', 15); + expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith( + 'rec-1', + 15 + ); expect(notifications.show).toHaveBeenCalledWith( - expect.objectContaining({ title: 'Recording extended', color: 'teal' }) + expect.objectContaining({ + title: 'Recording extended', + color: 'teal', + }) ); }); }); @@ -663,7 +745,10 @@ describe('RecordingCard', () => { render(); fireEvent.click(screen.getByText('+30 minutes')); await waitFor(() => { - expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith('rec-1', 30); + expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith( + 'rec-1', + 30 + ); }); }); @@ -673,12 +758,17 @@ describe('RecordingCard', () => { render(); fireEvent.click(screen.getByText('+1 hour')); await waitFor(() => { - expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith('rec-1', 60); + expect(RecordingCardUtils.extendRecordingById).toHaveBeenCalledWith( + 'rec-1', + 60 + ); }); }); it('shows error notification when extendRecordingById throws', async () => { - vi.mocked(RecordingCardUtils.extendRecordingById).mockRejectedValue(new Error('Network error')); + vi.mocked(RecordingCardUtils.extendRecordingById).mockRejectedValue( + new Error('Network error') + ); const recording = makeInProgress(); setupMocks({ recording }); render(); @@ -698,7 +788,11 @@ describe('RecordingCard', () => { makeRecording({ start_time: PAST, end_time: FUTURE, - custom_properties: { status: 'recording', program: { title: 'Live Show' }, file_url: '/f.ts' }, + custom_properties: { + status: 'recording', + program: { title: 'Live Show' }, + file_url: '/f.ts', + }, }); it('shows stop modal when stop button is clicked', () => { @@ -711,7 +805,9 @@ describe('RecordingCard', () => { fireEvent.click(stopButton); expect(screen.getByTestId('modal')).toBeInTheDocument(); - expect(screen.getByTestId('modal-title')).toHaveTextContent('Stop Recording'); + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Stop Recording' + ); }); it('closes stop modal when Go Back is clicked', () => { @@ -736,7 +832,9 @@ describe('RecordingCard', () => { fireEvent.click(screen.getAllByText('Stop Recording')[1]); await waitFor(() => { - expect(RecordingCardUtils.stopRecordingById).toHaveBeenCalledWith('rec-1'); + expect(RecordingCardUtils.stopRecordingById).toHaveBeenCalledWith( + 'rec-1' + ); expect(mockFetchRecordings).toHaveBeenCalled(); }); }); @@ -761,48 +859,71 @@ describe('RecordingCard', () => { describe('delete recording', () => { it('shows delete modal for a completed non-series recording', () => { setupMocks(); - render(); + render( + + ); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); expect(screen.getByTestId('modal')).toBeInTheDocument(); - expect(screen.getByTestId('modal-title')).toHaveTextContent('Delete Recording'); + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Delete Recording' + ); }); it('shows "Cancel Recording" title for upcoming recording delete', () => { const recording = makeRecording({ start_time: FUTURE, end_time: FUTURE, - custom_properties: { status: 'scheduled', program: { title: 'Future Show' } }, + custom_properties: { + status: 'scheduled', + program: { title: 'Future Show' }, + }, }); setupMocks({ recording }); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); - expect(screen.getByTestId('modal-title')).toHaveTextContent('Cancel Recording'); + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Cancel Recording' + ); }); it('calls removeRecording when delete is confirmed', async () => { setupMocks(); - render(); + render( + + ); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Delete')); await waitFor(() => { - expect(RecordingCardUtils.removeRecording).toHaveBeenCalledWith('rec-1'); + expect(RecordingCardUtils.removeRecording).toHaveBeenCalledWith( + 'rec-1' + ); }); }); it('closes delete modal after confirming', async () => { setupMocks(); - render(); + render( + + ); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Delete')); @@ -813,9 +934,13 @@ describe('RecordingCard', () => { it('closes delete modal on Go Back click', () => { setupMocks(); - render(); + render( + + ); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Go Back')); @@ -842,10 +967,14 @@ describe('RecordingCard', () => { setupMocks({ recording }); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); - expect(screen.getByTestId('modal-title')).toHaveTextContent('Cancel Series'); + expect(screen.getByTestId('modal-title')).toHaveTextContent( + 'Cancel Series' + ); }); it('calls deleteRecordingById when "Only this upcoming" is clicked', async () => { @@ -853,12 +982,16 @@ describe('RecordingCard', () => { const { mockFetchRecordings } = setupMocks({ recording }); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Only this upcoming')); await waitFor(() => { - expect(RecordingCardUtils.deleteRecordingById).toHaveBeenCalledWith('rec-1'); + expect(RecordingCardUtils.deleteRecordingById).toHaveBeenCalledWith( + 'rec-1' + ); expect(mockFetchRecordings).toHaveBeenCalled(); }); }); @@ -868,7 +1001,9 @@ describe('RecordingCard', () => { const { mockFetchRecordings } = setupMocks({ recording }); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Entire series + rule')); @@ -883,7 +1018,9 @@ describe('RecordingCard', () => { setupMocks({ recording }); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); fireEvent.click(screen.getByText('Only this upcoming')); @@ -918,13 +1055,18 @@ describe('RecordingCard', () => { _group_count: 3, start_time: FUTURE, end_time: FUTURE, - custom_properties: { status: 'scheduled', program: { title: 'Series' } }, + custom_properties: { + status: 'scheduled', + program: { title: 'Series' }, + }, }); const { mockFetchRecordings } = setupMocks({ recording }); mockFetchRecordings.mockRejectedValue(new Error('network')); render(); - const deleteButton = screen.getByTestId('icon-square-x').closest('button'); + const deleteButton = screen + .getByTestId('icon-square-x') + .closest('button'); fireEvent.click(deleteButton); await expect( @@ -932,4 +1074,4 @@ describe('RecordingCard', () => { ).resolves.not.toThrow(); }); }); -}); \ No newline at end of file +}); diff --git a/frontend/src/components/cards/__tests__/SeriesCard.test.jsx b/frontend/src/components/cards/__tests__/SeriesCard.test.jsx index fb95c882..49b8524f 100644 --- a/frontend/src/components/cards/__tests__/SeriesCard.test.jsx +++ b/frontend/src/components/cards/__tests__/SeriesCard.test.jsx @@ -30,7 +30,13 @@ vi.mock('@mantine/core', async () => ({ ), Stack: ({ children, gap }) =>
{children}
, Text: ({ children, size, fw, c, lineClamp, style }) => ( - + {children} ), @@ -38,6 +44,7 @@ vi.mock('@mantine/core', async () => ({ // ── lucide-react ─────────────────────────────────────────────────────────────── vi.mock('lucide-react', () => ({ + ListOrdered: () => , Calendar: () => , Play: () => , Star: () => , @@ -78,7 +85,12 @@ describe('SeriesCard', () => { }); it('renders a fallback image when poster_url is missing', () => { - render(); + render( + + ); const img = screen.getByRole('img'); expect(img).toBeInTheDocument(); }); @@ -105,7 +117,12 @@ describe('SeriesCard', () => { it('renders play icon', () => { //this only renders when logo.url is missing, but we want to test that the icon itself renders correctly - render(); + render( + + ); expect(screen.getByTestId('icon-play')).toBeInTheDocument(); }); @@ -119,27 +136,52 @@ describe('SeriesCard', () => { describe('optional fields', () => { it('does not crash when year is missing', () => { - render(); + render( + + ); expect(screen.getByTestId('series-card')).toBeInTheDocument(); }); it('does not crash when rating is missing', () => { - render(); + render( + + ); expect(screen.getByTestId('series-card')).toBeInTheDocument(); }); it('does not crash when genre is missing', () => { - render(); + render( + + ); expect(screen.getByTestId('series-card')).toBeInTheDocument(); }); it('does not crash when description is missing', () => { - render(); + render( + + ); expect(screen.getByTestId('series-card')).toBeInTheDocument(); }); it('does not crash when seasons is missing', () => { - render(); + render( + + ); expect(screen.getByTestId('series-card')).toBeInTheDocument(); }); @@ -161,4 +203,4 @@ describe('SeriesCard', () => { expect(onClick).toHaveBeenCalledWith(series); }); }); -}); \ No newline at end of file +}); diff --git a/frontend/src/components/cards/__tests__/StreamConnectionCard.test.jsx b/frontend/src/components/cards/__tests__/StreamConnectionCard.test.jsx index babcf1e6..2f006244 100644 --- a/frontend/src/components/cards/__tests__/StreamConnectionCard.test.jsx +++ b/frontend/src/components/cards/__tests__/StreamConnectionCard.test.jsx @@ -95,7 +95,12 @@ vi.mock('@mantine/core', () => ({ Center: ({ children }) =>
{children}
, Group: ({ children }) =>
{children}
, Progress: ({ value, size, color }) => ( -
+
), Select: ({ value, onChange, label, data, disabled, placeholder }) => (