mirror of
https://github.com/Dispatcharr/Dispatcharr.git
synced 2026-07-17 16:50:53 +00:00
fix(proxy): improve resource cleanup and connection management
Some checks are pending
CI Pipeline / prepare (push) Waiting to run
CI Pipeline / docker (amd64, ubuntu-24.04) (push) Blocked by required conditions
CI Pipeline / docker (arm64, ubuntu-24.04-arm) (push) Blocked by required conditions
CI Pipeline / create-manifest (push) Blocked by required conditions
Build and Push Multi-Arch Docker Image / build-and-push (push) Waiting to run
Some checks are pending
CI Pipeline / prepare (push) Waiting to run
CI Pipeline / docker (amd64, ubuntu-24.04) (push) Blocked by required conditions
CI Pipeline / docker (arm64, ubuntu-24.04-arm) (push) Blocked by required conditions
CI Pipeline / create-manifest (push) Blocked by required conditions
Build and Push Multi-Arch Docker Image / build-and-push (push) Waiting to run
- Added calls to close_old_connections in various components to ensure proper resource cleanup and prevent connection leaks.
This commit is contained in:
SergeantPanda
parent
67239d921d
commit
8f40f6065f
6 changed files with 140 additions and 118 deletions
|
|
@ -690,6 +690,9 @@ class StreamManager:
|
|||
self.transcode_cmd = [arg for arg in self.transcode_cmd if self.user_agent not in arg and 'user-agent' not in arg.lower() and 'user_agent' not in arg.lower()]
|
||||
logger.debug(f"Removed user_agent parameters from UDP stream command for channel: {self.channel_id}")
|
||||
|
||||
# Profile lookup is done; release the pool slot before a long-lived ffmpeg process.
|
||||
close_old_connections()
|
||||
|
||||
logger.debug(f"Starting transcode process: {self.transcode_cmd} for channel: {self.channel_id}")
|
||||
|
||||
import os as _os
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import time
|
|||
import gevent
|
||||
from apps.channels.models import Channel, Stream
|
||||
from core.utils import log_system_event
|
||||
from django.db import close_old_connections
|
||||
from ...server import ProxyServer
|
||||
from ...redis_keys import RedisKeys
|
||||
from ...constants import ChannelMetadataField
|
||||
|
|
@ -331,46 +332,49 @@ class FMP4StreamGenerator:
|
|||
# ------------------------------------------------------------------
|
||||
|
||||
def _cleanup(self):
|
||||
elapsed = time.time() - self.stream_start_time
|
||||
proxy_server = ProxyServer.get_instance()
|
||||
try:
|
||||
elapsed = time.time() - self.stream_start_time
|
||||
proxy_server = ProxyServer.get_instance()
|
||||
|
||||
# Release stream allocation if last client (mirrors StreamGenerator)
|
||||
if proxy_server.redis_client:
|
||||
try:
|
||||
meta_key = RedisKeys.channel_metadata(self.channel_id)
|
||||
stream_id_bytes = proxy_server.redis_client.hget(
|
||||
meta_key, ChannelMetadataField.STREAM_ID
|
||||
)
|
||||
if stream_id_bytes:
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_count = proxy_server.client_managers[
|
||||
self.channel_id
|
||||
].get_total_client_count()
|
||||
if client_count <= 1 and proxy_server.am_i_owner(self.channel_id):
|
||||
try:
|
||||
# Release stream allocation if last client (mirrors StreamGenerator)
|
||||
if proxy_server.redis_client:
|
||||
try:
|
||||
meta_key = RedisKeys.channel_metadata(self.channel_id)
|
||||
stream_id_bytes = proxy_server.redis_client.hget(
|
||||
meta_key, ChannelMetadataField.STREAM_ID
|
||||
)
|
||||
if stream_id_bytes:
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_count = proxy_server.client_managers[
|
||||
self.channel_id
|
||||
].get_total_client_count()
|
||||
if client_count <= 1 and proxy_server.am_i_owner(self.channel_id):
|
||||
try:
|
||||
obj = Channel.objects.get(uuid=self.channel_id)
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
obj = Stream.objects.get(stream_hash=self.channel_id)
|
||||
obj.release_stream()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[{self.client_id}] Error releasing stream: {e}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error in stream release check: {e}")
|
||||
try:
|
||||
obj = Channel.objects.get(uuid=self.channel_id)
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
obj = Stream.objects.get(stream_hash=self.channel_id)
|
||||
obj.release_stream()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[{self.client_id}] Error releasing stream: {e}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error in stream release check: {e}")
|
||||
|
||||
# Remove from MAIN ClientManager - this is what triggers handle_client_disconnect
|
||||
# and the zero-clients → stop_channel path, same as TS clients.
|
||||
local_clients = 0
|
||||
total_clients = 0
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_manager = proxy_server.client_managers[self.channel_id]
|
||||
local_clients = client_manager.remove_client(self.client_id)
|
||||
total_clients = client_manager.get_total_client_count()
|
||||
# Remove from MAIN ClientManager - this is what triggers handle_client_disconnect
|
||||
# and the zero-clients → stop_channel path, same as TS clients.
|
||||
local_clients = 0
|
||||
total_clients = 0
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_manager = proxy_server.client_managers[self.channel_id]
|
||||
local_clients = client_manager.remove_client(self.client_id)
|
||||
total_clients = client_manager.get_total_client_count()
|
||||
|
||||
logger.info(
|
||||
f"[{self.client_id}] fMP4 client disconnected after {elapsed:.2f}s "
|
||||
f"({self.bytes_sent / 1024:.1f} KB sent, "
|
||||
f"local: {local_clients}, total: {total_clients})"
|
||||
)
|
||||
logger.info(
|
||||
f"[{self.client_id}] fMP4 client disconnected after {elapsed:.2f}s "
|
||||
f"({self.bytes_sent / 1024:.1f} KB sent, "
|
||||
f"local: {local_clients}, total: {total_clients})"
|
||||
)
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import gevent
|
|||
from apps.proxy.config import TSConfig as Config
|
||||
from apps.channels.models import Channel, Stream
|
||||
from core.utils import log_system_event
|
||||
from django.db import close_old_connections
|
||||
from ...server import ProxyServer
|
||||
from ...utils import create_ts_packet, get_logger
|
||||
from ...redis_keys import RedisKeys
|
||||
|
|
@ -590,59 +591,62 @@ class StreamGenerator:
|
|||
|
||||
def _cleanup(self):
|
||||
"""Clean up resources and report final statistics."""
|
||||
# Client cleanup
|
||||
elapsed = time.time() - self.stream_start_time
|
||||
local_clients = 0
|
||||
total_clients = 0
|
||||
proxy_server = ProxyServer.get_instance()
|
||||
try:
|
||||
# Client cleanup
|
||||
elapsed = time.time() - self.stream_start_time
|
||||
local_clients = 0
|
||||
total_clients = 0
|
||||
proxy_server = ProxyServer.get_instance()
|
||||
|
||||
# Release M3U profile stream allocation if this is the last client
|
||||
stream_released = False
|
||||
if proxy_server.redis_client:
|
||||
try:
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_count = proxy_server.client_managers[self.channel_id].get_total_client_count()
|
||||
# Pool slots are global; the last client on any worker must release.
|
||||
if client_count <= 1:
|
||||
try:
|
||||
# Release M3U profile stream allocation if this is the last client
|
||||
stream_released = False
|
||||
if proxy_server.redis_client:
|
||||
try:
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_count = proxy_server.client_managers[self.channel_id].get_total_client_count()
|
||||
# Pool slots are global; the last client on any worker must release.
|
||||
if client_count <= 1:
|
||||
try:
|
||||
obj = Channel.objects.get(uuid=self.channel_id)
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
obj = Stream.objects.get(stream_hash=self.channel_id)
|
||||
stream_released = obj.release_stream()
|
||||
if stream_released:
|
||||
logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}")
|
||||
else:
|
||||
logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error checking stream data for release: {e}")
|
||||
try:
|
||||
obj = Channel.objects.get(uuid=self.channel_id)
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
obj = Stream.objects.get(stream_hash=self.channel_id)
|
||||
stream_released = obj.release_stream()
|
||||
if stream_released:
|
||||
logger.debug(f"[{self.client_id}] Released stream for channel {self.channel_id}")
|
||||
else:
|
||||
logger.warning(f"[{self.client_id}] release_stream found no keys for channel {self.channel_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error releasing stream for channel {self.channel_id}: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{self.client_id}] Error checking stream data for release: {e}")
|
||||
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_manager = proxy_server.client_managers[self.channel_id]
|
||||
if self.client_id in client_manager.clients:
|
||||
local_clients = client_manager.remove_client(self.client_id)
|
||||
else:
|
||||
local_clients = client_manager.get_client_count()
|
||||
total_clients = client_manager.get_total_client_count()
|
||||
logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})")
|
||||
if self.channel_id in proxy_server.client_managers:
|
||||
client_manager = proxy_server.client_managers[self.channel_id]
|
||||
if self.client_id in client_manager.clients:
|
||||
local_clients = client_manager.remove_client(self.client_id)
|
||||
else:
|
||||
local_clients = client_manager.get_client_count()
|
||||
total_clients = client_manager.get_total_client_count()
|
||||
logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})")
|
||||
|
||||
# Log client disconnect event
|
||||
try:
|
||||
log_system_event(
|
||||
'client_disconnect',
|
||||
channel_id=self.channel_id,
|
||||
channel_name=self.channel_name,
|
||||
client_ip=self.client_ip,
|
||||
client_id=self.client_id,
|
||||
user_agent=self.client_user_agent[:100] if self.client_user_agent else None,
|
||||
duration=round(elapsed, 2),
|
||||
bytes_sent=self.bytes_sent,
|
||||
username=self.user.username if self.user else None
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Could not log client disconnect event: {e}")
|
||||
# Log client disconnect event
|
||||
try:
|
||||
log_system_event(
|
||||
'client_disconnect',
|
||||
channel_id=self.channel_id,
|
||||
channel_name=self.channel_name,
|
||||
client_ip=self.client_ip,
|
||||
client_id=self.client_id,
|
||||
user_agent=self.client_user_agent[:100] if self.client_user_agent else None,
|
||||
duration=round(elapsed, 2),
|
||||
bytes_sent=self.bytes_sent,
|
||||
username=self.user.username if self.user else None
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Could not log client disconnect event: {e}")
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
||||
|
||||
def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False, user=None, buffer=None):
|
||||
|
|
|
|||
|
|
@ -760,6 +760,8 @@ class ProxyServer:
|
|||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Could not log channel start event: {e}")
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
||||
# Create client manager with channel_id, redis_client AND worker_id (only if not already exists)
|
||||
if channel_id not in self.client_managers:
|
||||
|
|
@ -2072,42 +2074,45 @@ class ProxyServer:
|
|||
"""Clean up all Redis keys for a channel more efficiently"""
|
||||
total_deleted = 0
|
||||
|
||||
# Release the M3U profile slot while channel_stream / metadata still exist.
|
||||
# Scanning live:channel keys first deletes metadata and breaks release_stream()
|
||||
# fallback, leaving profile_connections counters stuck (e.g. profile_connections:70 = 1).
|
||||
try:
|
||||
channel = Channel.objects.get(uuid=channel_id)
|
||||
if not channel.release_stream():
|
||||
logger.debug(f"Channel {channel_id}: release_stream found no keys to clean")
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
# Release the M3U profile slot while channel_stream / metadata still exist.
|
||||
# Scanning live:channel keys first deletes metadata and breaks release_stream()
|
||||
# fallback, leaving profile_connections counters stuck (e.g. profile_connections:70 = 1).
|
||||
try:
|
||||
stream = Stream.objects.get(stream_hash=channel_id)
|
||||
if not stream.release_stream():
|
||||
logger.debug(f"Stream {channel_id}: release_stream found no keys to clean")
|
||||
except (Stream.DoesNotExist, Exception):
|
||||
logger.debug(f"No Channel or Stream found for {channel_id}")
|
||||
channel = Channel.objects.get(uuid=channel_id)
|
||||
if not channel.release_stream():
|
||||
logger.debug(f"Channel {channel_id}: release_stream found no keys to clean")
|
||||
except (Channel.DoesNotExist, Exception):
|
||||
try:
|
||||
stream = Stream.objects.get(stream_hash=channel_id)
|
||||
if not stream.release_stream():
|
||||
logger.debug(f"Stream {channel_id}: release_stream found no keys to clean")
|
||||
except (Stream.DoesNotExist, Exception):
|
||||
logger.debug(f"No Channel or Stream found for {channel_id}")
|
||||
|
||||
if self.redis_client:
|
||||
try:
|
||||
patterns = [
|
||||
f"live:channel:{channel_id}:*",
|
||||
RedisKeys.events_channel(channel_id),
|
||||
]
|
||||
if self.redis_client:
|
||||
try:
|
||||
patterns = [
|
||||
f"live:channel:{channel_id}:*",
|
||||
RedisKeys.events_channel(channel_id),
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
cursor = 0
|
||||
while True:
|
||||
cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100)
|
||||
if keys:
|
||||
self.redis_client.delete(*keys)
|
||||
total_deleted += len(keys)
|
||||
for pattern in patterns:
|
||||
cursor = 0
|
||||
while True:
|
||||
cursor, keys = self.redis_client.scan(cursor, match=pattern, count=100)
|
||||
if keys:
|
||||
self.redis_client.delete(*keys)
|
||||
total_deleted += len(keys)
|
||||
|
||||
if cursor == 0:
|
||||
break
|
||||
if cursor == 0:
|
||||
break
|
||||
|
||||
logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}")
|
||||
logger.info(f"Cleaned up {total_deleted} Redis keys for channel {channel_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning Redis keys for channel {channel_id}: {e}")
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
||||
return total_deleted
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue