Merge branch 'main' into EPG-Region-Update

This commit is contained in:
Dispatcharr 2025-03-19 22:06:43 -05:00
commit ec4c3484c8
12 changed files with 1125 additions and 618 deletions

View file

@ -2,6 +2,8 @@ import logging
import time
import re
from . import proxy_server
from .redis_keys import RedisKeys
from .constants import TS_PACKET_SIZE
logger = logging.getLogger("ts_proxy")
@ -9,22 +11,15 @@ class ChannelStatus:
def get_detailed_channel_info(channel_id):
# Get channel metadata
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata = proxy_server.redis_client.hgetall(metadata_key)
if not metadata:
return None
# Get detailed info - existing implementation
# Get channel metadata
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if not metadata:
return None
# Basic channel info
buffer_index_value = proxy_server.redis_client.get(f"ts_proxy:channel:{channel_id}:buffer:index")
buffer_index_key = RedisKeys.buffer_index(channel_id)
buffer_index_value = proxy_server.redis_client.get(buffer_index_key)
info = {
'channel_id': channel_id,
@ -33,8 +28,6 @@ class ChannelStatus:
'profile': metadata.get(b'profile', b'unknown').decode('utf-8'),
'started_at': metadata.get(b'init_time', b'0').decode('utf-8'),
'owner': metadata.get(b'owner', b'unknown').decode('utf-8'),
# Properly decode the buffer index value
'buffer_index': int(buffer_index_value.decode('utf-8')) if buffer_index_value else 0,
}
@ -50,7 +43,7 @@ class ChannelStatus:
info['uptime'] = time.time() - created_at
# Get client information
client_set_key = f"ts_proxy:channel:{channel_id}:clients"
client_set_key = RedisKeys.clients(channel_id)
client_ids = proxy_server.redis_client.smembers(client_set_key)
clients = []
@ -97,7 +90,7 @@ class ChannelStatus:
# Check if the keys exist before getting
for i in range(info['buffer_index']-sample_chunks+1, info['buffer_index']+1):
chunk_key = f"ts_proxy:channel:{channel_id}:buffer:chunk:{i}"
chunk_key = RedisKeys.buffer_chunk(channel_id, i)
# Check if key exists first
if proxy_server.redis_client.exists(chunk_key):
@ -135,9 +128,9 @@ class ChannelStatus:
buffer_stats['total_sample_bytes'] = total_data
# Add TS packet analysis
total_ts_packets = total_data // 188
total_ts_packets = total_data // TS_PACKET_SIZE
buffer_stats['estimated_ts_packets'] = total_ts_packets
buffer_stats['is_ts_aligned'] = all(size % 188 == 0 for size in chunk_sizes)
buffer_stats['is_ts_aligned'] = all(size % TS_PACKET_SIZE == 0 for size in chunk_sizes)
else:
# If no chunks found, scan for keys to help debug
all_buffer_keys = []
@ -161,7 +154,7 @@ class ChannelStatus:
buffer_stats['diagnostics']['exception'] = str(e)
# Add TTL information to see if chunks are expiring
chunk_ttl_key = f"ts_proxy:channel:{channel_id}:buffer:chunk:{info['buffer_index']}"
chunk_ttl_key = RedisKeys.buffer_chunk(channel_id, info['buffer_index'])
chunk_ttl = proxy_server.redis_client.ttl(chunk_ttl_key)
buffer_stats['latest_chunk_ttl'] = chunk_ttl
@ -182,17 +175,18 @@ class ChannelStatus:
# Function for basic channel info (used for all channels summary)
def get_basic_channel_info(channel_id):
# Get channel metadata
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if not metadata:
return None
# Basic channel info only - omit diagnostics and details
buffer_index_value = proxy_server.redis_client.get(f"ts_proxy:channel:{channel_id}:buffer:index")
buffer_index_key = RedisKeys.buffer_index(channel_id)
buffer_index_value = proxy_server.redis_client.get(buffer_index_key)
# Count clients (using efficient count method)
client_set_key = f"ts_proxy:channel:{channel_id}:clients"
client_set_key = RedisKeys.clients(channel_id)
client_count = proxy_server.redis_client.scard(client_set_key) or 0
# Calculate uptime
@ -218,7 +212,6 @@ class ChannelStatus:
# Get concise client information
clients = []
client_set_key = f"ts_proxy:channel:{channel_id}:clients"
client_ids = proxy_server.redis_client.smembers(client_set_key)
# Process only if we have clients and keep it limited

View file

@ -6,6 +6,9 @@ import time
import json
from typing import Set, Optional
from apps.proxy.config import TSConfig as Config
from .constants import EventType
from .config_helper import ConfigHelper
from .redis_keys import RedisKeys
logger = logging.getLogger("ts_proxy")
@ -21,9 +24,9 @@ class ClientManager:
self.worker_id = worker_id # Store worker ID as instance variable
# STANDARDIZED KEYS: Move client set under channel namespace
self.client_set_key = f"ts_proxy:channel:{channel_id}:clients"
self.client_ttl = getattr(Config, 'CLIENT_RECORD_TTL', 60)
self.heartbeat_interval = getattr(Config, 'CLIENT_HEARTBEAT_INTERVAL', 10)
self.client_set_key = RedisKeys.clients(channel_id)
self.client_ttl = ConfigHelper.get('CLIENT_RECORD_TTL', 60)
self.heartbeat_interval = ConfigHelper.get('CLIENT_HEARTBEAT_INTERVAL', 10)
self.last_heartbeat_time = {}
# Start heartbeat thread for local clients
@ -143,7 +146,7 @@ class ClientManager:
self._registered_clients.add(client_id)
# FIX: Consistent key naming - note the 's' in 'clients'
# Use a function to get the client key
client_key = f"ts_proxy:channel:{self.channel_id}:clients:{client_id}"
# Prepare client data
@ -172,13 +175,14 @@ class ClientManager:
self.redis_client.expire(self.client_set_key, self.client_ttl)
# Clear any initialization timer
self.redis_client.delete(f"ts_proxy:channel:{self.channel_id}:init_time")
init_key = f"ts_proxy:channel:{self.channel_id}:init_time"
self.redis_client.delete(init_key)
self._notify_owner_of_activity()
# Publish client connected event with user agent
event_data = {
"event": "client_connected",
"event": EventType.CLIENT_CONNECTED, # Use constant instead of string
"channel_id": self.channel_id,
"client_id": client_id,
"worker_id": self.worker_id or "unknown",
@ -192,7 +196,7 @@ class ClientManager:
logger.debug(f"No user agent provided for client {client_id}")
self.redis_client.publish(
f"ts_proxy:events:{self.channel_id}",
RedisKeys.events_channel(self.channel_id), # Use RedisKeys instead of string
json.dumps(event_data)
)
@ -233,21 +237,21 @@ class ClientManager:
logger.warning(f"Last client removed: {client_id} - channel may shut down soon")
# Trigger disconnect time tracking even if we're not the owner
disconnect_key = f"ts_proxy:channel:{self.channel_id}:last_client_disconnect_time"
disconnect_key = RedisKeys.last_client_disconnect(self.channel_id)
self.redis_client.setex(disconnect_key, 60, str(time.time()))
self._notify_owner_of_activity()
# Publish client disconnected event
event_data = json.dumps({
"event": "client_disconnected",
"event": EventType.CLIENT_DISCONNECTED, # Use constant instead of string
"channel_id": self.channel_id,
"client_id": client_id,
"worker_id": self.worker_id or "unknown",
"timestamp": time.time(),
"remaining_clients": remaining
})
self.redis_client.publish(f"ts_proxy:events:{self.channel_id}", event_data)
self.redis_client.publish(RedisKeys.events_channel(self.channel_id), event_data)
total_clients = self.get_total_client_count()
logger.info(f"Client disconnected: {client_id} (local: {len(self.clients)}, total: {total_clients})")

View file

@ -0,0 +1,70 @@
"""
Helper module to access configuration values with proper defaults.
"""
import logging
from apps.proxy.config import TSConfig as Config
logger = logging.getLogger("ts_proxy")
class ConfigHelper:
"""
Helper class for accessing configuration values with sensible defaults.
This simplifies code and ensures consistent defaults across the application.
"""
@staticmethod
def get(name, default=None):
"""Get a configuration value with a default fallback"""
return getattr(Config, name, default)
# Commonly used configuration values
@staticmethod
def connection_timeout():
"""Get connection timeout in seconds"""
return ConfigHelper.get('CONNECTION_TIMEOUT', 10)
@staticmethod
def client_wait_timeout():
"""Get client wait timeout in seconds"""
return ConfigHelper.get('CLIENT_WAIT_TIMEOUT', 30)
@staticmethod
def stream_timeout():
"""Get stream timeout in seconds"""
return ConfigHelper.get('STREAM_TIMEOUT', 60)
@staticmethod
def channel_shutdown_delay():
"""Get channel shutdown delay in seconds"""
return ConfigHelper.get('CHANNEL_SHUTDOWN_DELAY', 5)
@staticmethod
def initial_behind_chunks():
"""Get number of chunks to start behind"""
return ConfigHelper.get('INITIAL_BEHIND_CHUNKS', 10)
@staticmethod
def keepalive_interval():
"""Get keepalive interval in seconds"""
return ConfigHelper.get('KEEPALIVE_INTERVAL', 0.5)
@staticmethod
def cleanup_check_interval():
"""Get cleanup check interval in seconds"""
return ConfigHelper.get('CLEANUP_CHECK_INTERVAL', 3)
@staticmethod
def redis_chunk_ttl():
"""Get Redis chunk TTL in seconds"""
return ConfigHelper.get('REDIS_CHUNK_TTL', 60)
@staticmethod
def chunk_size():
"""Get chunk size in bytes"""
return ConfigHelper.get('CHUNK_SIZE', 8192)
@staticmethod
def max_retries():
"""Get maximum retry attempts"""
return ConfigHelper.get('MAX_RETRIES', 3)

View file

@ -0,0 +1,42 @@
"""
Constants used throughout the TS Proxy application.
Centralizing constants makes it easier to maintain and modify them.
"""
# Redis related constants
REDIS_KEY_PREFIX = "ts_proxy"
REDIS_TTL_DEFAULT = 3600 # 1 hour
REDIS_TTL_SHORT = 60 # 1 minute
REDIS_TTL_MEDIUM = 300 # 5 minutes
# Channel states
class ChannelState:
INITIALIZING = "initializing"
CONNECTING = "connecting"
WAITING_FOR_CLIENTS = "waiting_for_clients"
ACTIVE = "active"
ERROR = "error"
STOPPING = "stopping"
STOPPED = "stopped"
# Event types
class EventType:
STREAM_SWITCH = "stream_switch"
STREAM_SWITCHED = "stream_switched"
CHANNEL_STOP = "channel_stop"
CHANNEL_STOPPED = "channel_stopped"
CLIENT_CONNECTED = "client_connected"
CLIENT_DISCONNECTED = "client_disconnected"
CLIENT_STOP = "client_stop"
# Stream types
class StreamType:
HLS = "hls"
TS = "ts"
UNKNOWN = "unknown"
# TS packet constants
TS_PACKET_SIZE = 188
TS_SYNC_BYTE = 0x47
NULL_PID_HIGH = 0x1F
NULL_PID_LOW = 0xFF

View file

@ -0,0 +1,75 @@
"""
Defines Redis key patterns used throughout the TS proxy service.
Centralizing these key patterns makes it easier to maintain and change them if needed.
"""
class RedisKeys:
@staticmethod
def channel_metadata(channel_id):
"""Key for channel metadata hash"""
return f"ts_proxy:channel:{channel_id}:metadata"
@staticmethod
def buffer_index(channel_id):
"""Key for tracking buffer index"""
return f"ts_proxy:channel:{channel_id}:buffer:index"
@staticmethod
def buffer_chunk(channel_id, chunk_index):
"""Key for specific buffer chunk"""
return f"ts_proxy:channel:{channel_id}:buffer:chunk:{chunk_index}"
@staticmethod
def buffer_chunk_prefix(channel_id):
"""Prefix for buffer chunks"""
return f"ts_proxy:channel:{channel_id}:buffer:chunk:"
@staticmethod
def channel_stopping(channel_id):
"""Key indicating channel is stopping"""
return f"ts_proxy:channel:{channel_id}:stopping"
@staticmethod
def client_stop(channel_id, client_id):
"""Key requesting client stop"""
return f"ts_proxy:channel:{channel_id}:client:{client_id}:stop"
@staticmethod
def events_channel(channel_id):
"""PubSub channel for events"""
return f"ts_proxy:events:{channel_id}"
@staticmethod
def switch_request(channel_id):
"""Key for stream switch request"""
return f"ts_proxy:channel:{channel_id}:switch_request"
@staticmethod
def channel_owner(channel_id):
"""Key for storing channel owner worker ID"""
return f"ts_proxy:channel:{channel_id}:owner"
@staticmethod
def clients(channel_id):
"""Key for set of client IDs"""
return f"ts_proxy:channel:{channel_id}:clients"
@staticmethod
def last_client_disconnect(channel_id):
"""Key for last client disconnect timestamp"""
return f"ts_proxy:channel:{channel_id}:last_client_disconnect_time"
@staticmethod
def connection_attempt(channel_id):
"""Key for connection attempt timestamp"""
return f"ts_proxy:channel:{channel_id}:connection_attempt_time"
@staticmethod
def last_data(channel_id):
"""Key for last data timestamp"""
return f"ts_proxy:channel:{channel_id}:last_data"
@staticmethod
def switch_status(channel_id):
"""Key for stream switch status"""
return f"ts_proxy:channel:{channel_id}:switch_status"

View file

@ -21,6 +21,9 @@ from apps.channels.models import Channel
from .stream_manager import StreamManager
from .stream_buffer import StreamBuffer
from .client_manager import ClientManager
from .redis_keys import RedisKeys
from .constants import ChannelState, EventType, StreamType
from .config_helper import ConfigHelper
logger = logging.getLogger("ts_proxy")
@ -87,25 +90,24 @@ class ProxyServer:
if channel_id and event_type:
# For owner, update client status immediately
if self.am_i_owner(channel_id):
if event_type == "client_connected":
logger.debug(f"Owner received client_connected event for channel {channel_id}")
if event_type == EventType.CLIENT_CONNECTED:
logger.debug(f"Owner received {EventType.CLIENT_CONNECTED} event for channel {channel_id}")
# Reset any disconnect timer
# RENAMED: no_clients_since → last_client_disconnect_time
disconnect_key = f"ts_proxy:channel:{channel_id}:last_client_disconnect_time"
disconnect_key = RedisKeys.last_client_disconnect(channel_id)
self.redis_client.delete(disconnect_key)
elif event_type == "client_disconnected":
logger.debug(f"Owner received client_disconnected event for channel {channel_id}")
elif event_type == EventType.CLIENT_DISCONNECTED:
logger.debug(f"Owner received {EventType.CLIENT_DISCONNECTED} event for channel {channel_id}")
# Check if any clients remain
if channel_id in self.client_managers:
# VERIFY REDIS CLIENT COUNT DIRECTLY
client_set_key = f"ts_proxy:channel:{channel_id}:clients"
client_set_key = RedisKeys.clients(channel_id)
total = self.redis_client.scard(client_set_key) or 0
if total == 0:
logger.debug(f"No clients left after disconnect event - stopping channel {channel_id}")
# Set the disconnect timer for other workers to see
disconnect_key = f"ts_proxy:channel:{channel_id}:last_client_disconnect_time"
disconnect_key = RedisKeys.last_client_disconnect(channel_id)
self.redis_client.setex(disconnect_key, 60, str(time.time()))
# Get configured shutdown delay or default
@ -126,8 +128,8 @@ class ProxyServer:
self.stop_channel(channel_id)
elif event_type == "stream_switch":
logger.info(f"Owner received stream switch request for channel {channel_id}")
elif event_type == EventType.STREAM_SWITCH:
logger.info(f"Owner received {EventType.STREAM_SWITCH} request for channel {channel_id}")
# Handle stream switch request
new_url = data.get("url")
user_agent = data.get("user_agent")
@ -135,13 +137,13 @@ class ProxyServer:
if new_url and channel_id in self.stream_managers:
# Update metadata in Redis
if self.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
self.redis_client.hset(metadata_key, "url", new_url)
if user_agent:
self.redis_client.hset(metadata_key, "user_agent", user_agent)
# Set switch status
status_key = f"ts_proxy:channel:{channel_id}:switch_status"
status_key = RedisKeys.switch_status(channel_id)
self.redis_client.set(status_key, "switching")
# Perform the stream switch
@ -153,7 +155,7 @@ class ProxyServer:
# Publish confirmation
switch_result = {
"event": "stream_switched",
"event": EventType.STREAM_SWITCHED, # Use constant instead of string
"channel_id": channel_id,
"success": True,
"url": new_url,
@ -172,7 +174,7 @@ class ProxyServer:
# Publish failure
switch_result = {
"event": "stream_switched",
"event": EventType.STREAM_SWITCHED,
"channel_id": channel_id,
"success": False,
"url": new_url,
@ -182,15 +184,15 @@ class ProxyServer:
f"ts_proxy:events:{channel_id}",
json.dumps(switch_result)
)
elif event_type == "channel_stop":
logger.info(f"Received channel stop event for channel {channel_id}")
elif event_type == EventType.CHANNEL_STOP:
logger.info(f"Received {EventType.CHANNEL_STOP} event for channel {channel_id}")
# First mark channel as stopping in Redis
if self.redis_client:
# Set stopping state in metadata
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
if self.redis_client.exists(metadata_key):
self.redis_client.hset(metadata_key, mapping={
"state": "stopping",
"state": ChannelState.STOPPING,
"state_changed_at": str(time.time())
})
@ -202,7 +204,7 @@ class ProxyServer:
# Acknowledge stop by publishing a response
stop_response = {
"event": "channel_stopped",
"event": EventType.CHANNEL_STOPPED,
"channel_id": channel_id,
"worker_id": self.worker_id,
"timestamp": time.time()
@ -211,7 +213,7 @@ class ProxyServer:
f"ts_proxy:events:{channel_id}",
json.dumps(stop_response)
)
elif event_type == "client_stop":
elif event_type == EventType.CLIENT_STOP:
client_id = data.get("client_id")
if client_id and channel_id:
logger.info(f"Received request to stop client {client_id} on channel {channel_id}")
@ -225,7 +227,7 @@ class ProxyServer:
# Set a Redis key for the generator to detect
if self.redis_client:
stop_key = f"ts_proxy:channel:{channel_id}:client:{client_id}:stop"
stop_key = RedisKeys.client_stop(channel_id, client_id)
self.redis_client.setex(stop_key, 30, "true") # 30 second TTL
logger.info(f"Set stop key for client {client_id}")
except Exception as e:
@ -246,7 +248,7 @@ class ProxyServer:
return None
try:
lock_key = f"ts_proxy:channel:{channel_id}:owner"
lock_key = RedisKeys.channel_owner(channel_id)
owner = self.redis_client.get(lock_key)
if owner:
return owner.decode('utf-8')
@ -267,7 +269,7 @@ class ProxyServer:
try:
# Create a lock key with proper namespace
lock_key = f"ts_proxy:channel:{channel_id}:owner"
lock_key = RedisKeys.channel_owner(channel_id)
# Use Redis SETNX for atomic locking - only succeeds if the key doesn't exist
acquired = self.redis_client.setnx(lock_key, self.worker_id)
@ -299,7 +301,7 @@ class ProxyServer:
return
try:
lock_key = f"ts_proxy:channel:{channel_id}:owner"
lock_key = RedisKeys.channel_owner(channel_id)
# Only delete if we're the current owner to prevent race conditions
current = self.redis_client.get(lock_key)
@ -315,7 +317,7 @@ class ProxyServer:
return False
try:
lock_key = f"ts_proxy:channel:{channel_id}:owner"
lock_key = RedisKeys.channel_owner(channel_id)
current = self.redis_client.get(lock_key)
# Only extend if we're still the owner
@ -348,7 +350,7 @@ class ProxyServer:
# First check if channel metadata already exists
existing_metadata = None
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
if self.redis_client:
existing_metadata = self.redis_client.hgetall(metadata_key)
@ -412,7 +414,7 @@ class ProxyServer:
"init_time": str(time.time()),
"last_active": str(time.time()),
"owner": self.worker_id,
"state": "initializing" # Only the owner sets this initial state
"state": ChannelState.INITIALIZING # Use constant instead of string literal
}
if channel_user_agent:
metadata["user_agent"] = channel_user_agent
@ -447,16 +449,16 @@ class ProxyServer:
# If we're the owner, we need to set the channel state rather than starting a grace period immediately
if self.am_i_owner(channel_id):
self.update_channel_state(channel_id, "connecting", {
self.update_channel_state(channel_id, ChannelState.CONNECTING, {
"init_time": str(time.time()),
"owner": self.worker_id
})
# Set connection attempt start time
attempt_key = f"ts_proxy:channel:{channel_id}:connection_attempt_time"
attempt_key = RedisKeys.connection_attempt(channel_id)
self.redis_client.setex(attempt_key, 60, str(time.time()))
logger.info(f"Channel {channel_id} in connecting state - will start grace period after connection")
logger.info(f"Channel {channel_id} in {ChannelState.CONNECTING} state - will start grace period after connection")
return True
except Exception as e:
@ -474,7 +476,7 @@ class ProxyServer:
# Check Redis using the standard key pattern
if self.redis_client:
# Primary check - look for channel metadata
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
# If metadata exists, return true
if self.redis_client.exists(metadata_key):
@ -482,9 +484,9 @@ class ProxyServer:
# Additional checks if metadata doesn't exist
additional_keys = [
f"ts_proxy:channel:{channel_id}:clients",
f"ts_proxy:channel:{channel_id}:buffer:index",
f"ts_proxy:channel:{channel_id}:owner"
RedisKeys.clients(channel_id),
RedisKeys.buffer_index(channel_id),
RedisKeys.channel_owner(channel_id)
]
for key in additional_keys:
@ -497,12 +499,12 @@ class ProxyServer:
"""Stop a channel with proper ownership handling"""
try:
logger.info(f"Stopping channel {channel_id}")
# First set a stopping key that clients will check
if self.redis_client:
stop_key = f"ts_proxy:channel:{channel_id}:stopping"
stop_key = RedisKeys.channel_stopping(channel_id)
self.redis_client.setex(stop_key, 10, "true")
# Only stop the actual stream manager if we're the owner
if self.am_i_owner(channel_id):
logger.info(f"This worker ({self.worker_id}) is the owner - closing provider connection")
@ -544,7 +546,7 @@ class ProxyServer:
if channel_id in self.stream_managers:
del self.stream_managers[channel_id]
logger.info(f"Removed stream manager for channel {channel_id}")
# Stop buffer and ensure all its timers are cancelled - SAFE CHECK HERE
if channel_id in self.stream_buffers:
buffer = self.stream_buffers[channel_id]
@ -555,7 +557,7 @@ class ProxyServer:
logger.debug(f"Buffer for channel {channel_id} properly stopped")
except Exception as e:
logger.error(f"Error stopping buffer: {e}")
# Save reference and check again before deleting
try:
if channel_id in self.stream_buffers: # Check again to prevent race conditions
@ -563,7 +565,7 @@ class ProxyServer:
logger.info(f"Removed stream buffer for channel {channel_id}")
except KeyError:
logger.debug(f"Buffer for channel {channel_id} already removed")
# Clean up client manager - SAFE CHECK HERE TOO
if channel_id in self.client_managers:
try:
@ -571,10 +573,10 @@ class ProxyServer:
logger.info(f"Removed client manager for channel {channel_id}")
except KeyError:
logger.debug(f"Client manager for channel {channel_id} already removed")
# Clean up Redis keys
self._clean_redis_keys(channel_id)
return True
except Exception as e:
logger.error(f"Error stopping channel {channel_id}: {e}", exc_info=True)
@ -594,8 +596,8 @@ class ProxyServer:
def _cleanup_channel(self, channel_id: str) -> None:
"""Remove channel resources"""
for collection in [self.stream_managers, self.stream_buffers,
self.client_managers, self.fetch_threads]:
# Removed reference to non-existent fetch_threads collection
for collection in [self.stream_managers, self.stream_buffers, self.client_managers]:
collection.pop(channel_id, None)
def shutdown(self) -> None:
@ -624,7 +626,7 @@ class ProxyServer:
# Get channel state from metadata hash
channel_state = "unknown"
if self.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
metadata = self.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
channel_state = metadata[b'state'].decode('utf-8')
@ -640,7 +642,7 @@ class ProxyServer:
logger.info(f"Channel {channel_id} has {total_clients} clients, state: {channel_state}")
# If in connecting or waiting_for_clients state, check grace period
if channel_state in ["connecting", "waiting_for_clients"]:
if channel_state in [ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS]:
# Get connection ready time from metadata
connection_ready_time = None
if metadata and b'connection_ready_time' in metadata:
@ -650,13 +652,13 @@ class ProxyServer:
pass
# If still connecting, give it more time
if channel_state == "connecting":
if channel_state == ChannelState.CONNECTING:
logger.debug(f"Channel {channel_id} still connecting - not checking for clients yet")
continue
# If waiting for clients, check grace period
if connection_ready_time:
grace_period = getattr(Config, 'CHANNEL_INIT_GRACE_PERIOD', 20)
grace_period = ConfigHelper.get('CHANNEL_INIT_GRACE_PERIOD', 20)
time_since_ready = time.time() - connection_ready_time
# Add this debug log
@ -678,15 +680,15 @@ class ProxyServer:
old_state = "unknown"
if metadata and b'state' in metadata:
old_state = metadata[b'state'].decode('utf-8')
if self.update_channel_state(channel_id, "active", {
if self.update_channel_state(channel_id, ChannelState.ACTIVE, {
"grace_period_ended_at": str(time.time()),
"clients_at_activation": str(total_clients)
}):
logger.info(f"Channel {channel_id} activated with {total_clients} clients after grace period")
# If active and no clients, start normal shutdown procedure
elif channel_state not in ["connecting", "waiting_for_clients"] and total_clients == 0:
elif channel_state not in [ChannelState.CONNECTING, ChannelState.WAITING_FOR_CLIENTS] and total_clients == 0:
# Check if there's a pending no-clients timeout
disconnect_key = f"ts_proxy:channel:{channel_id}:last_client_disconnect_time"
disconnect_key = RedisKeys.last_client_disconnect(channel_id)
disconnect_time = None
if self.redis_client:
@ -704,7 +706,7 @@ class ProxyServer:
if self.redis_client:
self.redis_client.setex(disconnect_key, 60, str(current_time))
logger.warning(f"No clients detected for channel {channel_id}, starting shutdown timer")
elif current_time - disconnect_time > getattr(Config, 'CHANNEL_SHUTDOWN_DELAY', 5):
elif current_time - disconnect_time > ConfigHelper.channel_shutdown_delay():
# We've had no clients for the shutdown delay period
logger.warning(f"No clients for {current_time - disconnect_time:.1f}s, stopping channel {channel_id}")
self.stop_channel(channel_id)
@ -712,7 +714,7 @@ class ProxyServer:
# Still in shutdown delay period
logger.debug(f"Channel {channel_id} shutdown timer: "
f"{current_time - disconnect_time:.1f}s of "
f"{getattr(Config, 'CHANNEL_SHUTDOWN_DELAY', 5)}s elapsed")
f"{ConfigHelper.channel_shutdown_delay()}s elapsed")
else:
# There are clients or we're still connecting - clear any disconnect timestamp
if self.redis_client:
@ -723,21 +725,21 @@ class ProxyServer:
# For channels we don't own, check if they've been stopped/cleaned up in Redis
if self.redis_client:
# Method 1: Check for stopping key
stop_key = f"ts_proxy:channel:{channel_id}:stopping"
stop_key = RedisKeys.channel_stopping(channel_id)
if self.redis_client.exists(stop_key):
logger.debug(f"Non-owner cleanup: Channel {channel_id} has stopping flag in Redis, cleaning up local resources")
self._cleanup_local_resources(channel_id)
continue
# Method 2: Check if owner still exists
owner_key = f"ts_proxy:channel:{channel_id}:owner"
owner_key = RedisKeys.channel_owner(channel_id)
if not self.redis_client.exists(owner_key):
logger.debug(f"Non-owner cleanup: Channel {channel_id} has no owner in Redis, cleaning up local resources")
self._cleanup_local_resources(channel_id)
continue
# Method 3: Check if metadata still exists
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
if not self.redis_client.exists(metadata_key):
logger.debug(f"Non-owner cleanup: Channel {channel_id} has no metadata in Redis, cleaning up local resources")
self._cleanup_local_resources(channel_id)
@ -752,12 +754,12 @@ class ProxyServer:
except Exception as e:
logger.error(f"Error in cleanup thread: {e}", exc_info=True)
time.sleep(getattr(Config, 'CLEANUP_CHECK_INTERVAL', 1))
time.sleep(ConfigHelper.cleanup_check_interval())
thread = threading.Thread(target=cleanup_task, daemon=True)
thread.name = "ts-proxy-cleanup"
thread.start()
logger.info(f"Started TS proxy cleanup thread (interval: {getattr(Config, 'CLEANUP_CHECK_INTERVAL', 3)}s)")
logger.info(f"Started TS proxy cleanup thread (interval: {ConfigHelper.cleanup_check_interval()}s)")
def _check_orphaned_channels(self):
"""Check for orphaned channels in Redis (owner worker crashed)"""
@ -782,7 +784,7 @@ class ProxyServer:
if not owner:
# Check if there are any clients
client_set_key = f"ts_proxy:channel:{channel_id}:clients"
client_set_key = RedisKeys.clients(channel_id)
client_count = self.redis_client.scard(client_set_key) or 0
if client_count > 0:
@ -811,7 +813,7 @@ class ProxyServer:
# Define key patterns to scan for
patterns = [
f"ts_proxy:channel:{channel_id}:*", # All channel keys
f"ts_proxy:events:{channel_id}" # Event channel
RedisKeys.events_channel(channel_id) # Event channel
]
total_deleted = 0
@ -843,7 +845,7 @@ class ProxyServer:
# Refresh registry entries for channels we own
for channel_id in list(self.stream_buffers.keys()):
# Use standard key pattern
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
# Update activity timestamp in metadata only
self.redis_client.hset(metadata_key, "last_active", str(time.time()))
@ -856,7 +858,7 @@ class ProxyServer:
return False
try:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
# Get current state for logging
current_state = None

View file

@ -0,0 +1,374 @@
"""
Channel service layer for handling business logic related to channel operations.
This separates business logic from HTTP handling in views.
"""
import logging
import time
import json
from django.shortcuts import get_object_or_404
from apps.channels.models import Channel
from apps.proxy.config import TSConfig as Config
from .. import proxy_server
from ..redis_keys import RedisKeys
from ..constants import EventType
logger = logging.getLogger("ts_proxy")
class ChannelService:
"""Service class for channel operations"""
@staticmethod
def initialize_channel(channel_id, stream_url, user_agent, transcode=False, profile_value=None):
"""
Initialize a channel with the given parameters.
Args:
channel_id: UUID of the channel
stream_url: URL of the stream
user_agent: User agent for the stream connection
transcode: Whether to transcode the stream
profile_value: Stream profile value to store in metadata
Returns:
bool: Success status
"""
success = proxy_server.initialize_channel(stream_url, channel_id, user_agent, transcode)
# Store additional metadata if initialization was successful
if success and proxy_server.redis_client and profile_value:
metadata_key = RedisKeys.channel_metadata(channel_id)
proxy_server.redis_client.hset(metadata_key, "profile", profile_value)
return success
@staticmethod
def change_stream_url(channel_id, new_url, user_agent=None):
"""
Change the URL of an existing stream.
Args:
channel_id: UUID of the channel
new_url: New stream URL
user_agent: Optional user agent to update
Returns:
dict: Result information including success status and diagnostics
"""
# Check if channel exists
in_local_managers = channel_id in proxy_server.stream_managers
in_local_buffers = channel_id in proxy_server.stream_buffers
# Check Redis for keys
redis_keys = None
if proxy_server.redis_client:
try:
# This is inefficient but used for diagnostics - in production would use more targeted checks
redis_keys = proxy_server.redis_client.keys(f"ts_proxy:*:{channel_id}*")
redis_keys = [k.decode('utf-8') for k in redis_keys] if redis_keys else []
except Exception as e:
logger.error(f"Error checking Redis keys: {e}")
# Check if channel exists using standard method
channel_exists = proxy_server.check_if_channel_exists(channel_id)
# Log detailed diagnostics
logger.info(f"Channel {channel_id} diagnostics: "
f"in_local_managers={in_local_managers}, "
f"in_local_buffers={in_local_buffers}, "
f"redis_keys_count={len(redis_keys) if redis_keys else 0}, "
f"channel_exists={channel_exists}")
if not channel_exists:
# Try to recover if Redis keys exist but channel check failed
if redis_keys:
logger.warning(f"Channel {channel_id} not detected but Redis keys exist. Forcing initialization.")
proxy_server.initialize_channel(new_url, channel_id, user_agent)
result = {
'status': 'recovered',
'message': 'Channel was recovered and initialized'
}
else:
logger.error(f"Channel {channel_id} not found in any worker or Redis")
return {
'status': 'error',
'message': 'Channel not found',
'diagnostics': {
'in_local_managers': in_local_managers,
'in_local_buffers': in_local_buffers,
'redis_keys': redis_keys,
}
}
else:
result = {'status': 'success'}
# Update metadata in Redis regardless of ownership
if proxy_server.redis_client:
try:
ChannelService._update_channel_metadata(channel_id, new_url, user_agent)
result['metadata_updated'] = True
except Exception as e:
logger.error(f"Error updating Redis metadata: {e}", exc_info=True)
result['metadata_updated'] = False
# If we're the owner, update directly
if proxy_server.am_i_owner(channel_id) and channel_id in proxy_server.stream_managers:
logger.info(f"This worker is the owner, changing stream URL for channel {channel_id}")
manager = proxy_server.stream_managers[channel_id]
old_url = manager.url
# Update the stream
success = manager.update_url(new_url)
logger.info(f"Stream URL changed from {old_url} to {new_url}, result: {success}")
result.update({
'direct_update': True,
'success': success,
'worker_id': proxy_server.worker_id
})
else:
# If we're not the owner, publish an event for the owner to pick up
logger.info(f"Not the owner, requesting URL change via Redis PubSub")
if proxy_server.redis_client:
ChannelService._publish_stream_switch_event(channel_id, new_url, user_agent)
result.update({
'direct_update': False,
'event_published': True,
'worker_id': proxy_server.worker_id
})
else:
result.update({
'direct_update': False,
'event_published': False,
'error': 'Redis not available for pubsub'
})
return result
@staticmethod
def stop_channel(channel_id):
"""
Stop a channel and release all resources.
Args:
channel_id: UUID of the channel
Returns:
dict: Result information including previous state if available
"""
# Check if channel exists
channel_exists = proxy_server.check_if_channel_exists(channel_id)
if not channel_exists:
logger.warning(f"Channel {channel_id} not found in any worker or Redis")
return {'status': 'error', 'message': 'Channel not found'}
# Get channel state information for result
channel_info = None
if proxy_server.redis_client:
metadata_key = RedisKeys.channel_metadata(channel_id)
try:
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
channel_info = {"state": state}
except Exception as e:
logger.error(f"Error fetching channel state: {e}")
# Broadcast stop event to all workers via PubSub
if proxy_server.redis_client:
ChannelService._publish_channel_stop_event(channel_id)
# Also stop locally to ensure this worker cleans up right away
local_result = proxy_server.stop_channel(channel_id)
else:
# No Redis, just stop locally
local_result = proxy_server.stop_channel(channel_id)
# Release the channel in the channel model if applicable
try:
channel = Channel.objects.get(uuid=channel_id)
channel.release_stream()
logger.info(f"Released channel {channel_id} stream allocation")
model_released = True
except Channel.DoesNotExist:
logger.warning(f"Could not find Channel model for UUID {channel_id}")
model_released = False
except Exception as e:
logger.error(f"Error releasing channel stream: {e}")
model_released = False
return {
'status': 'success',
'message': 'Channel stop request sent',
'channel_id': channel_id,
'previous_state': channel_info,
'model_released': model_released,
'local_stop_result': local_result
}
@staticmethod
def stop_client(channel_id, client_id):
"""
Stop a specific client connection.
Args:
channel_id: UUID of the channel
client_id: ID of the client to stop
Returns:
dict: Result information
"""
logger.info(f"Request to stop client {client_id} on channel {channel_id}")
# Set a Redis key for immediate detection
key_set = False
if proxy_server.redis_client:
stop_key = RedisKeys.client_stop(channel_id, client_id)
try:
proxy_server.redis_client.setex(stop_key, 30, "true") # 30 second TTL
logger.info(f"Set stop key for client {client_id}")
key_set = True
except Exception as e:
logger.error(f"Error setting client stop key: {e}")
# Check if channel exists
channel_exists = proxy_server.check_if_channel_exists(channel_id)
if not channel_exists:
logger.warning(f"Channel {channel_id} not found")
return {
'status': 'error',
'message': 'Channel not found',
'stop_key_set': key_set
}
# Try to stop locally if client is on this worker
local_client_stopped = False
if channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[channel_id]
with client_manager.lock:
if client_id in client_manager.clients:
client_manager.remove_client(client_id)
local_client_stopped = True
logger.info(f"Client {client_id} stopped locally on channel {channel_id}")
# If client wasn't found locally, broadcast stop event for other workers
event_published = False
if not local_client_stopped and proxy_server.redis_client:
try:
ChannelService._publish_client_stop_event(channel_id, client_id)
event_published = True
logger.info(f"Published stop request for client {client_id} on channel {channel_id}")
except Exception as e:
logger.error(f"Error publishing client stop event: {e}")
return {
'status': 'success',
'message': 'Client stop request processed',
'channel_id': channel_id,
'client_id': client_id,
'locally_processed': local_client_stopped,
'stop_key_set': key_set,
'event_published': event_published
}
# Helper methods for Redis operations
@staticmethod
def _update_channel_metadata(channel_id, url, user_agent=None):
"""Update channel metadata in Redis"""
if not proxy_server.redis_client:
return False
metadata_key = RedisKeys.channel_metadata(channel_id)
# First check if the key exists and what type it is
key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8')
logger.debug(f"Redis key {metadata_key} is of type: {key_type}")
# Use the appropriate method based on the key type
if key_type == 'hash':
proxy_server.redis_client.hset(metadata_key, "url", url)
if user_agent:
proxy_server.redis_client.hset(metadata_key, "user_agent", user_agent)
elif key_type == 'none': # Key doesn't exist yet
# Create new hash with all required fields
metadata = {"url": url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
else:
# If key exists with wrong type, delete it and recreate
proxy_server.redis_client.delete(metadata_key)
metadata = {"url": url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
# Set switch request flag to ensure all workers see it
switch_key = RedisKeys.switch_request(channel_id)
proxy_server.redis_client.setex(switch_key, 30, url) # 30 second TTL
logger.info(f"Updated metadata for channel {channel_id} in Redis")
return True
@staticmethod
def _publish_stream_switch_event(channel_id, new_url, user_agent=None):
"""Publish a stream switch event to Redis pubsub"""
if not proxy_server.redis_client:
return False
switch_request = {
"event": EventType.STREAM_SWITCH, # Use constant instead of string
"channel_id": channel_id,
"url": new_url,
"user_agent": user_agent,
"requester": proxy_server.worker_id,
"timestamp": time.time()
}
proxy_server.redis_client.publish(
RedisKeys.events_channel(channel_id),
json.dumps(switch_request)
)
return True
@staticmethod
def _publish_channel_stop_event(channel_id):
"""Publish a channel stop event to Redis pubsub"""
if not proxy_server.redis_client:
return False
stop_request = {
"event": EventType.CHANNEL_STOP, # Use constant instead of string
"channel_id": channel_id,
"requester_worker_id": proxy_server.worker_id,
"timestamp": time.time()
}
proxy_server.redis_client.publish(
RedisKeys.events_channel(channel_id),
json.dumps(stop_request)
)
logger.info(f"Published channel stop event for {channel_id}")
return True
@staticmethod
def _publish_client_stop_event(channel_id, client_id):
"""Publish a client stop event to Redis pubsub"""
if not proxy_server.redis_client:
return False
stop_request = {
"event": EventType.CLIENT_STOP, # Use constant instead of string
"channel_id": channel_id,
"client_id": client_id,
"requester_worker_id": proxy_server.worker_id,
"timestamp": time.time()
}
proxy_server.redis_client.publish(
RedisKeys.events_channel(channel_id),
json.dumps(stop_request)
)
return True

View file

@ -7,6 +7,9 @@ from collections import deque
from typing import Optional, Deque
import random
from apps.proxy.config import TSConfig as Config
from .redis_keys import RedisKeys
from .config_helper import ConfigHelper
from .constants import TS_PACKET_SIZE
logger = logging.getLogger("ts_proxy")
@ -18,13 +21,13 @@ class StreamBuffer:
self.redis_client = redis_client
self.lock = threading.Lock()
self.index = 0
self.TS_PACKET_SIZE = 188
self.TS_PACKET_SIZE = TS_PACKET_SIZE
# STANDARDIZED KEYS: Move buffer keys under channel namespace
self.buffer_index_key = f"ts_proxy:channel:{channel_id}:buffer:index"
self.buffer_prefix = f"ts_proxy:channel:{channel_id}:buffer:chunk:"
# STANDARDIZED KEYS: Use RedisKeys class instead of hardcoded patterns
self.buffer_index_key = RedisKeys.buffer_index(channel_id) if channel_id else ""
self.buffer_prefix = RedisKeys.buffer_chunk_prefix(channel_id) if channel_id else ""
self.chunk_ttl = getattr(Config, 'REDIS_CHUNK_TTL', 60)
self.chunk_ttl = ConfigHelper.redis_chunk_ttl()
# Initialize from Redis if available
if self.redis_client and channel_id:
@ -37,7 +40,7 @@ class StreamBuffer:
logger.error(f"Error initializing buffer from Redis: {e}")
self._write_buffer = bytearray()
self.target_chunk_size = getattr(Config, 'BUFFER_CHUNK_SIZE', 188 * 5644) # ~1MB default
self.target_chunk_size = ConfigHelper.get('BUFFER_CHUNK_SIZE', TS_PACKET_SIZE * 5644) # ~1MB default
# Track timers for proper cleanup
self.stopping = False
@ -57,7 +60,7 @@ class StreamBuffer:
combined_data = bytearray(self._partial_packet) + bytearray(chunk)
# Calculate complete packets
complete_packets_size = (len(combined_data) // 188) * 188
complete_packets_size = (len(combined_data) // self.TS_PACKET_SIZE) * self.TS_PACKET_SIZE
if complete_packets_size == 0:
# Not enough data for a complete packet
@ -82,7 +85,7 @@ class StreamBuffer:
# Write optimized chunk to Redis
if self.redis_client:
chunk_index = self.redis_client.incr(self.buffer_index_key)
chunk_key = f"{self.buffer_prefix}{chunk_index}"
chunk_key = RedisKeys.buffer_chunk(self.channel_id, chunk_index)
self.redis_client.setex(chunk_key, self.chunk_ttl, bytes(chunk_data))
# Update local tracking
@ -146,7 +149,7 @@ class StreamBuffer:
# Directly fetch from Redis using pipeline for efficiency
pipe = self.redis_client.pipeline()
for idx in range(start_id, end_id):
chunk_key = f"{self.buffer_prefix}{idx}"
chunk_key = RedisKeys.buffer_chunk(self.channel_id, idx)
pipe.get(chunk_key)
results = pipe.execute()
@ -200,7 +203,7 @@ class StreamBuffer:
# Directly fetch from Redis using pipeline
pipe = self.redis_client.pipeline()
for idx in range(start_id, end_id):
chunk_key = f"{self.buffer_prefix}{idx}"
chunk_key = RedisKeys.buffer_chunk(self.channel_id, idx)
pipe.get(chunk_key)
results = pipe.execute()
@ -222,7 +225,7 @@ class StreamBuffer:
"""Stop the buffer and cancel all timers"""
# Set stopping flag first to prevent new timer creation
self.stopping = True
# Cancel all pending timers
timers_cancelled = 0
for timer in list(self.fill_timers):
@ -232,10 +235,10 @@ class StreamBuffer:
timers_cancelled += 1
except Exception as e:
logger.error(f"Error canceling timer: {e}")
if timers_cancelled:
logger.info(f"Cancelled {timers_cancelled} buffer timers for channel {self.channel_id}")
# Clear timer list
self.fill_timers.clear()
@ -315,7 +318,7 @@ class StreamBuffer:
"""Schedule a timer and track it for proper cleanup"""
if self.stopping:
return None
timer = threading.Timer(delay, callback, args=args, kwargs=kwargs)
timer.daemon = True
timer.start()

View file

@ -0,0 +1,344 @@
"""
Stream generation and client-side handling for TS streams.
This module handles generating and delivering video streams to clients.
"""
import time
import logging
import threading
from apps.proxy.config import TSConfig as Config
from . import proxy_server
from .utils import create_ts_packet
from .redis_keys import RedisKeys
logger = logging.getLogger("ts_proxy")
class StreamGenerator:
"""
Handles generating streams for clients, including initialization,
data delivery, and cleanup.
"""
def __init__(self, channel_id, client_id, client_ip, client_user_agent, channel_initializing=False):
"""
Initialize the stream generator with client and channel details.
Args:
channel_id: The UUID of the channel to stream
client_id: Unique ID for this client connection
client_ip: Client's IP address
client_user_agent: User agent string from client
channel_initializing: Whether the channel is still initializing
"""
self.channel_id = channel_id
self.client_id = client_id
self.client_ip = client_ip
self.client_user_agent = client_user_agent
self.channel_initializing = channel_initializing
# Performance and state tracking
self.stream_start_time = time.time()
self.bytes_sent = 0
self.chunks_sent = 0
self.local_index = 0
self.consecutive_empty = 0
def generate(self):
"""
Generator function that produces the stream content for the client.
Handles initialization state, data delivery, and client disconnection.
Yields:
bytes: Chunks of TS stream data
"""
self.stream_start_time = time.time()
self.bytes_sent = 0
self.chunks_sent = 0
try:
logger.info(f"[{self.client_id}] Stream generator started, channel_ready={not self.channel_initializing}")
# First handle initialization if needed
if self.channel_initializing:
channel_ready = self._wait_for_initialization()
if not channel_ready:
# If initialization failed or timed out, we've already sent error packets
return
# Channel is now ready - start normal streaming
logger.info(f"[{self.client_id}] Channel {self.channel_id} ready, starting normal streaming")
# Reset start time for real streaming
self.stream_start_time = time.time()
# Setup streaming parameters and verify resources
if not self._setup_streaming():
return
# Main streaming loop
for chunk in self._stream_data_generator():
yield chunk
except Exception as e:
logger.error(f"[{self.client_id}] Stream error: {e}", exc_info=True)
finally:
self._cleanup()
def _wait_for_initialization(self):
"""Wait for channel initialization to complete, sending keepalive packets."""
initialization_start = time.time()
max_init_wait = getattr(Config, 'CLIENT_WAIT_TIMEOUT', 30)
keepalive_interval = 0.5
last_keepalive = 0
# While init is happening, send keepalive packets
while time.time() - initialization_start < max_init_wait:
# Check if initialization has completed
if proxy_server.redis_client:
metadata_key = RedisKeys.channel_metadata(self.channel_id)
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if state in ['waiting_for_clients', 'active']:
logger.info(f"[{self.client_id}] Channel {self.channel_id} now ready (state={state})")
return True
elif state in ['error', 'stopped']:
error_message = metadata.get(b'error_message', b'Unknown error').decode('utf-8')
logger.error(f"[{self.client_id}] Channel {self.channel_id} in error state: {state}, message: {error_message}")
# Send error packet before giving up
yield create_ts_packet('error', f"Error: {error_message}")
return False
else:
# Still initializing - send keepalive if needed
if time.time() - last_keepalive >= keepalive_interval:
status_msg = f"Initializing: {state}"
keepalive_packet = create_ts_packet('keepalive', status_msg)
logger.debug(f"[{self.client_id}] Sending keepalive packet during initialization, state={state}")
yield keepalive_packet
self.bytes_sent += len(keepalive_packet)
last_keepalive = time.time()
# Wait a bit before checking again
time.sleep(0.1)
# Timed out waiting
logger.warning(f"[{self.client_id}] Timed out waiting for initialization")
yield create_ts_packet('error', "Error: Initialization timeout")
return False
def _setup_streaming(self):
"""Setup streaming parameters and check resources."""
# Get buffer - stream manager may not exist in this worker
buffer = proxy_server.stream_buffers.get(self.channel_id)
stream_manager = proxy_server.stream_managers.get(self.channel_id)
if not buffer:
logger.error(f"[{self.client_id}] No buffer found for channel {self.channel_id}")
return False
# Client state tracking - use config for initial position
initial_behind = getattr(Config, 'INITIAL_BEHIND_CHUNKS', 10)
current_buffer_index = buffer.index
self.local_index = max(0, current_buffer_index - initial_behind)
# Store important objects as instance variables
self.buffer = buffer
self.stream_manager = stream_manager
self.last_yield_time = time.time()
self.empty_reads = 0
self.consecutive_empty = 0
self.is_owner_worker = proxy_server.am_i_owner(self.channel_id) if hasattr(proxy_server, 'am_i_owner') else True
logger.info(f"[{self.client_id}] Starting stream at index {self.local_index} (buffer at {buffer.index})")
return True
def _stream_data_generator(self):
"""Generate stream data chunks based on buffer contents."""
bytes_sent = 0
chunks_sent = 0
stream_start_time = time.time()
local_index = self.local_index
# Main streaming loop
while True:
# Check if resources still exist
if not self._check_resources():
break
# Get chunks at client's position using improved strategy
chunks, next_index = self.buffer.get_optimized_client_data(local_index)
if chunks:
yield from self._process_chunks(chunks, next_index, bytes_sent, chunks_sent, stream_start_time)
local_index = next_index
self.local_index = local_index
self.last_yield_time = time.time()
self.empty_reads = 0
self.consecutive_empty = 0
else:
# Handle no data condition (with possible keepalive packets)
self.empty_reads += 1
self.consecutive_empty += 1
if self._should_send_keepalive(local_index):
keepalive_packet = create_ts_packet('keepalive')
logger.debug(f"[{self.client_id}] Sending keepalive packet while waiting at buffer head")
yield keepalive_packet
bytes_sent += len(keepalive_packet)
self.last_yield_time = time.time()
self.consecutive_empty = 0 # Reset consecutive counter but keep total empty_reads
time.sleep(Config.KEEPALIVE_INTERVAL)
else:
# Standard wait with backoff
sleep_time = min(0.1 * self.consecutive_empty, 1.0)
time.sleep(sleep_time)
# Log empty reads periodically
if self.empty_reads % 50 == 0:
stream_status = "healthy" if (self.stream_manager and self.stream_manager.healthy) else "unknown"
logger.debug(f"[{self.client_id}] Waiting for chunks beyond {local_index} (buffer at {self.buffer.index}, stream: {stream_status})")
# Check for ghost clients
if self._is_ghost_client(local_index):
logger.warning(f"[{self.client_id}] Possible ghost client: buffer has advanced {self.buffer.index - local_index} chunks ahead but client stuck at {local_index}")
break
# Check for timeouts
if self._is_timeout():
break
def _check_resources(self):
"""Check if required resources still exist."""
# Enhanced resource checks
if self.channel_id not in proxy_server.stream_buffers:
logger.info(f"[{self.client_id}] Channel buffer no longer exists, terminating stream")
return False
if self.channel_id not in proxy_server.client_managers:
logger.info(f"[{self.client_id}] Client manager no longer exists, terminating stream")
return False
# Check if this specific client has been stopped (Redis keys, etc.)
if proxy_server.redis_client:
# Channel stop check
stop_key = RedisKeys.channel_stopping(self.channel_id)
if proxy_server.redis_client.exists(stop_key):
logger.info(f"[{self.client_id}] Detected channel stop signal, terminating stream")
return False
# Client stop check
client_stop_key = RedisKeys.client_stop(self.channel_id, self.client_id)
if proxy_server.redis_client.exists(client_stop_key):
logger.info(f"[{self.client_id}] Detected client stop signal, terminating stream")
return False
# Also check if client has been removed from client_manager
if self.channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[self.channel_id]
if self.client_id not in client_manager.clients:
logger.info(f"[{self.client_id}] Client no longer in client manager, terminating stream")
return False
return True
def _process_chunks(self, chunks, next_index, bytes_sent, chunks_sent, stream_start_time):
"""Process and yield chunks to the client."""
# Process and send chunks
total_size = sum(len(c) for c in chunks)
logger.debug(f"[{self.client_id}] Retrieved {len(chunks)} chunks ({total_size} bytes) from index {self.local_index+1} to {next_index}")
# Send the chunks to the client
for chunk in chunks:
try:
yield chunk
bytes_sent += len(chunk)
chunks_sent += 1
# Log every 100 chunks for visibility
if chunks_sent % 100 == 0:
elapsed = time.time() - stream_start_time
rate = bytes_sent / elapsed / 1024 if elapsed > 0 else 0
logger.info(f"[{self.client_id}] Stats: {chunks_sent} chunks, {bytes_sent/1024:.1f}KB, {rate:.1f}KB/s")
except Exception as e:
logger.error(f"[{self.client_id}] Error sending chunk to client: {e}")
raise # Re-raise to exit the generator
return bytes_sent, chunks_sent
def _should_send_keepalive(self, local_index):
"""Determine if a keepalive packet should be sent."""
# Check if we're caught up to buffer head
at_buffer_head = local_index >= self.buffer.index
# If we're at buffer head and no data is coming, send keepalive
stream_healthy = self.stream_manager.healthy if self.stream_manager else True
return at_buffer_head and not stream_healthy and self.consecutive_empty >= 5
def _is_ghost_client(self, local_index):
"""Check if this appears to be a ghost client (stuck but buffer advancing)."""
return self.consecutive_empty > 100 and self.buffer.index > local_index + 50
def _is_timeout(self):
"""Check if the stream has timed out."""
# Disconnect after long inactivity
if time.time() - self.last_yield_time > Config.STREAM_TIMEOUT:
if self.stream_manager and not self.stream_manager.healthy:
logger.warning(f"[{self.client_id}] No data for {Config.STREAM_TIMEOUT}s and stream unhealthy, disconnecting")
return True
elif not self.is_owner_worker and self.consecutive_empty > 100:
# Non-owner worker without data for too long
logger.warning(f"[{self.client_id}] Non-owner worker with no data for {Config.STREAM_TIMEOUT}s, disconnecting")
return True
return False
def _cleanup(self):
"""Clean up resources and report final statistics."""
# Client cleanup
elapsed = time.time() - self.stream_start_time
local_clients = 0
total_clients = 0
if self.channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[self.channel_id]
local_clients = client_manager.remove_client(self.client_id)
total_clients = client_manager.get_total_client_count()
logger.info(f"[{self.client_id}] Disconnected after {elapsed:.2f}s (local: {local_clients}, total: {total_clients})")
# Schedule channel shutdown if no clients left
self._schedule_channel_shutdown_if_needed(local_clients)
def _schedule_channel_shutdown_if_needed(self, local_clients):
"""
Schedule channel shutdown if there are no clients left and we're the owner.
"""
# If no clients left and we're the owner, schedule shutdown using the config value
if local_clients == 0 and proxy_server.am_i_owner(self.channel_id):
logger.info(f"No local clients left for channel {self.channel_id}, scheduling shutdown")
def delayed_shutdown():
# Use the config setting instead of hardcoded value
shutdown_delay = getattr(Config, 'CHANNEL_SHUTDOWN_DELAY', 5)
logger.info(f"Waiting {shutdown_delay}s before checking if channel should be stopped")
time.sleep(shutdown_delay)
# After delay, check global client count
if self.channel_id in proxy_server.client_managers:
total = proxy_server.client_managers[self.channel_id].get_total_client_count()
if total == 0:
logger.info(f"Shutting down channel {self.channel_id} as no clients connected")
proxy_server.stop_channel(self.channel_id)
else:
logger.info(f"Not shutting down channel {self.channel_id}, {total} clients still connected")
shutdown_thread = threading.Thread(target=delayed_shutdown)
shutdown_thread.daemon = True
shutdown_thread.start()
def create_stream_generator(channel_id, client_id, client_ip, client_user_agent, channel_initializing=False):
"""
Factory function to create a new stream generator.
Returns a function that can be passed to StreamingHttpResponse.
"""
generator = StreamGenerator(channel_id, client_id, client_ip, client_user_agent, channel_initializing)
return generator.generate

View file

@ -3,6 +3,7 @@
import threading
import logging
import time
import socket
import requests
import subprocess
from typing import Optional, List
@ -13,6 +14,9 @@ from apps.m3u.models import M3UAccount, M3UAccountProfile
from core.models import UserAgent, CoreSettings
from .stream_buffer import StreamBuffer
from .utils import detect_stream_type
from .redis_keys import RedisKeys
from .constants import ChannelState, EventType, StreamType, TS_PACKET_SIZE
from .config_helper import ConfigHelper
logger = logging.getLogger("ts_proxy")
@ -27,7 +31,7 @@ class StreamManager:
self.running = True
self.connected = False
self.retry_count = 0
self.max_retries = Config.MAX_RETRIES
self.max_retries = ConfigHelper.max_retries()
self.current_response = None
self.current_session = None
self.url_switching = False
@ -43,8 +47,8 @@ class StreamManager:
# Stream health monitoring
self.last_data_time = time.time()
self.healthy = True
self.health_check_interval = Config.HEALTH_CHECK_INTERVAL
self.chunk_size = getattr(Config, 'CHUNK_SIZE', 8192)
self.health_check_interval = ConfigHelper.get('HEALTH_CHECK_INTERVAL', 5)
self.chunk_size = ConfigHelper.chunk_size()
# Add to your __init__ method
self._buffer_check_timers = []
@ -84,7 +88,7 @@ class StreamManager:
try:
# Check stream type before connecting
stream_type = detect_stream_type(self.url)
if self.transcode == False and stream_type == 'hls':
if self.transcode == False and stream_type == StreamType.HLS:
logger.info(f"Detected HLS stream: {self.url}")
logger.info(f"HLS streams will be handled with FFmpeg for now - future version will support HLS natively")
# Enable transcoding for HLS streams
@ -192,7 +196,7 @@ class StreamManager:
# Update last data timestamp in Redis
if hasattr(self.buffer, 'redis_client') and self.buffer.redis_client:
last_data_key = f"ts_proxy:channel:{self.buffer.channel_id}:last_data"
last_data_key = RedisKeys.last_data(self.buffer.channel_id)
self.buffer.redis_client.set(last_data_key, str(time.time()), ex=60)
except (AttributeError, ConnectionError) as e:
if self.stop_requested:
@ -411,16 +415,19 @@ class StreamManager:
logger.debug(f"Error closing session: {e}")
self.current_session = None
# Keep backward compatibility - let's create an alias to the new method
def _close_socket(self):
"""Backward compatibility wrapper for _close_connection"""
if self.current_response:
return self._close_connection()
"""Close socket and transcode resources as needed"""
# First try to use _close_connection for HTTP resources
if self.current_response or self.current_session:
self._close_connection()
return
# Otherwise handle socket and transcode resources
if self.socket:
try:
self.socket.close()
except Exception as e:
logging.debug(f"Error closing socket: {e}")
logger.debug(f"Error closing socket: {e}")
pass
self.socket = None
@ -431,7 +438,7 @@ class StreamManager:
self.transcode_process.terminate()
self.transcode_process.wait()
except Exception as e:
logging.debug(f"Error terminating transcode process: {e}")
logger.debug(f"Error terminating transcode process: {e}")
pass
self.transcode_process = None
@ -466,7 +473,7 @@ class StreamManager:
# Update last data timestamp in Redis if successful
if success and hasattr(self.buffer, 'redis_client') and self.buffer.redis_client:
last_data_key = f"ts_proxy:channel:{self.buffer.channel_id}:last_data"
last_data_key = RedisKeys.last_data(self.buffer.channel_id)
self.buffer.redis_client.set(last_data_key, str(time.time()), ex=60)
return True
@ -491,7 +498,7 @@ class StreamManager:
if channel_id and redis_client:
current_time = str(time.time())
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
# Check current state first
current_state = None
@ -503,16 +510,16 @@ class StreamManager:
logger.error(f"Error checking current state: {e}")
# Only update if not already past connecting
if not current_state or current_state in ["initializing", "connecting"]:
if not current_state or current_state in [ChannelState.INITIALIZING, ChannelState.CONNECTING]:
# NEW CODE: Check if buffer has enough chunks
current_buffer_index = getattr(self.buffer, 'index', 0)
initial_chunks_needed = getattr(Config, 'INITIAL_BEHIND_CHUNKS', 10)
initial_chunks_needed = ConfigHelper.initial_behind_chunks()
if current_buffer_index < initial_chunks_needed:
# Not enough buffer yet - set to connecting state if not already
if current_state != "connecting":
if current_state != ChannelState.CONNECTING:
update_data = {
"state": "connecting",
"state": ChannelState.CONNECTING,
"state_changed_at": current_time
}
redis_client.hset(metadata_key, mapping=update_data)
@ -526,7 +533,7 @@ class StreamManager:
# We have enough buffer, proceed with state change
update_data = {
"state": "waiting_for_clients",
"state": ChannelState.WAITING_FOR_CLIENTS,
"connection_ready_time": current_time,
"state_changed_at": current_time,
"buffer_chunks": str(current_buffer_index)
@ -534,8 +541,8 @@ class StreamManager:
redis_client.hset(metadata_key, mapping=update_data)
# Get configured grace period or default
grace_period = getattr(Config, 'CHANNEL_INIT_GRACE_PERIOD', 20)
logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'}waiting_for_clients with {current_buffer_index} buffer chunks")
grace_period = ConfigHelper.get('CHANNEL_INIT_GRACE_PERIOD', 20)
logger.info(f"STREAM MANAGER: Updated channel {channel_id} state: {current_state or 'None'}{ChannelState.WAITING_FOR_CLIENTS} with {current_buffer_index} buffer chunks")
logger.info(f"Started initial connection grace period ({grace_period}s) for channel {channel_id}")
else:
logger.debug(f"Not changing state: channel {channel_id} already in {current_state} state")

View file

@ -34,4 +34,47 @@ def detect_stream_type(url):
return 'hls'
# Default to TS
return 'ts'
return 'ts'
def get_client_ip(request):
"""
Extract client IP address from request.
Handles cases where request is behind a proxy by checking X-Forwarded-For.
"""
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR')
return ip
def create_ts_packet(packet_type='null', message=None):
"""
Create a Transport Stream (TS) packet for various purposes.
Args:
packet_type (str): Type of packet - 'null', 'error', 'keepalive', etc.
message (str): Optional message to include in packet payload
Returns:
bytes: A properly formatted 188-byte TS packet
"""
packet = bytearray(188)
# TS packet header
packet[0] = 0x47 # Sync byte
# PID - Use different PIDs based on packet type
if packet_type == 'error':
packet[1] = 0x1F # PID high bits
packet[2] = 0xFF # PID low bits
else: # null/keepalive packets
packet[1] = 0x1F # PID high bits (null packet)
packet[2] = 0xFF # PID low bits (null packet)
# Add message to payload if provided
if message:
msg_bytes = message.encode('utf-8')
packet[4:4+min(len(msg_bytes), 180)] = msg_bytes[:180]
return bytes(packet)

View file

@ -5,30 +5,27 @@ import random
import re
from django.http import StreamingHttpResponse, JsonResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods, require_GET
from django.shortcuts import get_object_or_404
from apps.proxy.config import TSConfig as Config
from . import proxy_server
from .channel_status import ChannelStatus
from .stream_generator import create_stream_generator
from .utils import get_client_ip
from .redis_keys import RedisKeys
import logging
from apps.channels.models import Channel, Stream
from apps.m3u.models import M3UAccount, M3UAccountProfile
from core.models import UserAgent, CoreSettings, PROXY_PROFILE_NAME
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from .constants import ChannelState, EventType, StreamType
from .config_helper import ConfigHelper
from .services.channel_service import ChannelService
# Configure logging properly
logger = logging.getLogger("ts_proxy")
def get_client_ip(request):
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR')
return ip
@api_view(['GET'])
def stream_ts(request, channel_id):
"""Stream TS data to client with immediate response and keep-alive packets during initialization"""
@ -97,12 +94,12 @@ def stream_ts(request, channel_id):
else:
transcode = True
# Initialize channel with the stream's user agent (not the client's)
success = proxy_server.initialize_channel(stream_url, channel_id, stream_user_agent, transcode)
if proxy_server.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
profile_value = str(stream_profile)
proxy_server.redis_client.hset(metadata_key, "profile", profile_value)
# Initialize channel using the service
profile_value = str(stream_profile)
success = ChannelService.initialize_channel(
channel_id, stream_url, stream_user_agent, transcode, profile_value
)
if not success:
return JsonResponse({'error': 'Failed to initialize channel'}, status=500)
@ -111,8 +108,9 @@ def stream_ts(request, channel_id):
manager = proxy_server.stream_managers.get(channel_id)
if manager:
wait_start = time.time()
timeout = ConfigHelper.connection_timeout()
while not manager.connected:
if time.time() - wait_start > Config.CONNECTION_TIMEOUT:
if time.time() - wait_start > timeout:
proxy_server.stop_channel(channel_id)
return JsonResponse({'error': 'Connection timeout'}, status=504)
if not manager.should_retry():
@ -134,7 +132,7 @@ def stream_ts(request, channel_id):
stream_user_agent = None # Initialize the variable
if proxy_server.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata_key = RedisKeys.channel_metadata(channel_id)
url_bytes = proxy_server.redis_client.hget(metadata_key, "url")
ua_bytes = proxy_server.redis_client.hget(metadata_key, "user_agent")
profile_bytes = proxy_server.redis_client.hget(metadata_key, "profile")
@ -168,291 +166,18 @@ def stream_ts(request, channel_id):
client_manager.add_client(client_id, client_ip, client_user_agent)
logger.info(f"[{client_id}] Client registered with channel {channel_id}")
# Define a single generate function
def generate():
stream_start_time = time.time()
bytes_sent = 0
chunks_sent = 0
# Create a stream generator for this client
generate = create_stream_generator(
channel_id, client_id, client_ip, client_user_agent, channel_initializing
)
# Keep track of initialization state
initialization_start = time.time()
max_init_wait = getattr(Config, 'CLIENT_WAIT_TIMEOUT', 30)
channel_ready = not channel_initializing
keepalive_interval = 0.5
last_keepalive = 0
try:
logger.info(f"[{client_id}] Stream generator started, channel_ready={channel_ready}")
# Wait for initialization to complete if needed
if not channel_ready:
# While init is happening, send keepalive packets
while time.time() - initialization_start < max_init_wait:
# Check if initialization has completed
if proxy_server.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
if state in ['waiting_for_clients', 'active']:
logger.info(f"[{client_id}] Channel {channel_id} now ready (state={state})")
channel_ready = True
break
elif state in ['error', 'stopped']:
error_message = metadata.get(b'error_message', b'Unknown error').decode('utf-8')
logger.error(f"[{client_id}] Channel {channel_id} in error state: {state}, message: {error_message}")
# Send error in a comment TS packet before giving up
error_packet = bytearray(188)
error_packet[0] = 0x47 # Sync byte
error_packet[1] = 0x1F # PID high bits
error_packet[2] = 0xFF # PID low bits
error_msg = f"Error: {error_message}".encode('utf-8')
error_packet[4:4+min(len(error_msg), 180)] = error_msg[:180]
yield bytes(error_packet)
return
else:
# Still initializing - send keepalive if needed
if time.time() - last_keepalive >= keepalive_interval:
keepalive_packet = bytearray(188)
keepalive_packet[0] = 0x47 # Sync byte
keepalive_packet[1] = 0x1F # PID high bits (null packet)
keepalive_packet[2] = 0xFF # PID low bits (null packet)
# Add status info in packet payload (will be ignored by players)
status_msg = f"Initializing: {state}".encode('utf-8')
keepalive_packet[4:4+min(len(status_msg), 180)] = status_msg[:180]
logger.debug(f"[{client_id}] Sending keepalive packet during initialization, state={state}")
yield bytes(keepalive_packet)
bytes_sent += len(keepalive_packet)
last_keepalive = time.time()
# Wait a bit before checking again (don't send too many keepalives)
time.sleep(0.1)
# Check if we timed out waiting
if not channel_ready:
logger.warning(f"[{client_id}] Timed out waiting for initialization")
error_packet = bytearray(188)
error_packet[0] = 0x47 # Sync byte
error_packet[1] = 0x1F # PID high bits
error_packet[2] = 0xFF # PID low bits
error_msg = f"Error: Initialization timeout".encode('utf-8')
error_packet[4:4+min(len(error_msg), 180)] = error_msg[:180]
yield bytes(error_packet)
return
# Channel is now ready - original streaming code goes here
logger.info(f"[{client_id}] Channel {channel_id} ready, starting normal streaming")
# Reset start time for real streaming
stream_start_time = time.time()
# Get buffer - stream manager may not exist in this worker
buffer = proxy_server.stream_buffers.get(channel_id)
stream_manager = proxy_server.stream_managers.get(channel_id)
if not buffer:
logger.error(f"[{client_id}] No buffer found for channel {channel_id}")
return
# Client state tracking - use config for initial position
initial_behind = getattr(Config, 'INITIAL_BEHIND_CHUNKS', 10)
current_buffer_index = buffer.index
local_index = max(0, current_buffer_index - initial_behind)
logger.debug(f"[{client_id}] Buffer at {current_buffer_index}, starting {initial_behind} chunks behind at index {local_index}")
initial_position = local_index
last_yield_time = time.time()
empty_reads = 0
bytes_sent = 0
chunks_sent = 0
stream_start_time = time.time()
consecutive_empty = 0 # Track consecutive empty reads
# Timing parameters from config
ts_packet_size = 188
target_bitrate = Config.TARGET_BITRATE
packets_per_second = target_bitrate / (8 * ts_packet_size)
logger.info(f"[{client_id}] Starting stream at index {local_index} (buffer at {buffer.index})")
# Check if we're the owner worker
is_owner_worker = proxy_server.am_i_owner(channel_id) if hasattr(proxy_server, 'am_i_owner') else True
# Main streaming loop
while True:
# Enhanced resource checks
if channel_id not in proxy_server.stream_buffers:
logger.info(f"[{client_id}] Channel buffer no longer exists, terminating stream")
break
if channel_id not in proxy_server.client_managers:
logger.info(f"[{client_id}] Client manager no longer exists, terminating stream")
break
# Check if this specific client has been stopped
if proxy_server.redis_client:
# Channel stop check
stop_key = f"ts_proxy:channel:{channel_id}:stopping"
if proxy_server.redis_client.exists(stop_key):
logger.info(f"[{client_id}] Detected channel stop signal, terminating stream")
break
# Client stop check - NEW
client_stop_key = f"ts_proxy:channel:{channel_id}:client:{client_id}:stop"
if proxy_server.redis_client.exists(client_stop_key):
logger.info(f"[{client_id}] Detected client stop signal, terminating stream")
break
# Also check if client has been removed from client_manager
if channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[channel_id]
if client_id not in client_manager.clients:
logger.info(f"[{client_id}] Client no longer in client manager, terminating stream")
break
# Get chunks at client's position using improved strategy
chunks, next_index = buffer.get_optimized_client_data(local_index)
if chunks:
empty_reads = 0
consecutive_empty = 0
# Process and send chunks
total_size = sum(len(c) for c in chunks)
logger.debug(f"[{client_id}] Retrieved {len(chunks)} chunks ({total_size} bytes) from index {local_index+1} to {next_index}")
# CRITICAL FIX: Actually send the chunks to the client
for chunk in chunks:
try:
# This is the crucial line that was likely missing
yield chunk
bytes_sent += len(chunk)
chunks_sent += 1
# Log every 100 chunks for visibility
if chunks_sent % 100 == 0:
elapsed = time.time() - stream_start_time
rate = bytes_sent / elapsed / 1024 if elapsed > 0 else 0
logger.info(f"[{client_id}] Stats: {chunks_sent} chunks, {bytes_sent/1024:.1f}KB, {rate:.1f}KB/s")
except Exception as e:
logger.error(f"[{client_id}] Error sending chunk to client: {e}")
raise # Re-raise to exit the generator
# Update index after successfully sending all chunks
local_index = next_index
last_yield_time = time.time()
else:
# No chunks available
empty_reads += 1
consecutive_empty += 1
# Check if we're caught up to buffer head
at_buffer_head = local_index >= buffer.index
# If we're at buffer head and no data is coming, send keepalive
# Only check stream manager health if it exists
stream_healthy = stream_manager.healthy if stream_manager else True
if at_buffer_head and not stream_healthy and consecutive_empty >= 5:
# Create a null TS packet as keepalive (188 bytes filled with padding)
# This prevents VLC from hitting EOF
keepalive_packet = bytearray(188)
keepalive_packet[0] = 0x47 # Sync byte
keepalive_packet[1] = 0x1F # PID high bits (null packet)
keepalive_packet[2] = 0xFF # PID low bits (null packet)
logger.debug(f"[{client_id}] Sending keepalive packet while waiting at buffer head")
yield bytes(keepalive_packet)
bytes_sent += len(keepalive_packet)
last_yield_time = time.time()
consecutive_empty = 0 # Reset consecutive counter but keep total empty_reads
time.sleep(Config.KEEPALIVE_INTERVAL)
else:
# Standard wait
sleep_time = min(0.1 * consecutive_empty, 1.0) # Progressive backoff up to 1s
time.sleep(sleep_time)
# Log empty reads periodically
if empty_reads % 50 == 0:
stream_status = "healthy" if (stream_manager and stream_manager.healthy) else "unknown"
logger.debug(f"[{client_id}] Waiting for chunks beyond {local_index} (buffer at {buffer.index}, stream: {stream_status})")
# CRITICAL FIX: Check for client disconnect during wait periods
# Django/WSGI might not immediately detect disconnections, but we can check periodically
if consecutive_empty > 10: # After some number of empty reads
if hasattr(request, 'META') and request.META.get('wsgi.input'):
try:
# Try to check if the connection is still alive
available = request.META['wsgi.input'].read(0)
if available is None: # Connection closed
logger.info(f"[{client_id}] Detected client disconnect during wait")
break
except Exception:
# Error reading from connection, likely closed
logger.info(f"[{client_id}] Connection error, client likely disconnected")
break
# Disconnect after long inactivity
# For non-owner workers, we're more lenient with timeout
if time.time() - last_yield_time > Config.STREAM_TIMEOUT:
if stream_manager and not stream_manager.healthy:
logger.warning(f"[{client_id}] No data for {Config.STREAM_TIMEOUT}s and stream unhealthy, disconnecting")
break
elif not is_owner_worker and consecutive_empty > 100:
# Non-owner worker without data for too long
logger.warning(f"[{client_id}] Non-owner worker with no data for {Config.STREAM_TIMEOUT}s, disconnecting")
break
# ADD THIS: Check if worker has more recent chunks but still stuck
# This can indicate the client is disconnected but we're not detecting it
if consecutive_empty > 100 and buffer.index > local_index + 50:
logger.warning(f"[{client_id}] Possible ghost client: buffer has advanced {buffer.index - local_index} chunks ahead but client stuck at {local_index}")
break
except Exception as e:
logger.error(f"[{client_id}] Stream error: {e}", exc_info=True)
finally:
# Client cleanup
elapsed = time.time() - stream_start_time
local_clients = 0
if channel_id in proxy_server.client_managers:
local_clients = proxy_server.client_managers[channel_id].remove_client(client_id)
total_clients = proxy_server.client_managers[channel_id].get_total_client_count()
logger.info(f"[{client_id}] Disconnected after {elapsed:.2f}s, {bytes_sent/1024:.1f}KB in {chunks_sent} chunks (local: {local_clients}, total: {total_clients})")
# If no clients left and we're the owner, schedule shutdown using the config value
if local_clients == 0 and proxy_server.am_i_owner(channel_id):
logger.info(f"No local clients left for channel {channel_id}, scheduling shutdown")
def delayed_shutdown():
# Use the config setting instead of hardcoded value
shutdown_delay = getattr(Config, 'CHANNEL_SHUTDOWN_DELAY', 5)
logger.info(f"Waiting {shutdown_delay}s before checking if channel should be stopped")
time.sleep(shutdown_delay)
# After delay, check global client count
if channel_id in proxy_server.client_managers:
total = proxy_server.client_managers[channel_id].get_total_client_count()
if total == 0:
logger.info(f"Shutting down channel {channel_id} as no clients connected")
proxy_server.stop_channel(channel_id)
else:
logger.info(f"Not shutting down channel {channel_id}, {total} clients still connected")
shutdown_thread = threading.Thread(target=delayed_shutdown)
shutdown_thread.daemon = True
shutdown_thread.start()
# IMPORTANT: Return the StreamingHttpResponse from the main function
# Return the StreamingHttpResponse from the main function
response = StreamingHttpResponse(
streaming_content=generate(),
content_type='video/mp2t'
)
response['Cache-Control'] = 'no-cache'
return response # This now properly returns from stream_ts
return response
except Exception as e:
logger.error(f"Error in stream_ts: {e}", exc_info=True)
@ -473,91 +198,17 @@ def change_stream(request, channel_id):
logger.info(f"Attempting to change stream URL for channel {channel_id} to {new_url}")
# Enhanced channel detection
in_local_managers = channel_id in proxy_server.stream_managers
in_local_buffers = channel_id in proxy_server.stream_buffers
# Use the service layer instead of direct implementation
result = ChannelService.change_stream_url(channel_id, new_url, user_agent)
# First check Redis directly before using our wrapper method
redis_keys = None
if proxy_server.redis_client:
try:
redis_keys = proxy_server.redis_client.keys(f"ts_proxy:*:{channel_id}*")
redis_keys = [k.decode('utf-8') for k in redis_keys] if redis_keys else []
except Exception as e:
logger.error(f"Error checking Redis keys: {e}")
if result.get('status') == 'error':
return JsonResponse({
'error': result.get('message', 'Unknown error'),
'diagnostics': result.get('diagnostics', {})
}, status=404)
# Now use our standard check
channel_exists = proxy_server.check_if_channel_exists(channel_id)
# Log detailed diagnostics
logger.info(f"Channel {channel_id} diagnostics: "
f"in_local_managers={in_local_managers}, "
f"in_local_buffers={in_local_buffers}, "
f"redis_keys_count={len(redis_keys) if redis_keys else 0}, "
f"channel_exists={channel_exists}")
if not channel_exists:
# If channel doesn't exist but we found Redis keys, force initialize it
if redis_keys:
logger.warning(f"Channel {channel_id} not detected by check_if_channel_exists but Redis keys exist. Forcing initialization.")
proxy_server.initialize_channel(new_url, channel_id, user_agent)
else:
logger.error(f"Channel {channel_id} not found in any worker or Redis")
return JsonResponse({
'error': 'Channel not found',
'diagnostics': {
'in_local_managers': in_local_managers,
'in_local_buffers': in_local_buffers,
'redis_keys': redis_keys,
}
}, status=404)
# Update metadata in Redis regardless of ownership - this ensures URL is updated
# even if the owner worker is handling another request
if proxy_server.redis_client:
try:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
# First check if the key exists and what type it is
key_type = proxy_server.redis_client.type(metadata_key).decode('utf-8')
logger.debug(f"Redis key {metadata_key} is of type: {key_type}")
# Use the appropriate method based on the key type
if key_type == 'hash':
proxy_server.redis_client.hset(metadata_key, "url", new_url)
if user_agent:
proxy_server.redis_client.hset(metadata_key, "user_agent", user_agent)
elif key_type == 'none': # Key doesn't exist yet
# Create new hash with all required fields
metadata = {"url": new_url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
else:
# If key exists with wrong type, delete it and recreate
proxy_server.redis_client.delete(metadata_key)
metadata = {"url": new_url}
if user_agent:
metadata["user_agent"] = user_agent
proxy_server.redis_client.hset(metadata_key, mapping=metadata)
# Set switch request flag to ensure all workers see it
switch_key = f"ts_proxy:channel:{channel_id}:switch_request"
proxy_server.redis_client.setex(switch_key, 30, new_url) # 30 second TTL
logger.info(f"Updated metadata for channel {channel_id} in Redis")
except Exception as e:
logger.error(f"Error updating Redis metadata: {e}", exc_info=True)
# If we're the owner, update directly
if proxy_server.am_i_owner(channel_id) and channel_id in proxy_server.stream_managers:
logger.info(f"This worker is the owner, changing stream URL for channel {channel_id}")
manager = proxy_server.stream_managers[channel_id]
old_url = manager.url
# Update the stream
result = manager.update_url(new_url)
logger.info(f"Stream URL changed from {old_url} to {new_url}, result: {result}")
# Format response based on whether it was a direct update or event-based
if result.get('direct_update'):
return JsonResponse({
'message': 'Stream URL updated',
'channel': channel_id,
@ -565,25 +216,7 @@ def change_stream(request, channel_id):
'owner': True,
'worker_id': proxy_server.worker_id
})
# If we're not the owner, publish an event for the owner to pick up
else:
logger.info(f"This worker is not the owner, requesting URL change via Redis PubSub")
# Publish switch request event
switch_request = {
"event": "stream_switch",
"channel_id": channel_id,
"url": new_url,
"user_agent": user_agent,
"requester": proxy_server.worker_id,
"timestamp": time.time()
}
proxy_server.redis_client.publish(
f"ts_proxy:events:{channel_id}",
json.dumps(switch_request)
)
return JsonResponse({
'message': 'Stream URL change requested',
'channel': channel_id,
@ -653,61 +286,16 @@ def stop_channel(request, channel_id):
try:
logger.info(f"Request to stop channel {channel_id} received")
# Check if channel exists
channel_exists = proxy_server.check_if_channel_exists(channel_id)
if not channel_exists:
logger.warning(f"Channel {channel_id} not found in any worker or Redis")
return JsonResponse({'error': 'Channel not found'}, status=404)
# Use the service layer instead of direct implementation
result = ChannelService.stop_channel(channel_id)
# Get channel state information for response
channel_info = None
if proxy_server.redis_client:
metadata_key = f"ts_proxy:channel:{channel_id}:metadata"
try:
metadata = proxy_server.redis_client.hgetall(metadata_key)
if metadata and b'state' in metadata:
state = metadata[b'state'].decode('utf-8')
channel_info = {"state": state}
except Exception as e:
logger.error(f"Error fetching channel state: {e}")
# Broadcast stop event to all workers via PubSub
if proxy_server.redis_client:
stop_request = {
"event": "channel_stop",
"channel_id": channel_id,
"requester_worker_id": proxy_server.worker_id,
"timestamp": time.time()
}
# Publish the stop event
proxy_server.redis_client.publish(
f"ts_proxy:events:{channel_id}",
json.dumps(stop_request)
)
logger.info(f"Published channel stop event for {channel_id}")
# Also stop locally to ensure this worker cleans up right away
result = proxy_server.stop_channel(channel_id)
else:
# No Redis, just stop locally
result = proxy_server.stop_channel(channel_id)
# Release the channel in the channel model if applicable
try:
channel = Channel.objects.get(uuid=channel_id)
channel.release_stream()
logger.info(f"Released channel {channel_id} stream allocation")
except Channel.DoesNotExist:
logger.warning(f"Could not find Channel model for UUID {channel_id}")
except Exception as e:
logger.error(f"Error releasing channel stream: {e}")
if result.get('status') == 'error':
return JsonResponse({'error': result.get('message', 'Unknown error')}, status=404)
return JsonResponse({
'message': 'Channel stop request sent',
'channel_id': channel_id,
'previous_state': channel_info
'previous_state': result.get('previous_state')
})
except Exception as e:
@ -727,55 +315,17 @@ def stop_client(request, channel_id):
if not client_id:
return JsonResponse({'error': 'No client_id provided'}, status=400)
logger.info(f"Request to stop client {client_id} on channel {channel_id}")
# Use the service layer instead of direct implementation
result = ChannelService.stop_client(channel_id, client_id)
# Set a Redis key for the generator to detect regardless of whether client is local
if proxy_server.redis_client:
stop_key = f"ts_proxy:channel:{channel_id}:client:{client_id}:stop"
proxy_server.redis_client.setex(stop_key, 30, "true") # 30 second TTL
logger.info(f"Set stop key for client {client_id}")
# Check if channel exists
channel_exists = proxy_server.check_if_channel_exists(channel_id)
if not channel_exists:
logger.warning(f"Channel {channel_id} not found")
return JsonResponse({'error': 'Channel not found'}, status=404)
# Two-part approach:
# 1. Handle locally if client is on this worker
# 2. Use events to inform other workers if needed
local_client_stopped = False
if channel_id in proxy_server.client_managers:
client_manager = proxy_server.client_managers[channel_id]
# Use the existing remove_client method directly
with client_manager.lock:
if client_id in client_manager.clients:
client_manager.remove_client(client_id)
local_client_stopped = True
logger.info(f"Client {client_id} stopped locally on channel {channel_id}")
# If client wasn't found locally, broadcast stop event for other workers
if not local_client_stopped and proxy_server.redis_client:
stop_request = {
"event": "client_stop",
"channel_id": channel_id,
"client_id": client_id,
"requester_worker_id": proxy_server.worker_id,
"timestamp": time.time()
}
proxy_server.redis_client.publish(
f"ts_proxy:events:{channel_id}",
json.dumps(stop_request)
)
logger.info(f"Published stop request for client {client_id} on channel {channel_id}")
if result.get('status') == 'error':
return JsonResponse({'error': result.get('message')}, status=404)
return JsonResponse({
'message': 'Client stop request processed',
'channel_id': channel_id,
'client_id': client_id,
'locally_processed': local_client_stopped
'locally_processed': result.get('locally_processed', False)
})
except json.JSONDecodeError: