Dispatcharr/apps/channels/tests/test_ts_proxy_teardown.py

"""Tests for multi-worker channel teardown coordination."""
import time
from unittest.mock import ANY, MagicMock, patch

from django.test import TestCase

from apps.proxy.live_proxy.constants import ChannelMetadataField, ChannelState
from apps.proxy.live_proxy.input.buffer import StreamBuffer
from apps.proxy.live_proxy.input.manager import StreamManager
from apps.proxy.live_proxy.redis_keys import RedisKeys
from apps.proxy.live_proxy.server import ProxyServer
from apps.proxy.live_proxy.services.channel_service import ChannelService


CHANNEL_ID = "00000000-0000-0000-0000-000000000099"


def _configure_ownership_pipeline(
    redis,
    *,
    stop_exists=0,
    metadata_exists=1,
    client_count=0,
    owner=None,
    disconnect=None,
    state=None,
):
    pipe = MagicMock()
    redis.pipeline.return_value = pipe
    pipe.execute.return_value = (
        stop_exists,
        metadata_exists,
        client_count,
        owner,
        disconnect,
        state,
    )
    return pipe


def _mock_proxy_server(redis_client=None):
    server = MagicMock()
    server.redis_client = redis_client or MagicMock()
    server._stopping_channels = set()
    return server


class ChannelTeardownAvailabilityTests(TestCase):
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_teardown_active_when_stopping_key_exists(self, mock_get_instance):
        redis = MagicMock()
        redis.exists.side_effect = lambda key: key == RedisKeys.channel_stopping(CHANNEL_ID)
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID))

    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_teardown_active_when_metadata_state_is_stopping(self, mock_get_instance):
        redis = MagicMock()
        redis.exists.return_value = False
        redis.hget.return_value = ChannelState.STOPPING.encode()
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID))

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay")
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_shutdown_pending_within_delay_window(self, mock_get_instance, mock_delay):
        mock_delay.return_value = 5
        redis = MagicMock()
        redis.exists.return_value = False
        redis.get.return_value = str(time.time() - 2).encode()
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertTrue(ChannelService.is_shutdown_pending(CHANNEL_ID))
        self.assertFalse(ChannelService.is_channel_unavailable_for_new_clients(CHANNEL_ID))

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay")
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_cancel_pending_shutdown_clears_disconnect_key(self, mock_get_instance, mock_delay):
        mock_delay.return_value = 30
        redis = MagicMock()
        redis.exists.side_effect = lambda key: "last_client_disconnect" in key
        redis.get.return_value = None
        redis.hget.return_value = ChannelState.ACTIVE.encode()
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertTrue(ChannelService.cancel_pending_shutdown(CHANNEL_ID))
        redis.delete.assert_any_call(RedisKeys.last_client_disconnect(CHANNEL_ID))

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay")
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_cancel_pending_shutdown_skips_during_active_stop(self, mock_get_instance, mock_delay):
        mock_delay.return_value = 30
        redis = MagicMock()
        redis.exists.return_value = True
        server = _mock_proxy_server(redis)
        server._stopping_channels = {CHANNEL_ID}
        mock_get_instance.return_value = server

        self.assertFalse(ChannelService.cancel_pending_shutdown(CHANNEL_ID))
        redis.delete.assert_not_called()

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay")
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_cancel_pending_shutdown_skips_real_teardown_without_grace(self, mock_get_instance, mock_delay):
        mock_delay.return_value = 30
        redis = MagicMock()
        redis.exists.side_effect = lambda key: "stopping" in key
        redis.get.return_value = None
        redis.hget.return_value = ChannelState.STOPPING.encode()
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertFalse(ChannelService.cancel_pending_shutdown(CHANNEL_ID))
        redis.hset.assert_not_called()
        redis.delete.assert_not_called()

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.channel_shutdown_delay")
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_shutdown_pending_expired_after_delay(self, mock_get_instance, mock_delay):
        mock_delay.return_value = 5
        redis = MagicMock()
        redis.exists.return_value = False
        redis.get.return_value = str(time.time() - 10).encode()
        mock_get_instance.return_value = _mock_proxy_server(redis)

        self.assertFalse(ChannelService.is_shutdown_pending(CHANNEL_ID))


class ClientManagerAddClientTests(TestCase):
    @patch("apps.proxy.live_proxy.services.channel_service.ChannelService.cancel_pending_shutdown", return_value=False)
    @patch("apps.proxy.live_proxy.client_manager.send_websocket_update")
    def test_add_client_stores_ip_and_user_agent_in_redis(self, _mock_ws, _mock_cancel):
        from apps.proxy.live_proxy.client_manager import ClientManager

        redis = MagicMock()
        cm = ClientManager(CHANNEL_ID, redis_client=redis, worker_id="worker-1")
        cm.proxy_server = MagicMock()

        result = cm.add_client(
            "client-1",
            "10.0.2.163",
            user_agent="VLC/3.0.21",
        )

        self.assertEqual(result, 1)
        mapping = redis.hset.call_args[1]["mapping"]
        self.assertEqual(mapping["ip_address"], "10.0.2.163")
        self.assertEqual(mapping["user_agent"], "VLC/3.0.21")


class LocalStreamActivityTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server.stream_buffers = {}
        server.client_managers = {}
        server.profile_managers = {}
        server.profile_buffers = {}
        server._live_stream_managers = {}
        server._stopping_channels = set()
        server.redis_client = MagicMock()
        server.stop_all_output_formats = MagicMock()
        server.stop_all_output_profiles = MagicMock()
        return server

    @patch.object(ProxyServer, "_join_stream_thread")
    def test_stop_local_stream_activity_stops_live_manager(self, mock_join):
        server = self._make_server()
        manager = MagicMock()
        server._live_stream_managers[CHANNEL_ID] = manager

        server._stop_local_stream_activity(CHANNEL_ID)

        manager.stop.assert_called_once()
        mock_join.assert_called_once_with(CHANNEL_ID)
        self.assertNotIn(CHANNEL_ID, server._live_stream_managers)


class OrphanMetadataCleanupTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server.stream_buffers = {}
        server.client_managers = {}
        server.profile_managers = {}
        server.profile_buffers = {}
        server._live_stream_managers = {}
        server._stopping_channels = set()
        server.redis_client = MagicMock()
        return server

    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_stop_local_stream_activity")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True)
    def test_orphan_metadata_stops_local_processes_before_redis(
        self, mock_has_upstream, mock_stop_local, mock_clean_redis
    ):
        server = self._make_server()
        metadata_key = RedisKeys.channel_metadata(CHANNEL_ID)
        server.redis_client.keys.return_value = [metadata_key.encode()]
        server.redis_client.hgetall.return_value = {
            b"owner": b"",
            b"state": b"unknown",
        }
        server.redis_client.exists.return_value = False
        server.redis_client.scard.return_value = 0

        server._check_orphaned_metadata()

        mock_has_upstream.assert_called_with(CHANNEL_ID)
        mock_stop_local.assert_called_once_with(CHANNEL_ID)
        mock_clean_redis.assert_called_once_with(CHANNEL_ID)

    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_broadcast_upstream_stop")
    @patch.object(ProxyServer, "_stop_local_stream_activity")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=False)
    def test_orphan_metadata_remote_channel_broadcasts_stop(
        self, mock_has_upstream, mock_stop_local, mock_broadcast, mock_clean_redis
    ):
        server = self._make_server()
        metadata_key = RedisKeys.channel_metadata(CHANNEL_ID)
        server.redis_client.keys.return_value = [metadata_key.encode()]
        server.redis_client.hgetall.return_value = {b"owner": b"", b"state": b"unknown"}
        server.redis_client.exists.return_value = False
        server.redis_client.scard.return_value = 0

        server._check_orphaned_metadata()

        mock_broadcast.assert_called_once_with(CHANNEL_ID)
        mock_stop_local.assert_not_called()
        mock_clean_redis.assert_called_once_with(CHANNEL_ID)


class OrphanChannelCleanupTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server.stream_buffers = {}
        server.client_managers = {}
        server.profile_managers = {}
        server.profile_buffers = {}
        server._live_stream_managers = {}
        server._stopping_channels = set()
        server.redis_client = MagicMock()
        server.get_channel_owner = MagicMock(return_value=None)
        return server

    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_stop_local_stream_activity")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True)
    def test_orphan_channel_stops_local_before_redis(
        self, mock_has_upstream, mock_stop_local, mock_clean_redis
    ):
        server = self._make_server()
        metadata_key = RedisKeys.channel_metadata(CHANNEL_ID)
        server.redis_client.keys.return_value = [metadata_key.encode()]
        server.redis_client.scard.return_value = 0

        server._check_orphaned_channels()

        mock_stop_local.assert_called_once_with(CHANNEL_ID)
        mock_clean_redis.assert_called_once_with(CHANNEL_ID)


class StreamManagerOwnershipTests(TestCase):
    def test_still_owner_false_when_different_worker(self):
        buffer = MagicMock()
        _configure_ownership_pipeline(
            buffer.redis_client,
            owner=b"worker-b",
        )
        manager = StreamManager(
            CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a"
        )

        self.assertFalse(manager._still_owner())

    def test_still_owner_true_when_owner_lock_expired_but_not_stopping(self):
        buffer = MagicMock()
        _configure_ownership_pipeline(
            buffer.redis_client,
            client_count=0,
            owner=None,
            state=ChannelState.CONNECTING.encode(),
        )
        manager = StreamManager(
            CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a"
        )

        self.assertTrue(manager._still_owner())

    def test_still_owner_false_when_channel_stopping_key_set(self):
        buffer = MagicMock()
        _configure_ownership_pipeline(
            buffer.redis_client,
            stop_exists=1,
        )
        manager = StreamManager(
            CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a"
        )

        self.assertFalse(manager._still_owner())

    def test_update_bytes_skipped_after_ownership_lost(self):
        buffer = MagicMock()
        _configure_ownership_pipeline(
            buffer.redis_client,
            owner=b"other-worker",
        )
        manager = StreamManager(
            CHANNEL_ID, "http://example/stream", buffer, worker_id="worker-a"
        )
        manager.bytes_processed = 1000

        manager._update_bytes_processed(500)

        buffer.redis_client.hincrby.assert_not_called()


class StreamBufferStopTests(TestCase):
    def test_stop_discards_local_data_without_redis_writes(self):
        redis = MagicMock()
        buffer = StreamBuffer(channel_id=CHANNEL_ID, redis_client=redis)
        buffer._write_buffer = bytearray(b"x" * 376)

        buffer.stop()

        self.assertTrue(buffer.stopping)
        self.assertEqual(len(buffer._write_buffer), 0)
        redis.incr.assert_not_called()
        redis.setex.assert_not_called()
        redis.delete.assert_not_called()


class StopChannelTeardownTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server.stream_buffers = {}
        server.client_managers = {}
        server.profile_managers = {}
        server.profile_buffers = {}
        server._live_stream_managers = {}
        server._stopping_channels = set()
        server._stopping_since = {}
        server.redis_client = MagicMock()
        server.redis_client.exists.return_value = False
        server.am_i_owner = MagicMock(return_value=False)
        server._collect_channel_stop_event_data = MagicMock(return_value=None)
        server.release_ownership = MagicMock()
        return server

    @patch.object(ProxyServer, "_spawn_channel_stop_event")
    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_stop_local_stream_activity")
    def test_stop_channel_cleans_redis_before_blocking_local_stop(
        self, mock_stop_local, mock_clean_redis, mock_spawn_event
    ):
        server = self._make_server()
        call_order = []

        def stop_local(channel_id):
            call_order.append("local")

        def clean_redis(channel_id):
            call_order.append("redis")

        mock_stop_local.side_effect = stop_local
        mock_clean_redis.side_effect = clean_redis

        server.stop_channel(CHANNEL_ID)

        self.assertEqual(call_order, ["redis", "local"])
        mock_spawn_event.assert_called_once_with(None)

    @patch.object(ProxyServer, "_spawn_channel_stop_event")
    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_stop_local_stream_activity", side_effect=RuntimeError("boom"))
    def test_stop_channel_cleans_redis_in_finally_when_local_stop_fails(
        self, mock_stop_local, mock_clean_redis, mock_spawn_event
    ):
        server = self._make_server()

        result = server.stop_channel(CHANNEL_ID)

        self.assertFalse(result)
        mock_clean_redis.assert_called_once_with(CHANNEL_ID)
        mock_spawn_event.assert_not_called()
        self.assertNotIn(CHANNEL_ID, server._stopping_channels)

    @patch.object(ProxyServer, "_spawn_channel_stop_event")
    @patch.object(ProxyServer, "_clean_redis_keys")
    @patch.object(ProxyServer, "_stop_local_stream_activity")
    def test_stop_channel_owner_releases_after_redis_cleanup(
        self, mock_stop_local, mock_clean_redis, mock_spawn_event
    ):
        server = self._make_server()
        server.am_i_owner.return_value = True
        stop_data = {"channel_id": CHANNEL_ID}
        server._collect_channel_stop_event_data.return_value = stop_data
        call_order = []

        def stop_local(channel_id):
            call_order.append("local")

        def release(channel_id):
            call_order.append("release")

        def clean_redis(channel_id):
            call_order.append("redis")

        mock_stop_local.side_effect = stop_local
        server.release_ownership.side_effect = release
        mock_clean_redis.side_effect = clean_redis

        server.stop_channel(CHANNEL_ID)

        self.assertEqual(call_order, ["redis", "release", "local"])
        server._collect_channel_stop_event_data.assert_called_once_with(CHANNEL_ID)
        mock_spawn_event.assert_called_once_with(stop_data)


class CleanRedisKeysOrderTests(TestCase):
    @patch("apps.proxy.live_proxy.server.Stream.objects.get")
    @patch("apps.proxy.live_proxy.server.Channel.objects.get")
    def test_clean_redis_keys_releases_profile_slot_before_live_keys_deleted(
        self, mock_channel_get, mock_stream_get
    ):
        from apps.channels.models import Channel, Stream

        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.redis_client = MagicMock()
        call_order = []

        channel = MagicMock()
        channel.release_stream.return_value = True

        def channel_get(uuid):
            call_order.append("release")
            return channel

        mock_channel_get.side_effect = channel_get
        mock_stream_get.side_effect = Stream.DoesNotExist

        channel_key = f"live:channel:{CHANNEL_ID}:input:buffer:index".encode()

        def scan(cursor, match=None, count=100):
            call_order.append("redis")
            if match == f"live:channel:{CHANNEL_ID}:*":
                return (0, [channel_key])
            return (0, [])

        server.redis_client.scan.side_effect = scan

        server._clean_redis_keys(CHANNEL_ID)

        self.assertEqual(call_order, ["release", "redis", "redis"])
        channel.release_stream.assert_called_once()
        server.redis_client.delete.assert_called_once_with(channel_key)


class LocalUpstreamActivityTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server.stream_buffers = {}
        server.client_managers = {}
        server._live_stream_managers = {}
        return server

    def test_upstream_activity_excludes_reader_only_client_manager(self):
        server = self._make_server()
        server.client_managers[CHANNEL_ID] = MagicMock()
        server.stream_buffers[CHANNEL_ID] = MagicMock()
        self.assertFalse(server._has_local_upstream_activity(CHANNEL_ID))

    def test_upstream_activity_from_live_registry(self):
        server = self._make_server()
        server._live_stream_managers[CHANNEL_ID] = MagicMock()
        self.assertTrue(server._has_local_upstream_activity(CHANNEL_ID))


class ClientDisconnectOwnershipTests(TestCase):
    def test_last_client_triggers_stop_when_upstream_active_without_owner_lock(self):
        from apps.proxy.live_proxy.client_manager import ClientManager

        proxy_server = MagicMock()
        proxy_server.worker_id = "testhost:1"
        proxy_server.am_i_owner.return_value = False
        proxy_server._has_local_upstream_activity.return_value = True
        proxy_server.extend_ownership.return_value = False

        redis = MagicMock()
        redis.hget.return_value = b"viewer"
        redis.scard.return_value = 0

        manager = ClientManager(
            channel_id=CHANNEL_ID,
            redis_client=redis,
            worker_id="testhost:1",
        )
        manager.proxy_server = proxy_server
        manager.clients = {"client-1"}
        manager.client_set_key = RedisKeys.clients(CHANNEL_ID)
        manager._notify_owner_of_activity = MagicMock()
        manager._trigger_stats_update = MagicMock()
        manager.get_total_client_count = MagicMock(return_value=0)
        proxy_server._spawn_on_hub = MagicMock()

        manager.remove_client("client-1")

        proxy_server._spawn_on_hub.assert_called_once_with(
            proxy_server.handle_client_disconnect, CHANNEL_ID
        )


class TeardownActiveLocalStopTests(TestCase):
    @patch("apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance")
    def test_teardown_active_when_local_stop_in_progress(self, mock_get_instance):
        server = MagicMock()
        server._stopping_channels = {CHANNEL_ID}
        server.redis_client = MagicMock()
        server.redis_client.exists.return_value = False
        mock_get_instance.return_value = server

        self.assertTrue(ChannelService.is_channel_teardown_active(CHANNEL_ID))


class HandleClientDisconnectUpstreamFirstTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.client_managers = {CHANNEL_ID: MagicMock()}
        server.stream_managers = {CHANNEL_ID: MagicMock()}
        server._live_stream_managers = {}
        server.profile_managers = {}
        server.output_managers = {}
        server.redis_client = MagicMock()
        server.redis_client.scard.return_value = 0
        server._stopping_channels = set()
        return server

    @patch.object(ProxyServer, "_coordinated_stop_channel")
    @patch.object(ProxyServer, "_stop_upstream_before_redis_cleanup")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True)
    def test_last_client_uses_coordinated_stop_only(
        self, _mock_upstream, mock_stop_upstream, mock_coordinated
    ):
        server = self._make_server()

        with patch(
            "apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay",
            return_value=0,
        ):
            server.handle_client_disconnect(CHANNEL_ID)

        mock_stop_upstream.assert_not_called()
        mock_coordinated.assert_called_once_with(CHANNEL_ID)


class ShutdownDelayWaitTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            with patch.object(ProxyServer, "_start_cleanup_thread"):
                server = ProxyServer()
        server.redis_client = MagicMock()
        server.redis_client.scard.return_value = 0
        return server

    @patch("apps.proxy.live_proxy.server.gevent.sleep")
    @patch("apps.proxy.live_proxy.server.time.time", return_value=1000.0)
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    def test_aborts_when_disconnect_key_deleted(self, _mock_delay, _mock_time, mock_sleep):
        server = self._make_server()
        server.redis_client.get.side_effect = [b"1000.0", None]

        result = server._wait_for_shutdown_delay(CHANNEL_ID)

        self.assertFalse(result)
        self.assertGreaterEqual(mock_sleep.call_count, 1)

    @patch("apps.proxy.live_proxy.server.gevent.sleep")
    @patch("apps.proxy.live_proxy.server.time.time", return_value=1000.0)
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    def test_aborts_when_clients_reconnect(self, _mock_delay, _mock_time, _mock_sleep):
        server = self._make_server()
        server.redis_client.get.return_value = b"1000.0"
        server.redis_client.scard.side_effect = [0, 1]

        result = server._wait_for_shutdown_delay(CHANNEL_ID)

        self.assertFalse(result)
        server.redis_client.delete.assert_called_with(
            RedisKeys.last_client_disconnect(CHANNEL_ID)
        )

    @patch("apps.proxy.live_proxy.server.gevent.sleep")
    @patch("apps.proxy.live_proxy.server.time.time")
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    def test_timer_resets_when_disconnect_timestamp_updated(
        self, _mock_delay, mock_time, mock_sleep
    ):
        server = self._make_server()
        disconnect_key = RedisKeys.last_client_disconnect(CHANNEL_ID)
        current_time = [1000.0]
        disconnect_timestamp = [1000.0]
        poll_count = [0]

        mock_time.side_effect = lambda: current_time[0]

        def get_side_effect(key):
            poll_count[0] += 1
            if poll_count[0] >= 3:
                disconnect_timestamp[0] = 1020.0
            if key == disconnect_key:
                return str(disconnect_timestamp[0]).encode()
            return None

        server.redis_client.get.side_effect = get_side_effect

        def advance_sleep(duration):
            current_time[0] += duration

        mock_sleep.side_effect = advance_sleep

        result = server._wait_for_shutdown_delay(CHANNEL_ID)

        self.assertTrue(result)
        self.assertGreaterEqual(current_time[0], 1050.0)

    @patch.object(ProxyServer, "_coordinated_stop_channel")
    @patch.object(ProxyServer, "_wait_for_shutdown_delay", return_value=True)
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    def test_handle_client_disconnect_uses_polling_wait(
        self, _mock_delay, mock_wait, mock_coordinated
    ):
        server = HandleClientDisconnectUpstreamFirstTests()._make_server()
        server.redis_client.get.return_value = b"1700000000.0"

        server.handle_client_disconnect(CHANNEL_ID)

        mock_wait.assert_called_once_with(CHANNEL_ID)
        mock_coordinated.assert_called_once_with(CHANNEL_ID)

    @patch.object(ProxyServer, "_coordinated_stop_channel")
    @patch.object(ProxyServer, "_wait_for_shutdown_delay", return_value=False)
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    def test_handle_client_disconnect_skips_stop_when_wait_aborted(
        self, _mock_delay, _mock_wait, mock_coordinated
    ):
        server = HandleClientDisconnectUpstreamFirstTests()._make_server()
        server.redis_client.get.return_value = b"1700000000.0"

        server.handle_client_disconnect(CHANNEL_ID)

        mock_coordinated.assert_not_called()


class InitWaitAbortTests(TestCase):
    def _make_generator(self):
        from apps.proxy.live_proxy.output.ts.generator import StreamGenerator

        return StreamGenerator(
            CHANNEL_ID,
            "client-1",
            "127.0.0.1",
            "test-agent",
            channel_initializing=True,
        )

    def test_abort_when_client_removed_locally(self):
        generator = self._make_generator()
        server = MagicMock()
        client_manager = MagicMock()
        client_manager.clients = set()
        server.client_managers = {CHANNEL_ID: client_manager}
        server.redis_client = MagicMock()

        self.assertEqual(generator._init_wait_abort_reason(server, time.time()), "client_gone")

    @patch("apps.proxy.live_proxy.output.ts.generator.ConfigHelper.channel_init_grace_period", return_value=10)
    def test_abort_when_connect_stalled_without_buffer(self, _mock_grace):
        generator = self._make_generator()
        server = MagicMock()
        client_manager = MagicMock()
        client_manager.clients = {"client-1"}
        server.client_managers = {CHANNEL_ID: client_manager}
        buffer = MagicMock()
        buffer.index = 0
        server.stream_buffers = {CHANNEL_ID: buffer}
        server.redis_client = MagicMock()
        server.redis_client.hget.return_value = b"connecting"
        server.redis_client.get.return_value = None

        started = time.time() - 11
        self.assertEqual(generator._init_wait_abort_reason(server, started), "stalled")

    @patch("apps.proxy.live_proxy.output.ts.generator.ConfigHelper.channel_init_grace_period", return_value=30)
    def test_no_stall_abort_within_init_grace_period(self, _mock_grace):
        generator = self._make_generator()
        server = MagicMock()
        client_manager = MagicMock()
        client_manager.clients = {"client-1"}
        server.client_managers = {CHANNEL_ID: client_manager}
        buffer = MagicMock()
        buffer.index = 0
        server.stream_buffers = {CHANNEL_ID: buffer}
        server.redis_client = MagicMock()
        server.redis_client.hget.return_value = b"connecting"
        server.redis_client.get.return_value = None

        started = time.time() - 15
        self.assertIsNone(generator._init_wait_abort_reason(server, started))

    @patch("apps.proxy.live_proxy.output.ts.generator.ConfigHelper.channel_init_grace_period", return_value=30)
    def test_stall_abort_after_init_grace_period(self, _mock_grace):
        generator = self._make_generator()
        server = MagicMock()
        client_manager = MagicMock()
        client_manager.clients = {"client-1"}
        server.client_managers = {CHANNEL_ID: client_manager}
        buffer = MagicMock()
        buffer.index = 0
        server.stream_buffers = {CHANNEL_ID: buffer}
        server.redis_client = MagicMock()
        server.redis_client.hget.return_value = b"connecting"
        server.redis_client.get.return_value = None

        started = time.time() - 31
        self.assertEqual(generator._init_wait_abort_reason(server, started), "stalled")


class PromoteChannelWhenBufferReadyTests(TestCase):
    def _mock_proxy(self, redis_client):
        proxy_server = MagicMock()
        proxy_server.redis_client = redis_client
        return patch(
            "apps.proxy.live_proxy.services.channel_service.ProxyServer.get_instance",
            return_value=proxy_server,
        ), proxy_server

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.initial_behind_chunks", return_value=4)
    def test_buffer_ready_with_clients_becomes_active(self, _mock_chunks):
        redis = MagicMock()
        redis.hget.return_value = ChannelState.CONNECTING.encode()
        redis.get.return_value = b"4"
        redis.scard.return_value = 2

        ctx, proxy_server = self._mock_proxy(redis)
        with ctx:
            result = ChannelService.promote_channel_when_buffer_ready(CHANNEL_ID)

        self.assertEqual(result, ChannelState.ACTIVE)
        proxy_server.update_channel_state.assert_called_once()
        args = proxy_server.update_channel_state.call_args[0]
        self.assertEqual(args[1], ChannelState.ACTIVE)
        self.assertEqual(args[2]["clients_at_activation"], "2")

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.initial_behind_chunks", return_value=4)
    def test_buffer_ready_without_clients_becomes_waiting(self, _mock_chunks):
        redis = MagicMock()
        redis.hget.return_value = ChannelState.CONNECTING.encode()
        redis.get.return_value = b"5"
        redis.scard.return_value = 0

        ctx, proxy_server = self._mock_proxy(redis)
        with ctx:
            result = ChannelService.promote_channel_when_buffer_ready(CHANNEL_ID)

        self.assertEqual(result, ChannelState.WAITING_FOR_CLIENTS)
        proxy_server.update_channel_state.assert_called_once_with(
            CHANNEL_ID,
            ChannelState.WAITING_FOR_CLIENTS,
            {
                ChannelMetadataField.CONNECTION_READY_TIME: ANY,
                ChannelMetadataField.BUFFER_CHUNKS: "5",
            },
        )

    @patch("apps.proxy.live_proxy.services.channel_service.ConfigHelper.initial_behind_chunks", return_value=4)
    def test_buffer_not_ready_does_not_promote(self, _mock_chunks):
        redis = MagicMock()
        redis.hget.return_value = ChannelState.CONNECTING.encode()
        redis.get.return_value = b"2"

        ctx, proxy_server = self._mock_proxy(redis)
        with ctx:
            result = ChannelService.promote_channel_when_buffer_ready(CHANNEL_ID)

        self.assertIsNone(result)
        proxy_server.update_channel_state.assert_not_called()

    def test_waiting_for_clients_with_clients_becomes_active(self):
        redis = MagicMock()

        def hget_side_effect(key, field):
            if field == ChannelMetadataField.STATE:
                return ChannelState.WAITING_FOR_CLIENTS.encode()
            if field == ChannelMetadataField.CONNECTION_READY_TIME:
                return b"1700000000.0"
            return None

        redis.hget.side_effect = hget_side_effect
        redis.scard.return_value = 1

        ctx, proxy_server = self._mock_proxy(redis)
        with ctx:
            result = ChannelService.promote_channel_when_buffer_ready(CHANNEL_ID)

        self.assertEqual(result, ChannelState.ACTIVE)
        proxy_server.update_channel_state.assert_called_once_with(
            CHANNEL_ID,
            ChannelState.ACTIVE,
            {"clients_at_activation": "1"},
        )


class UpstreamStopBroadcastTests(TestCase):
    def _make_server(self):
        with patch("apps.proxy.live_proxy.server.RedisClient.get_client", return_value=MagicMock()):
            server = ProxyServer()
        server.worker_id = "testhost:1"
        server.stream_managers = {}
        server._live_stream_managers = {}
        server.redis_client = MagicMock()
        return server

    @patch.object(ProxyServer, "_stop_local_stream_activity")
    @patch.object(ProxyServer, "_broadcast_upstream_stop")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=True)
    def test_local_upstream_stops_locally_without_broadcast(
        self, _mock_has, mock_broadcast, mock_stop_local
    ):
        server = self._make_server()
        server._stop_upstream_before_redis_cleanup(CHANNEL_ID)
        mock_stop_local.assert_called_once_with(CHANNEL_ID)
        mock_broadcast.assert_not_called()

    @patch.object(ProxyServer, "_stop_local_stream_activity")
    @patch.object(ProxyServer, "_broadcast_upstream_stop")
    @patch.object(ProxyServer, "_has_local_upstream_activity", return_value=False)
    def test_orphan_cleanup_broadcasts_when_no_local_upstream(
        self, _mock_has, mock_broadcast, mock_stop_local
    ):
        server = self._make_server()
        server._stop_upstream_before_redis_cleanup(CHANNEL_ID)
        mock_broadcast.assert_called_once_with(CHANNEL_ID)
        mock_stop_local.assert_not_called()


class StreamManagerStillOwnerTests(TestCase):
    def _make_manager(self, redis_client):
        buffer = MagicMock()
        buffer.redis_client = redis_client
        buffer.channel_id = CHANNEL_ID
        manager = StreamManager(
            CHANNEL_ID,
            "http://example/stream.ts",
            buffer,
            worker_id="testhost:1",
        )
        return manager

    def test_stops_when_metadata_removed(self):
        redis = MagicMock()
        _configure_ownership_pipeline(redis, metadata_exists=0)
        manager = self._make_manager(redis)
        self.assertFalse(manager._still_owner())

    def test_keeps_running_during_connecting_before_client_registered(self):
        redis = MagicMock()
        _configure_ownership_pipeline(
            redis,
            client_count=0,
            owner=b"testhost:1",
            state=ChannelState.CONNECTING.encode(),
        )
        manager = self._make_manager(redis)
        self.assertTrue(manager._still_owner())

    def test_stops_after_disconnect_when_shutdown_delay_is_zero(self):
        redis = MagicMock()
        _configure_ownership_pipeline(
            redis,
            client_count=0,
            owner=b"testhost:1",
            disconnect=b"1700000000.0",
            state=ChannelState.ACTIVE.encode(),
        )
        manager = self._make_manager(redis)
        with patch(
            "apps.proxy.live_proxy.input.manager.ConfigHelper.channel_shutdown_delay",
            return_value=0,
        ):
            self.assertFalse(manager._still_owner())

    def test_keeps_running_during_shutdown_delay(self):
        redis = MagicMock()
        _configure_ownership_pipeline(
            redis,
            client_count=0,
            owner=b"testhost:1",
            disconnect=str(time.time()).encode(),
            state=ChannelState.ACTIVE.encode(),
        )
        manager = self._make_manager(redis)
        with patch(
            "apps.proxy.live_proxy.input.manager.ConfigHelper.channel_shutdown_delay",
            return_value=5,
        ):
            self.assertTrue(manager._still_owner())


class PreActiveNoClientsTimeoutTests(TestCase):
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5)
    def test_buffer_ready_uses_client_wait_period(self, _mock_client_wait):
        should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop(
            connection_ready_time=1000.0,
            start_time=900.0,
            now=1006.0,
        )
        self.assertTrue(should_stop)
        self.assertEqual(timeout, 5)
        self.assertEqual(reason, "client_wait")

    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5)
    def test_buffer_ready_within_client_wait_period(self, _mock_client_wait):
        should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop(
            connection_ready_time=1000.0,
            start_time=900.0,
            now=1003.0,
        )
        self.assertFalse(should_stop)
        self.assertEqual(timeout, 5)
        self.assertEqual(reason, "client_wait")

    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_init_grace_period", return_value=60)
    def test_startup_uses_init_grace_period(self, _mock_init_grace):
        should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop(
            connection_ready_time=None,
            start_time=1000.0,
            now=1070.0,
        )
        self.assertTrue(should_stop)
        self.assertEqual(timeout, 60)
        self.assertEqual(reason, "startup")

    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_init_grace_period", return_value=60)
    def test_startup_within_init_grace_period(self, _mock_init_grace):
        should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop(
            connection_ready_time=None,
            start_time=1000.0,
            now=1030.0,
        )
        self.assertFalse(should_stop)
        self.assertEqual(timeout, 60)
        self.assertEqual(reason, "startup")

    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30)
    @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5)
    def test_buffer_ready_does_not_use_shutdown_delay(self, mock_client_wait, mock_shutdown_delay):
        should_stop, _, reason = ProxyServer._pre_active_no_clients_should_stop(
            connection_ready_time=1000.0,
            start_time=900.0,
            now=1006.0,
        )
        self.assertTrue(should_stop)
        self.assertEqual(reason, "client_wait")
        mock_client_wait.assert_called_once()
        mock_shutdown_delay.assert_not_called()