From 391c3412d2a750bf70bf668dec5e2f01f451c789 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 28 Jun 2026 11:31:21 -0500 Subject: [PATCH] refactor(proxy): Add new client connect grace period setting and update channel initialization grace period defaults. The `channel_client_wait_period` is introduced to manage channel lifetimes before client connections, while the `channel_init_grace_period` default is increased to improve failover handling. --- CHANGELOG.md | 12 ++ apps/channels/tests/test_ts_proxy_teardown.py | 59 ++++++++ apps/proxy/config.py | 16 +- apps/proxy/live_proxy/config_helper.py | 5 + apps/proxy/live_proxy/server.py | 54 ++++--- .../live_proxy/tests/test_proxy_settings.py | 143 ++++++++++++++++++ core/api_views.py | 3 +- .../0026_add_channel_client_wait_period.py | 51 +++++++ core/models.py | 3 +- core/serializers.py | 14 +- .../forms/settings/ProxySettingsForm.jsx | 9 +- frontend/src/constants.js | 7 +- .../forms/settings/ProxySettingsFormUtils.js | 3 +- .../__tests__/ProxySettingsFormUtils.test.js | 4 +- 14 files changed, 349 insertions(+), 34 deletions(-) create mode 100644 apps/proxy/live_proxy/tests/test_proxy_settings.py create mode 100644 core/migrations/0026_add_channel_client_wait_period.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ea23e1dd..03a9bcaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **New proxy setting: Client Connect Grace Period (`channel_client_wait_period`, default 5s).** Restores the grace-period behaviour that was unintentionally dropped in 0.27.1. When a channel's buffer becomes ready but no client has connected yet (e.g. after an API warmup), this controls how long to keep the channel alive before tearing down. + +### Changed + +- **Proxy grace-period settings are now split into three distinct timeouts.** In 0.27.1, `channel_init_grace_period` was repurposed as the channel startup/failover timeout (replacing a hardcoded 10s). That left API-warmup "wait for the first client" behaviour tied to `channel_shutdown_delay` (default 0s), so warmed-up channels stopped almost immediately. Behaviour is now: + - **`channel_init_grace_period` (default 60s, max 300s):** how long the proxy may spend connecting and cycling failover streams before giving up on startup. + - **`channel_client_wait_period` (default 5s):** how long a ready channel with no viewers stays up waiting for the first client (the original grace-period use case). + - **`channel_shutdown_delay` (default 0s):** delay after the last client disconnects only; no longer applies to warmup. +- **Migration 0026 bumps `channel_init_grace_period` to 60s when the stored value is below 60.** Existing installs on the old 5s default (or any custom value under 60) are raised automatically. Values already at 60s or higher are unchanged. If you previously raised `channel_shutdown_delay` to keep warmed-up channels alive, set `channel_client_wait_period` instead. + ## [0.27.1] - 2026-06-25 ### Security diff --git a/apps/channels/tests/test_ts_proxy_teardown.py b/apps/channels/tests/test_ts_proxy_teardown.py index ef78e576..b76918ac 100644 --- a/apps/channels/tests/test_ts_proxy_teardown.py +++ b/apps/channels/tests/test_ts_proxy_teardown.py @@ -923,3 +923,62 @@ class StreamManagerStillOwnerTests(TestCase): return_value=5, ): self.assertTrue(manager._still_owner()) + + +class PreActiveNoClientsTimeoutTests(TestCase): + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5) + def test_buffer_ready_uses_client_wait_period(self, _mock_client_wait): + should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop( + connection_ready_time=1000.0, + start_time=900.0, + now=1006.0, + ) + self.assertTrue(should_stop) + self.assertEqual(timeout, 5) + self.assertEqual(reason, "client_wait") + + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5) + def test_buffer_ready_within_client_wait_period(self, _mock_client_wait): + should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop( + connection_ready_time=1000.0, + start_time=900.0, + now=1003.0, + ) + self.assertFalse(should_stop) + self.assertEqual(timeout, 5) + self.assertEqual(reason, "client_wait") + + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_init_grace_period", return_value=60) + def test_startup_uses_init_grace_period(self, _mock_init_grace): + should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop( + connection_ready_time=None, + start_time=1000.0, + now=1070.0, + ) + self.assertTrue(should_stop) + self.assertEqual(timeout, 60) + self.assertEqual(reason, "startup") + + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_init_grace_period", return_value=60) + def test_startup_within_init_grace_period(self, _mock_init_grace): + should_stop, timeout, reason = ProxyServer._pre_active_no_clients_should_stop( + connection_ready_time=None, + start_time=1000.0, + now=1030.0, + ) + self.assertFalse(should_stop) + self.assertEqual(timeout, 60) + self.assertEqual(reason, "startup") + + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_shutdown_delay", return_value=30) + @patch("apps.proxy.live_proxy.server.ConfigHelper.channel_client_wait_period", return_value=5) + def test_buffer_ready_does_not_use_shutdown_delay(self, mock_client_wait, mock_shutdown_delay): + should_stop, _, reason = ProxyServer._pre_active_no_clients_should_stop( + connection_ready_time=1000.0, + start_time=900.0, + now=1006.0, + ) + self.assertTrue(should_stop) + self.assertEqual(reason, "client_wait") + mock_client_wait.assert_called_once() + mock_shutdown_delay.assert_not_called() diff --git a/apps/proxy/config.py b/apps/proxy/config.py index c7ba5628..48d9fc9b 100644 --- a/apps/proxy/config.py +++ b/apps/proxy/config.py @@ -42,7 +42,8 @@ class BaseConfig: "buffering_speed": 1.0, "redis_chunk_ttl": 60, "channel_shutdown_delay": 0, - "channel_init_grace_period": 5, + "channel_init_grace_period": 60, + "channel_client_wait_period": 5, "new_client_behind_seconds": 5, } @@ -135,7 +136,13 @@ class TSConfig(BaseConfig): def get_channel_init_grace_period(cls): """Max seconds to wait for initial buffer fill during channel startup.""" settings = cls.get_proxy_settings() - return settings.get("channel_init_grace_period", 5) + return settings.get("channel_init_grace_period", 60) + + @classmethod + def get_channel_client_wait_period(cls): + """Seconds to keep a ready channel alive waiting for the first client to connect.""" + settings = cls.get_proxy_settings() + return settings.get("channel_client_wait_period", 5) # Dynamic property access for these settings @property @@ -154,5 +161,6 @@ class TSConfig(BaseConfig): def CHANNEL_INIT_GRACE_PERIOD(self): return self.get_channel_init_grace_period() - - + @property + def CHANNEL_CLIENT_WAIT_PERIOD(self): + return self.get_channel_client_wait_period() diff --git a/apps/proxy/live_proxy/config_helper.py b/apps/proxy/live_proxy/config_helper.py index d151a8a8..846b0153 100644 --- a/apps/proxy/live_proxy/config_helper.py +++ b/apps/proxy/live_proxy/config_helper.py @@ -110,6 +110,11 @@ class ConfigHelper: """Max seconds to wait for initial buffer fill during channel startup.""" return Config.get_channel_init_grace_period() + @staticmethod + def channel_client_wait_period(): + """Seconds to keep a ready channel alive waiting for the first client to connect.""" + return Config.get_channel_client_wait_period() + @staticmethod def chunk_timeout(): """ diff --git a/apps/proxy/live_proxy/server.py b/apps/proxy/live_proxy/server.py index fd54fc24..97c5b68b 100644 --- a/apps/proxy/live_proxy/server.py +++ b/apps/proxy/live_proxy/server.py @@ -910,6 +910,26 @@ class ProxyServer: delay = ConfigHelper.channel_shutdown_delay() return max(int(delay * 2), 60) + @staticmethod + def _pre_active_no_clients_should_stop(connection_ready_time, start_time, now=None): + """ + Decide whether a pre-active channel with zero clients should be stopped. + + Returns (should_stop, timeout_seconds, reason) where reason is + 'client_wait' (buffer ready, waiting for first viewer) or 'startup' + (still connecting / filling buffer). + """ + now = now if now is not None else time.time() + if connection_ready_time: + elapsed = now - connection_ready_time + timeout = ConfigHelper.channel_client_wait_period() + return elapsed > timeout, timeout, "client_wait" + if start_time: + elapsed = now - start_time + timeout = ConfigHelper.channel_init_grace_period() + return elapsed > timeout, timeout, "startup" + return False, None, None + def _wait_for_shutdown_delay(self, channel_id): """ Wait until shutdown_delay has elapsed since the Redis disconnect @@ -1799,31 +1819,27 @@ class ProxyServer: start_time = connection_attempt_time or init_time if start_time: - # Check which timeout to apply based on channel lifecycle - if connection_ready_time: - # Already reached ready - use shutdown_delay - time_since_ready = time.time() - connection_ready_time - shutdown_delay = ConfigHelper.channel_shutdown_delay() - - if time_since_ready > shutdown_delay: + should_stop, timeout, reason = ( + self._pre_active_no_clients_should_stop( + connection_ready_time, + start_time, + ) + ) + if should_stop: + if reason == "client_wait": + time_since_ready = time.time() - connection_ready_time logger.warning( f"Channel {channel_id} in {channel_state} state with 0 clients for {time_since_ready:.1f}s " - f"(after reaching ready, shutdown_delay: {shutdown_delay}s) - stopping channel" + f"(buffer ready, no client connected, client_wait_period: {timeout}s) - stopping channel" ) - self._coordinated_stop_channel(channel_id) - continue - else: - # Never reached ready - use grace_period timeout - time_since_start = time.time() - start_time - connecting_timeout = ConfigHelper.channel_init_grace_period() - - if time_since_start > connecting_timeout: + else: + time_since_start = time.time() - start_time logger.warning( f"Channel {channel_id} stuck in {channel_state} state for {time_since_start:.1f}s " - f"with no clients (timeout: {connecting_timeout}s) - stopping channel due to upstream issues" + f"with no clients (timeout: {timeout}s) - stopping channel due to upstream issues" ) - self._coordinated_stop_channel(channel_id) - continue + self._coordinated_stop_channel(channel_id) + continue elif ( channel_state == ChannelState.WAITING_FOR_CLIENTS and total_clients > 0 diff --git a/apps/proxy/live_proxy/tests/test_proxy_settings.py b/apps/proxy/live_proxy/tests/test_proxy_settings.py new file mode 100644 index 00000000..e821f1cb --- /dev/null +++ b/apps/proxy/live_proxy/tests/test_proxy_settings.py @@ -0,0 +1,143 @@ +"""Tests for proxy settings defaults, serializer validation, and migration 0026.""" + +from importlib import import_module +from unittest.mock import patch + +from django.apps import apps +from django.test import SimpleTestCase, TestCase + +from apps.proxy.config import TSConfig +from core.models import CoreSettings +from core.serializers import ProxySettingsSerializer + +MIGRATION_0026 = import_module("core.migrations.0026_add_channel_client_wait_period") + + +class TSConfigProxySettingsDefaultsTests(SimpleTestCase): + @patch.object(TSConfig, "get_proxy_settings", return_value={}) + def test_channel_init_grace_period_default(self, _mock_settings): + self.assertEqual(TSConfig.get_channel_init_grace_period(), 60) + + @patch.object(TSConfig, "get_proxy_settings", return_value={}) + def test_channel_client_wait_period_default(self, _mock_settings): + self.assertEqual(TSConfig.get_channel_client_wait_period(), 5) + + @patch.object( + TSConfig, + "get_proxy_settings", + return_value={ + "channel_init_grace_period": 120, + "channel_client_wait_period": 15, + }, + ) + def test_settings_override_db_values(self, _mock_settings): + self.assertEqual(TSConfig.get_channel_init_grace_period(), 120) + self.assertEqual(TSConfig.get_channel_client_wait_period(), 15) + + +class ProxySettingsSerializerTests(SimpleTestCase): + def _valid_payload(self, **overrides): + payload = { + "buffering_timeout": 15, + "buffering_speed": 1.0, + "redis_chunk_ttl": 60, + "channel_shutdown_delay": 0, + "channel_init_grace_period": 60, + "channel_client_wait_period": 5, + "new_client_behind_seconds": 5, + } + payload.update(overrides) + return payload + + def test_accepts_new_client_wait_period(self): + serializer = ProxySettingsSerializer(data=self._valid_payload()) + self.assertTrue(serializer.is_valid(), serializer.errors) + self.assertEqual(serializer.validated_data["channel_client_wait_period"], 5) + + def test_init_grace_period_allows_up_to_300(self): + serializer = ProxySettingsSerializer( + data=self._valid_payload(channel_init_grace_period=300) + ) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_init_grace_period_rejects_above_300(self): + serializer = ProxySettingsSerializer( + data=self._valid_payload(channel_init_grace_period=301) + ) + self.assertFalse(serializer.is_valid()) + self.assertIn("channel_init_grace_period", serializer.errors) + + +class CoreSettingsProxyDefaultsTests(TestCase): + def test_get_proxy_settings_defaults_when_missing(self): + CoreSettings.objects.filter(key="proxy_settings").delete() + defaults = CoreSettings.get_proxy_settings() + self.assertEqual(defaults["channel_init_grace_period"], 60) + self.assertEqual(defaults["channel_client_wait_period"], 5) + + +class Migration0026ProxySettingsTests(TestCase): + def _run_migration_forward(self): + MIGRATION_0026.add_channel_client_wait_period(apps, None) + + def _set_proxy_settings(self, value): + settings_obj, _ = CoreSettings.objects.get_or_create( + key="proxy_settings", + defaults={"name": "Proxy Settings", "value": value}, + ) + settings_obj.value = value + settings_obj.save(update_fields=["value"]) + return settings_obj + + def test_bumps_legacy_init_grace_and_adds_client_wait(self): + settings_obj = self._set_proxy_settings( + { + "buffering_timeout": 15, + "buffering_speed": 1.0, + "redis_chunk_ttl": 60, + "channel_shutdown_delay": 0, + "channel_init_grace_period": 5, + "new_client_behind_seconds": 5, + } + ) + + self._run_migration_forward() + settings_obj.refresh_from_db() + + self.assertEqual(settings_obj.value["channel_init_grace_period"], 60) + self.assertEqual(settings_obj.value["channel_client_wait_period"], 5) + + def test_bumps_init_grace_below_new_default(self): + settings_obj = self._set_proxy_settings( + { + "buffering_timeout": 15, + "buffering_speed": 1.0, + "redis_chunk_ttl": 60, + "channel_shutdown_delay": 0, + "channel_init_grace_period": 45, + "new_client_behind_seconds": 5, + } + ) + + self._run_migration_forward() + settings_obj.refresh_from_db() + + self.assertEqual(settings_obj.value["channel_init_grace_period"], 60) + + def test_preserves_init_grace_at_or_above_new_default(self): + settings_obj = self._set_proxy_settings( + { + "buffering_timeout": 15, + "buffering_speed": 1.0, + "redis_chunk_ttl": 60, + "channel_shutdown_delay": 0, + "channel_init_grace_period": 90, + "new_client_behind_seconds": 5, + } + ) + + self._run_migration_forward() + settings_obj.refresh_from_db() + + self.assertEqual(settings_obj.value["channel_init_grace_period"], 90) + self.assertEqual(settings_obj.value["channel_client_wait_period"], 5) diff --git a/core/api_views.py b/core/api_views.py index ec66d69a..ee7d1db1 100644 --- a/core/api_views.py +++ b/core/api_views.py @@ -225,7 +225,8 @@ class ProxySettingsViewSet(viewsets.ViewSet): "buffering_speed": 1.0, "redis_chunk_ttl": 60, "channel_shutdown_delay": 0, - "channel_init_grace_period": 5, + "channel_init_grace_period": 60, + "channel_client_wait_period": 5, "new_client_behind_seconds": 5, } settings_obj, created = CoreSettings.objects.get_or_create( diff --git a/core/migrations/0026_add_channel_client_wait_period.py b/core/migrations/0026_add_channel_client_wait_period.py new file mode 100644 index 00000000..6119eb66 --- /dev/null +++ b/core/migrations/0026_add_channel_client_wait_period.py @@ -0,0 +1,51 @@ +from django.db import migrations + +PROXY_SETTINGS_KEY = "proxy_settings" + + +def add_channel_client_wait_period(apps, schema_editor): + CoreSettings = apps.get_model("core", "CoreSettings") + + try: + obj = CoreSettings.objects.get(key=PROXY_SETTINGS_KEY) + except CoreSettings.DoesNotExist: + return + + value = obj.value if isinstance(obj.value, dict) else {} + + # Add the new client-connect grace period default. + value.setdefault("channel_client_wait_period", 5) + + # channel_init_grace_period was repurposed in 0.27.1 as the channel startup + # timeout (replacing a hardcoded 10s). Values below the new 60s default are + # too short when a channel has many failover streams to cycle through. + current_init = value.get("channel_init_grace_period", 5) + if current_init < 60: + value["channel_init_grace_period"] = 60 + + obj.value = value + obj.save() + + +def remove_channel_client_wait_period(apps, schema_editor): + CoreSettings = apps.get_model("core", "CoreSettings") + + try: + obj = CoreSettings.objects.get(key=PROXY_SETTINGS_KEY) + except CoreSettings.DoesNotExist: + return + + value = obj.value if isinstance(obj.value, dict) else {} + value.pop("channel_client_wait_period", None) + obj.value = value + obj.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0025_move_preferred_region_and_auto_import_to_system_settings"), + ] + + operations = [ + migrations.RunPython(add_channel_client_wait_period, remove_channel_client_wait_period), + ] diff --git a/core/models.py b/core/models.py index 60be2f0e..ad9677c6 100644 --- a/core/models.py +++ b/core/models.py @@ -394,7 +394,8 @@ class CoreSettings(models.Model): "buffering_speed": 1.0, "redis_chunk_ttl": 60, "channel_shutdown_delay": 0, - "channel_init_grace_period": 5, + "channel_init_grace_period": 60, + "channel_client_wait_period": 5, "new_client_behind_seconds": 5, }) diff --git a/core/serializers.py b/core/serializers.py index a0f27baf..1398b194 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -96,7 +96,8 @@ class ProxySettingsSerializer(serializers.Serializer): buffering_speed = serializers.FloatField(min_value=0.1, max_value=10.0) redis_chunk_ttl = serializers.IntegerField(min_value=10, max_value=3600) channel_shutdown_delay = serializers.IntegerField(min_value=0, max_value=300) - channel_init_grace_period = serializers.IntegerField(min_value=0, max_value=60) + channel_init_grace_period = serializers.IntegerField(min_value=0, max_value=300) + channel_client_wait_period = serializers.IntegerField(min_value=0, max_value=300, required=False, default=5) new_client_behind_seconds = serializers.IntegerField(min_value=0, max_value=120, required=False, default=5) def validate_buffering_timeout(self, value): @@ -120,9 +121,16 @@ class ProxySettingsSerializer(serializers.Serializer): return value def validate_channel_init_grace_period(self, value): - if value < 0 or value > 60: + if value < 0 or value > 300: raise serializers.ValidationError( - "Channel initialization timeout must be between 0 and 60 seconds" + "Channel initialization timeout must be between 0 and 300 seconds" + ) + return value + + def validate_channel_client_wait_period(self, value): + if value < 0 or value > 300: + raise serializers.ValidationError( + "Client connect grace period must be between 0 and 300 seconds" ) return value diff --git a/frontend/src/components/forms/settings/ProxySettingsForm.jsx b/frontend/src/components/forms/settings/ProxySettingsForm.jsx index 52440769..a3a9600a 100644 --- a/frontend/src/components/forms/settings/ProxySettingsForm.jsx +++ b/frontend/src/components/forms/settings/ProxySettingsForm.jsx @@ -24,6 +24,7 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { 'redis_chunk_ttl', 'channel_shutdown_delay', 'channel_init_grace_period', + 'channel_client_wait_period', 'new_client_behind_seconds', ].includes(key); }; @@ -37,9 +38,11 @@ const ProxySettingsOptions = React.memo(({ proxySettingsForm }) => { ? 3600 : key === 'channel_shutdown_delay' ? 300 - : key === 'new_client_behind_seconds' - ? 120 - : 60; + : key === 'channel_client_wait_period' + ? 300 + : key === 'new_client_behind_seconds' + ? 120 + : 300; }; return ( <> diff --git a/frontend/src/constants.js b/frontend/src/constants.js index b130553e..fba0d7c6 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -49,13 +49,18 @@ export const PROXY_SETTINGS_OPTIONS = { channel_shutdown_delay: { label: 'Channel Shutdown Delay', description: - 'Delay in seconds before shutting down a channel after last client disconnects', + 'Delay in seconds before shutting down a channel after the last client disconnects', }, channel_init_grace_period: { label: 'Channel Initialization Timeout', description: 'Maximum seconds to wait for the initial buffer to fill while a channel is connecting. Channels that never receive enough buffered data are stopped after this limit.', }, + channel_client_wait_period: { + label: 'Client Connect Grace Period', + description: + 'Seconds to keep a ready channel alive when no client has connected yet (e.g. after an API warmup). Once a client connects the channel becomes active; if none connect within this window the channel stops.', + }, new_client_behind_seconds: { label: 'New Client Buffer (seconds)', description: diff --git a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js index eddbba91..ccbabdeb 100644 --- a/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js +++ b/frontend/src/utils/forms/settings/ProxySettingsFormUtils.js @@ -13,7 +13,8 @@ export const getProxySettingDefaults = () => { buffering_speed: 1.0, redis_chunk_ttl: 60, channel_shutdown_delay: 0, - channel_init_grace_period: 5, + channel_init_grace_period: 60, + channel_client_wait_period: 5, new_client_behind_seconds: 5, }; }; diff --git a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js index 21c48269..3c17c42e 100644 --- a/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js +++ b/frontend/src/utils/forms/settings/__tests__/ProxySettingsFormUtils.test.js @@ -60,7 +60,8 @@ describe('ProxySettingsFormUtils', () => { buffering_speed: 1.0, redis_chunk_ttl: 60, channel_shutdown_delay: 0, - channel_init_grace_period: 5, + channel_init_grace_period: 60, + channel_client_wait_period: 5, new_client_behind_seconds: 5, }); }); @@ -81,6 +82,7 @@ describe('ProxySettingsFormUtils', () => { expect(typeof result.redis_chunk_ttl).toBe('number'); expect(typeof result.channel_shutdown_delay).toBe('number'); expect(typeof result.channel_init_grace_period).toBe('number'); + expect(typeof result.channel_client_wait_period).toBe('number'); expect(typeof result.new_client_behind_seconds).toBe('number'); }); });