From 6cd6d3ef9a74b1100b00bf83ef09d0d2889ce0b3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 16 Apr 2026 17:15:54 -0500 Subject: [PATCH] =?UTF-8?q?Enhancement:=20-=20**Output=20bitrate=20DB=20pe?= =?UTF-8?q?rsistence**:=20the=20`ffmpeg=5Foutput=5Fbitrate`=20stat=20is=20?= =?UTF-8?q?no=20longer=20written=20to=20the=20database=20on=20every=20FFmp?= =?UTF-8?q?eg=20stats=20tick=20(~2/second).=20Instead,=20a=20local=20expon?= =?UTF-8?q?ential=20moving=20average=20(EMA,=20=CE=B1=3D0.1)=20accumulates?= =?UTF-8?q?=20readings=20continuously.=20The=20first=2010=20samples=20(~5?= =?UTF-8?q?=20seconds)=20are=20discarded=20as=20warmup=20to=20avoid=20poll?= =?UTF-8?q?uting=20the=20average=20with=20FFmpeg's=20unstable=20ramp-up=20?= =?UTF-8?q?values.=20After=20warmup,=20the=20smoothed=20value=20is=20flush?= =?UTF-8?q?ed=20to=20the=20database=20at=20most=20once=20every=2030=20seco?= =?UTF-8?q?nds,=20and=20a=20final=20flush=20occurs=20when=20the=20stream?= =?UTF-8?q?=20stops=20but=20only=20if=20the=20EMA=20has=20been=20seeded=20?= =?UTF-8?q?(i.e.=20the=20stream=20ran=20past=20warmup).=20Streams=20that?= =?UTF-8?q?=20stop=20during=20warmup=20leave=20the=20existing=20database?= =?UTF-8?q?=20value=20untouched,=20preserving=20previously=20accurate=20me?= =?UTF-8?q?asurements=20when=20channel-hopping.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + apps/proxy/ts_proxy/stream_manager.py | 55 ++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b45dfa5..231af9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Output bitrate DB persistence**: the `ffmpeg_output_bitrate` stat is no longer written to the database on every FFmpeg stats tick (~2/second). Instead, a local exponential moving average (EMA, α=0.1) accumulates readings continuously. The first 10 samples (~5 seconds) are discarded as warmup to avoid polluting the average with FFmpeg's unstable ramp-up values. After warmup, the smoothed value is flushed to the database at most once every 30 seconds, and a final flush occurs when the stream stops but only if the EMA has been seeded (i.e. the stream ran past warmup). Streams that stop during warmup leave the existing database value untouched, preserving previously accurate measurements when channel-hopping. - Performance: `generate_m3u`, `generate_epg`, and `xc_get_live_streams` now use `select_related('channel_group', 'logo')` (or `select_related('logo')` for EPG) on every Channel queryset in `apps/output/views.py`. Previously each channel in the loop triggered a separate database query for its `logo` and `channel_group` foreign keys; with the JOIN-based prefetch this is reduced to a single query per request. On deployments with ~2 000 channels, `xc_get_live_streams` response time drops from ~2.5–4 s to ~250–450 ms. (Closes #1127) — Thanks [@xBOBxSAGETx](https://github.com/xBOBxSAGETx) - Performance: `generate_epg` now uses `select_related('epg_data__epg_source')` on all EPG channel querysets, eliminating N+1 database queries for `EPGSource` traversal per channel (~15 s improvement on ~2000-channel deployments; total EPG generation time dropped from ~87 s to ~72 s in benchmarks). - Performance: `xc_get_epg` now uses `select_related('epg_data__epg_source')` on all three channel fetch paths. Previously each request triggered 2 additional queries to resolve `channel.epg_data` and `channel.epg_data.epg_source`. diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index 7b07c744..d22fc7d0 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -123,6 +123,12 @@ class StreamManager: # Add HTTP reader thread property self.http_reader = None + # Output bitrate smoothing / throttled DB persistence + self._smoothed_output_bitrate = None + self._last_bitrate_db_save_time = 0 + self._bitrate_db_save_interval = 30 # seconds between DB writes + self._bitrate_warmup_samples = 10 # discard first N samples while EMA stabilizes (~5s) + def _create_session(self): """Create and configure requests session with optimal settings""" session = requests.Session() @@ -773,13 +779,25 @@ class StreamManager: if any(x is not None for x in [ffmpeg_speed, ffmpeg_fps, actual_fps, ffmpeg_output_bitrate]): self._update_ffmpeg_stats_in_redis(ffmpeg_speed, ffmpeg_fps, actual_fps, ffmpeg_output_bitrate) - # Also save ffmpeg_output_bitrate to database if we have stream_id + # Update local EMA and periodically flush to database if ffmpeg_output_bitrate is not None and self.current_stream_id: - from .services.channel_service import ChannelService - ChannelService._update_stream_stats_in_db( - self.current_stream_id, - ffmpeg_output_bitrate=ffmpeg_output_bitrate - ) + if self._bitrate_warmup_samples > 0: + # Discard early samples from the EMA + self._bitrate_warmup_samples -= 1 + else: + if self._smoothed_output_bitrate is None: + self._smoothed_output_bitrate = ffmpeg_output_bitrate + else: + self._smoothed_output_bitrate = 0.9 * self._smoothed_output_bitrate + 0.1 * ffmpeg_output_bitrate + + now = time.time() + if now - self._last_bitrate_db_save_time >= self._bitrate_db_save_interval: + from .services.channel_service import ChannelService + ChannelService._update_stream_stats_in_db( + self.current_stream_id, + ffmpeg_output_bitrate=round(self._smoothed_output_bitrate, 1) + ) + self._last_bitrate_db_save_time = now # Fix the f-string formatting actual_fps_str = f"{actual_fps:.1f}" if actual_fps is not None else "N/A" @@ -1057,6 +1075,20 @@ class StreamManager: # Set running to false to ensure thread exits self.running = False + # Flush the final bitrate to DB on stop only if warmup completed and we have + # a meaningful EMA. Short previews / channel hops that die during warmup do NOT + # write anything, preserving any previously correct value in the database. + if self._smoothed_output_bitrate is not None and self.current_stream_id: + final_bitrate = self._smoothed_output_bitrate + try: + from .services.channel_service import ChannelService + ChannelService._update_stream_stats_in_db( + self.current_stream_id, + ffmpeg_output_bitrate=round(final_bitrate, 1) + ) + except Exception as e: + logger.debug(f"Error flushing final bitrate to DB for channel {self.channel_id}: {e}") + def update_url(self, new_url, stream_id=None, m3u_profile_id=None): """Update stream URL and reconnect with proper cleanup for both HTTP and transcode sessions""" if new_url == self.url: @@ -1114,6 +1146,11 @@ class StreamManager: self.url = new_url self.connected = False + # Reset bitrate EMA on every URL change so stale data never carries over + self._smoothed_output_bitrate = None + self._last_bitrate_db_save_time = 0 + self._bitrate_warmup_samples = 10 + # Update stream ID if provided if stream_id: old_stream_id = self.current_stream_id @@ -1151,7 +1188,7 @@ class StreamManager: logger.error(f"Error during URL update for channel {self.channel_id}: {e}", exc_info=True) return False finally: - # CRITICAL FIX: Always reset the URL switching flag when done, whether successful or not + # Always reset the URL switching flag when done, whether successful or not self.url_switching = False logger.info(f"Stream switch completed for channel {self.channel_id}") @@ -1656,7 +1693,7 @@ class StreamManager: new_user_agent = stream_info['user_agent'] new_transcode = stream_info['transcode'] - # CRITICAL FIX: Check if the new URL is the same as current URL + # Check if the new URL is the same as current URL # This can happen when current_stream_id is None and we accidentally select the same stream if new_url == self.url: logger.warning(f"Stream ID {stream_id} generates the same URL as current stream ({new_url}). " @@ -1665,7 +1702,7 @@ class StreamManager: logger.info(f"Switching from URL {self.url} to {new_url} for channel {self.channel_id}") - # IMPORTANT: Just update the URL, don't stop the channel or release resources + # Just update the URL, don't stop the channel or release resources switch_result = self.update_url(new_url, stream_id, profile_id) if not switch_result: logger.error(f"Failed to update URL for stream ID {stream_id} for channel {self.channel_id}")