diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df32e44..9afda4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A second Celery worker is now started alongside the default one, configured with `--pool=threads --concurrency=20` and bound to a dedicated `dvr` queue. Threads fit this workload because every blocking call in `run_recording` releases the GIL (sleep, subprocess wait, file / DB / Redis I/O) and FFmpeg itself runs in a separate OS process. A `task_routes` entry in `dispatcharr/celery.py` routes `apps.channels.tasks.run_recording` to the `dvr` queue regardless of how it is dispatched (Celery Beat `PeriodicTask`, `.delay()`, etc.). The default prefork worker now subscribes only to the `celery` queue (`-Q celery -n default@%h --autoscale=6,1`), so recordings can no longer starve background tasks and background tasks can no longer delay recordings. - Net result: up to 20 concurrent recordings, zero startup delay regardless of EPG / M3U refresh activity, and a memory cost of roughly 80 to 120 MB for the second always-on worker process. Each additional concurrent recording costs nearly nothing on top because all 20 threads share that single process. Applied to AIO (`docker/uwsgi.ini`), dev (`docker/uwsgi.dev.ini`), debug (`docker/uwsgi.debug.ini`), and the modular celery container (`docker/entrypoint.celery.sh`). - **Added database index on `Stream.name`**. The stream list default sort is `ORDER BY name`, but the column had no index, causing full table scans that spilled to disk (observed at ~38 MB temp file / ~800 ms) on large stream libraries. `db_index=True` is now set on the field, letting PostgreSQL satisfy the sort via an index scan with no disk spill. (Fixes #1209) +- **Streams table now only refetches what changed**. Loading the Channels page fired `/streams/`, `/streams/ids/`, and `/streams/filter-options/` together via `Promise.all`, and the trio refired on every pagination, sort, or filter change. The IDs list and filter options don't depend on page or sort order, so they were re-pulled needlessly (~440 KB per pagination round trip on a 70k-stream library). `fetchData` was split into a `fetchPageData` for the visible rows (depends on page + sort + filters) plus dedicated effects for the IDs and filter-options endpoints (filters only). Initial-load timing is unchanged (still parallelized), but every subsequent paginate or sort toggle now hits only `/streams/`. ### Changed diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index 0197631b..626e4530 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -202,6 +202,9 @@ const StreamsTable = ({ onReady }) => { const fetchVersionRef = useRef(0); // Track fetch version to prevent stale updates const lastFetchParamsRef = useRef(null); // Track last fetch params to prevent duplicate requests const fetchInProgressRef = useRef(false); // Track if a fetch is currently in progress + const initialDataCountRef = useRef(null); // First page count, kept in a ref so the page fetcher doesn't recreate when set + const lastIdsParamsRef = useRef(null); // De-dupe StrictMode double-fire of the IDs fetch + const lastFilterOptionsParamsRef = useRef(null); // De-dupe StrictMode double-fire of the filter-options fetch // Channel creation modal state (bulk) const [channelNumberingModalOpen, setChannelNumberingModalOpen] = @@ -586,16 +589,30 @@ const StreamsTable = ({ onReady }) => { })); }; - const fetchData = useCallback( + // Build a URLSearchParams object containing only the filter portion of the + // query. Page-rows fetches add page/page_size/ordering on top of this. + const buildFilterParams = useCallback(() => { + const params = new URLSearchParams(); + Object.entries(debouncedFilters).forEach(([key, value]) => { + if (typeof value === 'boolean') { + if (value) params.append(key, 'true'); + } else if (value !== null && value !== undefined && value !== '') { + params.append(key, String(value)); + } + }); + return params; + }, [debouncedFilters]); + + // Fetch the visible page of stream rows. Depends on pagination, sorting, + // and filters. + const fetchPageData = useCallback( async ({ showLoader = true } = {}) => { - const params = new URLSearchParams(); + const params = buildFilterParams(); params.append('page', pagination.pageIndex + 1); params.append('page_size', pagination.pageSize); - // Apply sorting if (sorting.length > 0) { const columnId = sorting[0].id; - // Map frontend column IDs to backend field names const fieldMapping = { name: 'name', group: 'channel_group__name', @@ -607,15 +624,6 @@ const StreamsTable = ({ onReady }) => { params.append('ordering', `${sortDirection}${sortField}`); } - // Apply debounced filters; send boolean filters as 'true' when set - Object.entries(debouncedFilters).forEach(([key, value]) => { - if (typeof value === 'boolean') { - if (value) params.append(key, 'true'); - } else if (value !== null && value !== undefined && value !== '') { - params.append(key, String(value)); - } - }); - const paramsString = params.toString(); // Skip if same fetch is already in progress (prevents StrictMode double-fetch) @@ -626,7 +634,6 @@ const StreamsTable = ({ onReady }) => { return; } - // Increment fetch version to track this specific fetch request const currentFetchVersion = ++fetchVersionRef.current; lastFetchParamsRef.current = paramsString; fetchInProgressRef.current = true; @@ -636,45 +643,19 @@ const StreamsTable = ({ onReady }) => { } try { - const [result, ids, filterOptions] = await Promise.all([ - API.queryStreamsTable(params), - API.getAllStreamIds(params), - API.getStreamFilterOptions(params), - ]); + const result = await API.queryStreamsTable(params); fetchInProgressRef.current = false; - // Skip state updates if a newer fetch has been initiated if (currentFetchVersion !== fetchVersionRef.current) { return; } - setAllRowIds(ids); - - // Set filtered options based on current filters - // Ensure groupOptions is always an array of valid strings - if (filterOptions && typeof filterOptions === 'object') { - setGroupOptions( - (filterOptions.groups || []) - .filter((group) => group != null && group !== '') - .map((group) => String(group)) - ); - // Ensure m3uOptions is always an array of valid objects - setM3uOptions( - (filterOptions.m3u_accounts || []) - .filter((m3u) => m3u && m3u.id != null && m3u.name) - .map((m3u) => ({ - label: String(m3u.name), - value: String(m3u.id), - })) - ); - } - - if (initialDataCount === null) { + if (initialDataCountRef.current === null) { + initialDataCountRef.current = result.count; setInitialDataCount(result.count); } - // Signal that initial data load is complete if (!hasSignaledReady.current && onReady) { hasSignaledReady.current = true; onReady(); @@ -682,14 +663,12 @@ const StreamsTable = ({ onReady }) => { } catch (error) { fetchInProgressRef.current = false; - // Skip logging if a newer fetch has been initiated if (currentFetchVersion !== fetchVersionRef.current) { return; } console.error('Error fetching data:', error); } - // Skip state updates if a newer fetch has been initiated if (currentFetchVersion !== fetchVersionRef.current) { return; } @@ -699,7 +678,7 @@ const StreamsTable = ({ onReady }) => { setIsLoading(false); } }, - [pagination, sorting, debouncedFilters, onReady] + [pagination, sorting, buildFilterParams, onReady] ); // Bulk creation: create channels from selected streams asynchronously @@ -1320,19 +1299,73 @@ const StreamsTable = ({ onReady }) => { * useEffects */ useEffect(() => { - // Load data independently, don't wait for logos or other data - fetchData(); - }, [fetchData]); + // Load page rows independently, don't wait for logos or other data + fetchPageData(); + }, [fetchPageData]); - // Refetch data when video player closes to update stream stats + // The full ID list and filter options only depend on filters, not pagination + // or sort order, so they get their own effects to avoid refetching on every + // page change or sort toggle. + useEffect(() => { + const params = buildFilterParams(); + const paramsString = params.toString(); + if (lastIdsParamsRef.current === paramsString) { + return; + } + lastIdsParamsRef.current = paramsString; + let cancelled = false; + (async () => { + const ids = await API.getAllStreamIds(params); + if (!cancelled && ids) { + setAllRowIds(ids); + } + })(); + return () => { + cancelled = true; + }; + }, [buildFilterParams, setAllRowIds]); + + useEffect(() => { + const params = buildFilterParams(); + const paramsString = params.toString(); + if (lastFilterOptionsParamsRef.current === paramsString) { + return; + } + lastFilterOptionsParamsRef.current = paramsString; + let cancelled = false; + (async () => { + const filterOptions = await API.getStreamFilterOptions(params); + if (cancelled || !filterOptions || typeof filterOptions !== 'object') { + return; + } + setGroupOptions( + (filterOptions.groups || []) + .filter((group) => group != null && group !== '') + .map((group) => String(group)) + ); + setM3uOptions( + (filterOptions.m3u_accounts || []) + .filter((m3u) => m3u && m3u.id != null && m3u.name) + .map((m3u) => ({ + label: String(m3u.name), + value: String(m3u.id), + })) + ); + })(); + return () => { + cancelled = true; + }; + }, [buildFilterParams]); + + // Refetch page rows when video player closes to update stream stats const prevVideoVisible = useRef(false); useEffect(() => { if (prevVideoVisible.current && !videoIsVisible) { // Video was closed, refetch to get updated stream stats - fetchData({ showLoader: false }); + fetchPageData({ showLoader: false }); } prevVideoVisible.current = videoIsVisible; - }, [videoIsVisible, fetchData]); + }, [videoIsVisible, fetchPageData]); useEffect(() => { if (