Enhancement: Optimize StreamsTable data fetching by reducing unnecessary requests and improving filter handling

CHANGELOG.md

@@ -63,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - A second Celery worker is now started alongside the default one, configured with `--pool=threads --concurrency=20` and bound to a dedicated `dvr` queue. Threads fit this workload because every blocking call in `run_recording` releases the GIL (sleep, subprocess wait, file / DB / Redis I/O) and FFmpeg itself runs in a separate OS process. A `task_routes` entry in `dispatcharr/celery.py` routes `apps.channels.tasks.run_recording` to the `dvr` queue regardless of how it is dispatched (Celery Beat `PeriodicTask`, `.delay()`, etc.). The default prefork worker now subscribes only to the `celery` queue (`-Q celery -n default@%h --autoscale=6,1`), so recordings can no longer starve background tasks and background tasks can no longer delay recordings.
   - Net result: up to 20 concurrent recordings, zero startup delay regardless of EPG / M3U refresh activity, and a memory cost of roughly 80 to 120 MB for the second always-on worker process. Each additional concurrent recording costs nearly nothing on top because all 20 threads share that single process. Applied to AIO (`docker/uwsgi.ini`), dev (`docker/uwsgi.dev.ini`), debug (`docker/uwsgi.debug.ini`), and the modular celery container (`docker/entrypoint.celery.sh`).
 - **Added database index on `Stream.name`**. The stream list default sort is `ORDER BY name`, but the column had no index, causing full table scans that spilled to disk (observed at ~38 MB temp file / ~800 ms) on large stream libraries. `db_index=True` is now set on the field, letting PostgreSQL satisfy the sort via an index scan with no disk spill. (Fixes #1209)
+- **Streams table now only refetches what changed**. Loading the Channels page fired `/streams/`, `/streams/ids/`, and `/streams/filter-options/` together via `Promise.all`, and the trio refired on every pagination, sort, or filter change. The IDs list and filter options don't depend on page or sort order, so they were re-pulled needlessly (~440 KB per pagination round trip on a 70k-stream library). `fetchData` was split into a `fetchPageData` for the visible rows (depends on page + sort + filters) plus dedicated effects for the IDs and filter-options endpoints (filters only). Initial-load timing is unchanged (still parallelized), but every subsequent paginate or sort toggle now hits only `/streams/`.
 
 ### Changed
 

frontend/src/components/tables/StreamsTable.jsx

@@ -202,6 +202,9 @@ const StreamsTable = ({ onReady }) => {
   const fetchVersionRef = useRef(0); // Track fetch version to prevent stale updates
   const lastFetchParamsRef = useRef(null); // Track last fetch params to prevent duplicate requests
   const fetchInProgressRef = useRef(false); // Track if a fetch is currently in progress
+  const initialDataCountRef = useRef(null); // First page count, kept in a ref so the page fetcher doesn't recreate when set
+  const lastIdsParamsRef = useRef(null); // De-dupe StrictMode double-fire of the IDs fetch
+  const lastFilterOptionsParamsRef = useRef(null); // De-dupe StrictMode double-fire of the filter-options fetch
 
   // Channel creation modal state (bulk)
   const [channelNumberingModalOpen, setChannelNumberingModalOpen] =
@@ -586,16 +589,30 @@ const StreamsTable = ({ onReady }) => {
     }));
   };
 
-  const fetchData = useCallback(
+  // Build a URLSearchParams object containing only the filter portion of the
+  // query. Page-rows fetches add page/page_size/ordering on top of this.
+  const buildFilterParams = useCallback(() => {
+    const params = new URLSearchParams();
+    Object.entries(debouncedFilters).forEach(([key, value]) => {
+      if (typeof value === 'boolean') {
+        if (value) params.append(key, 'true');
+      } else if (value !== null && value !== undefined && value !== '') {
+        params.append(key, String(value));
+      }
+    });
+    return params;
+  }, [debouncedFilters]);
+
+  // Fetch the visible page of stream rows. Depends on pagination, sorting,
+  // and filters.
+  const fetchPageData = useCallback(
     async ({ showLoader = true } = {}) => {
-      const params = new URLSearchParams();
+      const params = buildFilterParams();
       params.append('page', pagination.pageIndex + 1);
       params.append('page_size', pagination.pageSize);
 
-      // Apply sorting
       if (sorting.length > 0) {
         const columnId = sorting[0].id;
-        // Map frontend column IDs to backend field names
         const fieldMapping = {
           name: 'name',
           group: 'channel_group__name',
@@ -607,15 +624,6 @@ const StreamsTable = ({ onReady }) => {
         params.append('ordering', `${sortDirection}${sortField}`);
       }
 
-      // Apply debounced filters; send boolean filters as 'true' when set
-      Object.entries(debouncedFilters).forEach(([key, value]) => {
-        if (typeof value === 'boolean') {
-          if (value) params.append(key, 'true');
-        } else if (value !== null && value !== undefined && value !== '') {
-          params.append(key, String(value));
-        }
-      });
-
       const paramsString = params.toString();
 
       // Skip if same fetch is already in progress (prevents StrictMode double-fetch)
@@ -626,7 +634,6 @@ const StreamsTable = ({ onReady }) => {
         return;
       }
 
-      // Increment fetch version to track this specific fetch request
       const currentFetchVersion = ++fetchVersionRef.current;
       lastFetchParamsRef.current = paramsString;
       fetchInProgressRef.current = true;
@@ -636,45 +643,19 @@ const StreamsTable = ({ onReady }) => {
       }
 
       try {
-        const [result, ids, filterOptions] = await Promise.all([
-          API.queryStreamsTable(params),
-          API.getAllStreamIds(params),
-          API.getStreamFilterOptions(params),
-        ]);
+        const result = await API.queryStreamsTable(params);
 
         fetchInProgressRef.current = false;
 
-        // Skip state updates if a newer fetch has been initiated
         if (currentFetchVersion !== fetchVersionRef.current) {
           return;
         }
 
-        setAllRowIds(ids);
-
-        // Set filtered options based on current filters
-        // Ensure groupOptions is always an array of valid strings
-        if (filterOptions && typeof filterOptions === 'object') {
-          setGroupOptions(
-            (filterOptions.groups || [])
-              .filter((group) => group != null && group !== '')
-              .map((group) => String(group))
-          );
-          // Ensure m3uOptions is always an array of valid objects
-          setM3uOptions(
-            (filterOptions.m3u_accounts || [])
-              .filter((m3u) => m3u && m3u.id != null && m3u.name)
-              .map((m3u) => ({
-                label: String(m3u.name),
-                value: String(m3u.id),
-              }))
-          );
-        }
-
-        if (initialDataCount === null) {
+        if (initialDataCountRef.current === null) {
+          initialDataCountRef.current = result.count;
           setInitialDataCount(result.count);
         }
 
-        // Signal that initial data load is complete
         if (!hasSignaledReady.current && onReady) {
           hasSignaledReady.current = true;
           onReady();
@@ -682,14 +663,12 @@ const StreamsTable = ({ onReady }) => {
       } catch (error) {
         fetchInProgressRef.current = false;
 
-        // Skip logging if a newer fetch has been initiated
         if (currentFetchVersion !== fetchVersionRef.current) {
           return;
         }
         console.error('Error fetching data:', error);
       }
 
-      // Skip state updates if a newer fetch has been initiated
       if (currentFetchVersion !== fetchVersionRef.current) {
         return;
       }
@@ -699,7 +678,7 @@ const StreamsTable = ({ onReady }) => {
         setIsLoading(false);
       }
     },
-    [pagination, sorting, debouncedFilters, onReady]
+    [pagination, sorting, buildFilterParams, onReady]
   );
 
   // Bulk creation: create channels from selected streams asynchronously
@@ -1320,19 +1299,73 @@ const StreamsTable = ({ onReady }) => {
    * useEffects
    */
   useEffect(() => {
-    // Load data independently, don't wait for logos or other data
-    fetchData();
-  }, [fetchData]);
+    // Load page rows independently, don't wait for logos or other data
+    fetchPageData();
+  }, [fetchPageData]);
 
-  // Refetch data when video player closes to update stream stats
+  // The full ID list and filter options only depend on filters, not pagination
+  // or sort order, so they get their own effects to avoid refetching on every
+  // page change or sort toggle.
+  useEffect(() => {
+    const params = buildFilterParams();
+    const paramsString = params.toString();
+    if (lastIdsParamsRef.current === paramsString) {
+      return;
+    }
+    lastIdsParamsRef.current = paramsString;
+    let cancelled = false;
+    (async () => {
+      const ids = await API.getAllStreamIds(params);
+      if (!cancelled && ids) {
+        setAllRowIds(ids);
+      }
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [buildFilterParams, setAllRowIds]);
+
+  useEffect(() => {
+    const params = buildFilterParams();
+    const paramsString = params.toString();
+    if (lastFilterOptionsParamsRef.current === paramsString) {
+      return;
+    }
+    lastFilterOptionsParamsRef.current = paramsString;
+    let cancelled = false;
+    (async () => {
+      const filterOptions = await API.getStreamFilterOptions(params);
+      if (cancelled || !filterOptions || typeof filterOptions !== 'object') {
+        return;
+      }
+      setGroupOptions(
+        (filterOptions.groups || [])
+          .filter((group) => group != null && group !== '')
+          .map((group) => String(group))
+      );
+      setM3uOptions(
+        (filterOptions.m3u_accounts || [])
+          .filter((m3u) => m3u && m3u.id != null && m3u.name)
+          .map((m3u) => ({
+            label: String(m3u.name),
+            value: String(m3u.id),
+          }))
+      );
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [buildFilterParams]);
+
+  // Refetch page rows when video player closes to update stream stats
   const prevVideoVisible = useRef(false);
   useEffect(() => {
     if (prevVideoVisible.current && !videoIsVisible) {
       // Video was closed, refetch to get updated stream stats
-      fetchData({ showLoader: false });
+      fetchPageData({ showLoader: false });
     }
     prevVideoVisible.current = videoIsVisible;
-  }, [videoIsVisible, fetchData]);
+  }, [videoIsVisible, fetchPageData]);
 
   useEffect(() => {
     if (