From 94c8bd36bfffc6d361ff6f252037ccda9c1854c7 Mon Sep 17 00:00:00 2001 From: Jonathan Caicedo Date: Mon, 25 May 2026 11:12:46 -0400 Subject: [PATCH 1/5] perf: replace per-request DB connections with geventpool --- dispatcharr/settings.py | 12 ++++++++---- pyproject.toml | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index e8818108..1b103ccf 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -111,7 +111,7 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries -DATABASE_CONN_MAX_AGE = 0 # Close after each request; gevent makes per-greenlet connections +DATABASE_CONN_MAX_AGE = None # Managed by django-db-geventpool; None = persistent pool connections # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -221,13 +221,17 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "django_db_geventpool.backends.postgresql", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), "HOST": os.environ.get("POSTGRES_HOST", "localhost"), "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), "CONN_MAX_AGE": DATABASE_CONN_MAX_AGE, + "OPTIONS": { + "MAX_CONNS": 20, # Per-worker pool size; 4 workers × 20 = 80 total < pg max_connections=100 + "REUSE_CONNS": 10, # Connections to keep warm between requests + }, } } @@ -238,9 +242,9 @@ else: ("POSTGRES_SSL_KEY", POSTGRES_SSL_KEY), ], "PostgreSQL") - DATABASES["default"]["OPTIONS"] = { + DATABASES["default"]["OPTIONS"].update({ "sslmode": POSTGRES_SSL_MODE, - } + }) if POSTGRES_SSL_CA_CERT: DATABASES["default"]["OPTIONS"]["sslrootcert"] = POSTGRES_SSL_CA_CERT if POSTGRES_SSL_CERT: diff --git a/pyproject.toml b/pyproject.toml index db7efc78..fe7112f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "yt-dlp", "gevent==26.4.0", "psycogreen", + "django-db-geventpool", "daphne", "uwsgi", "django-cors-headers", From d6f837238f4a1242d7f5c3c8c7f0b40d4e3d3601 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:27:11 -0500 Subject: [PATCH 2/5] Enhancement: Update psycopg from 2 to 3, remove psycogreen as it's no longer needed with psycopg3. --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe7112f3..a003bc5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.13" dynamic = ["version"] dependencies = [ "Django==6.0.5", - "psycopg2-binary==2.9.12", + "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", "requests==2.33.1", @@ -18,7 +18,6 @@ dependencies = [ "python-vlc", "yt-dlp", "gevent==26.4.0", - "psycogreen", "django-db-geventpool", "daphne", "uwsgi", From 80b72d8cfa93b9e430e8ec1317f9064a797dde33 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:27:52 -0500 Subject: [PATCH 3/5] Enhancement: Update psycopg2 to 3. --- core/management/commands/dropdb.py | 7 +++--- core/tests.py | 8 ++++--- dispatcharr/gevent_patch.py | 35 +++++++++--------------------- docker/uwsgi.debug.ini | 4 ++-- docker/uwsgi.dev.ini | 4 ++-- docker/uwsgi.ini | 4 ++-- docker/uwsgi.modular.ini | 4 ++-- 7 files changed, 26 insertions(+), 40 deletions(-) diff --git a/core/management/commands/dropdb.py b/core/management/commands/dropdb.py index d61b9891..09455366 100644 --- a/core/management/commands/dropdb.py +++ b/core/management/commands/dropdb.py @@ -1,6 +1,6 @@ import sys -import psycopg2 -from psycopg2 import sql +import psycopg +from psycopg import sql from django.core.management.base import BaseCommand from django.conf import settings from django.db import connection @@ -38,8 +38,7 @@ class Command(BaseCommand): maintenance_db = 'postgres' try: self.stdout.write("Connecting to maintenance database...") - conn = psycopg2.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, **ssl_kwargs) - conn.autocommit = True + conn = psycopg.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, autocommit=True, **ssl_kwargs) cur = conn.cursor() self.stdout.write(f"Dropping database '{db_name}'...") cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name))) diff --git a/core/tests.py b/core/tests.py index b4770f92..9c758e76 100644 --- a/core/tests.py +++ b/core/tests.py @@ -153,7 +153,7 @@ class EpgIgnoreListsTest(TestCase): class DropDBCommandTlsTest(TestCase): - """Verify dropdb management command passes TLS parameters to psycopg2.""" + """Verify dropdb management command passes TLS parameters to psycopg.""" databases = [] _DB_WITH_TLS = { @@ -184,7 +184,7 @@ class DropDBCommandTlsTest(TestCase): } } - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_passes_ssl_kwargs_when_tls_enabled(self, _inp, _conn, mock_connect): @@ -199,13 +199,14 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, sslmode='verify-full', sslrootcert='/certs/ca.crt', sslcert='/certs/client.crt', sslkey='/certs/client.key', ) - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_no_ssl_kwargs_when_tls_disabled(self, _inp, _conn, mock_connect): @@ -220,4 +221,5 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, ) diff --git a/dispatcharr/gevent_patch.py b/dispatcharr/gevent_patch.py index 86ac92ca..1c9f9727 100644 --- a/dispatcharr/gevent_patch.py +++ b/dispatcharr/gevent_patch.py @@ -1,24 +1,20 @@ """ Loaded via uWSGI's `import = dispatcharr.gevent_patch` directive. -Two things happen here: +gevent stdlib monkey-patching - replaces blocking socket/threading/os +primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` +in the ini should already have done this; calling it again is a safe no-op if +it did, and a necessary fallback if it didn't (e.g. older uWSGI build). -1. gevent stdlib monkey-patching - replaces blocking socket/threading/os - primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` - in the ini should already have done this; calling it again is a safe no-op if - it did, and a necessary fallback if it didn't (e.g. older uWSGI build). - -2. psycogreen - installs a wait callback on psycopg2 so libpq yields to the - gevent hub during I/O instead of blocking the OS thread. - -Without (1), `async_to_sync(channel_layer.group_send)` in send_websocket_update +Without this, `async_to_sync(channel_layer.group_send)` in send_websocket_update calls epoll_wait() directly, which blocks the OS thread and freezes all greenlets -on the worker until the call returns. With (1), select.epoll is replaced by -monkey-patching, which breaks asyncio event loop creation in threadpool threads. +on the worker until the call returns. With monkey-patching, select.epoll is +replaced, which breaks asyncio event loop creation in threadpool threads. send_websocket_update therefore uses a synchronous Redis path in gevent workers instead of asyncio - see _gevent_ws_send() in core/utils.py. -Without (2), psycopg2 network calls pin the worker during slow/stalled queries. +psycopg3 uses Python's socket layer for I/O, so monkey.patch_all() provides +gevent compatibility without any additional driver patching. Celery and Daphne run in separate daemon processes and do not load this module. @@ -40,15 +36,4 @@ if not monkey.is_module_patched("socket"): else: print("[gevent_patch] gevent stdlib monkey-patching already active.", flush=True) -try: - from psycogreen.gevent import patch_psycopg - patch_psycopg() - print("[gevent_patch] psycogreen: psycopg2 patched for gevent.", flush=True) -except ImportError: - print( - "[gevent_patch] WARNING: psycogreen not installed - " - "psycopg2 will block the gevent hub during DB I/O. " - "Run: uv pip install psycogreen", - file=sys.stderr, - flush=True, - ) + diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 9a65db66..d5f577b3 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -42,8 +42,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 852a5114..a481952d 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -45,8 +45,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 76aaebdf..8e8f08dc 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -48,8 +48,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.modular.ini b/docker/uwsgi.modular.ini index 290081b0..7124b10d 100644 --- a/docker/uwsgi.modular.ini +++ b/docker/uwsgi.modular.ini @@ -44,8 +44,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch From a2656184e844f147ffe59c0f58a00203d86619b8 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:31:50 -0500 Subject: [PATCH 4/5] perf/fix: migrate DB driver to psycopg3, fix pool settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch engine to postgresql_psycopg3 (PR had wrong engine name, and we're migrating from psycopg2 to psycopg3 as part of this work) - Set CONN_MAX_AGE=0 (required by geventpool; PR had None) - Reduce MAX_CONNS 20→8, REUSE_CONNS 10→3 to avoid exhausting pg max_connections across 4 uWSGI workers + Celery - Add pool=False to explicitly disable Django's native psycopg3 pool --- dispatcharr/settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 1b103ccf..6a6856db 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -111,7 +111,7 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries -DATABASE_CONN_MAX_AGE = None # Managed by django-db-geventpool; None = persistent pool connections +DATABASE_CONN_MAX_AGE = 0 # geventpool intercepts close(); pool handles reuse # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -221,7 +221,7 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django_db_geventpool.backends.postgresql", + "ENGINE": "django_db_geventpool.backends.postgresql_psycopg3", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), @@ -229,8 +229,9 @@ else: "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), "CONN_MAX_AGE": DATABASE_CONN_MAX_AGE, "OPTIONS": { - "MAX_CONNS": 20, # Per-worker pool size; 4 workers × 20 = 80 total < pg max_connections=100 - "REUSE_CONNS": 10, # Connections to keep warm between requests + "MAX_CONNS": 8, # Per-worker pool size; 4 workers × 8 = 32 total < pg max_connections=100 + "REUSE_CONNS": 3, # Connections to keep warm between requests + "pool": False, # Disable Django's native psycopg3 pool; geventpool manages connections }, } } From f6acbb0c987c53a0c41285f4e44feba1f0c17c16 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Thu, 28 May 2026 19:42:06 -0500 Subject: [PATCH 5/5] changelog: Update changelog for connection pooling pr. --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 406e7b59..0fad82a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Docker base image now uses a multi-stage build.** The builder stage installs compilers and dev headers (`gcc`, `g++`, `gfortran`, `build-essential`, `libopenblas-dev`, `libpcre3-dev`, `python3.13-dev`, `ninja-build`) to create the virtual environment and compile the legacy NumPy wheel (cpu-baseline=none for old hardware). The final runtime image starts from the same ffmpeg base, copies only the prebuilt venv and wheel from the builder, and installs only the runtime libraries needed at runtime - eliminating compiler binaries from production containers and reducing final image size. — Thanks [@kensac](https://github.com/kensac) -- **`libpq-dev` removed from both build and runtime stages.** The project uses `psycopg2-binary`, which bundles its own statically linked copy of libpq inside the wheel. No system libpq headers or shared library are needed at build or runtime. +- **`libpq-dev` removed from both build and runtime stages.** The project uses `psycopg[binary]` (psycopg3), which bundles its own statically linked copy of libpq inside the wheel. No system libpq headers or shared library are needed at build or runtime. +- **Database driver upgraded from `psycopg2` to `psycopg3` (`psycopg[binary]`).** psycopg3 rewrote its network I/O layer in Python, so `gevent`'s `monkey.patch_all()` makes it gevent-cooperative without any additional patching. The `psycogreen` dependency and its driver-patching block in `gevent_patch.py` have been removed. Django's native psycopg3 connection pool is explicitly disabled (`pool: False`) so `django-db-geventpool` remains in sole control of connection lifecycle. The `dropdb` management command was updated to the `psycopg` API (`psycopg.connect`, `psycopg.sql`). ### Performance +- **Database connections are now managed by a persistent per-worker pool.** Previously each request opened a fresh TCP connection to PostgreSQL, paid a full authentication handshake, and closed the connection at request end. `django-db-geventpool` now maintains a pool of warm connections per uWSGI worker; requests borrow a connection and return it when done, eliminating connection-setup overhead on every request. Pool size is bounded (`MAX_CONNS=8` per worker, `REUSE_CONNS=3` warm connections kept idle) to stay comfortably within PostgreSQL's default `max_connections=100` across all uWSGI workers, Celery workers, and Daphne. — Thanks [@JCBird1012](https://github.com/JCBird1012) - **Reduced Redis round-trips on the Stats page channel status endpoint.** `get_basic_channel_info` was making up to 6 individual `HGET` calls per connected client plus a redundant `HGET` for `TOTAL_BYTES` (already present in the preceding `HGETALL` result). Client metadata is now fetched with a single `HMGET` per client, and `TOTAL_BYTES` is read from the already-fetched hash. Under load with many active streams this significantly reduces the time each uWSGI worker holds the GIL servicing the stats endpoint, reducing the chance of concurrent requests from other pages timing out with a 503. The same `HGET`-to-`HMGET` consolidation was applied to `stream_ts` and `get_user_active_connections`. The Stats page frontend was also fixed to fire the initial fetch only once on mount (previously two `useEffect` hooks both triggered an immediate fetch on load). — Thanks [@JCBird1012](https://github.com/JCBird1012) ### Fixed