diff --git a/CHANGELOG.md b/CHANGELOG.md index 406e7b59..0fad82a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Docker base image now uses a multi-stage build.** The builder stage installs compilers and dev headers (`gcc`, `g++`, `gfortran`, `build-essential`, `libopenblas-dev`, `libpcre3-dev`, `python3.13-dev`, `ninja-build`) to create the virtual environment and compile the legacy NumPy wheel (cpu-baseline=none for old hardware). The final runtime image starts from the same ffmpeg base, copies only the prebuilt venv and wheel from the builder, and installs only the runtime libraries needed at runtime - eliminating compiler binaries from production containers and reducing final image size. — Thanks [@kensac](https://github.com/kensac) -- **`libpq-dev` removed from both build and runtime stages.** The project uses `psycopg2-binary`, which bundles its own statically linked copy of libpq inside the wheel. No system libpq headers or shared library are needed at build or runtime. +- **`libpq-dev` removed from both build and runtime stages.** The project uses `psycopg[binary]` (psycopg3), which bundles its own statically linked copy of libpq inside the wheel. No system libpq headers or shared library are needed at build or runtime. +- **Database driver upgraded from `psycopg2` to `psycopg3` (`psycopg[binary]`).** psycopg3 rewrote its network I/O layer in Python, so `gevent`'s `monkey.patch_all()` makes it gevent-cooperative without any additional patching. The `psycogreen` dependency and its driver-patching block in `gevent_patch.py` have been removed. Django's native psycopg3 connection pool is explicitly disabled (`pool: False`) so `django-db-geventpool` remains in sole control of connection lifecycle. The `dropdb` management command was updated to the `psycopg` API (`psycopg.connect`, `psycopg.sql`). ### Performance +- **Database connections are now managed by a persistent per-worker pool.** Previously each request opened a fresh TCP connection to PostgreSQL, paid a full authentication handshake, and closed the connection at request end. `django-db-geventpool` now maintains a pool of warm connections per uWSGI worker; requests borrow a connection and return it when done, eliminating connection-setup overhead on every request. Pool size is bounded (`MAX_CONNS=8` per worker, `REUSE_CONNS=3` warm connections kept idle) to stay comfortably within PostgreSQL's default `max_connections=100` across all uWSGI workers, Celery workers, and Daphne. — Thanks [@JCBird1012](https://github.com/JCBird1012) - **Reduced Redis round-trips on the Stats page channel status endpoint.** `get_basic_channel_info` was making up to 6 individual `HGET` calls per connected client plus a redundant `HGET` for `TOTAL_BYTES` (already present in the preceding `HGETALL` result). Client metadata is now fetched with a single `HMGET` per client, and `TOTAL_BYTES` is read from the already-fetched hash. Under load with many active streams this significantly reduces the time each uWSGI worker holds the GIL servicing the stats endpoint, reducing the chance of concurrent requests from other pages timing out with a 503. The same `HGET`-to-`HMGET` consolidation was applied to `stream_ts` and `get_user_active_connections`. The Stats page frontend was also fixed to fire the initial fetch only once on mount (previously two `useEffect` hooks both triggered an immediate fetch on load). — Thanks [@JCBird1012](https://github.com/JCBird1012) ### Fixed diff --git a/core/management/commands/dropdb.py b/core/management/commands/dropdb.py index d61b9891..09455366 100644 --- a/core/management/commands/dropdb.py +++ b/core/management/commands/dropdb.py @@ -1,6 +1,6 @@ import sys -import psycopg2 -from psycopg2 import sql +import psycopg +from psycopg import sql from django.core.management.base import BaseCommand from django.conf import settings from django.db import connection @@ -38,8 +38,7 @@ class Command(BaseCommand): maintenance_db = 'postgres' try: self.stdout.write("Connecting to maintenance database...") - conn = psycopg2.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, **ssl_kwargs) - conn.autocommit = True + conn = psycopg.connect(dbname=maintenance_db, user=user, password=password, host=host, port=port, autocommit=True, **ssl_kwargs) cur = conn.cursor() self.stdout.write(f"Dropping database '{db_name}'...") cur.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name))) diff --git a/core/tests.py b/core/tests.py index b4770f92..9c758e76 100644 --- a/core/tests.py +++ b/core/tests.py @@ -153,7 +153,7 @@ class EpgIgnoreListsTest(TestCase): class DropDBCommandTlsTest(TestCase): - """Verify dropdb management command passes TLS parameters to psycopg2.""" + """Verify dropdb management command passes TLS parameters to psycopg.""" databases = [] _DB_WITH_TLS = { @@ -184,7 +184,7 @@ class DropDBCommandTlsTest(TestCase): } } - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_passes_ssl_kwargs_when_tls_enabled(self, _inp, _conn, mock_connect): @@ -199,13 +199,14 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, sslmode='verify-full', sslrootcert='/certs/ca.crt', sslcert='/certs/client.crt', sslkey='/certs/client.key', ) - @patch('core.management.commands.dropdb.psycopg2.connect') + @patch('core.management.commands.dropdb.psycopg.connect') @patch('core.management.commands.dropdb.connection') @patch('builtins.input', return_value='yes') def test_dropdb_no_ssl_kwargs_when_tls_disabled(self, _inp, _conn, mock_connect): @@ -220,4 +221,5 @@ class DropDBCommandTlsTest(TestCase): mock_connect.assert_called_once_with( dbname='postgres', user='testuser', password='testpass', host='localhost', port=5432, + autocommit=True, ) diff --git a/dispatcharr/gevent_patch.py b/dispatcharr/gevent_patch.py index 86ac92ca..1c9f9727 100644 --- a/dispatcharr/gevent_patch.py +++ b/dispatcharr/gevent_patch.py @@ -1,24 +1,20 @@ """ Loaded via uWSGI's `import = dispatcharr.gevent_patch` directive. -Two things happen here: +gevent stdlib monkey-patching - replaces blocking socket/threading/os +primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` +in the ini should already have done this; calling it again is a safe no-op if +it did, and a necessary fallback if it didn't (e.g. older uWSGI build). -1. gevent stdlib monkey-patching - replaces blocking socket/threading/os - primitives with gevent-cooperative versions. `gevent-early-monkey-patch = true` - in the ini should already have done this; calling it again is a safe no-op if - it did, and a necessary fallback if it didn't (e.g. older uWSGI build). - -2. psycogreen - installs a wait callback on psycopg2 so libpq yields to the - gevent hub during I/O instead of blocking the OS thread. - -Without (1), `async_to_sync(channel_layer.group_send)` in send_websocket_update +Without this, `async_to_sync(channel_layer.group_send)` in send_websocket_update calls epoll_wait() directly, which blocks the OS thread and freezes all greenlets -on the worker until the call returns. With (1), select.epoll is replaced by -monkey-patching, which breaks asyncio event loop creation in threadpool threads. +on the worker until the call returns. With monkey-patching, select.epoll is +replaced, which breaks asyncio event loop creation in threadpool threads. send_websocket_update therefore uses a synchronous Redis path in gevent workers instead of asyncio - see _gevent_ws_send() in core/utils.py. -Without (2), psycopg2 network calls pin the worker during slow/stalled queries. +psycopg3 uses Python's socket layer for I/O, so monkey.patch_all() provides +gevent compatibility without any additional driver patching. Celery and Daphne run in separate daemon processes and do not load this module. @@ -40,15 +36,4 @@ if not monkey.is_module_patched("socket"): else: print("[gevent_patch] gevent stdlib monkey-patching already active.", flush=True) -try: - from psycogreen.gevent import patch_psycopg - patch_psycopg() - print("[gevent_patch] psycogreen: psycopg2 patched for gevent.", flush=True) -except ImportError: - print( - "[gevent_patch] WARNING: psycogreen not installed - " - "psycopg2 will block the gevent hub during DB I/O. " - "Run: uv pip install psycogreen", - file=sys.stderr, - flush=True, - ) + diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index e8818108..6a6856db 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -111,7 +111,7 @@ XC_PROFILE_REFRESH_DELAY = float(os.environ.get('XC_PROFILE_REFRESH_DELAY', '2.5 # Database optimization settings DATABASE_STATEMENT_TIMEOUT = 300 # Seconds before timing out long-running queries -DATABASE_CONN_MAX_AGE = 0 # Close after each request; gevent makes per-greenlet connections +DATABASE_CONN_MAX_AGE = 0 # geventpool intercepts close(); pool handles reuse # Disable atomic requests for performance-sensitive views ATOMIC_REQUESTS = False @@ -221,13 +221,18 @@ if os.getenv("DB_ENGINE", None) == "sqlite": else: DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql", + "ENGINE": "django_db_geventpool.backends.postgresql_psycopg3", "NAME": os.environ.get("POSTGRES_DB", "dispatcharr"), "USER": os.environ.get("POSTGRES_USER", "dispatch"), "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "secret"), "HOST": os.environ.get("POSTGRES_HOST", "localhost"), "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), "CONN_MAX_AGE": DATABASE_CONN_MAX_AGE, + "OPTIONS": { + "MAX_CONNS": 8, # Per-worker pool size; 4 workers × 8 = 32 total < pg max_connections=100 + "REUSE_CONNS": 3, # Connections to keep warm between requests + "pool": False, # Disable Django's native psycopg3 pool; geventpool manages connections + }, } } @@ -238,9 +243,9 @@ else: ("POSTGRES_SSL_KEY", POSTGRES_SSL_KEY), ], "PostgreSQL") - DATABASES["default"]["OPTIONS"] = { + DATABASES["default"]["OPTIONS"].update({ "sslmode": POSTGRES_SSL_MODE, - } + }) if POSTGRES_SSL_CA_CERT: DATABASES["default"]["OPTIONS"]["sslrootcert"] = POSTGRES_SSL_CA_CERT if POSTGRES_SSL_CERT: diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 9a65db66..d5f577b3 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -42,8 +42,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 852a5114..a481952d 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -45,8 +45,8 @@ gevent = 100 # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 76aaebdf..8e8f08dc 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -48,8 +48,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/docker/uwsgi.modular.ini b/docker/uwsgi.modular.ini index 290081b0..7124b10d 100644 --- a/docker/uwsgi.modular.ini +++ b/docker/uwsgi.modular.ini @@ -44,8 +44,8 @@ gevent = 400 # Each unused greenlet costs ~2-4KB of memory # Patch the stdlib (socket, threading, time, ...) before any app code # loads so blocking calls yield to the gevent hub. Without this, a single -# blocking psycopg2 / requests / DNS call freezes every greenlet on the -# worker. The companion module greens psycopg2 specifically. +# blocking requests / DNS call freezes every greenlet on the +# worker. psycopg3 uses Python's socket layer, so no additional patching is needed. gevent-early-monkey-patch = true import = dispatcharr.gevent_patch diff --git a/pyproject.toml b/pyproject.toml index db7efc78..a003bc5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.13" dynamic = ["version"] dependencies = [ "Django==6.0.5", - "psycopg2-binary==2.9.12", + "psycopg[binary]", "celery[redis]==5.6.3", "djangorestframework==3.17.1", "requests==2.33.1", @@ -18,7 +18,7 @@ dependencies = [ "python-vlc", "yt-dlp", "gevent==26.4.0", - "psycogreen", + "django-db-geventpool", "daphne", "uwsgi", "django-cors-headers",