fix: replace fork()-based subprocess with posix_spawn at all uWSGI spawn sites
Some checks failed
CI Pipeline / prepare (push) Waiting to run
CI Pipeline / docker (amd64, ubuntu-24.04) (push) Blocked by required conditions
CI Pipeline / docker (arm64, ubuntu-24.04-arm) (push) Blocked by required conditions
CI Pipeline / create-manifest (push) Blocked by required conditions
Build and Push Multi-Arch Docker Image / build-and-push (push) Waiting to run
Frontend Tests / test (push) Waiting to run
Base Image Build / prepare (push) Has been cancelled
Base Image Build / docker (amd64, ubuntu-24.04) (push) Has been cancelled
Base Image Build / docker (arm64, ubuntu-24.04-arm) (push) Has been cancelled
Base Image Build / create-manifest (push) Has been cancelled

gevent registers a pthread_atfork handler that never yields. Any
subprocess.Popen/run call in a uWSGI greenlet hangs indefinitely at
fork() before the child process even starts.

- dispatcharr/gevent_patch.py: new; monkey.patch_all() + psycogreen,
  loaded via uWSGI import= in all four ini files
- live_proxy/input/manager.py: posix_spawn + _SpawnedProcess; removed
  dead forwarder code
- live_proxy/input/http_streamer.py: O_NONBLOCK on relay pipe + EAGAIN
  retry (blocking write to a full pipe stalls the gevent hub)
- live_proxy/utils.py: new posix_spawn_proc() helper with O_NONBLOCK
  stdin, shared by both output managers
- live_proxy/output/fmp4/manager.py, output/profile/manager.py:
  posix_spawn_proc(); _write_all() treats EAGAIN (None) as cooperative
  select.select wait instead of fatal error
- core/views.py (stream_view): posix_spawn; fixed pre-existing bug
  where return StreamingHttpResponse(...) was indented inside
  stream_generator, making the success path always return None
- connect/handlers/script.py: _posix_run() with posix_spawn +
  cooperative select reads + non-blocking waitpid; fixes deadlock when
  a script integration fires on a uWSGI-context event (client_connect
  fires in the live proxy, not Celery)
This commit is contained in:
SergeantPanda 2026-05-15 16:45:09 -05:00
parent c33d456743
commit c68249c314
11 changed files with 581 additions and 275 deletions

View file

@ -1,11 +1,89 @@
# connect/handlers/script.py
import os
import select as _select
import signal as _signal
import stat
import subprocess
import time
from django.conf import settings
from .base import IntegrationHandler
def _waitpid_nonblocking(pid, timeout_s=2.0):
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
try:
wpid, status = os.waitpid(pid, os.WNOHANG)
except ChildProcessError:
return -1
if wpid == pid:
return os.WEXITSTATUS(status) if os.WIFEXITED(status) else -os.WTERMSIG(status)
time.sleep(0.01)
return -1
def _posix_run(path, env, timeout):
stdout_r, stdout_w = os.pipe()
stderr_r, stderr_w = os.pipe()
devnull_r = os.open(os.devnull, os.O_RDONLY)
file_actions = [
(os.POSIX_SPAWN_DUP2, devnull_r, 0),
(os.POSIX_SPAWN_DUP2, stdout_w, 1),
(os.POSIX_SPAWN_DUP2, stderr_w, 2),
(os.POSIX_SPAWN_CLOSE, devnull_r),
(os.POSIX_SPAWN_CLOSE, stdout_w),
(os.POSIX_SPAWN_CLOSE, stderr_w),
(os.POSIX_SPAWN_CLOSE, stdout_r),
(os.POSIX_SPAWN_CLOSE, stderr_r),
]
try:
pid = os.posix_spawn(path, [path], env, file_actions=file_actions)
except Exception:
for fd in (stdout_r, stdout_w, stderr_r, stderr_w, devnull_r):
try:
os.close(fd)
except OSError:
pass
raise
for fd in (devnull_r, stdout_w, stderr_w):
os.close(fd)
out, err = [], []
done = set()
deadline = time.monotonic() + timeout
timed_out = False
try:
while len(done) < 2:
remaining = deadline - time.monotonic()
if remaining <= 0:
timed_out = True
try:
os.kill(pid, _signal.SIGKILL)
except ProcessLookupError:
pass
break
fds = [fd for fd in (stdout_r, stderr_r) if fd not in done]
readable, _, _ = _select.select(fds, [], [], min(remaining, 0.5))
for fd in readable:
data = os.read(fd, 8192)
if data:
(out if fd == stdout_r else err).append(data)
else:
done.add(fd)
finally:
for fd in (stdout_r, stderr_r):
try:
os.close(fd)
except OSError:
pass
rc = _waitpid_nonblocking(pid)
if timed_out:
raise TimeoutError(f"Script exceeded {timeout}s")
return rc, b"".join(out).decode("utf-8", errors="replace"), b"".join(err).decode("utf-8", errors="replace")
def _is_path_allowed(real_path: str) -> bool:
# Ensure path is within one of the allowed directories
for base in getattr(settings, "CONNECT_ALLOWED_SCRIPT_DIRS", []):
@ -56,26 +134,17 @@ class ScriptHandler(IntegrationHandler):
timeout = getattr(settings, "CONNECT_SCRIPT_TIMEOUT", 10)
max_out = getattr(settings, "CONNECT_SCRIPT_MAX_OUTPUT", 65536)
result = subprocess.run(
[real_path],
capture_output=True,
text=True,
env=env,
timeout=timeout,
cwd=os.path.dirname(real_path) or None,
)
rc, stdout, stderr = _posix_run(real_path, env, timeout)
# Truncate outputs to avoid excessive memory/logging
stdout = result.stdout or ""
stderr = result.stderr or ""
if len(stdout) > max_out:
stdout = stdout[:max_out] + "... [truncated]"
if len(stderr) > max_out:
stderr = stderr[:max_out] + "... [truncated]"
return {
"exit_code": result.returncode,
"exit_code": rc,
"stdout": stdout,
"stderr": stderr,
"success": result.returncode == 0,
"success": rc == 0,
}