From 6ce387b0b07f02da54389abf37b407cee1e830cf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 2 Jun 2025 18:03:32 -0500 Subject: [PATCH] Auto-scales Celery based on demand. Should lower overall memory and CPU usage while allowing for high cpu demand tasks to complete quickly. Closes #111 --- dispatcharr/settings.py | 9 --------- docker/uwsgi.debug.ini | 2 +- docker/uwsgi.dev.ini | 2 +- docker/uwsgi.ini | 2 +- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 4e1e0d55..06084b49 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -199,15 +199,6 @@ CELERY_BROKER_TRANSPORT_OPTIONS = { CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' -# Memory management settings -#CELERY_WORKER_MAX_TASKS_PER_CHILD = 10 # Restart worker after 10 tasks to free memory -#CELERY_WORKER_PREFETCH_MULTIPLIER = 1 # Don't prefetch tasks - process one at a time -#CELERY_TASK_ACKS_LATE = True # Only acknowledge tasks after they're processed -#CELERY_TASK_TIME_LIMIT = 3600 # 1 hour time limit per task -#CELERY_TASK_SOFT_TIME_LIMIT = 3540 # Soft limit 60 seconds before hard limit -#CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True # Cancel tasks if connection lost -#CELERY_TASK_IGNORE_RESULT = True # Don't store results unless explicitly needed - CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler" CELERY_BEAT_SCHEDULE = { 'fetch-channel-statuses': { diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 6ca855f3..e049df87 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -8,7 +8,7 @@ exec-before = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker --concurrency=4 +attach-daemon = celery -A dispatcharr worker --autoscale=6,1 attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index f3e5238e..7e50f2ef 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -10,7 +10,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker --concurrency=4 +attach-daemon = celery -A dispatcharr worker --autoscale=6,1 attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 32eb6e3c..b35ea5bf 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -10,7 +10,7 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker --concurrency=4 +attach-daemon = celery -A dispatcharr worker --autoscale=6,1 attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application