EPG processing enhancements. Celery memory management.

dispatcharr/celery.py

@@ -2,6 +2,7 @@
 import os
 from celery import Celery
 import logging
+from celery.signals import task_postrun  # Add import for signals
 
 # Initialize with defaults before Django settings are loaded
 DEFAULT_LOG_LEVEL = 'DEBUG'
@@ -48,6 +49,24 @@ app.conf.update(
     worker_task_log_format='%(asctime)s %(levelname)s %(task_name)s: %(message)s',
 )
 
+# Add memory cleanup after task completion
+@task_postrun.connect  # Use the imported signal
+def cleanup_task_memory(**kwargs):
+    """Clean up memory after each task completes"""
+    import gc
+    # Force garbage collection
+    gc.collect()
+
+    # Log memory usage if psutil is installed
+    try:
+        import psutil
+        process = psutil.Process()
+        if hasattr(process, 'memory_info'):
+            mem = process.memory_info().rss / (1024 * 1024)
+            print(f"Memory usage after task: {mem:.2f} MB")
+    except (ImportError, Exception):
+        pass
+
 @app.on_after_configure.connect
 def setup_celery_logging(**kwargs):
     # Use our directly determined log level

dispatcharr/settings.py

@@ -199,6 +199,15 @@ CELERY_BROKER_TRANSPORT_OPTIONS = {
 CELERY_ACCEPT_CONTENT = ['json']
 CELERY_TASK_SERIALIZER = 'json'
 
+# Memory management settings
+#CELERY_WORKER_MAX_TASKS_PER_CHILD = 10  # Restart worker after 10 tasks to free memory
+#CELERY_WORKER_PREFETCH_MULTIPLIER = 1   # Don't prefetch tasks - process one at a time
+#CELERY_TASK_ACKS_LATE = True            # Only acknowledge tasks after they're processed
+#CELERY_TASK_TIME_LIMIT = 3600           # 1 hour time limit per task
+#CELERY_TASK_SOFT_TIME_LIMIT = 3540      # Soft limit 60 seconds before hard limit
+#CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True  # Cancel tasks if connection lost
+#CELERY_TASK_IGNORE_RESULT = True        # Don't store results unless explicitly needed
+
 CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
 CELERY_BEAT_SCHEDULE = {
     'fetch-channel-statuses': {