From 6afccadf74eedc88afd44010149df2f15d21d053 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 12:00:57 -0500 Subject: [PATCH 01/55] Delete celery tasks properly during m3u account deletion. --- apps/m3u/signals.py | 23 +++++++++++++--- apps/m3u/tasks.py | 65 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/apps/m3u/signals.py b/apps/m3u/signals.py index e5951ff2..467d0069 100644 --- a/apps/m3u/signals.py +++ b/apps/m3u/signals.py @@ -2,9 +2,12 @@ from django.db.models.signals import post_save, post_delete, pre_save from django.dispatch import receiver from .models import M3UAccount -from .tasks import refresh_single_m3u_account, refresh_m3u_groups +from .tasks import refresh_single_m3u_account, refresh_m3u_groups, delete_m3u_refresh_task_by_id from django_celery_beat.models import PeriodicTask, IntervalSchedule import json +import logging + +logger = logging.getLogger(__name__) @receiver(post_save, sender=M3UAccount) def refresh_account_on_save(sender, instance, created, **kwargs): @@ -65,9 +68,21 @@ def delete_refresh_task(sender, instance, **kwargs): """ Delete the associated Celery Beat periodic task when a Channel is deleted. """ - if instance.refresh_task: - instance.refresh_task.interval.delete() - instance.refresh_task.delete() + try: + # First try the foreign key relationship to find the task ID + task = None + if instance.refresh_task: + logger.info(f"Found task via foreign key: {instance.refresh_task.id} for M3UAccount {instance.id}") + task = instance.refresh_task + + # Use the helper function to delete the task + if task: + delete_m3u_refresh_task_by_id(instance.id) + else: + # Otherwise use the helper function + delete_m3u_refresh_task_by_id(instance.id) + except Exception as e: + logger.error(f"Error in delete_refresh_task signal handler: {str(e)}", exc_info=True) @receiver(pre_save, sender=M3UAccount) def update_status_on_active_change(sender, instance, **kwargs): diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 8fb16ba5..7bc549e0 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -733,6 +733,60 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): return extinf_data, groups +def delete_m3u_refresh_task_by_id(account_id): + """ + Delete the periodic task associated with an M3U account ID. + Can be called directly or from the post_delete signal. + Returns True if a task was found and deleted, False otherwise. + """ + try: + task = None + task_name = f"m3u_account-refresh-{account_id}" + + # Look for task by name + try: + from django_celery_beat.models import PeriodicTask, IntervalSchedule + task = PeriodicTask.objects.get(name=task_name) + logger.info(f"Found task by name: {task.id} for M3UAccount {account_id}") + except PeriodicTask.DoesNotExist: + logger.warning(f"No PeriodicTask found with name {task_name}") + return False + + # Now delete the task and its interval + if task: + # Store interval info before deleting the task + interval_id = None + if hasattr(task, 'interval') and task.interval: + interval_id = task.interval.id + + # Count how many TOTAL tasks use this interval (including this one) + tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() + logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") + + # Delete the task first + task_id = task.id + task.delete() + logger.info(f"Successfully deleted periodic task {task_id}") + + # Now check if we should delete the interval + # We only delete if it was the ONLY task using this interval + if interval_id and tasks_with_same_interval == 1: + try: + interval = IntervalSchedule.objects.get(id=interval_id) + logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") + interval.delete() + logger.info(f"Successfully deleted interval {interval_id}") + except IntervalSchedule.DoesNotExist: + logger.warning(f"Interval {interval_id} no longer exists") + elif interval_id: + logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") + + return True + return False + except Exception as e: + logger.error(f"Error deleting periodic task for M3UAccount {account_id}: {str(e)}", exc_info=True) + return False + @shared_task def refresh_single_m3u_account(account_id): """Splits M3U processing into chunks and dispatches them as parallel tasks.""" @@ -758,8 +812,17 @@ def refresh_single_m3u_account(account_id): filters = list(account.filters.all()) except M3UAccount.DoesNotExist: + # The M3U account doesn't exist, so delete the periodic task if it exists + logger.warning(f"M3U account with ID {account_id} not found, but task was triggered. Cleaning up orphaned task.") + + # Call the helper function to delete the task + if delete_m3u_refresh_task_by_id(account_id): + logger.info(f"Successfully cleaned up orphaned task for M3U account {account_id}") + else: + logger.info(f"No orphaned task found for M3U account {account_id}") + release_task_lock('refresh_single_m3u_account', account_id) - return f"M3UAccount with ID={account_id} not found or inactive." + return f"M3UAccount with ID={account_id} not found or inactive, task cleaned up" # Fetch M3U lines and handle potential issues extinf_data = [] From 25fc69d453f747d477269aca81432fee6e25955c Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 12:06:43 -0500 Subject: [PATCH 02/55] Properly disable celery task if m3u is disabled. --- apps/m3u/signals.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/m3u/signals.py b/apps/m3u/signals.py index 467d0069..d014ac92 100644 --- a/apps/m3u/signals.py +++ b/apps/m3u/signals.py @@ -31,14 +31,17 @@ def create_or_update_refresh_task(sender, instance, **kwargs): period=IntervalSchedule.HOURS ) + # Task should be enabled only if refresh_interval != 0 AND account is active + should_be_enabled = (instance.refresh_interval != 0) and instance.is_active + # First check if the task already exists to avoid validation errors try: task = PeriodicTask.objects.get(name=task_name) # Task exists, just update it updated_fields = [] - if task.enabled != (instance.refresh_interval != 0): - task.enabled = instance.refresh_interval != 0 + if task.enabled != should_be_enabled: + task.enabled = should_be_enabled updated_fields.append("enabled") if task.interval != interval: @@ -59,7 +62,7 @@ def create_or_update_refresh_task(sender, instance, **kwargs): interval=interval, task="apps.m3u.tasks.refresh_single_m3u_account", kwargs=json.dumps({"account_id": instance.id}), - enabled=instance.refresh_interval != 0, + enabled=should_be_enabled, ) M3UAccount.objects.filter(id=instance.id).update(refresh_task=refresh_task) From 178dc61e94d746a3fca02a0814e71087610b9976 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 12:20:29 -0500 Subject: [PATCH 03/55] Better error handling for debug wrapper when timeouts occur. --- scripts/debug_wrapper.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/debug_wrapper.py b/scripts/debug_wrapper.py index 2339fe87..eb71d43d 100644 --- a/scripts/debug_wrapper.py +++ b/scripts/debug_wrapper.py @@ -50,9 +50,14 @@ try: # Don't wait for connection, just set up the debugging session logger.info("Initializing debugpy on 0.0.0.0:5678...") try: + # Set socket timeout using the existing DEBUG_TIMEOUT variable + import socket + socket.setdefaulttimeout(DEBUG_TIMEOUT) # Use connect instead of listen to avoid the adapter process debugpy.listen(("0.0.0.0", 5678)) logger.info("debugpy now listening on 0.0.0.0:5678") + # Reset socket timeout + socket.setdefaulttimeout(None) if WAIT_FOR_DEBUGGER: logger.info(f"Waiting for debugger to attach (timeout: {DEBUG_TIMEOUT}s)...") @@ -65,9 +70,22 @@ try: logger.info("Debugger attached!") else: logger.info(f"Debugger not attached after {DEBUG_TIMEOUT}s, continuing anyway...") + except RuntimeError as re: + if "timed out waiting for adapter to connect" in str(re): + logger.warning(f"debugpy.listen timed out after {DEBUG_TIMEOUT}s. This is normal in some environments.") + logger.info("Continuing without debugging...") + else: + logger.error(f"RuntimeError with debugpy.listen: {re}", exc_info=True) + logger.info("Continuing without debugging...") + except socket.timeout: + logger.warning(f"Socket timeout after {DEBUG_TIMEOUT}s while initializing debugpy.") + logger.info("Continuing without debugging...") except Exception as e: logger.error(f"Error with debugpy.listen: {e}", exc_info=True) logger.info("Continuing without debugging...") + finally: + # Ensure socket timeout is always reset + socket.setdefaulttimeout(None) except ImportError: logger.error("debugpy not installed, continuing without debugging support") From 04cbfa5f26b512326c99b3e358e63c6c2dc69910 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 12:54:52 -0500 Subject: [PATCH 04/55] Build arm test. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbfd8d71..4a60ac49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: with: context: . push: ${{ github.event_name != 'pull_request' }} - platforms: linux/amd64 # Fast build - amd64 only + platforms: linux/amd64,linux/arm64 tags: | ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.meta.outputs.branch_tag }} ghcr.io/${{ steps.meta.outputs.repo_owner }}/${{ steps.meta.outputs.repo_name }}:${{ steps.version.outputs.version }}-${{ steps.timestamp.outputs.timestamp }} From f762e1b923579c3703b18f4a0d33923577021fe6 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 13:44:49 -0500 Subject: [PATCH 05/55] Capture and display transcode strd err. --- apps/proxy/ts_proxy/stream_manager.py | 43 ++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/apps/proxy/ts_proxy/stream_manager.py b/apps/proxy/ts_proxy/stream_manager.py index 3bc288ac..7d158c09 100644 --- a/apps/proxy/ts_proxy/stream_manager.py +++ b/apps/proxy/ts_proxy/stream_manager.py @@ -102,6 +102,9 @@ class StreamManager: self.last_bytes_update = time.time() self.bytes_update_interval = 5 # Update Redis every 5 seconds + # Add stderr reader thread property + self.stderr_reader_thread = None + def _create_session(self): """Create and configure requests session with optimal settings""" session = requests.Session() @@ -333,13 +336,17 @@ class StreamManager: self.transcode_cmd = stream_profile.build_command(self.url, self.user_agent) logger.debug(f"Starting transcode process: {self.transcode_cmd}") + # Modified to capture stderr instead of discarding it self.transcode_process = subprocess.Popen( self.transcode_cmd, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, # Suppress error logs + stderr=subprocess.PIPE, # Capture stderr instead of discarding it bufsize=188 * 64 # Buffer optimized for TS packets ) + # Start a thread to read stderr + self._start_stderr_reader() + # Set flag that transcoding process is active self.transcode_process_active = True @@ -358,6 +365,40 @@ class StreamManager: self._close_socket() return False + def _start_stderr_reader(self): + """Start a thread to read stderr from the transcode process""" + if self.transcode_process and self.transcode_process.stderr: + self.stderr_reader_thread = threading.Thread( + target=self._read_stderr, + daemon=True # Use daemon thread so it doesn't block program exit + ) + self.stderr_reader_thread.start() + logger.debug(f"Started stderr reader thread for channel {self.channel_id}") + + def _read_stderr(self): + """Read and log stderr output from the transcode process""" + try: + if not self.transcode_process or not self.transcode_process.stderr: + logger.warning(f"No stderr to read for channel {self.channel_id}") + return + + for line in iter(self.transcode_process.stderr.readline, b''): + if not line: + break + + # Decode the line and strip whitespace + error_line = line.decode('utf-8', errors='replace').strip() + + # Skip empty lines + if not error_line: + continue + + # Log all stderr output as debug messages + logger.debug(f"Transcode stderr [{self.channel_id}]: {error_line}") + + except Exception as e: + logger.error(f"Error reading transcode stderr: {e}") + def _establish_http_connection(self): """Establish a direct HTTP connection to the stream""" try: From aff93591fd708d1a1ed494aad244a7b0b42a5a97 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 17:19:46 -0500 Subject: [PATCH 06/55] Detect hardware acceleration capabilities and provide suggestions. --- docker/docker-compose.aio.yml | 16 +++ docker/docker-compose.yml | 15 +++ docker/entrypoint.sh | 1 + docker/init/04-check-gpu.sh | 220 ++++++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 docker/init/04-check-gpu.sh diff --git a/docker/docker-compose.aio.yml b/docker/docker-compose.aio.yml index 77b9bec1..95834915 100644 --- a/docker/docker-compose.aio.yml +++ b/docker/docker-compose.aio.yml @@ -13,6 +13,22 @@ services: - DISPATCHARR_ENV=aio - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 + # Optional for hardware acceleration + #group_add: + # - video + # #- render # Uncomment if your GPU requires it + #devices: + # - /dev/dri:/dev/dri # For Intel/AMD GPU acceleration (VA-API) + # Uncomment the following lines for NVIDIA GPU support + # NVidia GPU support (requires NVIDIA Container Toolkit) + #deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] + volumes: dispatcharr_data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e6a06603..4c45dd59 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,6 +14,21 @@ services: - POSTGRES_PASSWORD=secret - REDIS_HOST=redis - CELERY_BROKER_URL=redis://redis:6379/0 + # Optional for hardware acceleration + #group_add: + # - video + # #- render # Uncomment if your GPU requires it + #devices: + # - /dev/dri:/dev/dri # For Intel/AMD GPU acceleration (VA-API) + # Uncomment the following lines for NVIDIA GPU support + # NVidia GPU support (requires NVIDIA Container Toolkit) + #deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] celery: image: dispatcharr/dispatcharr:alpha-v1 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d2afb3a3..bbd2b9ba 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -82,6 +82,7 @@ echo "Starting init process..." . /app/docker/init/01-user-setup.sh . /app/docker/init/02-postgres.sh . /app/docker/init/03-init-dispatcharr.sh +. /app/docker/init/04-check-gpu.sh # Start PostgreSQL echo "Starting Postgres..." diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh new file mode 100644 index 00000000..a5b294f3 --- /dev/null +++ b/docker/init/04-check-gpu.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +echo "🔍 Checking for GPU acceleration devices..." + +# Helper function for device access checks +check_dev() { + local dev=$1 + if [ -e "$dev" ]; then + if [ -r "$dev" ] && [ -w "$dev" ]; then + echo "✅ Device $dev is accessible." + else + echo "⚠ī¸ Device $dev exists but is not accessible. Check permissions or container runtime options." + fi + else + echo "ℹī¸ Device $dev does not exist." + fi +} + +# Check Intel/AMD VAAPI devices +echo "🔍 Checking for Intel/AMD (VAAPI) devices..." +for dev in /dev/dri/renderD* /dev/dri/card*; do + [ -e "$dev" ] && check_dev "$dev" +done + +# Check NVIDIA device nodes +echo "🔍 Checking for NVIDIA devices..." +NVIDIA_FOUND=false +for dev in /dev/nvidia*; do + [ -e "$dev" ] && NVIDIA_FOUND=true && check_dev "$dev" +done +if [ "$NVIDIA_FOUND" = false ]; then + echo "ℹī¸ No NVIDIA device nodes found under /dev." +fi + +# Check group membership for GPU access - context-aware based on hardware +echo "🔍 Checking user group memberships..." +VIDEO_GID=$(getent group video | cut -d: -f3) +RENDER_GID=$(getent group render | cut -d: -f3) +NVIDIA_CONTAINER_TOOLKIT_FOUND=false + +# Check if NVIDIA Container Toolkit is present through environment or CLI tool +if command -v nvidia-container-cli >/dev/null 2>&1; then + NVIDIA_CONTAINER_TOOLKIT_FOUND=true +# Check for environment variables set by NVIDIA Container Runtime +elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + NVIDIA_CONTAINER_TOOLKIT_FOUND=true + echo "✅ NVIDIA Container Toolkit detected (via environment variables)." + echo " The container is properly configured with Docker Compose's 'driver: nvidia' syntax." +fi + +# For NVIDIA GPUs with Container Toolkit, video group is optional +if [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then + if [ -n "$VIDEO_GID" ] && id -G | grep -qw "$VIDEO_GID"; then + echo "✅ User is in the 'video' group (GID $VIDEO_GID)." + echo " Note: With NVIDIA Container Toolkit properly configured, this is usually not required." + elif [ -n "$VIDEO_GID" ]; then + echo "ℹī¸ User is not in the 'video' group, but NVIDIA Container Toolkit is present." + echo " This is typically fine as the Container Toolkit handles device permissions." + fi +# For other GPU types (or NVIDIA without Toolkit), video/render group is important +else + if [ -n "$VIDEO_GID" ]; then + if id -G | grep -qw "$VIDEO_GID"; then + echo "✅ User is in the 'video' group (GID $VIDEO_GID)." + else + echo "⚠ī¸ User is NOT in the 'video' group (GID $VIDEO_GID). Hardware acceleration may not work." + fi + elif [ -n "$RENDER_GID" ]; then + if id -G | grep -qw "$RENDER_GID"; then + echo "✅ User is in the 'render' group (GID $RENDER_GID)." + else + echo "⚠ī¸ User is NOT in the 'render' group (GID $RENDER_GID). Hardware acceleration may not work." + fi + else + echo "⚠ī¸ Neither 'video' nor 'render' groups found on this system." + fi +fi + +# Check NVIDIA Container Toolkit support +echo "🔍 Checking NVIDIA container runtime support..." +if command -v nvidia-container-cli >/dev/null 2>&1; then + echo "✅ NVIDIA Container Toolkit detected (nvidia-container-cli found)." + + if nvidia-container-cli info >/dev/null 2>&1; then + echo "✅ NVIDIA container runtime is functional." + else + echo "⚠ī¸ nvidia-container-cli found, but 'info' command failed. Runtime may be misconfigured." + fi +elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + echo "✅ NVIDIA Container Toolkit detected through environment variables." + echo " Your Docker Compose configuration with 'driver: nvidia' and 'capabilities: [gpu]' is working correctly." + echo " This is the modern, recommended way to use NVIDIA GPUs with containers." +else + echo "ℹī¸ NVIDIA Container Toolkit not detected." + + # Only show this message if NVIDIA devices are found but toolkit is missing + if [ "$NVIDIA_FOUND" = true ]; then + echo "ℹī¸ You appear to be using direct device passthrough for NVIDIA GPU access." + echo " This method works, but consider using Docker Compose's 'deploy' configuration:" + echo " deploy:" + echo " resources:" + echo " reservations:" + echo " devices:" + echo " - driver: nvidia" + echo " count: all" + echo " capabilities: [gpu]" + fi +fi + +# Run nvidia-smi if available +if command -v nvidia-smi >/dev/null 2>&1; then + echo "🔍 Running nvidia-smi to verify GPU visibility..." + if nvidia-smi >/dev/null 2>&1; then + echo "✅ nvidia-smi successful - GPU is accessible to container!" + echo " This confirms hardware acceleration should be available to FFmpeg." + else + echo "⚠ī¸ nvidia-smi command failed. GPU may not be properly mapped into container." + fi +else + echo "ℹī¸ nvidia-smi not installed or not in PATH." +fi + +# Show relevant environment variables with contextual suggestions +echo "🔍 Checking GPU-related environment variables..." + +# Set flags based on device detection +DRI_DEVICES_FOUND=false +for dev in /dev/dri/renderD* /dev/dri/card*; do + if [ -e "$dev" ]; then + DRI_DEVICES_FOUND=true + break + fi +done + +# Give contextual suggestions based on detected hardware +if [ "$DRI_DEVICES_FOUND" = true ]; then + if [ -n "$LIBVA_DRIVER_NAME" ]; then + echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" + else + echo "💡 Consider setting LIBVA_DRIVER_NAME to 'i965' (Intel) or 'radeonsi' (AMD) for VAAPI acceleration" + fi +fi + +if [ "$NVIDIA_FOUND" = true ]; then + if [ -n "$NVIDIA_VISIBLE_DEVICES" ]; then + echo "ℹī¸ NVIDIA_VISIBLE_DEVICES is set to '$NVIDIA_VISIBLE_DEVICES'" + else + echo "💡 Consider setting NVIDIA_VISIBLE_DEVICES to 'all' or specific indices (e.g., '0,1')" + fi + + if [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + echo "ℹī¸ NVIDIA_DRIVER_CAPABILITIES is set to '$NVIDIA_DRIVER_CAPABILITIES'" + else + echo "💡 Consider setting NVIDIA_DRIVER_CAPABILITIES to 'all' or 'compute,video,utility' for full functionality" + fi + + if [ -n "$CUDA_VISIBLE_DEVICES" ]; then + echo "ℹī¸ CUDA_VISIBLE_DEVICES is set to '$CUDA_VISIBLE_DEVICES'" + fi +fi + +# Check FFmpeg hardware acceleration support +echo "🔍 Checking FFmpeg hardware acceleration capabilities..." +if command -v ffmpeg >/dev/null 2>&1; then + HWACCEL=$(ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -v "Hardware acceleration methods:" || echo "None found") + echo "Available FFmpeg hardware acceleration methods:" + echo "$HWACCEL" +else + echo "⚠ī¸ FFmpeg not found in PATH." +fi + +# Provide a final summary of the hardware acceleration setup +echo "📋 ===================== SUMMARY =====================" + +# Identify which GPU type is active and working +if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VISIBLE_DEVICES" ]); then + echo "🔰 NVIDIA GPU: ACTIVE" + if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then + echo "✅ NVIDIA Container Toolkit: CONFIGURED CORRECTLY" + elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + echo "✅ NVIDIA Docker configuration: USING MODERN DEPLOYMENT" + else + echo "⚠ī¸ NVIDIA setup method: DIRECT DEVICE MAPPING (functional but not optimal)" + fi + # Display FFmpeg NVIDIA acceleration methods + if echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then + echo "✅ FFmpeg NVIDIA acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" + fi +elif [ "$DRI_DEVICES_FOUND" = true ]; then + # Intel/AMD detection + if [ -n "$LIBVA_DRIVER_NAME" ]; then + echo "🔰 ${LIBVA_DRIVER_NAME^^} GPU: ACTIVE" + else + echo "🔰 INTEL/AMD GPU: ACTIVE" + fi + + # Check group membership + if [ -n "$VIDEO_GID" ] && id -G | grep -qw "$VIDEO_GID"; then + echo "✅ Video group membership: CORRECT" + elif [ -n "$RENDER_GID" ] && id -G | grep -qw "$RENDER_GID"; then + echo "✅ Render group membership: CORRECT" + else + echo "⚠ī¸ Group membership: MISSING (may cause permission issues)" + fi + + # Display FFmpeg VAAPI acceleration method + if echo "$HWACCEL" | grep -q "vaapi"; then + echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" + fi +else + echo "❌ NO GPU ACCELERATION DETECTED" + echo "⚠ī¸ Hardware acceleration is unavailable or misconfigured" +fi + +echo "📋 ==================================================" +echo "✅ GPU detection script complete." From 18bc4220776593d632152f9ed5cbd221b927b15f Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 17:37:39 -0500 Subject: [PATCH 07/55] Better detection. --- docker/init/04-check-gpu.sh | 62 +++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index a5b294f3..f01b5b8c 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -37,15 +37,19 @@ echo "🔍 Checking user group memberships..." VIDEO_GID=$(getent group video | cut -d: -f3) RENDER_GID=$(getent group render | cut -d: -f3) NVIDIA_CONTAINER_TOOLKIT_FOUND=false +NVIDIA_ENV_MISMATCH=false # Check if NVIDIA Container Toolkit is present through environment or CLI tool -if command -v nvidia-container-cli >/dev/null 2>&1; then +# IMPORTANT: Only mark as found if both env vars AND actual NVIDIA devices exist +if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then NVIDIA_CONTAINER_TOOLKIT_FOUND=true -# Check for environment variables set by NVIDIA Container Runtime -elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then +# Check for environment variables set by NVIDIA Container Runtime, but only if NVIDIA hardware exists +elif [ "$NVIDIA_FOUND" = true ] && [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then NVIDIA_CONTAINER_TOOLKIT_FOUND=true echo "✅ NVIDIA Container Toolkit detected (via environment variables)." echo " The container is properly configured with Docker Compose's 'driver: nvidia' syntax." +elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ] && [ "$NVIDIA_FOUND" = false ]; then + NVIDIA_ENV_MISMATCH=true fi # For NVIDIA GPUs with Container Toolkit, video group is optional @@ -78,7 +82,7 @@ fi # Check NVIDIA Container Toolkit support echo "🔍 Checking NVIDIA container runtime support..." -if command -v nvidia-container-cli >/dev/null 2>&1; then +if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then echo "✅ NVIDIA Container Toolkit detected (nvidia-container-cli found)." if nvidia-container-cli info >/dev/null 2>&1; then @@ -86,24 +90,21 @@ if command -v nvidia-container-cli >/dev/null 2>&1; then else echo "⚠ī¸ nvidia-container-cli found, but 'info' command failed. Runtime may be misconfigured." fi -elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then +elif [ "$NVIDIA_FOUND" = true ] && [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then echo "✅ NVIDIA Container Toolkit detected through environment variables." echo " Your Docker Compose configuration with 'driver: nvidia' and 'capabilities: [gpu]' is working correctly." - echo " This is the modern, recommended way to use NVIDIA GPUs with containers." else - echo "ℹī¸ NVIDIA Container Toolkit not detected." - - # Only show this message if NVIDIA devices are found but toolkit is missing - if [ "$NVIDIA_FOUND" = true ]; then - echo "ℹī¸ You appear to be using direct device passthrough for NVIDIA GPU access." - echo " This method works, but consider using Docker Compose's 'deploy' configuration:" - echo " deploy:" - echo " resources:" - echo " reservations:" - echo " devices:" - echo " - driver: nvidia" - echo " count: all" - echo " capabilities: [gpu]" + if [ "$NVIDIA_ENV_MISMATCH" = true ]; then + echo "⚠ī¸ NVIDIA environment variables detected but no NVIDIA hardware found." + echo " Your Docker Compose has NVIDIA configuration (driver: nvidia, capabilities: [gpu])," + echo " but actual NVIDIA devices are not available. Instead found Intel/AMD devices." + echo " Update your Docker Compose to match your actual hardware." + elif [ "$NVIDIA_FOUND" = true ]; then + echo "ℹī¸ NVIDIA devices found but Container Toolkit not detected." + echo " You appear to be using direct device passthrough for NVIDIA GPU access." + echo " This method works, but consider using Docker Compose's 'deploy' configuration." + else + echo "ℹī¸ No NVIDIA GPU hardware or toolkit detected." fi fi @@ -137,7 +138,28 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then if [ -n "$LIBVA_DRIVER_NAME" ]; then echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" else - echo "💡 Consider setting LIBVA_DRIVER_NAME to 'i965' (Intel) or 'radeonsi' (AMD) for VAAPI acceleration" + # Check if we can detect the GPU type + if command -v lspci >/dev/null 2>&1; then + if lspci | grep -q "Intel Corporation.*VGA" | grep -i "HD Graphics"; then + # Newer Intel GPUs (Gen12+/Tiger Lake and newer) + if lspci | grep -q "Intel Corporation.*VGA" | grep -E "Xe|Alchemist|Tiger Lake|Gen1[2-9]"; then + echo "💡 Consider setting LIBVA_DRIVER_NAME=iHD for this modern Intel GPU" + else + # Older Intel GPUs + echo "💡 Consider setting LIBVA_DRIVER_NAME=i965 for this Intel GPU" + fi + elif lspci | grep -q "Advanced Micro Devices.*VGA"; then + echo "💡 Consider setting LIBVA_DRIVER_NAME=radeonsi for this AMD GPU" + else + echo "ℹī¸ Auto-detection of VAAPI driver is usually reliable, but if you have issues:" + echo " - For newer Intel GPUs: LIBVA_DRIVER_NAME=iHD" + echo " - For older Intel GPUs: LIBVA_DRIVER_NAME=i965" + echo " - For AMD GPUs: LIBVA_DRIVER_NAME=radeonsi" + fi + else + echo "ℹī¸ Intel/AMD GPU detected. If VAAPI doesn't work automatically, you may need to set LIBVA_DRIVER_NAME" + echo " based on your specific hardware (iHD for newer Intel, i965 for older Intel, radeonsi for AMD)" + fi fi fi From 2ddc6beb1588b4360eda48108c287bd52e2831c9 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 18:31:16 -0500 Subject: [PATCH 08/55] Better messaging since nvidia variables will always exist in our container. --- docker/init/04-check-gpu.sh | 50 ++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index f01b5b8c..06583b69 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -82,30 +82,46 @@ fi # Check NVIDIA Container Toolkit support echo "🔍 Checking NVIDIA container runtime support..." -if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then - echo "✅ NVIDIA Container Toolkit detected (nvidia-container-cli found)." + +# More reliable detection of NVIDIA Container Runtime +NVIDIA_RUNTIME_ACTIVE=false + +# Method 1: Check for nvidia-container-cli tool +if command -v nvidia-container-cli >/dev/null 2>&1; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime detected (nvidia-container-cli found)." if nvidia-container-cli info >/dev/null 2>&1; then echo "✅ NVIDIA container runtime is functional." else echo "⚠ī¸ nvidia-container-cli found, but 'info' command failed. Runtime may be misconfigured." fi -elif [ "$NVIDIA_FOUND" = true ] && [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then - echo "✅ NVIDIA Container Toolkit detected through environment variables." - echo " Your Docker Compose configuration with 'driver: nvidia' and 'capabilities: [gpu]' is working correctly." +fi + +# Method 2: Check for NVIDIA Container Runtime specific files +if [ -e "/dev/.nv" ] || [ -e "/.nv" ] || [ -e "/.nvidia-container-runtime" ]; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime files detected." +fi + +# Method 3: Check cgroup information for NVIDIA +if grep -q "nvidia" /proc/self/cgroup 2>/dev/null; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime cgroups detected." +fi + +# Final verdict based on hardware AND runtime +if [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_RUNTIME_ACTIVE" = true ]; then + echo "✅ NVIDIA Container Runtime is properly configured with hardware access." +elif [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_RUNTIME_ACTIVE" = false ]; then + echo "ℹī¸ NVIDIA GPU detected, but using direct device passthrough instead of Container Runtime." + echo " This works but consider using the 'deploy: resources: reservations: devices:' method in docker-compose." +elif [ "$NVIDIA_FOUND" = false ] && [ "$NVIDIA_RUNTIME_ACTIVE" = true ]; then + echo "⚠ī¸ NVIDIA Container Runtime appears to be configured, but no NVIDIA devices found." + echo " Check that your host has NVIDIA drivers installed and GPUs are properly passed to the container." else - if [ "$NVIDIA_ENV_MISMATCH" = true ]; then - echo "⚠ī¸ NVIDIA environment variables detected but no NVIDIA hardware found." - echo " Your Docker Compose has NVIDIA configuration (driver: nvidia, capabilities: [gpu])," - echo " but actual NVIDIA devices are not available. Instead found Intel/AMD devices." - echo " Update your Docker Compose to match your actual hardware." - elif [ "$NVIDIA_FOUND" = true ]; then - echo "ℹī¸ NVIDIA devices found but Container Toolkit not detected." - echo " You appear to be using direct device passthrough for NVIDIA GPU access." - echo " This method works, but consider using Docker Compose's 'deploy' configuration." - else - echo "ℹī¸ No NVIDIA GPU hardware or toolkit detected." - fi + # No need to show NVIDIA environment variable warnings if they're default in the container + echo "ℹī¸ Using Intel/AMD GPU hardware for acceleration." fi # Run nvidia-smi if available From 23b678bb03d053757d2cf8402bf87481e29bfdbd Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Fri, 9 May 2025 20:06:28 -0500 Subject: [PATCH 09/55] Add pciutils for better hardware detection. --- docker/DispatcharrBase | 2 +- docker/init/04-check-gpu.sh | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index 58741715..8b65595f 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ python3.13 python3.13-dev python3.13-venv \ python-is-python3 python3-pip \ libpcre3 libpcre3-dev libpq-dev procps \ - build-essential gcc \ + build-essential gcc pciutils \ nginx streamlink \ && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index 06583b69..e983018d 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -151,6 +151,16 @@ done # Give contextual suggestions based on detected hardware if [ "$DRI_DEVICES_FOUND" = true ]; then + # Detect Intel/AMD GPU model + if command -v lspci >/dev/null 2>&1; then + GPU_INFO=$(lspci -nn | grep -i "VGA\|Display" | head -1) + if [ -n "$GPU_INFO" ]; then + echo "🔍 Detected GPU: $GPU_INFO" + # Extract model for cleaner display in summary + GPU_MODEL=$(echo "$GPU_INFO" | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //' | sed 's/Technologies //') + fi + fi + if [ -n "$LIBVA_DRIVER_NAME" ]; then echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" else @@ -180,6 +190,15 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then fi if [ "$NVIDIA_FOUND" = true ]; then + # Try to get NVIDIA GPU model info + if command -v nvidia-smi >/dev/null 2>&1; then + NVIDIA_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1) + if [ -n "$NVIDIA_MODEL" ]; then + echo "🔍 Detected NVIDIA GPU: $NVIDIA_MODEL" + GPU_MODEL=$NVIDIA_MODEL + fi + fi + if [ -n "$NVIDIA_VISIBLE_DEVICES" ]; then echo "ℹī¸ NVIDIA_VISIBLE_DEVICES is set to '$NVIDIA_VISIBLE_DEVICES'" else @@ -227,8 +246,10 @@ if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VI echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" fi elif [ "$DRI_DEVICES_FOUND" = true ]; then - # Intel/AMD detection - if [ -n "$LIBVA_DRIVER_NAME" ]; then + # Intel/AMD detection with model if available + if [ -n "$GPU_MODEL" ]; then + echo "🔰 GPU: $GPU_MODEL" + elif [ -n "$LIBVA_DRIVER_NAME" ]; then echo "🔰 ${LIBVA_DRIVER_NAME^^} GPU: ACTIVE" else echo "🔰 INTEL/AMD GPU: ACTIVE" From 9c9e546f809728a5f6f776e550da7a75ffa2e34d Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 10 May 2025 08:40:53 -0400 Subject: [PATCH 10/55] websockets behind auth, cleaned up errors and bad state handling in websocket.jsx --- dispatcharr/asgi.py | 7 +- dispatcharr/consumers.py | 8 +- dispatcharr/jwt_ws_auth.py | 36 ++++++ frontend/src/WebSocket.jsx | 219 +++++++++++++++++++++++++------------ 4 files changed, 197 insertions(+), 73 deletions(-) create mode 100644 dispatcharr/jwt_ws_auth.py diff --git a/dispatcharr/asgi.py b/dispatcharr/asgi.py index 5e60f635..45923fe0 100644 --- a/dispatcharr/asgi.py +++ b/dispatcharr/asgi.py @@ -1,14 +1,17 @@ +import django import os from django.core.asgi import get_asgi_application from channels.routing import ProtocolTypeRouter, URLRouter -from channels.auth import AuthMiddlewareStack import dispatcharr.routing os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dispatcharr.settings") +django.setup() + +from .jwt_ws_auth import JWTAuthMiddleware application = ProtocolTypeRouter({ "http": get_asgi_application(), - "websocket": AuthMiddlewareStack( + "websocket": JWTAuthMiddleware( URLRouter(dispatcharr.routing.websocket_urlpatterns) ), }) diff --git a/dispatcharr/consumers.py b/dispatcharr/consumers.py index f7d4a47c..4e21bdae 100644 --- a/dispatcharr/consumers.py +++ b/dispatcharr/consumers.py @@ -6,9 +6,15 @@ logger = logging.getLogger(__name__) class MyWebSocketConsumer(AsyncWebsocketConsumer): async def connect(self): + self.room_name = "updates" + + user = self.scope["user"] + if not user.is_authenticated: + await self.close() + return + try: await self.accept() - self.room_name = "updates" await self.channel_layer.group_add(self.room_name, self.channel_name) # Send a connection confirmation to the client with consistent format await self.send(text_data=json.dumps({ diff --git a/dispatcharr/jwt_ws_auth.py b/dispatcharr/jwt_ws_auth.py new file mode 100644 index 00000000..3c7afeab --- /dev/null +++ b/dispatcharr/jwt_ws_auth.py @@ -0,0 +1,36 @@ +from urllib.parse import parse_qs +from channels.middleware import BaseMiddleware +from channels.db import database_sync_to_async +from rest_framework_simplejwt.tokens import UntypedToken +from django.contrib.auth.models import AnonymousUser +from django.contrib.auth import get_user_model +from rest_framework_simplejwt.exceptions import InvalidToken, TokenError +from rest_framework_simplejwt.authentication import JWTAuthentication + +User = get_user_model() + +@database_sync_to_async +def get_user(validated_token): + try: + jwt_auth = JWTAuthentication() + user = jwt_auth.get_user(validated_token) + return user + except: + return AnonymousUser() + +class JWTAuthMiddleware(BaseMiddleware): + async def __call__(self, scope, receive, send): + try: + # Extract the token from the query string + query_string = parse_qs(scope["query_string"].decode()) + token = query_string.get("token", [None])[0] + + if token is not None: + validated_token = JWTAuthentication().get_validated_token(token) + scope["user"] = await get_user(validated_token) + else: + scope["user"] = AnonymousUser() + except (InvalidToken, TokenError): + scope["user"] = AnonymousUser() + + return await super().__call__(scope, receive, send) diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index d12665fd..5e80f2f9 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -7,7 +7,6 @@ import React, { useMemo, useCallback, } from 'react'; -import useStreamsStore from './store/streams'; import { notifications } from '@mantine/notifications'; import useChannelsStore from './store/channels'; import usePlaylistsStore from './store/playlists'; @@ -15,8 +14,9 @@ import useEPGsStore from './store/epgs'; import { Box, Button, Stack, Alert, Group } from '@mantine/core'; import API from './api'; import useSettingsStore from './store/settings'; +import useAuthStore from './store/auth'; -export const WebsocketContext = createContext([false, () => { }, null]); +export const WebsocketContext = createContext([false, () => {}, null]); export const WebsocketProvider = ({ children }) => { const [isReady, setIsReady] = useState(false); @@ -28,10 +28,21 @@ export const WebsocketProvider = ({ children }) => { const maxReconnectAttempts = 5; const initialBackoffDelay = 1000; // 1 second initial delay const env_mode = useSettingsStore((s) => s.environment.env_mode); + const accessToken = useAuthStore((s) => s.accessToken); + + const epgs = useEPGsStore((s) => s.epgs); + const updateEPG = useEPGsStore((s) => s.updateEPG); + const updateEPGProgress = useEPGsStore((s) => s.updateEPGProgress); + + const playlists = usePlaylistsStore((s) => s.playlists); + const updatePlaylist = usePlaylistsStore((s) => s.updatePlaylist); // Calculate reconnection delay with exponential backoff const getReconnectDelay = useCallback(() => { - return Math.min(initialBackoffDelay * Math.pow(1.5, reconnectAttempts), 30000); // max 30 seconds + return Math.min( + initialBackoffDelay * Math.pow(1.5, reconnectAttempts), + 30000 + ); // max 30 seconds }, [reconnectAttempts]); // Clear any existing reconnect timers @@ -50,15 +61,15 @@ export const WebsocketProvider = ({ children }) => { // In development mode, connect directly to the WebSocket server on port 8001 if (env_mode === 'dev') { - return `${protocol}//${host}:8001/ws/`; + return `${protocol}//${host}:8001/ws/?token=${accessToken}`; } else { // In production mode, use the same port as the main application // This allows nginx to handle the WebSocket forwarding return appPort - ? `${protocol}//${host}:${appPort}/ws/` - : `${protocol}//${host}/ws/`; + ? `${protocol}//${host}:${appPort}/ws/?token=${accessToken}` + : `${protocol}//${host}/ws/?token=${accessToken}`; } - }, [env_mode]); + }, [env_mode, accessToken]); // Function to handle websocket connection const connectWebSocket = useCallback(() => { @@ -76,12 +87,14 @@ export const WebsocketProvider = ({ children }) => { try { ws.current.close(); } catch (e) { - console.warn("Error closing existing WebSocket:", e); + console.warn('Error closing existing WebSocket:', e); } } try { - console.log(`Attempting WebSocket connection (attempt ${reconnectAttempts + 1}/${maxReconnectAttempts})...`); + console.log( + `Attempting WebSocket connection (attempt ${reconnectAttempts + 1}/${maxReconnectAttempts})...` + ); // Use the function to get the correct WebSocket URL const wsUrl = getWebSocketUrl(); @@ -91,42 +104,50 @@ export const WebsocketProvider = ({ children }) => { const socket = new WebSocket(wsUrl); socket.onopen = () => { - console.log("WebSocket connected successfully"); + console.log('WebSocket connected successfully'); setIsReady(true); setConnectionError(null); setReconnectAttempts(0); }; socket.onerror = (error) => { - console.error("WebSocket connection error:", error); + console.error('WebSocket connection error:', error); // Don't show error notification on initial page load, // only show it after a connection was established then lost if (reconnectAttempts > 0 || isReady) { - setConnectionError("Failed to connect to WebSocket server."); + setConnectionError('Failed to connect to WebSocket server.'); } else { - console.log("Initial connection attempt failed, will retry..."); + console.log('Initial connection attempt failed, will retry...'); } }; socket.onclose = (event) => { - console.warn("WebSocket connection closed", event); + console.warn('WebSocket connection closed', event); setIsReady(false); // Only attempt reconnect if we haven't reached max attempts if (reconnectAttempts < maxReconnectAttempts) { const delay = getReconnectDelay(); - setConnectionError(`Connection lost. Reconnecting in ${Math.ceil(delay / 1000)} seconds...`); - console.log(`Scheduling reconnect in ${delay}ms (attempt ${reconnectAttempts + 1}/${maxReconnectAttempts})...`); + setConnectionError( + `Connection lost. Reconnecting in ${Math.ceil(delay / 1000)} seconds...` + ); + console.log( + `Scheduling reconnect in ${delay}ms (attempt ${reconnectAttempts + 1}/${maxReconnectAttempts})...` + ); // Store timer reference so we can cancel it if needed reconnectTimerRef.current = setTimeout(() => { - setReconnectAttempts(prev => prev + 1); + setReconnectAttempts((prev) => prev + 1); connectWebSocket(); }, delay); } else { - setConnectionError("Maximum reconnection attempts reached. Please reload the page."); - console.error("Maximum reconnection attempts reached. WebSocket connection failed."); + setConnectionError( + 'Maximum reconnection attempts reached. Please reload the page.' + ); + console.error( + 'Maximum reconnection attempts reached. WebSocket connection failed.' + ); } }; @@ -137,7 +158,10 @@ export const WebsocketProvider = ({ children }) => { // Handle connection_established event if (parsedEvent.type === 'connection_established') { - console.log('WebSocket connection established:', parsedEvent.data?.message); + console.log( + 'WebSocket connection established:', + parsedEvent.data?.message + ); // Don't need to do anything else for this event type return; } @@ -167,8 +191,9 @@ export const WebsocketProvider = ({ children }) => { // Update the playlist status whenever we receive a status update // Not just when progress is 100% or status is pending_setup if (parsedEvent.data.status && parsedEvent.data.account) { - const playlistsState = usePlaylistsStore.getState(); - const playlist = playlistsState.playlists.find(p => p.id === parsedEvent.data.account); + const playlist = playlists.find( + (p) => p.id === parsedEvent.data.account + ); if (playlist) { // When we receive a "success" status with 100% progress, this is a completed refresh @@ -176,15 +201,19 @@ export const WebsocketProvider = ({ children }) => { const updateData = { ...playlist, status: parsedEvent.data.status, - last_message: parsedEvent.data.message || playlist.last_message + last_message: + parsedEvent.data.message || playlist.last_message, }; // Update the timestamp when we complete a successful refresh - if (parsedEvent.data.status === 'success' && parsedEvent.data.progress === 100) { + if ( + parsedEvent.data.status === 'success' && + parsedEvent.data.progress === 100 + ) { updateData.updated_at = new Date().toISOString(); } - playlistsState.updatePlaylist(updateData); + updatePlaylist(updateData); } } break; @@ -201,12 +230,11 @@ export const WebsocketProvider = ({ children }) => { // If source_id is provided, update that specific EPG's status if (parsedEvent.data.source_id) { - const epgsState = useEPGsStore.getState(); - const epg = epgsState.epgs[parsedEvent.data.source_id]; + const epg = epgs[parsedEvent.data.source_id]; if (epg) { - epgsState.updateEPG({ + updateEPG({ ...epg, - status: 'success' + status: 'success', }); } } @@ -221,13 +249,19 @@ export const WebsocketProvider = ({ children }) => { }); // Check if we have associations data and use the more efficient batch API - if (parsedEvent.data.associations && parsedEvent.data.associations.length > 0) { + if ( + parsedEvent.data.associations && + parsedEvent.data.associations.length > 0 + ) { API.batchSetEPG(parsedEvent.data.associations); } break; case 'm3u_profile_test': - setProfilePreview(parsedEvent.data.search_preview, parsedEvent.data.result); + setProfilePreview( + parsedEvent.data.search_preview, + parsedEvent.data.result + ); break; case 'recording_started': @@ -254,13 +288,12 @@ export const WebsocketProvider = ({ children }) => { // Update EPG status in store if (parsedEvent.data.source_id) { - const epgsState = useEPGsStore.getState(); - const epg = epgsState.epgs[parsedEvent.data.source_id]; + const epg = epgs[parsedEvent.data.source_id]; if (epg) { - epgsState.updateEPG({ + updateEPG({ ...epg, status: 'error', - last_message: parsedEvent.data.message + last_message: parsedEvent.data.message, }); } } @@ -268,28 +301,33 @@ export const WebsocketProvider = ({ children }) => { case 'epg_refresh': // Update the store with progress information - const epgsState = useEPGsStore.getState(); - epgsState.updateEPGProgress(parsedEvent.data); + updateEPGProgress(parsedEvent.data); // If we have source_id/account info, update the EPG source status if (parsedEvent.data.source_id || parsedEvent.data.account) { - const sourceId = parsedEvent.data.source_id || parsedEvent.data.account; - const epg = epgsState.epgs[sourceId]; + const sourceId = + parsedEvent.data.source_id || parsedEvent.data.account; + const epg = epgs[sourceId]; if (epg) { // Check for any indication of an error (either via status or error field) - const hasError = parsedEvent.data.status === "error" || + const hasError = + parsedEvent.data.status === 'error' || !!parsedEvent.data.error || - (parsedEvent.data.message && parsedEvent.data.message.toLowerCase().includes("error")); + (parsedEvent.data.message && + parsedEvent.data.message.toLowerCase().includes('error')); if (hasError) { // Handle error state - const errorMessage = parsedEvent.data.error || parsedEvent.data.message || "Unknown error occurred"; + const errorMessage = + parsedEvent.data.error || + parsedEvent.data.message || + 'Unknown error occurred'; - epgsState.updateEPG({ + updateEPG({ ...epg, status: 'error', - last_message: errorMessage + last_message: errorMessage, }); // Show notification for the error @@ -301,14 +339,15 @@ export const WebsocketProvider = ({ children }) => { } // Update status on completion only if no errors else if (parsedEvent.data.progress === 100) { - epgsState.updateEPG({ + updateEPG({ ...epg, status: parsedEvent.data.status || 'success', - last_message: parsedEvent.data.message || epg.last_message + last_message: + parsedEvent.data.message || epg.last_message, }); // Only show success notification if we've finished parsing programs and had no errors - if (parsedEvent.data.action === "parsing_programs") { + if (parsedEvent.data.action === 'parsing_programs') { notifications.show({ title: 'EPG Processing Complete', message: 'EPG data has been updated successfully', @@ -323,29 +362,41 @@ export const WebsocketProvider = ({ children }) => { break; default: - console.error(`Unknown websocket event type: ${parsedEvent.data?.type}`); + console.error( + `Unknown websocket event type: ${parsedEvent.data?.type}` + ); break; } } catch (error) { - console.error('Error processing WebSocket message:', error, event.data); + console.error( + 'Error processing WebSocket message:', + error, + event.data + ); } }; ws.current = socket; } catch (error) { - console.error("Error creating WebSocket connection:", error); + console.error('Error creating WebSocket connection:', error); setConnectionError(`WebSocket error: ${error.message}`); // Schedule a reconnect if we haven't reached max attempts if (reconnectAttempts < maxReconnectAttempts) { const delay = getReconnectDelay(); reconnectTimerRef.current = setTimeout(() => { - setReconnectAttempts(prev => prev + 1); + setReconnectAttempts((prev) => prev + 1); connectWebSocket(); }, delay); } } - }, [reconnectAttempts, clearReconnectTimer, getReconnectDelay, getWebSocketUrl, isReady]); + }, [ + reconnectAttempts, + clearReconnectTimer, + getReconnectDelay, + getWebSocketUrl, + isReady, + ]); // Initial connection and cleanup useEffect(() => { @@ -355,7 +406,7 @@ export const WebsocketProvider = ({ children }) => { clearReconnectTimer(); // Clear any pending reconnect timers if (ws.current) { - console.log("Closing WebSocket connection due to component unmount"); + console.log('Closing WebSocket connection due to component unmount'); ws.current.onclose = null; // Remove handlers to avoid reconnection ws.current.close(); ws.current = null; @@ -364,7 +415,6 @@ export const WebsocketProvider = ({ children }) => { }, [connectWebSocket, clearReconnectTimer]); const setChannelStats = useChannelsStore((s) => s.setChannelStats); - const fetchChannelGroups = useChannelsStore((s) => s.fetchChannelGroups); const fetchPlaylists = usePlaylistsStore((s) => s.fetchPlaylists); const setRefreshProgress = usePlaylistsStore((s) => s.setRefreshProgress); const setProfilePreview = usePlaylistsStore((s) => s.setProfilePreview); @@ -377,22 +427,51 @@ export const WebsocketProvider = ({ children }) => { return ( - {connectionError && !isReady && reconnectAttempts >= maxReconnectAttempts && ( - - {connectionError} - - - )} - {connectionError && !isReady && reconnectAttempts < maxReconnectAttempts && reconnectAttempts > 0 && ( - - {connectionError} - - )} + {connectionError && + !isReady && + reconnectAttempts >= maxReconnectAttempts && ( + + {connectionError} + + + )} + {connectionError && + !isReady && + reconnectAttempts < maxReconnectAttempts && + reconnectAttempts > 0 && ( + + {connectionError} + + )} {children} ); From 2dba3aca2415ee274c97117967ede64f7803dfc2 Mon Sep 17 00:00:00 2001 From: dekzter Date: Sat, 10 May 2025 09:39:03 -0400 Subject: [PATCH 11/55] fixed dependencies for onEdit callback --- .../src/components/tables/StreamsTable.jsx | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/tables/StreamsTable.jsx b/frontend/src/components/tables/StreamsTable.jsx index 7fccc681..10c3b410 100644 --- a/frontend/src/components/tables/StreamsTable.jsx +++ b/frontend/src/components/tables/StreamsTable.jsx @@ -84,7 +84,7 @@ const StreamRowActions = ({ const onEdit = useCallback(() => { editStream(row.original); - }, [row.original.id, editStream]); + }, [row.original, editStream]); const onDelete = useCallback(() => { deleteStream(row.original.id); @@ -169,7 +169,7 @@ const StreamRowActions = ({ ); }; -const StreamsTable = ({ }) => { +const StreamsTable = ({}) => { const theme = useMantineTheme(); /** @@ -399,7 +399,14 @@ const StreamsTable = ({ }) => { } setIsLoading(false); - }, [pagination, sorting, debouncedFilters, groupsLoaded, channelGroups, fetchChannelGroups]); + }, [ + pagination, + sorting, + debouncedFilters, + groupsLoaded, + channelGroups, + fetchChannelGroups, + ]); // Bulk creation: create channels from selected streams in one API call const createChannelsFromStreams = async () => { @@ -592,7 +599,14 @@ const StreamsTable = ({ }) => { ); } }, - [selectedChannelIds, channelSelectionStreams, theme, editStream, deleteStream, handleWatchStream] + [ + selectedChannelIds, + channelSelectionStreams, + theme, + editStream, + deleteStream, + handleWatchStream, + ] ); const table = useTable({ @@ -675,10 +689,10 @@ const StreamsTable = ({ }) => { style={ selectedStreamIds.length > 0 && selectedChannelIds.length === 1 ? { - borderWidth: '1px', - borderColor: theme.tailwind.green[5], - color: 'white', - } + borderWidth: '1px', + borderColor: theme.tailwind.green[5], + color: 'white', + } : undefined } disabled={ From d3615e1a6628cef38ed0b9440ffc4d7c58b17990 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 09:29:06 -0500 Subject: [PATCH 12/55] Huge overhaul of logging. More standardized and we are now capturing logs from celery task and sening to console. Also adds a new environmental variable: DISPATCHARR_LOG_LEVEL, log levels available: TRACE, DEBUG, INFO, WARNING, ERROR, CRITICAL --- core/apps.py | 13 +++++++ core/tasks.py | 66 +++++++++++++++++++++++++++++------ dispatcharr/celery.py | 20 +++++++++++ dispatcharr/settings.py | 77 +++++++++++++++++++++++++++++++++++++++++ docker/uwsgi.debug.ini | 2 +- 5 files changed, 167 insertions(+), 11 deletions(-) diff --git a/core/apps.py b/core/apps.py index 63c883ca..96ccfb3b 100644 --- a/core/apps.py +++ b/core/apps.py @@ -2,6 +2,19 @@ from django.apps import AppConfig from django.conf import settings import os, logging +# Define TRACE level (5 is below DEBUG which is 10) +TRACE = 5 +logging.addLevelName(TRACE, "TRACE") + +# Add trace method to the Logger class +def trace(self, message, *args, **kwargs): + """Log a message with TRACE level (more detailed than DEBUG)""" + if self.isEnabledFor(TRACE): + self._log(TRACE, message, args, **kwargs) + +# Add the trace method to the Logger class +logging.Logger.trace = trace + class CoreConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'core' diff --git a/core/tasks.py b/core/tasks.py index 64a89c7a..3668a873 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -33,6 +33,8 @@ last_known_data = {} _last_log_times = {} # Don't repeat similar log messages more often than this (in seconds) LOG_THROTTLE_SECONDS = 300 # 5 minutes +# Track if this is the first scan since startup +_first_scan_completed = False @shared_task def beat_periodic_task(): @@ -52,6 +54,7 @@ def throttled_log(logger_method, message, key=None, *args, **kwargs): @shared_task def scan_and_process_files(): + global _first_scan_completed redis_client = RedisClient.get_client() now = time.time() @@ -84,7 +87,11 @@ def scan_and_process_files(): # Check if this file is already in the database existing_m3u = M3UAccount.objects.filter(file_path=filepath).exists() if existing_m3u: - logger.debug(f"Skipping {filename}: Already exists in database") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Already exists in database") + else: + logger.debug(f"Skipping {filename}: Already exists in database") redis_client.set(redis_key, mtime, ex=REDIS_TTL) m3u_skipped += 1 continue @@ -94,13 +101,21 @@ def scan_and_process_files(): # File too new — probably still being written if age < MIN_AGE_SECONDS: - logger.debug(f"Skipping {filename}: Too new (age={age}s)") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Too new (age={age}s)") + else: + logger.debug(f"Skipping {filename}: Too new (age={age}s)") m3u_skipped += 1 continue # Skip if we've already processed this mtime if stored_mtime and float(stored_mtime) >= mtime: - logger.debug(f"Skipping {filename}: Already processed this version") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Already processed this version") + else: + logger.debug(f"Skipping {filename}: Already processed this version") m3u_skipped += 1 continue @@ -112,7 +127,11 @@ def scan_and_process_files(): redis_client.set(redis_key, mtime, ex=REDIS_TTL) if not m3u_account.is_active: - logger.debug(f"Skipping {filename}: M3U account is inactive") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: M3U account is inactive") + else: + logger.debug(f"Skipping {filename}: M3U account is inactive") m3u_skipped += 1 continue @@ -147,12 +166,20 @@ def scan_and_process_files(): filepath = os.path.join(EPG_WATCH_DIR, filename) if not os.path.isfile(filepath): - logger.debug(f"Skipping {filename}: Not a file") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Not a file") + else: + logger.debug(f"Skipping {filename}: Not a file") epg_skipped += 1 continue if not filename.endswith('.xml') and not filename.endswith('.gz'): - logger.debug(f"Skipping {filename}: Not an XML or GZ file") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Not an XML or GZ file") + else: + logger.debug(f"Skipping {filename}: Not an XML or GZ file") epg_skipped += 1 continue @@ -166,7 +193,11 @@ def scan_and_process_files(): # Check if this file is already in the database existing_epg = EPGSource.objects.filter(file_path=filepath).exists() if existing_epg: - logger.debug(f"Skipping {filename}: Already exists in database") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Already exists in database") + else: + logger.debug(f"Skipping {filename}: Already exists in database") redis_client.set(redis_key, mtime, ex=REDIS_TTL) epg_skipped += 1 continue @@ -176,13 +207,21 @@ def scan_and_process_files(): # File too new — probably still being written if age < MIN_AGE_SECONDS: - logger.debug(f"Skipping {filename}: Too new, possibly still being written (age={age}s)") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Too new, possibly still being written (age={age}s)") + else: + logger.debug(f"Skipping {filename}: Too new, possibly still being written (age={age}s)") epg_skipped += 1 continue # Skip if we've already processed this mtime if stored_mtime and float(stored_mtime) >= mtime: - logger.debug(f"Skipping {filename}: Already processed this version") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: Already processed this version") + else: + logger.debug(f"Skipping {filename}: Already processed this version") epg_skipped += 1 continue @@ -200,7 +239,11 @@ def scan_and_process_files(): redis_client.set(redis_key, mtime, ex=REDIS_TTL) if not epg_source.is_active: - logger.debug(f"Skipping {filename}: EPG source is marked as inactive") + # Use trace level if not first scan + if _first_scan_completed: + logger.trace(f"Skipping {filename}: EPG source is marked as inactive") + else: + logger.debug(f"Skipping {filename}: EPG source is marked as inactive") epg_skipped += 1 continue @@ -215,6 +258,9 @@ def scan_and_process_files(): logger.debug(f"EPG processing complete: {epg_processed} processed, {epg_skipped} skipped, {epg_errors} errors") + # Mark that the first scan is complete + _first_scan_completed = True + def fetch_channel_stats(): redis_client = RedisClient.get_client() diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 1ff4fe09..f9e2525d 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -1,8 +1,28 @@ # dispatcharr/celery.py import os from celery import Celery +import logging os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') app = Celery("dispatcharr") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() + +# Configure Celery logging +app.conf.update( + worker_log_level='DEBUG', + worker_log_format='%(asctime)s %(levelname)s %(name)s: %(message)s', + beat_log_level='DEBUG', + worker_hijack_root_logger=False, + worker_task_log_format='%(asctime)s %(levelname)s %(task_name)s: %(message)s', +) + +# Set only specific log messages to DEBUG level +# This maintains user configurability for all other loggers +@app.on_after_configure.connect +def setup_celery_logging(**kwargs): + # Only set specific loggers to DEBUG that handle the routine messages + # we want to suppress from INFO level + logging.getLogger('celery.beat').getChild('Scheduler').setLevel(logging.DEBUG) + logging.getLogger('celery.worker.strategy').setLevel(logging.DEBUG) + logging.getLogger('celery.app.trace').setLevel(logging.DEBUG) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 96bda89b..1b4337ec 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -232,3 +232,80 @@ PROXY_SETTINGS = { 'REDIS_CHUNK_TTL': 60, # How long to keep chunks in Redis (seconds) } } + +# Map log level names to their numeric values +LOG_LEVEL_MAP = { + 'TRACE': 5, + 'DEBUG': 10, + 'INFO': 20, + 'WARNING': 30, + 'ERROR': 40, + 'CRITICAL': 50 +} + +# Get log level from environment variable, default to INFO if not set +LOG_LEVEL_NAME = os.environ.get('DISPATCHARR_LOG_LEVEL', 'INFO').upper() +LOG_LEVEL = LOG_LEVEL_MAP.get(LOG_LEVEL_NAME, 20) # Default to INFO (20) if invalid + +# Add this to your existing LOGGING configuration or create one if it doesn't exist +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '{asctime} {levelname} {name} {message}', + 'style': '{', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'verbose', + 'level': 5, # Always allow TRACE level messages through the handler + }, + }, + 'loggers': { + 'core.tasks': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use environment-configured level + 'propagate': False, # Don't propagate to root logger to avoid duplicate logs + }, + 'apps.proxy': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use environment-configured level + 'propagate': False, # Don't propagate to root logger + }, + # Add parent logger for all app modules + 'apps': { + 'handlers': ['console'], + 'level': LOG_LEVEL, + 'propagate': False, + }, + # Celery loggers to capture task execution messages + 'celery': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use configured log level for Celery logs + 'propagate': False, + }, + 'celery.task': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use configured log level for task-specific logs + 'propagate': False, + }, + 'celery.worker': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use configured log level for worker logs + 'propagate': False, + }, + 'celery.beat': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use configured log level for scheduler logs + 'propagate': False, + }, + # Add any other loggers you need to capture TRACE logs from + }, + 'root': { + 'handlers': ['console'], + 'level': LOG_LEVEL, # Use user-configured level instead of hardcoded 'INFO' + }, +} diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 5ab5e6d6..6acbdac3 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -9,7 +9,7 @@ exec-before = python /app/scripts/wait_for_redis.py attach-daemon = redis-server ; Then start other services attach-daemon = celery -A dispatcharr worker -l debug -attach-daemon = celery -A dispatcharr beat -l info +attach-daemon = celery -A dispatcharr beat -l debug attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev From 24fba3c2b1e8f632474c8388fb089e91a4317287 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 09:58:57 -0500 Subject: [PATCH 13/55] Change some celery tasks from info to debug. --- dispatcharr/celery.py | 33 ++++++++++++++++++++++++--------- docker/uwsgi.debug.ini | 4 ++-- docker/uwsgi.dev.ini | 4 ++-- docker/uwsgi.ini | 4 ++-- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index f9e2525d..28b94263 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -2,6 +2,7 @@ import os from celery import Celery import logging +from django.conf import settings # Import Django settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') app = Celery("dispatcharr") @@ -10,19 +11,33 @@ app.autodiscover_tasks() # Configure Celery logging app.conf.update( - worker_log_level='DEBUG', + worker_log_level=settings.LOG_LEVEL_NAME, # Use same log level from environment worker_log_format='%(asctime)s %(levelname)s %(name)s: %(message)s', - beat_log_level='DEBUG', + beat_log_level=settings.LOG_LEVEL_NAME, # Use same log level from environment worker_hijack_root_logger=False, worker_task_log_format='%(asctime)s %(levelname)s %(task_name)s: %(message)s', ) -# Set only specific log messages to DEBUG level -# This maintains user configurability for all other loggers @app.on_after_configure.connect def setup_celery_logging(**kwargs): - # Only set specific loggers to DEBUG that handle the routine messages - # we want to suppress from INFO level - logging.getLogger('celery.beat').getChild('Scheduler').setLevel(logging.DEBUG) - logging.getLogger('celery.worker.strategy').setLevel(logging.DEBUG) - logging.getLogger('celery.app.trace').setLevel(logging.DEBUG) + # Check if user has set logging to INFO level + if settings.LOG_LEVEL_NAME.upper() == 'INFO': + # Get the specific loggers that output the noisy INFO messages + for logger_name in ['celery.app.trace', 'celery.beat', 'celery.worker.strategy', 'celery.beat.Scheduler']: + # Create a custom filter to suppress specific messages + logger = logging.getLogger(logger_name) + + # Add a custom filter to completely filter out the repetitive messages + class SuppressFilter(logging.Filter): + def filter(self, record): + # Return False to completely suppress these specific patterns when at INFO level + if ( + "succeeded in" in getattr(record, 'msg', '') or + "Scheduler: Sending due task" in getattr(record, 'msg', '') or + "received" in getattr(record, 'msg', '') + ): + return False # Don't log these messages at all + return True # Log all other messages + + # Add the filter to each logger + logger.addFilter(SuppressFilter()) diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 6acbdac3..9cd4ccdf 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -8,8 +8,8 @@ exec-before = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker -l debug -attach-daemon = celery -A dispatcharr beat -l debug +attach-daemon = celery -A dispatcharr worker +attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 0aae9516..455111d3 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -8,8 +8,8 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker -l debug --concurrency=4 -attach-daemon = celery -A dispatcharr beat -l debug +attach-daemon = celery -A dispatcharr worker --concurrency=4 +attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application attach-daemon = cd /app/frontend && npm run dev diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 89d6e8a1..75d4de3f 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -8,8 +8,8 @@ exec-pre = python /app/scripts/wait_for_redis.py ; Start Redis first attach-daemon = redis-server ; Then start other services -attach-daemon = celery -A dispatcharr worker -l info --concurrency=4 -attach-daemon = celery -A dispatcharr beat -l error +attach-daemon = celery -A dispatcharr worker --concurrency=4 +attach-daemon = celery -A dispatcharr beat attach-daemon = daphne -b 0.0.0.0 -p 8001 dispatcharr.asgi:application # Core settings From ecd9d146c685dc3a96476bff085f53ac5b55782b Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 10:29:17 -0500 Subject: [PATCH 14/55] Changed several logging levels. --- apps/m3u/tasks.py | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 7bc549e0..5a0066cd 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -244,7 +244,7 @@ def process_groups(account, groups): group_objs = [] groups_to_create = [] for group_name, custom_props in groups.items(): - logger.info(f"Handling group: {group_name}") + logger.debug(f"Handling group: {group_name}") if (group_name not in existing_groups) and (group_name not in SKIP_EXTS): groups_to_create.append(ChannelGroup( name=group_name, @@ -253,9 +253,9 @@ def process_groups(account, groups): group_objs.append(existing_groups[group_name]) if groups_to_create: - logger.info(f"Creating {len(groups_to_create)} groups") + logger.debug(f"Creating {len(groups_to_create)} groups") created = ChannelGroup.bulk_create_and_fetch(groups_to_create) - logger.info(f"Created {len(created)} groups") + logger.debug(f"Created {len(created)} groups") group_objs.extend(created) relations = [] @@ -301,14 +301,14 @@ def process_xc_category(account_id, batch, groups, hash_keys): continue try: - logger.info(f"Fetching streams for XC category: {group_name} (ID: {props['xc_id']})") + logger.debug(f"Fetching streams for XC category: {group_name} (ID: {props['xc_id']})") streams = xc_client.get_live_category_streams(props['xc_id']) if not streams: logger.warning(f"No streams found for XC category {group_name} (ID: {props['xc_id']})") continue - logger.info(f"Found {len(streams)} streams for category {group_name}") + logger.debug(f"Found {len(streams)} streams for category {group_name}") for stream in streams: name = stream["name"] @@ -550,7 +550,7 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): if account.account_type == M3UAccount.Types.XC: # Log detailed information about the account logger.info(f"Processing XC account {account_id} with URL: {account.server_url}") - logger.info(f"Username: {account.username}, Has password: {'Yes' if account.password else 'No'}") + logger.debug(f"Username: {account.username}, Has password: {'Yes' if account.password else 'No'}") # Validate required fields if not account.server_url: @@ -582,7 +582,7 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): # User agent handling - completely rewritten try: # Debug the user agent issue - logger.info(f"Getting user agent for account {account.id}") + logger.debug(f"Getting user agent for account {account.id}") # Use a hardcoded user agent string to avoid any issues with object structure user_agent_string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" @@ -593,20 +593,20 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): ua_obj = UserAgent.objects.get(id=account.user_agent_id) if ua_obj and hasattr(ua_obj, 'user_agent') and ua_obj.user_agent: user_agent_string = ua_obj.user_agent - logger.info(f"Using user agent from account: {user_agent_string}") + logger.debug(f"Using user agent from account: {user_agent_string}") else: # Get default user agent from CoreSettings default_ua_id = CoreSettings.get_default_user_agent_id() - logger.info(f"Default user agent ID from settings: {default_ua_id}") + logger.debug(f"Default user agent ID from settings: {default_ua_id}") if default_ua_id: ua_obj = UserAgent.objects.get(id=default_ua_id) if ua_obj and hasattr(ua_obj, 'user_agent') and ua_obj.user_agent: user_agent_string = ua_obj.user_agent - logger.info(f"Using default user agent: {user_agent_string}") + logger.debug(f"Using default user agent: {user_agent_string}") except Exception as e: logger.warning(f"Error getting user agent, using fallback: {str(e)}") - logger.info(f"Final user agent string: {user_agent_string}") + logger.debug(f"Final user agent string: {user_agent_string}") except Exception as e: user_agent_string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" logger.warning(f"Exception in user agent handling, using fallback: {str(e)}") @@ -629,9 +629,9 @@ def refresh_m3u_groups(account_id, use_cache=False, full_refresh=False): # Authenticate with detailed error handling try: - logger.info(f"Authenticating with XC server {server_url}") + logger.debug(f"Authenticating with XC server {server_url}") auth_result = xc_client.authenticate() - logger.info(f"Authentication response: {auth_result}") + logger.debug(f"Authentication response: {auth_result}") except Exception as e: error_msg = f"Failed to authenticate with XC server: {str(e)}" logger.error(error_msg) @@ -747,7 +747,7 @@ def delete_m3u_refresh_task_by_id(account_id): try: from django_celery_beat.models import PeriodicTask, IntervalSchedule task = PeriodicTask.objects.get(name=task_name) - logger.info(f"Found task by name: {task.id} for M3UAccount {account_id}") + logger.debug(f"Found task by name: {task.id} for M3UAccount {account_id}") except PeriodicTask.DoesNotExist: logger.warning(f"No PeriodicTask found with name {task_name}") return False @@ -761,25 +761,25 @@ def delete_m3u_refresh_task_by_id(account_id): # Count how many TOTAL tasks use this interval (including this one) tasks_with_same_interval = PeriodicTask.objects.filter(interval_id=interval_id).count() - logger.info(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") + logger.debug(f"Interval {interval_id} is used by {tasks_with_same_interval} tasks total") # Delete the task first task_id = task.id task.delete() - logger.info(f"Successfully deleted periodic task {task_id}") + logger.debug(f"Successfully deleted periodic task {task_id}") # Now check if we should delete the interval # We only delete if it was the ONLY task using this interval if interval_id and tasks_with_same_interval == 1: try: interval = IntervalSchedule.objects.get(id=interval_id) - logger.info(f"Deleting interval schedule {interval_id} (not shared with other tasks)") + logger.debug(f"Deleting interval schedule {interval_id} (not shared with other tasks)") interval.delete() - logger.info(f"Successfully deleted interval {interval_id}") + logger.debug(f"Successfully deleted interval {interval_id}") except IntervalSchedule.DoesNotExist: logger.warning(f"Interval {interval_id} no longer exists") elif interval_id: - logger.info(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") + logger.debug(f"Not deleting interval {interval_id} as it's shared with {tasks_with_same_interval-1} other tasks") return True return False @@ -802,7 +802,7 @@ def refresh_single_m3u_account(account_id): try: account = M3UAccount.objects.get(id=account_id, is_active=True) if not account.is_active: - logger.info(f"Account {account_id} is not active, skipping.") + logger.debug(f"Account {account_id} is not active, skipping.") release_task_lock('refresh_single_m3u_account', account_id) return @@ -819,7 +819,7 @@ def refresh_single_m3u_account(account_id): if delete_m3u_refresh_task_by_id(account_id): logger.info(f"Successfully cleaned up orphaned task for M3U account {account_id}") else: - logger.info(f"No orphaned task found for M3U account {account_id}") + logger.debug(f"No orphaned task found for M3U account {account_id}") release_task_lock('refresh_single_m3u_account', account_id) return f"M3UAccount with ID={account_id} not found or inactive, task cleaned up" @@ -840,7 +840,7 @@ def refresh_single_m3u_account(account_id): try: logger.info(f"Calling refresh_m3u_groups for account {account_id}") result = refresh_m3u_groups(account_id, full_refresh=True) - logger.info(f"refresh_m3u_groups result: {result}") + logger.debug(f"refresh_m3u_groups result: {result}") # Check for completely empty result or missing groups if not result or result[1] is None: @@ -908,7 +908,7 @@ def refresh_single_m3u_account(account_id): task_group = group(process_m3u_batch.s(account_id, batch, existing_groups, hash_keys) for batch in batches) else: # For XC accounts, get the groups with their custom properties containing xc_id - logger.info(f"Processing XC account with groups: {existing_groups}") + logger.debug(f"Processing XC account with groups: {existing_groups}") # Get the ChannelGroupM3UAccount entries with their custom_properties channel_group_relationships = ChannelGroupM3UAccount.objects.filter( @@ -929,7 +929,7 @@ def refresh_single_m3u_account(account_id): 'xc_id': custom_props['xc_id'], 'channel_group_id': group_id } - logger.info(f"Added group {group_name} with xc_id {custom_props['xc_id']}") + logger.debug(f"Added group {group_name} with xc_id {custom_props['xc_id']}") else: logger.warning(f"No xc_id found in custom properties for group {group_name}") except (json.JSONDecodeError, KeyError) as e: From 67aca64420a5a3ff831a1f0d5a995e8494242bb7 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 13:25:03 -0500 Subject: [PATCH 15/55] Properly set EV for all profiles so uWSGI daemons can see it. --- dispatcharr/celery.py | 68 +++++++++++++++++++++++++++++++++++------ dispatcharr/settings.py | 12 +++++++- docker/entrypoint.sh | 24 ++++++++++++++- 3 files changed, 92 insertions(+), 12 deletions(-) diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 28b94263..2fe9e6ec 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -2,35 +2,74 @@ import os from celery import Celery import logging -from django.conf import settings # Import Django settings + +# Initialize with defaults before Django settings are loaded +DEFAULT_LOG_LEVEL = 'DEBUG' + +# Try multiple sources for log level in order of preference +def get_effective_log_level(): + # 1. Direct environment variable + env_level = os.environ.get('DISPATCHARR_LOG_LEVEL', '').upper() + if env_level and not env_level.startswith('$(') and not env_level.startswith('%('): + return env_level + + # 2. Check temp file that may have been created by settings.py + try: + if os.path.exists('/tmp/dispatcharr_log_level'): + with open('/tmp/dispatcharr_log_level', 'r') as f: + file_level = f.read().strip().upper() + if file_level: + return file_level + except: + pass + + # 3. Fallback to default + return DEFAULT_LOG_LEVEL + +# Get effective log level before Django loads +effective_log_level = get_effective_log_level() +print(f"Celery using effective log level: {effective_log_level}") os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dispatcharr.settings') app = Celery("dispatcharr") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() +# Use environment variable for log level with fallback to INFO +CELERY_LOG_LEVEL = os.environ.get('DISPATCHARR_LOG_LEVEL', 'INFO').upper() +print(f"Celery using log level from environment: {CELERY_LOG_LEVEL}") + # Configure Celery logging app.conf.update( - worker_log_level=settings.LOG_LEVEL_NAME, # Use same log level from environment + worker_log_level=effective_log_level, worker_log_format='%(asctime)s %(levelname)s %(name)s: %(message)s', - beat_log_level=settings.LOG_LEVEL_NAME, # Use same log level from environment + beat_log_level=effective_log_level, worker_hijack_root_logger=False, worker_task_log_format='%(asctime)s %(levelname)s %(task_name)s: %(message)s', ) @app.on_after_configure.connect def setup_celery_logging(**kwargs): - # Check if user has set logging to INFO level - if settings.LOG_LEVEL_NAME.upper() == 'INFO': - # Get the specific loggers that output the noisy INFO messages - for logger_name in ['celery.app.trace', 'celery.beat', 'celery.worker.strategy', 'celery.beat.Scheduler']: - # Create a custom filter to suppress specific messages - logger = logging.getLogger(logger_name) + # Use our directly determined log level + log_level = effective_log_level + print(f"Celery configuring loggers with level: {log_level}") + # Get the specific loggers that output potentially noisy messages + for logger_name in ['celery.app.trace', 'celery.beat', 'celery.worker.strategy', 'celery.beat.Scheduler']: + logger = logging.getLogger(logger_name) + + # Remove any existing filters first (in case this runs multiple times) + for filter in logger.filters[:]: + if hasattr(filter, '__class__') and filter.__class__.__name__ == 'SuppressFilter': + logger.removeFilter(filter) + + # For INFO level - add a filter to hide repetitive messages + # For DEBUG/TRACE level - don't filter (show everything) + if log_level == 'INFO': # Add a custom filter to completely filter out the repetitive messages class SuppressFilter(logging.Filter): def filter(self, record): - # Return False to completely suppress these specific patterns when at INFO level + # Return False to completely suppress these specific patterns if ( "succeeded in" in getattr(record, 'msg', '') or "Scheduler: Sending due task" in getattr(record, 'msg', '') or @@ -41,3 +80,12 @@ def setup_celery_logging(**kwargs): # Add the filter to each logger logger.addFilter(SuppressFilter()) + + # Set all Celery loggers to the configured level + # This ensures they respect TRACE/DEBUG when set + try: + numeric_level = getattr(logging, log_level) + logger.setLevel(numeric_level) + except (AttributeError, TypeError): + # If the log level string is invalid, default to DEBUG + logger.setLevel(logging.DEBUG) diff --git a/dispatcharr/settings.py b/dispatcharr/settings.py index 1b4337ec..8f3921c0 100644 --- a/dispatcharr/settings.py +++ b/dispatcharr/settings.py @@ -244,7 +244,17 @@ LOG_LEVEL_MAP = { } # Get log level from environment variable, default to INFO if not set -LOG_LEVEL_NAME = os.environ.get('DISPATCHARR_LOG_LEVEL', 'INFO').upper() +# Add debugging output to see exactly what's being detected +env_log_level = os.environ.get('DISPATCHARR_LOG_LEVEL', '') +print(f"Environment DISPATCHARR_LOG_LEVEL detected as: '{env_log_level}'") + +if not env_log_level: + print("No DISPATCHARR_LOG_LEVEL found in environment, using default INFO") + LOG_LEVEL_NAME = 'INFO' +else: + LOG_LEVEL_NAME = env_log_level.upper() + print(f"Setting log level to: {LOG_LEVEL_NAME}") + LOG_LEVEL = LOG_LEVEL_MAP.get(LOG_LEVEL_NAME, 20) # Default to INFO (20) if invalid # Add this to your existing LOGGING configuration or create one if it doesn't exist diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bbd2b9ba..3d2161d5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -49,6 +49,14 @@ else echo "đŸ“Ļ Dispatcharr version: ${DISPATCHARR_VERSION}" fi +# Set log level with default if not provided +export DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL:-info} +echo "Environment DISPATCHARR_LOG_LEVEL detected as: '${DISPATCHARR_LOG_LEVEL}'" +echo "Setting log level to: ${DISPATCHARR_LOG_LEVEL}" + +# Also make the log level available in /etc/environment for all login shells +#grep -q "DISPATCHARR_LOG_LEVEL" /etc/environment || echo "DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL}" >> /etc/environment + # READ-ONLY - don't let users change these export POSTGRES_DIR=/data/db @@ -65,12 +73,24 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export POSTGRES_PORT=$POSTGRES_PORT" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_ENV=$DISPATCHARR_ENV" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_DEBUG=$DISPATCHARR_DEBUG" >> /etc/profile.d/dispatcharr.sh + echo "export DISPATCHARR_LOG_LEVEL=$DISPATCHARR_LOG_LEVEL" >> /etc/profile.d/dispatcharr.sh echo "export REDIS_HOST=$REDIS_HOST" >> /etc/profile.d/dispatcharr.sh echo "export REDIS_DB=$REDIS_DB" >> /etc/profile.d/dispatcharr.sh echo "export POSTGRES_DIR=$POSTGRES_DIR" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh + + # Make sure we also set these variables in /etc/environment + # which is sourced even before /etc/profile.d scripts + for var in PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED POSTGRES_DB POSTGRES_USER \ + POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ + DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ + DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP; do + if [[ -n "${!var}" ]]; then + grep -q "^${var}=" /etc/environment || echo "${var}=${!var}" >> /etc/environment + fi + done fi chmod +x /etc/profile.d/dispatcharr.sh @@ -127,7 +147,9 @@ else uwsgi_file="/app/docker/uwsgi.ini" fi -su - $POSTGRES_USER -c "cd /app && uwsgi --ini $uwsgi_file &" +# Pass all environment variables to the uwsgi process +# The -p/--preserve-environment flag ensures all environment variables are passed through +su -p - $POSTGRES_USER -c "cd /app && uwsgi --ini $uwsgi_file &" uwsgi_pid=$(pgrep uwsgi | sort | head -n1) echo "✅ uwsgi started with PID $uwsgi_pid" pids+=("$uwsgi_pid") From d5b64a56d69d12365aca412e7cfe3324557bd1a2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 13:56:55 -0500 Subject: [PATCH 16/55] Smarter logging for file change imports. --- core/tasks.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/core/tasks.py b/core/tasks.py index 3668a873..a6bd80cf 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -57,11 +57,6 @@ def scan_and_process_files(): global _first_scan_completed redis_client = RedisClient.get_client() now = time.time() - - # Add debug logging for the auto-import setting - auto_import_value = CoreSettings.get_auto_import_mapped_files() - logger.debug(f"Auto-import mapped files setting value: '{auto_import_value}' (type: {type(auto_import_value).__name__})") - # Check if directories exist dirs_exist = all(os.path.exists(d) for d in [M3U_WATCH_DIR, EPG_WATCH_DIR]) if not dirs_exist: @@ -101,11 +96,7 @@ def scan_and_process_files(): # File too new — probably still being written if age < MIN_AGE_SECONDS: - # Use trace level if not first scan - if _first_scan_completed: - logger.trace(f"Skipping {filename}: Too new (age={age}s)") - else: - logger.debug(f"Skipping {filename}: Too new (age={age}s)") + logger.debug(f"Skipping {filename}: Too new (age={age}s)") m3u_skipped += 1 continue @@ -126,6 +117,13 @@ def scan_and_process_files(): redis_client.set(redis_key, mtime, ex=REDIS_TTL) + # More descriptive creation logging that includes active status + if created: + if m3u_account.is_active: + logger.info(f"Created new M3U account '{filename}' (active)") + else: + logger.info(f"Created new M3U account '{filename}' (inactive due to auto-import setting)") + if not m3u_account.is_active: # Use trace level if not first scan if _first_scan_completed: @@ -135,6 +133,10 @@ def scan_and_process_files(): m3u_skipped += 1 continue + # Log update for existing files (we've already logged creation above) + if not created: + logger.info(f"Detected update to existing M3U file: {filename}") + logger.info(f"Queueing refresh for M3U file: {filename}") refresh_single_m3u_account.delay(m3u_account.id) m3u_processed += 1 @@ -148,12 +150,12 @@ def scan_and_process_files(): }, ) - logger.debug(f"M3U processing complete: {m3u_processed} processed, {m3u_skipped} skipped, {len(m3u_files)} total") + logger.trace(f"M3U processing complete: {m3u_processed} processed, {m3u_skipped} skipped, {len(m3u_files)} total") # Process EPG files try: epg_files = os.listdir(EPG_WATCH_DIR) - logger.debug(f"Found {len(epg_files)} files in EPG directory") + logger.trace(f"Found {len(epg_files)} files in EPG directory") except Exception as e: logger.error(f"Error listing EPG directory: {e}") epg_files = [] @@ -232,12 +234,15 @@ def scan_and_process_files(): "is_active": CoreSettings.get_auto_import_mapped_files() in [True, "true", "True"], }) - # Add debug logging for created sources - if created: - logger.info(f"Created new EPG source '{filename}'") - redis_client.set(redis_key, mtime, ex=REDIS_TTL) + # More descriptive creation logging that includes active status + if created: + if epg_source.is_active: + logger.info(f"Created new EPG source '{filename}' (active)") + else: + logger.info(f"Created new EPG source '{filename}' (inactive due to auto-import setting)") + if not epg_source.is_active: # Use trace level if not first scan if _first_scan_completed: @@ -247,6 +252,10 @@ def scan_and_process_files(): epg_skipped += 1 continue + # Log update for existing files (we've already logged creation above) + if not created: + logger.info(f"Detected update to existing EPG file: {filename}") + logger.info(f"Queueing refresh for EPG file: {filename}") refresh_epg_data.delay(epg_source.id) # Trigger Celery task epg_processed += 1 @@ -256,7 +265,7 @@ def scan_and_process_files(): epg_errors += 1 continue - logger.debug(f"EPG processing complete: {epg_processed} processed, {epg_skipped} skipped, {epg_errors} errors") + logger.trace(f"EPG processing complete: {epg_processed} processed, {epg_skipped} skipped, {epg_errors} errors") # Mark that the first scan is complete _first_scan_completed = True From ee8cef5aa9d16c91f854147133f1b7c894f57f49 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 14:13:43 -0500 Subject: [PATCH 17/55] Cleaned up logging. --- apps/m3u/tasks.py | 4 ++-- dispatcharr/celery.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/m3u/tasks.py b/apps/m3u/tasks.py index 5a0066cd..f6b1044c 100644 --- a/apps/m3u/tasks.py +++ b/apps/m3u/tasks.py @@ -840,7 +840,7 @@ def refresh_single_m3u_account(account_id): try: logger.info(f"Calling refresh_m3u_groups for account {account_id}") result = refresh_m3u_groups(account_id, full_refresh=True) - logger.debug(f"refresh_m3u_groups result: {result}") + logger.trace(f"refresh_m3u_groups result: {result}") # Check for completely empty result or missing groups if not result or result[1] is None: @@ -1009,7 +1009,7 @@ def refresh_single_m3u_account(account_id): # Optionally remove completed task from the group to prevent processing it again result.remove(async_result) else: - logger.debug(f"Task is still running.") + logger.trace(f"Task is still running.") # Ensure all database transactions are committed before cleanup logger.info(f"All {total_batches} tasks completed, ensuring DB transactions are committed before cleanup") diff --git a/dispatcharr/celery.py b/dispatcharr/celery.py index 2fe9e6ec..a0ff2168 100644 --- a/dispatcharr/celery.py +++ b/dispatcharr/celery.py @@ -55,7 +55,7 @@ def setup_celery_logging(**kwargs): print(f"Celery configuring loggers with level: {log_level}") # Get the specific loggers that output potentially noisy messages - for logger_name in ['celery.app.trace', 'celery.beat', 'celery.worker.strategy', 'celery.beat.Scheduler']: + for logger_name in ['celery.app.trace', 'celery.beat', 'celery.worker.strategy', 'celery.beat.Scheduler', 'celery.pool']: logger = logging.getLogger(logger_name) # Remove any existing filters first (in case this runs multiple times) @@ -63,9 +63,8 @@ def setup_celery_logging(**kwargs): if hasattr(filter, '__class__') and filter.__class__.__name__ == 'SuppressFilter': logger.removeFilter(filter) - # For INFO level - add a filter to hide repetitive messages - # For DEBUG/TRACE level - don't filter (show everything) - if log_level == 'INFO': + # Add filtering for both INFO and DEBUG levels - only TRACE will show full logging + if log_level not in ['TRACE']: # Add a custom filter to completely filter out the repetitive messages class SuppressFilter(logging.Filter): def filter(self, record): @@ -73,7 +72,8 @@ def setup_celery_logging(**kwargs): if ( "succeeded in" in getattr(record, 'msg', '') or "Scheduler: Sending due task" in getattr(record, 'msg', '') or - "received" in getattr(record, 'msg', '') + "received" in getattr(record, 'msg', '') or + (logger_name == 'celery.pool' and "Apply" in getattr(record, 'msg', '')) ): return False # Don't log these messages at all return True # Log all other messages From ae2af82d1a9ba1931a9f6d59d78e56f61c7406b2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 14:33:52 -0500 Subject: [PATCH 18/55] Update docker compose to show new log level. --- docker/docker-compose.debug.yml | 1 + docker/docker-compose.dev.yml | 1 + docker/docker-compose.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docker/docker-compose.debug.yml b/docker/docker-compose.debug.yml index 40a87bfe..df9e5017 100644 --- a/docker/docker-compose.debug.yml +++ b/docker/docker-compose.debug.yml @@ -17,3 +17,4 @@ services: - DISPATCHARR_DEBUG=true - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 + - DISPATCHARR_LOG_LEVEL=trace diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 3b6f53df..434059e2 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -16,6 +16,7 @@ services: - DISPATCHARR_ENV=dev - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 + - DISPATCHARR_LOG_LEVEL=debug pgadmin: image: dpage/pgadmin4 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4c45dd59..59e21010 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,6 +14,7 @@ services: - POSTGRES_PASSWORD=secret - REDIS_HOST=redis - CELERY_BROKER_URL=redis://redis:6379/0 + - DISPATCHARR_LOG_LEVEL=info # Optional for hardware acceleration #group_add: # - video From 67b217897813b31a1995d62e079e9a3a32078ac9 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 14:44:47 -0500 Subject: [PATCH 19/55] Add LIBVA_DRIVER_PATH to environmental variables for all users. --- docker/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3d2161d5..34cc764f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -80,13 +80,14 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh + echo "export LIBVA_DRIVERS_PATH=/usr/local/lib/x86_64-linux-gnu/dri" >> /etc/profile.d/dispatcharr.sh # Make sure we also set these variables in /etc/environment # which is sourced even before /etc/profile.d scripts for var in PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED POSTGRES_DB POSTGRES_USER \ POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ - DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP; do + DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH; do if [[ -n "${!var}" ]]; then grep -q "^${var}=" /etc/environment || echo "${var}=${!var}" >> /etc/environment fi From 3a5dcab919a8993fc72fccce9f00336bfc354b85 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 15:07:38 -0500 Subject: [PATCH 20/55] Also add LIBVA_DRIVERS_PATH to global environment. --- docker/entrypoint.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 34cc764f..7f3421f5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -37,7 +37,7 @@ export POSTGRES_PORT=${POSTGRES_PORT:-5432} export REDIS_HOST=${REDIS_HOST:-localhost} export REDIS_DB=${REDIS_DB:-0} export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191} - +export LIBVA_DRIVERS_PATH='/usr/local/lib/x86_64-linux-gnu/dri' # Extract version information from version.py export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)") export DISPATCHARR_TIMESTAMP=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__timestamp__ or '')") @@ -88,8 +88,12 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH; do - if [[ -n "${!var}" ]]; then + # Check if the variable is set (exists) in the environment + if [ -n "${!var+x}" ]; then + # Add to /etc/environment if not already there grep -q "^${var}=" /etc/environment || echo "${var}=${!var}" >> /etc/environment + else + echo "Warning: Environment variable $var is not set" fi done fi From e59521ae94c5858ce26a0d80b4fed84f52c27457 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 15:29:25 -0500 Subject: [PATCH 21/55] Add Dispatch to video and render groups if they exist. --- docker/init/01-user-setup.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index 83530f10..94d2ebca 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -29,5 +29,16 @@ else fi fi +# Add user to video and render groups if they exist +if getent group video >/dev/null 2>&1; then + usermod -a -G video $POSTGRES_USER + echo "Added user $POSTGRES_USER to video group for hardware acceleration access" +fi + +if getent group render >/dev/null 2>&1; then + usermod -a -G render $POSTGRES_USER + echo "Added user $POSTGRES_USER to render group for GPU access" +fi + # Run nginx as specified user sed -i 's/user www-data;/user dispatch;/g' /etc/nginx/nginx.conf From 4afa3166ba541dab8b51de4bb320620986068b06 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 15:35:48 -0500 Subject: [PATCH 22/55] Improved group checking logic. --- docker/init/04-check-gpu.sh | 74 ++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index e983018d..586dc078 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -39,6 +39,48 @@ RENDER_GID=$(getent group render | cut -d: -f3) NVIDIA_CONTAINER_TOOLKIT_FOUND=false NVIDIA_ENV_MISMATCH=false +# Detect GPU type for smarter group membership requirements +INTEL_GPU_DETECTED=false +AMD_GPU_DETECTED=false +if command -v lspci >/dev/null 2>&1; then + if lspci | grep -q "Intel Corporation.*VGA"; then + INTEL_GPU_DETECTED=true + echo "ℹī¸ Intel GPU detected - video group membership is recommended." + elif lspci | grep -q "Advanced Micro Devices.*VGA"; then + AMD_GPU_DETECTED=true + echo "ℹī¸ AMD GPU detected - render group membership is recommended." + fi +fi + +# Explicitly check if $POSTGRES_USER is in the video/render groups based on detected hardware +echo "🔍 Verifying if $POSTGRES_USER user is in required groups..." +USER_IN_VIDEO_GROUP=false +USER_IN_RENDER_GROUP=false + +# For Intel GPUs or when GPU type can't be determined, check video group +if [ -n "$VIDEO_GID" ]; then + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then + USER_IN_VIDEO_GROUP=true + echo "✅ User $POSTGRES_USER is in the 'video' group." + elif [ "$INTEL_GPU_DETECTED" = true ] || ([ "$NVIDIA_FOUND" = false ] && [ "$DRI_DEVICES_FOUND" = true ]); then + echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'video' group - hardware acceleration for Intel GPUs may not work!" + else + echo "ℹī¸ User $POSTGRES_USER is not in the 'video' group - this is fine for NVIDIA GPUs with container toolkit." + fi +fi + +# For AMD GPUs, check render group +if [ -n "$RENDER_GID" ]; then + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "render"; then + USER_IN_RENDER_GROUP=true + echo "✅ User $POSTGRES_USER is in the 'render' group." + elif [ "$AMD_GPU_DETECTED" = true ]; then + echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'render' group - hardware acceleration for AMD GPUs may not work!" + else + echo "ℹī¸ User $POSTGRES_USER is not in the 'render' group - not needed for Intel/NVIDIA GPUs." + fi +fi + # Check if NVIDIA Container Toolkit is present through environment or CLI tool # IMPORTANT: Only mark as found if both env vars AND actual NVIDIA devices exist if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then @@ -255,13 +297,33 @@ elif [ "$DRI_DEVICES_FOUND" = true ]; then echo "🔰 INTEL/AMD GPU: ACTIVE" fi - # Check group membership - if [ -n "$VIDEO_GID" ] && id -G | grep -qw "$VIDEO_GID"; then - echo "✅ Video group membership: CORRECT" - elif [ -n "$RENDER_GID" ] && id -G | grep -qw "$RENDER_GID"; then - echo "✅ Render group membership: CORRECT" + # Check group membership based on detected GPU type + if [ "$INTEL_GPU_DETECTED" = true ]; then + if [ "$USER_IN_VIDEO_GROUP" = true ]; then + echo "✅ Video group membership for $POSTGRES_USER: CORRECT (required for Intel GPU)" + else + echo "⚠ī¸ Video group membership for $POSTGRES_USER: MISSING (required for Intel GPU)" + fi + elif [ "$AMD_GPU_DETECTED" = true ]; then + if [ "$USER_IN_RENDER_GROUP" = true ]; then + echo "✅ Render group membership for $POSTGRES_USER: CORRECT (required for AMD GPU)" + else + echo "⚠ī¸ Render group membership for $POSTGRES_USER: MISSING (required for AMD GPU)" + fi + elif [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = false ]; then + # For NVIDIA without container toolkit, check video group + if [ "$USER_IN_VIDEO_GROUP" = true ]; then + echo "✅ Video group membership for $POSTGRES_USER: CORRECT (helps with direct NVIDIA device access)" + else + echo "⚠ī¸ Video group membership for $POSTGRES_USER: MISSING (may affect direct NVIDIA device access)" + fi else - echo "⚠ī¸ Group membership: MISSING (may cause permission issues)" + # Generic case or NVIDIA with container toolkit (where group membership is less important) + if [ "$USER_IN_VIDEO_GROUP" = true ] || [ "$USER_IN_RENDER_GROUP" = true ]; then + echo "✅ GPU group membership for $POSTGRES_USER: CORRECT" + else + echo "ℹī¸ GPU group membership for $POSTGRES_USER: NOT REQUIRED (using NVIDIA Container Toolkit)" + fi fi # Display FFmpeg VAAPI acceleration method From decd54dba969f134269aeb5a987ceb8a646c7f30 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 15:45:08 -0500 Subject: [PATCH 23/55] Add NVIDIA device name to summary --- docker/init/04-check-gpu.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index 586dc078..9a245e0b 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -273,7 +273,12 @@ echo "📋 ===================== SUMMARY =====================" # Identify which GPU type is active and working if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VISIBLE_DEVICES" ]); then - echo "🔰 NVIDIA GPU: ACTIVE" + if [ -n "$NVIDIA_MODEL" ]; then + echo "🔰 NVIDIA GPU: $NVIDIA_MODEL" + else + echo "🔰 NVIDIA GPU: ACTIVE" + fi + if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then echo "✅ NVIDIA Container Toolkit: CONFIGURED CORRECTLY" elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then From 3c1157d330f9698eb989892cb94d45404765013e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 16:45:01 -0500 Subject: [PATCH 24/55] Enhance user group verification messages to include the username for clarity --- docker/init/04-check-gpu.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index 9a245e0b..30cbdbfd 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -96,26 +96,26 @@ fi # For NVIDIA GPUs with Container Toolkit, video group is optional if [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then - if [ -n "$VIDEO_GID" ] && id -G | grep -qw "$VIDEO_GID"; then - echo "✅ User is in the 'video' group (GID $VIDEO_GID)." + if [ -n "$VIDEO_GID" ] && id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then + echo "✅ User $POSTGRES_USER is in the 'video' group (GID $VIDEO_GID)." echo " Note: With NVIDIA Container Toolkit properly configured, this is usually not required." elif [ -n "$VIDEO_GID" ]; then - echo "ℹī¸ User is not in the 'video' group, but NVIDIA Container Toolkit is present." + echo "ℹī¸ User $POSTGRES_USER is not in the 'video' group, but NVIDIA Container Toolkit is present." echo " This is typically fine as the Container Toolkit handles device permissions." fi # For other GPU types (or NVIDIA without Toolkit), video/render group is important else if [ -n "$VIDEO_GID" ]; then - if id -G | grep -qw "$VIDEO_GID"; then - echo "✅ User is in the 'video' group (GID $VIDEO_GID)." + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then + echo "✅ User $POSTGRES_USER is in the 'video' group (GID $VIDEO_GID)." else - echo "⚠ī¸ User is NOT in the 'video' group (GID $VIDEO_GID). Hardware acceleration may not work." + echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'video' group (GID $VIDEO_GID). Hardware acceleration may not work." fi elif [ -n "$RENDER_GID" ]; then - if id -G | grep -qw "$RENDER_GID"; then - echo "✅ User is in the 'render' group (GID $RENDER_GID)." + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "render"; then + echo "✅ User $POSTGRES_USER is in the 'render' group (GID $RENDER_GID)." else - echo "⚠ī¸ User is NOT in the 'render' group (GID $RENDER_GID). Hardware acceleration may not work." + echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'render' group (GID $RENDER_GID). Hardware acceleration may not work." fi else echo "⚠ī¸ Neither 'video' nor 'render' groups found on this system." From 3a4631e9ad61a549e6114aa3f42fc02010d23f2d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 10 May 2025 16:51:42 -0500 Subject: [PATCH 25/55] Add extra note if model isn't detected. --- docker/init/04-check-gpu.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index 30cbdbfd..1410d0cb 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -276,7 +276,9 @@ if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VI if [ -n "$NVIDIA_MODEL" ]; then echo "🔰 NVIDIA GPU: $NVIDIA_MODEL" else - echo "🔰 NVIDIA GPU: ACTIVE" + echo "🔰 NVIDIA GPU: ACTIVE (model detection unavailable)" + echo "ℹī¸ Note: GPU model information couldn't be retrieved, but devices are present." + echo " This may be due to missing nvidia-smi tool or container limitations." fi if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then @@ -299,7 +301,9 @@ elif [ "$DRI_DEVICES_FOUND" = true ]; then elif [ -n "$LIBVA_DRIVER_NAME" ]; then echo "🔰 ${LIBVA_DRIVER_NAME^^} GPU: ACTIVE" else - echo "🔰 INTEL/AMD GPU: ACTIVE" + echo "🔰 INTEL/AMD GPU: ACTIVE (model detection unavailable)" + echo "ℹī¸ Note: Basic GPU drivers appear to be loaded (device nodes exist), but" + echo " couldn't identify specific model. This doesn't necessarily indicate a problem." fi # Check group membership based on detected GPU type From 3a06b12f2c3149222d71fcd927b32f9921f649db Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sun, 11 May 2025 12:45:57 -0500 Subject: [PATCH 26/55] Hopefully fixes QSV hardware acceleration. --- docker/entrypoint.sh | 4 +++- docker/init/01-user-setup.sh | 27 +++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7f3421f5..3f57f4bc 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -38,6 +38,7 @@ export REDIS_HOST=${REDIS_HOST:-localhost} export REDIS_DB=${REDIS_DB:-0} export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191} export LIBVA_DRIVERS_PATH='/usr/local/lib/x86_64-linux-gnu/dri' +export LIBVA_DRIVER_NAME=${LIBVA_DRIVER_NAME:-} # Extract version information from version.py export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)") export DISPATCHARR_TIMESTAMP=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__timestamp__ or '')") @@ -81,13 +82,14 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh echo "export LIBVA_DRIVERS_PATH=/usr/local/lib/x86_64-linux-gnu/dri" >> /etc/profile.d/dispatcharr.sh + echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh # Make sure we also set these variables in /etc/environment # which is sourced even before /etc/profile.d scripts for var in PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED POSTGRES_DB POSTGRES_USER \ POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ - DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH; do + DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH LIBVA_DRIVER_NAME; do # Check if the variable is set (exists) in the environment if [ -n "${!var+x}" ]; then # Add to /etc/environment if not already there diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index 94d2ebca..5683d115 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -29,16 +29,31 @@ else fi fi -# Add user to video and render groups if they exist +# Check if render group exists, if not create it with GID 109 +if getent group render >/dev/null 2>&1; then + current_gid=$(getent group render | cut -d: -f3) + if [ "$current_gid" != "109" ]; then + groupmod -g 109 render + echo "Changed render group GID from $current_gid to 109" + fi +else + groupadd -g 109 render + echo "Created render group with GID 109" +fi + +# Check if Postgres user is already in render group before adding +if id -nG "$POSTGRES_USER" | grep -qw "render"; then + echo "User $POSTGRES_USER is already in render group" +else + usermod -a -G render $POSTGRES_USER + echo "Added user $POSTGRES_USER to render group for GPU access" +fi + +# Add user to video group if it exists if getent group video >/dev/null 2>&1; then usermod -a -G video $POSTGRES_USER echo "Added user $POSTGRES_USER to video group for hardware acceleration access" fi -if getent group render >/dev/null 2>&1; then - usermod -a -G render $POSTGRES_USER - echo "Added user $POSTGRES_USER to render group for GPU access" -fi - # Run nginx as specified user sed -i 's/user www-data;/user dispatch;/g' /etc/nginx/nginx.conf From b85a5be0237c0c63acb84907c7246e938650df18 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 08:44:07 -0500 Subject: [PATCH 27/55] Add LD_LIBRARY_PATH to dispatch user and global. --- docker/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3f57f4bc..cbc3b435 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -38,6 +38,7 @@ export REDIS_HOST=${REDIS_HOST:-localhost} export REDIS_DB=${REDIS_DB:-0} export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191} export LIBVA_DRIVERS_PATH='/usr/local/lib/x86_64-linux-gnu/dri' +export LD_LIBRARY_PATH='/usr/local/lib' export LIBVA_DRIVER_NAME=${LIBVA_DRIVER_NAME:-} # Extract version information from version.py export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)") @@ -83,13 +84,14 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh echo "export LIBVA_DRIVERS_PATH=/usr/local/lib/x86_64-linux-gnu/dri" >> /etc/profile.d/dispatcharr.sh echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh + echo "export LD_LIBRARY_PATH=/usr/local/lib" >> /etc/profile.d/dispatcharr.sh # Make sure we also set these variables in /etc/environment # which is sourced even before /etc/profile.d scripts for var in PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED POSTGRES_DB POSTGRES_USER \ POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ - DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH LIBVA_DRIVER_NAME; do + DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH LIBVA_DRIVER_NAME LD_LIBRARY_PATH; do # Check if the variable is set (exists) in the environment if [ -n "${!var+x}" ]; then # Add to /etc/environment if not already there From 8dd3bd68789b6b307c38fd4a04c4b2237dbf0a39 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 09:10:38 -0500 Subject: [PATCH 28/55] Use variables in export. --- docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cbc3b435..7faf6979 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -82,9 +82,9 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh - echo "export LIBVA_DRIVERS_PATH=/usr/local/lib/x86_64-linux-gnu/dri" >> /etc/profile.d/dispatcharr.sh + echo "export LIBVA_DRIVERS_PATH=$LIBVA_DRIVERS_PATH" >> /etc/profile.d/dispatcharr.sh echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh - echo "export LD_LIBRARY_PATH=/usr/local/lib" >> /etc/profile.d/dispatcharr.sh + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile.d/dispatcharr.sh # Make sure we also set these variables in /etc/environment # which is sourced even before /etc/profile.d scripts From 87798f443430daf54dc0ea1d4ea84539e8a01d9a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 12:52:11 -0500 Subject: [PATCH 29/55] Only set LIBVA_DRIVER_NAME if user has it specified. Fixes but where FFmpeg wasn't auto detecting the correct driver for VAAPI and QSV. --- docker/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7faf6979..b3ff3509 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -39,7 +39,10 @@ export REDIS_DB=${REDIS_DB:-0} export DISPATCHARR_PORT=${DISPATCHARR_PORT:-9191} export LIBVA_DRIVERS_PATH='/usr/local/lib/x86_64-linux-gnu/dri' export LD_LIBRARY_PATH='/usr/local/lib' -export LIBVA_DRIVER_NAME=${LIBVA_DRIVER_NAME:-} +# Set LIBVA_DRIVER_NAME if user has specified it +if [ -v LIBVA_DRIVER_NAME ]; then + export LIBVA_DRIVER_NAME +fi # Extract version information from version.py export DISPATCHARR_VERSION=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__version__)") export DISPATCHARR_TIMESTAMP=$(python -c "import sys; sys.path.append('/app'); import version; print(version.__timestamp__ or '')") From da9a78c875c1d5940f8e4c0ae2a111cfb6e0f088 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 13:16:15 -0500 Subject: [PATCH 30/55] Only write LIBVA_DRIVER_NAME to dispatcharr.sh if it's set by the user. --- docker/entrypoint.sh | 7 ++++++- docker/init/04-check-gpu.sh | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b3ff3509..4f1ec081 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -86,7 +86,12 @@ if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh echo "export LIBVA_DRIVERS_PATH=$LIBVA_DRIVERS_PATH" >> /etc/profile.d/dispatcharr.sh - echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh + + # Only add LIBVA_DRIVER_NAME to profile if it's set + if [ -v LIBVA_DRIVER_NAME ]; then + echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh + fi + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile.d/dispatcharr.sh # Make sure we also set these variables in /etc/environment diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh index 1410d0cb..c75c5e8b 100644 --- a/docker/init/04-check-gpu.sh +++ b/docker/init/04-check-gpu.sh @@ -268,6 +268,20 @@ else echo "⚠ī¸ FFmpeg not found in PATH." fi +# Add hardware detection through lspci if available +NVIDIA_GPU_IN_LSPCI=false +if command -v lspci >/dev/null 2>&1; then + if lspci | grep -i "NVIDIA" | grep -i "VGA\|3D\|Display" >/dev/null; then + NVIDIA_GPU_IN_LSPCI=true + echo "🔍 NVIDIA GPU detected in hardware listing (lspci)." + if [ "$NVIDIA_FOUND" = false ]; then + echo "⚠ī¸ NVIDIA GPU exists in system but no /dev/nvidia* devices are mapped to the container." + echo " You have DRI devices mapped, but this provides limited functionality for NVIDIA GPUs." + echo " Consider using the NVIDIA Container Runtime for optimal performance." + fi + fi +fi + # Provide a final summary of the hardware acceleration setup echo "📋 ===================== SUMMARY =====================" @@ -294,6 +308,30 @@ if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VI else echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" fi +elif [ "$NVIDIA_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + # NVIDIA through DRI only (suboptimal but possible) + echo "🔰 NVIDIA GPU: DETECTED BUT SUBOPTIMALLY CONFIGURED" + echo "⚠ī¸ Your NVIDIA GPU is only accessible through DRI devices, not NVIDIA container runtime" + echo "💡 RECOMMENDATION: Use the proper NVIDIA container configuration:" + echo " deploy:" + echo " resources:" + echo " reservations:" + echo " devices:" + echo " - driver: nvidia" + echo " count: all" + echo " capabilities: [gpu]" + + if [ "$USER_IN_VIDEO_GROUP" = true ] || [ "$USER_IN_RENDER_GROUP" = true ]; then + echo "✅ GPU group membership for $POSTGRES_USER: CORRECT" + else + echo "⚠ī¸ GPU group membership for $POSTGRES_USER: MISSING (may affect DRI access)" + fi + + if echo "$HWACCEL" | grep -q "vaapi"; then + echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" + fi elif [ "$DRI_DEVICES_FOUND" = true ]; then # Intel/AMD detection with model if available if [ -n "$GPU_MODEL" ]; then From ade8a625bc48ae1d2f118deea94696dcbda434b0 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 13:57:22 -0500 Subject: [PATCH 31/55] Refactor environment variable handling in entrypoint.sh for consistency and efficiency --- docker/entrypoint.sh | 46 +++++++++++++------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4f1ec081..13c72728 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -67,41 +67,21 @@ export POSTGRES_DIR=/data/db # Global variables, stored so other users inherit them if [[ ! -f /etc/profile.d/dispatcharr.sh ]]; then - echo "export PATH=$PATH" >> /etc/profile.d/dispatcharr.sh - echo "export VIRTUAL_ENV=$VIRTUAL_ENV" >> /etc/profile.d/dispatcharr.sh - echo "export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE" >> /etc/profile.d/dispatcharr.sh - echo "export PYTHONUNBUFFERED=$PYTHONUNBUFFERED" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_DB=$POSTGRES_DB" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_USER=$POSTGRES_USER" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_PASSWORD=$POSTGRES_PASSWORD" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_HOST=$POSTGRES_HOST" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_PORT=$POSTGRES_PORT" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_ENV=$DISPATCHARR_ENV" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_DEBUG=$DISPATCHARR_DEBUG" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_LOG_LEVEL=$DISPATCHARR_LOG_LEVEL" >> /etc/profile.d/dispatcharr.sh - echo "export REDIS_HOST=$REDIS_HOST" >> /etc/profile.d/dispatcharr.sh - echo "export REDIS_DB=$REDIS_DB" >> /etc/profile.d/dispatcharr.sh - echo "export POSTGRES_DIR=$POSTGRES_DIR" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_PORT=$DISPATCHARR_PORT" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_VERSION=$DISPATCHARR_VERSION" >> /etc/profile.d/dispatcharr.sh - echo "export DISPATCHARR_TIMESTAMP=$DISPATCHARR_TIMESTAMP" >> /etc/profile.d/dispatcharr.sh - echo "export LIBVA_DRIVERS_PATH=$LIBVA_DRIVERS_PATH" >> /etc/profile.d/dispatcharr.sh + # Define all variables to process + variables=( + PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED + POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT + DISPATCHARR_ENV DISPATCHARR_DEBUG DISPATCHARR_LOG_LEVEL + REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT + DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH LIBVA_DRIVER_NAME LD_LIBRARY_PATH + ) - # Only add LIBVA_DRIVER_NAME to profile if it's set - if [ -v LIBVA_DRIVER_NAME ]; then - echo "export LIBVA_DRIVER_NAME=$LIBVA_DRIVER_NAME" >> /etc/profile.d/dispatcharr.sh - fi - - echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile.d/dispatcharr.sh - - # Make sure we also set these variables in /etc/environment - # which is sourced even before /etc/profile.d scripts - for var in PATH VIRTUAL_ENV DJANGO_SETTINGS_MODULE PYTHONUNBUFFERED POSTGRES_DB POSTGRES_USER \ - POSTGRES_PASSWORD POSTGRES_HOST POSTGRES_PORT DISPATCHARR_ENV DISPATCHARR_DEBUG \ - DISPATCHARR_LOG_LEVEL REDIS_HOST REDIS_DB POSTGRES_DIR DISPATCHARR_PORT \ - DISPATCHARR_VERSION DISPATCHARR_TIMESTAMP LIBVA_DRIVERS_PATH LIBVA_DRIVER_NAME LD_LIBRARY_PATH; do - # Check if the variable is set (exists) in the environment + # Process each variable for both profile.d and environment + for var in "${variables[@]}"; do + # Check if the variable is set in the environment if [ -n "${!var+x}" ]; then + # Add to profile.d + echo "export ${var}=${!var}" >> /etc/profile.d/dispatcharr.sh # Add to /etc/environment if not already there grep -q "^${var}=" /etc/environment || echo "${var}=${!var}" >> /etc/environment else From d25d57c1625baed0d8e6b079f68cbd3aa34d161a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 14:26:32 -0500 Subject: [PATCH 32/55] Add render group creation for hardware acceleration support in Dockerfile --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index ec24c818..0f997979 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,9 @@ ENV VIRTUAL_ENV=/dispatcharrpy ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /app +# Create render group for hardware acceleration support +RUN groupadd -r render || true + # Copy application code COPY . /app # Copy nginx configuration From 74152406d12f52ff96dd0099c66cb66a4b2ff833 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 15:57:25 -0500 Subject: [PATCH 33/55] Add Render group to base. --- docker/DispatcharrBase | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index 8b65595f..6efcb1fe 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -37,4 +37,9 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | \ tee /etc/apt/sources.list.d/pgdg.list && \ apt-get update && apt-get install -y postgresql-14 postgresql-contrib-14 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create render group for hardware acceleration support +RUN groupadd -r render || true + +ENTRYPOINT ["/app/docker/entrypoint.sh"] \ No newline at end of file From 9f1d38247225b5964c0ef79df440370aaf371b79 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 16:02:24 -0500 Subject: [PATCH 34/55] Don't force GID for render group. --- docker/Dockerfile | 1 + docker/init/01-user-setup.sh | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0f997979..17d995ce 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,6 +23,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /app # Create render group for hardware acceleration support +# Remove this later as it will be included in the base image RUN groupadd -r render || true # Copy application code diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index 5683d115..1df6fb52 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -30,23 +30,28 @@ else fi # Check if render group exists, if not create it with GID 109 +#if getent group render >/dev/null 2>&1; then +# current_gid=$(getent group render | cut -d: -f3) +# if [ "$current_gid" != "109" ]; then +# groupmod -g 109 render +# echo "Changed render group GID from $current_gid to 109" +# fi +#else +# groupadd -g 109 render +# echo "Created render group with GID 109" +#fi + +# Check if render group exists before trying to add user to it if getent group render >/dev/null 2>&1; then - current_gid=$(getent group render | cut -d: -f3) - if [ "$current_gid" != "109" ]; then - groupmod -g 109 render - echo "Changed render group GID from $current_gid to 109" + # Render group exists, check if user is already in it + if id -nG "$POSTGRES_USER" | grep -qw "render"; then + echo "User $POSTGRES_USER is already in render group" + else + usermod -a -G render $POSTGRES_USER + echo "Added user $POSTGRES_USER to render group for GPU access" fi else - groupadd -g 109 render - echo "Created render group with GID 109" -fi - -# Check if Postgres user is already in render group before adding -if id -nG "$POSTGRES_USER" | grep -qw "render"; then - echo "User $POSTGRES_USER is already in render group" -else - usermod -a -G render $POSTGRES_USER - echo "Added user $POSTGRES_USER to render group for GPU access" + echo "Render group does not exist, skipping adding user to render group" fi # Add user to video group if it exists From 397ec499fe53985351d8bb7faece34832d9a44d5 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 17:02:10 -0500 Subject: [PATCH 35/55] Assign GID 109 to render group --- docker/DispatcharrBase | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/DispatcharrBase b/docker/DispatcharrBase index 6efcb1fe..4360ced3 100644 --- a/docker/DispatcharrBase +++ b/docker/DispatcharrBase @@ -39,7 +39,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo apt-get update && apt-get install -y postgresql-14 postgresql-contrib-14 && \ apt-get clean && rm -rf /var/lib/apt/lists/* -# Create render group for hardware acceleration support -RUN groupadd -r render || true +# Create render group for hardware acceleration support with GID 109 +RUN groupadd -r -g 109 render || true ENTRYPOINT ["/app/docker/entrypoint.sh"] \ No newline at end of file From 634d16d4028a14ba3c2e945230d211a62495b544 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 17:23:10 -0500 Subject: [PATCH 36/55] Improved logic for assigned GID's and group memberships for video and render groups. --- docker/init/01-user-setup.sh | 59 ++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index 1df6fb52..aa478acb 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -29,35 +29,50 @@ else fi fi -# Check if render group exists, if not create it with GID 109 -#if getent group render >/dev/null 2>&1; then -# current_gid=$(getent group render | cut -d: -f3) -# if [ "$current_gid" != "109" ]; then -# groupmod -g 109 render -# echo "Changed render group GID from $current_gid to 109" -# fi -#else -# groupadd -g 109 render -# echo "Created render group with GID 109" -#fi +# Get the GID of /dev/dri/renderD128 on the host (must be mounted into container) +if [ -e "/dev/dri/renderD128" ]; then + HOST_RENDER_GID=$(stat -c '%g' /dev/dri/renderD128) -# Check if render group exists before trying to add user to it -if getent group render >/dev/null 2>&1; then - # Render group exists, check if user is already in it - if id -nG "$POSTGRES_USER" | grep -qw "render"; then - echo "User $POSTGRES_USER is already in render group" + # Check if this GID belongs to the video group + VIDEO_GID=$(getent group video 2>/dev/null | cut -d: -f3) + + if [ "$HOST_RENDER_GID" = "$VIDEO_GID" ]; then + echo "RenderD128 GID ($HOST_RENDER_GID) matches video group GID. Using video group for GPU access." + # Make sure POSTGRES_USER is in video group + if ! id -nG "$POSTGRES_USER" | grep -qw "video"; then + usermod -a -G video "$POSTGRES_USER" + echo "Added user $POSTGRES_USER to video group for GPU access" + fi else - usermod -a -G render $POSTGRES_USER - echo "Added user $POSTGRES_USER to render group for GPU access" + # We need to ensure render group exists with correct GID + if getent group render >/dev/null; then + CURRENT_RENDER_GID=$(getent group render | cut -d: -f3) + if [ "$CURRENT_RENDER_GID" != "$HOST_RENDER_GID" ]; then + echo "Changing render group GID from $CURRENT_RENDER_GID to $HOST_RENDER_GID" + groupmod -g "$HOST_RENDER_GID" render + fi + else + echo "Creating render group with GID $HOST_RENDER_GID" + groupadd -g "$HOST_RENDER_GID" render + fi + + # Make sure POSTGRES_USER is in render group + if ! id -nG "$POSTGRES_USER" | grep -qw "render"; then + usermod -a -G render "$POSTGRES_USER" + echo "Added user $POSTGRES_USER to render group for GPU access" + fi fi else - echo "Render group does not exist, skipping adding user to render group" + echo "Warning: /dev/dri/renderD128 not found. GPU acceleration may not be available." fi -# Add user to video group if it exists +# Always add user to video group for hardware acceleration if it exists +# (some systems use video group for general GPU access) if getent group video >/dev/null 2>&1; then - usermod -a -G video $POSTGRES_USER - echo "Added user $POSTGRES_USER to video group for hardware acceleration access" + if ! id -nG "$POSTGRES_USER" | grep -qw "video"; then + usermod -a -G video "$POSTGRES_USER" + echo "Added user $POSTGRES_USER to video group for hardware acceleration access" + fi fi # Run nginx as specified user From 0953e044b76228ec3a04939263b3440eba542a98 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 19:46:23 -0500 Subject: [PATCH 37/55] Huge improvement to hardware acceleration script. Renamed for accuracy. --- docker/entrypoint.sh | 2 +- docker/init/04-check-gpu.sh | 388 ---------------------- docker/init/04-check-hwaccel.sh | 552 ++++++++++++++++++++++++++++++++ 3 files changed, 553 insertions(+), 389 deletions(-) delete mode 100644 docker/init/04-check-gpu.sh create mode 100644 docker/init/04-check-hwaccel.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 13c72728..23daa73a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -99,7 +99,7 @@ echo "Starting init process..." . /app/docker/init/01-user-setup.sh . /app/docker/init/02-postgres.sh . /app/docker/init/03-init-dispatcharr.sh -. /app/docker/init/04-check-gpu.sh +. /app/docker/init/04-check-hwaccel.sh # Start PostgreSQL echo "Starting Postgres..." diff --git a/docker/init/04-check-gpu.sh b/docker/init/04-check-gpu.sh deleted file mode 100644 index c75c5e8b..00000000 --- a/docker/init/04-check-gpu.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/bin/bash - -echo "🔍 Checking for GPU acceleration devices..." - -# Helper function for device access checks -check_dev() { - local dev=$1 - if [ -e "$dev" ]; then - if [ -r "$dev" ] && [ -w "$dev" ]; then - echo "✅ Device $dev is accessible." - else - echo "⚠ī¸ Device $dev exists but is not accessible. Check permissions or container runtime options." - fi - else - echo "ℹī¸ Device $dev does not exist." - fi -} - -# Check Intel/AMD VAAPI devices -echo "🔍 Checking for Intel/AMD (VAAPI) devices..." -for dev in /dev/dri/renderD* /dev/dri/card*; do - [ -e "$dev" ] && check_dev "$dev" -done - -# Check NVIDIA device nodes -echo "🔍 Checking for NVIDIA devices..." -NVIDIA_FOUND=false -for dev in /dev/nvidia*; do - [ -e "$dev" ] && NVIDIA_FOUND=true && check_dev "$dev" -done -if [ "$NVIDIA_FOUND" = false ]; then - echo "ℹī¸ No NVIDIA device nodes found under /dev." -fi - -# Check group membership for GPU access - context-aware based on hardware -echo "🔍 Checking user group memberships..." -VIDEO_GID=$(getent group video | cut -d: -f3) -RENDER_GID=$(getent group render | cut -d: -f3) -NVIDIA_CONTAINER_TOOLKIT_FOUND=false -NVIDIA_ENV_MISMATCH=false - -# Detect GPU type for smarter group membership requirements -INTEL_GPU_DETECTED=false -AMD_GPU_DETECTED=false -if command -v lspci >/dev/null 2>&1; then - if lspci | grep -q "Intel Corporation.*VGA"; then - INTEL_GPU_DETECTED=true - echo "ℹī¸ Intel GPU detected - video group membership is recommended." - elif lspci | grep -q "Advanced Micro Devices.*VGA"; then - AMD_GPU_DETECTED=true - echo "ℹī¸ AMD GPU detected - render group membership is recommended." - fi -fi - -# Explicitly check if $POSTGRES_USER is in the video/render groups based on detected hardware -echo "🔍 Verifying if $POSTGRES_USER user is in required groups..." -USER_IN_VIDEO_GROUP=false -USER_IN_RENDER_GROUP=false - -# For Intel GPUs or when GPU type can't be determined, check video group -if [ -n "$VIDEO_GID" ]; then - if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then - USER_IN_VIDEO_GROUP=true - echo "✅ User $POSTGRES_USER is in the 'video' group." - elif [ "$INTEL_GPU_DETECTED" = true ] || ([ "$NVIDIA_FOUND" = false ] && [ "$DRI_DEVICES_FOUND" = true ]); then - echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'video' group - hardware acceleration for Intel GPUs may not work!" - else - echo "ℹī¸ User $POSTGRES_USER is not in the 'video' group - this is fine for NVIDIA GPUs with container toolkit." - fi -fi - -# For AMD GPUs, check render group -if [ -n "$RENDER_GID" ]; then - if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "render"; then - USER_IN_RENDER_GROUP=true - echo "✅ User $POSTGRES_USER is in the 'render' group." - elif [ "$AMD_GPU_DETECTED" = true ]; then - echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'render' group - hardware acceleration for AMD GPUs may not work!" - else - echo "ℹī¸ User $POSTGRES_USER is not in the 'render' group - not needed for Intel/NVIDIA GPUs." - fi -fi - -# Check if NVIDIA Container Toolkit is present through environment or CLI tool -# IMPORTANT: Only mark as found if both env vars AND actual NVIDIA devices exist -if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then - NVIDIA_CONTAINER_TOOLKIT_FOUND=true -# Check for environment variables set by NVIDIA Container Runtime, but only if NVIDIA hardware exists -elif [ "$NVIDIA_FOUND" = true ] && [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then - NVIDIA_CONTAINER_TOOLKIT_FOUND=true - echo "✅ NVIDIA Container Toolkit detected (via environment variables)." - echo " The container is properly configured with Docker Compose's 'driver: nvidia' syntax." -elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ] && [ "$NVIDIA_FOUND" = false ]; then - NVIDIA_ENV_MISMATCH=true -fi - -# For NVIDIA GPUs with Container Toolkit, video group is optional -if [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then - if [ -n "$VIDEO_GID" ] && id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then - echo "✅ User $POSTGRES_USER is in the 'video' group (GID $VIDEO_GID)." - echo " Note: With NVIDIA Container Toolkit properly configured, this is usually not required." - elif [ -n "$VIDEO_GID" ]; then - echo "ℹī¸ User $POSTGRES_USER is not in the 'video' group, but NVIDIA Container Toolkit is present." - echo " This is typically fine as the Container Toolkit handles device permissions." - fi -# For other GPU types (or NVIDIA without Toolkit), video/render group is important -else - if [ -n "$VIDEO_GID" ]; then - if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then - echo "✅ User $POSTGRES_USER is in the 'video' group (GID $VIDEO_GID)." - else - echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'video' group (GID $VIDEO_GID). Hardware acceleration may not work." - fi - elif [ -n "$RENDER_GID" ]; then - if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "render"; then - echo "✅ User $POSTGRES_USER is in the 'render' group (GID $RENDER_GID)." - else - echo "⚠ī¸ User $POSTGRES_USER is NOT in the 'render' group (GID $RENDER_GID). Hardware acceleration may not work." - fi - else - echo "⚠ī¸ Neither 'video' nor 'render' groups found on this system." - fi -fi - -# Check NVIDIA Container Toolkit support -echo "🔍 Checking NVIDIA container runtime support..." - -# More reliable detection of NVIDIA Container Runtime -NVIDIA_RUNTIME_ACTIVE=false - -# Method 1: Check for nvidia-container-cli tool -if command -v nvidia-container-cli >/dev/null 2>&1; then - NVIDIA_RUNTIME_ACTIVE=true - echo "✅ NVIDIA Container Runtime detected (nvidia-container-cli found)." - - if nvidia-container-cli info >/dev/null 2>&1; then - echo "✅ NVIDIA container runtime is functional." - else - echo "⚠ī¸ nvidia-container-cli found, but 'info' command failed. Runtime may be misconfigured." - fi -fi - -# Method 2: Check for NVIDIA Container Runtime specific files -if [ -e "/dev/.nv" ] || [ -e "/.nv" ] || [ -e "/.nvidia-container-runtime" ]; then - NVIDIA_RUNTIME_ACTIVE=true - echo "✅ NVIDIA Container Runtime files detected." -fi - -# Method 3: Check cgroup information for NVIDIA -if grep -q "nvidia" /proc/self/cgroup 2>/dev/null; then - NVIDIA_RUNTIME_ACTIVE=true - echo "✅ NVIDIA Container Runtime cgroups detected." -fi - -# Final verdict based on hardware AND runtime -if [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_RUNTIME_ACTIVE" = true ]; then - echo "✅ NVIDIA Container Runtime is properly configured with hardware access." -elif [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_RUNTIME_ACTIVE" = false ]; then - echo "ℹī¸ NVIDIA GPU detected, but using direct device passthrough instead of Container Runtime." - echo " This works but consider using the 'deploy: resources: reservations: devices:' method in docker-compose." -elif [ "$NVIDIA_FOUND" = false ] && [ "$NVIDIA_RUNTIME_ACTIVE" = true ]; then - echo "⚠ī¸ NVIDIA Container Runtime appears to be configured, but no NVIDIA devices found." - echo " Check that your host has NVIDIA drivers installed and GPUs are properly passed to the container." -else - # No need to show NVIDIA environment variable warnings if they're default in the container - echo "ℹī¸ Using Intel/AMD GPU hardware for acceleration." -fi - -# Run nvidia-smi if available -if command -v nvidia-smi >/dev/null 2>&1; then - echo "🔍 Running nvidia-smi to verify GPU visibility..." - if nvidia-smi >/dev/null 2>&1; then - echo "✅ nvidia-smi successful - GPU is accessible to container!" - echo " This confirms hardware acceleration should be available to FFmpeg." - else - echo "⚠ī¸ nvidia-smi command failed. GPU may not be properly mapped into container." - fi -else - echo "ℹī¸ nvidia-smi not installed or not in PATH." -fi - -# Show relevant environment variables with contextual suggestions -echo "🔍 Checking GPU-related environment variables..." - -# Set flags based on device detection -DRI_DEVICES_FOUND=false -for dev in /dev/dri/renderD* /dev/dri/card*; do - if [ -e "$dev" ]; then - DRI_DEVICES_FOUND=true - break - fi -done - -# Give contextual suggestions based on detected hardware -if [ "$DRI_DEVICES_FOUND" = true ]; then - # Detect Intel/AMD GPU model - if command -v lspci >/dev/null 2>&1; then - GPU_INFO=$(lspci -nn | grep -i "VGA\|Display" | head -1) - if [ -n "$GPU_INFO" ]; then - echo "🔍 Detected GPU: $GPU_INFO" - # Extract model for cleaner display in summary - GPU_MODEL=$(echo "$GPU_INFO" | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //' | sed 's/Technologies //') - fi - fi - - if [ -n "$LIBVA_DRIVER_NAME" ]; then - echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" - else - # Check if we can detect the GPU type - if command -v lspci >/dev/null 2>&1; then - if lspci | grep -q "Intel Corporation.*VGA" | grep -i "HD Graphics"; then - # Newer Intel GPUs (Gen12+/Tiger Lake and newer) - if lspci | grep -q "Intel Corporation.*VGA" | grep -E "Xe|Alchemist|Tiger Lake|Gen1[2-9]"; then - echo "💡 Consider setting LIBVA_DRIVER_NAME=iHD for this modern Intel GPU" - else - # Older Intel GPUs - echo "💡 Consider setting LIBVA_DRIVER_NAME=i965 for this Intel GPU" - fi - elif lspci | grep -q "Advanced Micro Devices.*VGA"; then - echo "💡 Consider setting LIBVA_DRIVER_NAME=radeonsi for this AMD GPU" - else - echo "ℹī¸ Auto-detection of VAAPI driver is usually reliable, but if you have issues:" - echo " - For newer Intel GPUs: LIBVA_DRIVER_NAME=iHD" - echo " - For older Intel GPUs: LIBVA_DRIVER_NAME=i965" - echo " - For AMD GPUs: LIBVA_DRIVER_NAME=radeonsi" - fi - else - echo "ℹī¸ Intel/AMD GPU detected. If VAAPI doesn't work automatically, you may need to set LIBVA_DRIVER_NAME" - echo " based on your specific hardware (iHD for newer Intel, i965 for older Intel, radeonsi for AMD)" - fi - fi -fi - -if [ "$NVIDIA_FOUND" = true ]; then - # Try to get NVIDIA GPU model info - if command -v nvidia-smi >/dev/null 2>&1; then - NVIDIA_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1) - if [ -n "$NVIDIA_MODEL" ]; then - echo "🔍 Detected NVIDIA GPU: $NVIDIA_MODEL" - GPU_MODEL=$NVIDIA_MODEL - fi - fi - - if [ -n "$NVIDIA_VISIBLE_DEVICES" ]; then - echo "ℹī¸ NVIDIA_VISIBLE_DEVICES is set to '$NVIDIA_VISIBLE_DEVICES'" - else - echo "💡 Consider setting NVIDIA_VISIBLE_DEVICES to 'all' or specific indices (e.g., '0,1')" - fi - - if [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then - echo "ℹī¸ NVIDIA_DRIVER_CAPABILITIES is set to '$NVIDIA_DRIVER_CAPABILITIES'" - else - echo "💡 Consider setting NVIDIA_DRIVER_CAPABILITIES to 'all' or 'compute,video,utility' for full functionality" - fi - - if [ -n "$CUDA_VISIBLE_DEVICES" ]; then - echo "ℹī¸ CUDA_VISIBLE_DEVICES is set to '$CUDA_VISIBLE_DEVICES'" - fi -fi - -# Check FFmpeg hardware acceleration support -echo "🔍 Checking FFmpeg hardware acceleration capabilities..." -if command -v ffmpeg >/dev/null 2>&1; then - HWACCEL=$(ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -v "Hardware acceleration methods:" || echo "None found") - echo "Available FFmpeg hardware acceleration methods:" - echo "$HWACCEL" -else - echo "⚠ī¸ FFmpeg not found in PATH." -fi - -# Add hardware detection through lspci if available -NVIDIA_GPU_IN_LSPCI=false -if command -v lspci >/dev/null 2>&1; then - if lspci | grep -i "NVIDIA" | grep -i "VGA\|3D\|Display" >/dev/null; then - NVIDIA_GPU_IN_LSPCI=true - echo "🔍 NVIDIA GPU detected in hardware listing (lspci)." - if [ "$NVIDIA_FOUND" = false ]; then - echo "⚠ī¸ NVIDIA GPU exists in system but no /dev/nvidia* devices are mapped to the container." - echo " You have DRI devices mapped, but this provides limited functionality for NVIDIA GPUs." - echo " Consider using the NVIDIA Container Runtime for optimal performance." - fi - fi -fi - -# Provide a final summary of the hardware acceleration setup -echo "📋 ===================== SUMMARY =====================" - -# Identify which GPU type is active and working -if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VISIBLE_DEVICES" ]); then - if [ -n "$NVIDIA_MODEL" ]; then - echo "🔰 NVIDIA GPU: $NVIDIA_MODEL" - else - echo "🔰 NVIDIA GPU: ACTIVE (model detection unavailable)" - echo "ℹī¸ Note: GPU model information couldn't be retrieved, but devices are present." - echo " This may be due to missing nvidia-smi tool or container limitations." - fi - - if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then - echo "✅ NVIDIA Container Toolkit: CONFIGURED CORRECTLY" - elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then - echo "✅ NVIDIA Docker configuration: USING MODERN DEPLOYMENT" - else - echo "⚠ī¸ NVIDIA setup method: DIRECT DEVICE MAPPING (functional but not optimal)" - fi - # Display FFmpeg NVIDIA acceleration methods - if echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then - echo "✅ FFmpeg NVIDIA acceleration: AVAILABLE" - else - echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" - fi -elif [ "$NVIDIA_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then - # NVIDIA through DRI only (suboptimal but possible) - echo "🔰 NVIDIA GPU: DETECTED BUT SUBOPTIMALLY CONFIGURED" - echo "⚠ī¸ Your NVIDIA GPU is only accessible through DRI devices, not NVIDIA container runtime" - echo "💡 RECOMMENDATION: Use the proper NVIDIA container configuration:" - echo " deploy:" - echo " resources:" - echo " reservations:" - echo " devices:" - echo " - driver: nvidia" - echo " count: all" - echo " capabilities: [gpu]" - - if [ "$USER_IN_VIDEO_GROUP" = true ] || [ "$USER_IN_RENDER_GROUP" = true ]; then - echo "✅ GPU group membership for $POSTGRES_USER: CORRECT" - else - echo "⚠ī¸ GPU group membership for $POSTGRES_USER: MISSING (may affect DRI access)" - fi - - if echo "$HWACCEL" | grep -q "vaapi"; then - echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" - else - echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" - fi -elif [ "$DRI_DEVICES_FOUND" = true ]; then - # Intel/AMD detection with model if available - if [ -n "$GPU_MODEL" ]; then - echo "🔰 GPU: $GPU_MODEL" - elif [ -n "$LIBVA_DRIVER_NAME" ]; then - echo "🔰 ${LIBVA_DRIVER_NAME^^} GPU: ACTIVE" - else - echo "🔰 INTEL/AMD GPU: ACTIVE (model detection unavailable)" - echo "ℹī¸ Note: Basic GPU drivers appear to be loaded (device nodes exist), but" - echo " couldn't identify specific model. This doesn't necessarily indicate a problem." - fi - - # Check group membership based on detected GPU type - if [ "$INTEL_GPU_DETECTED" = true ]; then - if [ "$USER_IN_VIDEO_GROUP" = true ]; then - echo "✅ Video group membership for $POSTGRES_USER: CORRECT (required for Intel GPU)" - else - echo "⚠ī¸ Video group membership for $POSTGRES_USER: MISSING (required for Intel GPU)" - fi - elif [ "$AMD_GPU_DETECTED" = true ]; then - if [ "$USER_IN_RENDER_GROUP" = true ]; then - echo "✅ Render group membership for $POSTGRES_USER: CORRECT (required for AMD GPU)" - else - echo "⚠ī¸ Render group membership for $POSTGRES_USER: MISSING (required for AMD GPU)" - fi - elif [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = false ]; then - # For NVIDIA without container toolkit, check video group - if [ "$USER_IN_VIDEO_GROUP" = true ]; then - echo "✅ Video group membership for $POSTGRES_USER: CORRECT (helps with direct NVIDIA device access)" - else - echo "⚠ī¸ Video group membership for $POSTGRES_USER: MISSING (may affect direct NVIDIA device access)" - fi - else - # Generic case or NVIDIA with container toolkit (where group membership is less important) - if [ "$USER_IN_VIDEO_GROUP" = true ] || [ "$USER_IN_RENDER_GROUP" = true ]; then - echo "✅ GPU group membership for $POSTGRES_USER: CORRECT" - else - echo "ℹī¸ GPU group membership for $POSTGRES_USER: NOT REQUIRED (using NVIDIA Container Toolkit)" - fi - fi - - # Display FFmpeg VAAPI acceleration method - if echo "$HWACCEL" | grep -q "vaapi"; then - echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" - else - echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" - fi -else - echo "❌ NO GPU ACCELERATION DETECTED" - echo "⚠ī¸ Hardware acceleration is unavailable or misconfigured" -fi - -echo "📋 ==================================================" -echo "✅ GPU detection script complete." diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh new file mode 100644 index 00000000..04eabf1e --- /dev/null +++ b/docker/init/04-check-hwaccel.sh @@ -0,0 +1,552 @@ +#!/bin/bash + +echo "🔍 Checking for GPU acceleration devices..." + +# Helper function for device access checks +check_dev() { + local dev=$1 + if [ -e "$dev" ]; then + if [ -r "$dev" ] && [ -w "$dev" ]; then + echo "✅ Device $dev is accessible." + else + echo "⚠ī¸ Device $dev exists but is not accessible. Check permissions or container runtime options." + fi + else + echo "ℹī¸ Device $dev does not exist." + fi +} + +# Initialize device detection flags +ANY_GPU_DEVICES_FOUND=false +DRI_DEVICES_FOUND=false +NVIDIA_FOUND=false +NVIDIA_GPU_IN_LSPCI=false +INTEL_GPU_IN_LSPCI=false +AMD_GPU_IN_LSPCI=false + +# Check for all GPU types in hardware via lspci +if command -v lspci >/dev/null 2>&1; then + # Check for NVIDIA GPUs + if lspci | grep -i "NVIDIA" | grep -i "VGA\|3D\|Display" >/dev/null; then + NVIDIA_GPU_IN_LSPCI=true + NVIDIA_MODEL=$(lspci | grep -i "NVIDIA" | grep -i "VGA\|3D\|Display" | head -1 | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //') + fi + + # Check for Intel GPUs - making sure it's not already detected as NVIDIA + if lspci | grep -i "Intel" | grep -v "NVIDIA" | grep -i "VGA\|3D\|Display" >/dev/null; then + INTEL_GPU_IN_LSPCI=true + INTEL_MODEL=$(lspci | grep -i "Intel" | grep -v "NVIDIA" | grep -i "VGA\|3D\|Display" | head -1 | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //') + fi + + # Check for AMD GPUs - making sure it's not already detected as NVIDIA or Intel + if lspci | grep -i "AMD\|ATI\|Advanced Micro Devices" | grep -v "NVIDIA\|Intel" | grep -i "VGA\|3D\|Display" >/dev/null; then + AMD_GPU_IN_LSPCI=true + AMD_MODEL=$(lspci | grep -i "AMD\|ATI\|Advanced Micro Devices" | grep -v "NVIDIA\|Intel" | grep -i "VGA\|3D\|Display" | head -1 | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //' | sed 's/Technologies //') + fi + + # Display detected GPU hardware + if [ "$NVIDIA_GPU_IN_LSPCI" = true ]; then + echo "🔍 Hardware detection: NVIDIA GPU ($NVIDIA_MODEL)" + fi + if [ "$INTEL_GPU_IN_LSPCI" = true ]; then + echo "🔍 Hardware detection: Intel GPU ($INTEL_MODEL)" + fi + if [ "$AMD_GPU_IN_LSPCI" = true ]; then + echo "🔍 Hardware detection: AMD GPU ($AMD_MODEL)" + fi +fi + +# Silently check for any GPU devices first +for dev in /dev/dri/renderD* /dev/dri/card* /dev/nvidia*; do + if [ -e "$dev" ]; then + ANY_GPU_DEVICES_FOUND=true + break + fi +done + +# Only if devices might exist, show detailed checks +if [ "$ANY_GPU_DEVICES_FOUND" = true ]; then + # Check Intel/AMD VAAPI devices + echo "🔍 Checking for VAAPI device nodes (Intel/AMD)..." + for dev in /dev/dri/renderD* /dev/dri/card*; do + if [ -e "$dev" ]; then + DRI_DEVICES_FOUND=true + check_dev "$dev" + fi + done + + # Check NVIDIA device nodes + echo "🔍 Checking for NVIDIA device nodes..." + for dev in /dev/nvidia*; do + if [ -e "$dev" ]; then + NVIDIA_FOUND=true + check_dev "$dev" + fi + done + + # Show GPU device availability messages + if [ "$NVIDIA_FOUND" = false ] && [ "$NVIDIA_GPU_IN_LSPCI" = true ]; then + echo "⚠ī¸ No NVIDIA device nodes available despite hardware detection." + echo " You may be able to use VAAPI for hardware acceleration, but NVENC/CUDA won't be available." + echo " For optimal performance, configure proper NVIDIA container runtime." + elif [ "$NVIDIA_FOUND" = false ]; then + echo "ℹī¸ No NVIDIA device nodes found under /dev." + fi + + # Check for Intel/AMD GPUs that might not be fully accessible + if [ "$DRI_DEVICES_FOUND" = false ] && [ "$INTEL_GPU_IN_LSPCI" = true ]; then + echo "⚠ī¸ Intel GPU detected in hardware but no DRI devices found." + echo " Hardware acceleration will not be available." + echo " Make sure /dev/dri/ devices are properly mapped to the container." + elif [ "$DRI_DEVICES_FOUND" = false ] && [ "$AMD_GPU_IN_LSPCI" = true ]; then + echo "⚠ī¸ AMD GPU detected in hardware but no DRI devices found." + echo " Hardware acceleration will not be available." + echo " Make sure /dev/dri/ devices are properly mapped to the container." + fi +else + # No GPU devices found, skip the detailed checks + echo "❌ No GPU acceleration devices detected in this container." + echo "ℹī¸ Checking for potential configuration issues..." + + # Check if the host might have GPUs that aren't passed to the container + if command -v lspci >/dev/null 2>&1; then + if lspci | grep -i "VGA\|3D\|Display" | grep -i "NVIDIA\|Intel\|AMD" >/dev/null; then + echo "⚠ī¸ Host system appears to have GPU hardware, but no devices are accessible to the container." + echo " - For NVIDIA GPUs: Ensure NVIDIA Container Runtime is configured properly" + echo " - For Intel/AMD GPUs: Verify that /dev/dri/ devices are passed to the container" + echo " - Check your Docker run command or docker-compose.yml for proper device mapping" + else + echo "ℹī¸ No GPU hardware detected on the host system. CPU-only transcoding will be used." + fi + else + echo "ℹī¸ Unable to check host GPU hardware (lspci not available). CPU-only transcoding will be used." + fi + + echo "📋 ==================================================" + echo "✅ GPU detection script complete. No GPUs available for hardware acceleration." + exit 0 +fi + +# Check group membership for GPU access - context-aware based on hardware +echo "🔍 Checking user group memberships and device access..." +VIDEO_GID=$(getent group video | cut -d: -f3) +RENDER_GID=$(getent group render | cut -d: -f3) +NVIDIA_CONTAINER_TOOLKIT_FOUND=false +NVIDIA_ENV_MISMATCH=false + +# Improved device access check function +check_user_device_access() { + local device=$1 + local user=$2 + if [ -e "$device" ]; then + if su -c "test -r $device && test -w $device" - $user 2>/dev/null; then + echo "✅ User $user has full access to $device" + return 0 + else + echo "⚠ī¸ User $user cannot access $device (permission denied)" + return 1 + fi + else + # Device doesn't exist, no need to report here + return 2 + fi +} + +# Direct device access verification for DRI (Intel/AMD) +echo "🔍 Verifying if $POSTGRES_USER has direct access to GPU devices..." +HAS_DRI_ACCESS=false +DRI_ACCESS_COUNT=0 +DRI_DEVICE_COUNT=0 + +for dev in /dev/dri/renderD* /dev/dri/card*; do + if [ -e "$dev" ]; then + DRI_DEVICE_COUNT=$((DRI_DEVICE_COUNT + 1)) + if check_user_device_access "$dev" "$POSTGRES_USER"; then + DRI_ACCESS_COUNT=$((DRI_ACCESS_COUNT + 1)) + HAS_DRI_ACCESS=true + fi + fi +done + +# Direct device access verification for NVIDIA +HAS_NVIDIA_ACCESS=false +NVIDIA_ACCESS_COUNT=0 +NVIDIA_DEVICE_COUNT=0 + +for dev in /dev/nvidia*; do + if [ -e "$dev" ]; then + NVIDIA_DEVICE_COUNT=$((NVIDIA_DEVICE_COUNT + 1)) + if check_user_device_access "$dev" "$POSTGRES_USER"; then + NVIDIA_ACCESS_COUNT=$((NVIDIA_ACCESS_COUNT + 1)) + HAS_NVIDIA_ACCESS=true + fi + fi +done + +# Summary of device access +if [ $DRI_DEVICE_COUNT -gt 0 ]; then + if [ $DRI_ACCESS_COUNT -eq $DRI_DEVICE_COUNT ]; then + echo "✅ User $POSTGRES_USER has access to all DRI devices ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI hardware acceleration should work properly." + else + echo "⚠ī¸ User $POSTGRES_USER has limited access to DRI devices ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI hardware acceleration may not work properly." + echo " Consider adding $POSTGRES_USER to the 'video' and/or 'render' groups." + fi +fi + +if [ $NVIDIA_DEVICE_COUNT -gt 0 ]; then + if [ $NVIDIA_ACCESS_COUNT -eq $NVIDIA_DEVICE_COUNT ]; then + echo "✅ User $POSTGRES_USER has access to all NVIDIA devices ($NVIDIA_ACCESS_COUNT/$NVIDIA_DEVICE_COUNT)" + echo " NVIDIA hardware acceleration should work properly." + else + echo "⚠ī¸ User $POSTGRES_USER has limited access to NVIDIA devices ($NVIDIA_ACCESS_COUNT/$NVIDIA_DEVICE_COUNT)" + echo " NVIDIA hardware acceleration may not work properly." + if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = false ]; then + echo " Consider adding $POSTGRES_USER to the 'video' group or use NVIDIA Container Toolkit." + fi + fi +fi + +# Check for traditional group memberships (as additional information) +USER_IN_VIDEO_GROUP=false +USER_IN_RENDER_GROUP=false + +if [ -n "$VIDEO_GID" ]; then + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "video"; then + USER_IN_VIDEO_GROUP=true + echo "ℹī¸ User $POSTGRES_USER is in the 'video' group (GID $VIDEO_GID)." + fi +fi + +if [ -n "$RENDER_GID" ]; then + if id -nG "$POSTGRES_USER" 2>/dev/null | grep -qw "render"; then + USER_IN_RENDER_GROUP=true + echo "ℹī¸ User $POSTGRES_USER is in the 'render' group (GID $RENDER_GID)." + fi +fi + +# Check if NVIDIA Container Toolkit is present through environment or CLI tool +# IMPORTANT: Only mark as found if both env vars AND actual NVIDIA devices exist +if [ "$NVIDIA_FOUND" = true ] && command -v nvidia-container-cli >/dev/null 2>&1; then + NVIDIA_CONTAINER_TOOLKIT_FOUND=true +# Check for environment variables set by NVIDIA Container Runtime, but only if NVIDIA hardware exists +elif [ "$NVIDIA_FOUND" = true ] && [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + NVIDIA_CONTAINER_TOOLKIT_FOUND=true + echo "✅ NVIDIA Container Toolkit detected (via environment variables)." + echo " The container is properly configured with Docker Compose's 'driver: nvidia' syntax." +elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ] && [ "$NVIDIA_FOUND" = false ]; then + NVIDIA_ENV_MISMATCH=true +fi + +# Removed duplicate video group checks here - consolidated into the earlier checks that include GID + +# Check NVIDIA Container Toolkit support +echo "🔍 Checking NVIDIA container runtime support..." + +# More reliable detection of NVIDIA Container Runtime +NVIDIA_RUNTIME_ACTIVE=false + +# Method 1: Check for nvidia-container-cli tool +if command -v nvidia-container-cli >/dev/null 2>&1; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime detected (nvidia-container-cli found)." + + if nvidia-container-cli info >/dev/null 2>&1; then + echo "✅ NVIDIA container runtime is functional." + else + echo "⚠ī¸ nvidia-container-cli found, but 'info' command failed. Runtime may be misconfigured." + fi +fi + +# Method 2: Check for NVIDIA Container Runtime specific files +if [ -e "/dev/.nv" ] || [ -e "/.nv" ] || [ -e "/.nvidia-container-runtime" ]; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime files detected." +fi + +# Method 3: Check cgroup information for NVIDIA +if grep -q "nvidia" /proc/self/cgroup 2>/dev/null; then + NVIDIA_RUNTIME_ACTIVE=true + echo "✅ NVIDIA Container Runtime cgroups detected." +fi + +# Final verdict based on hardware AND runtime with improved messaging +if [ "$NVIDIA_FOUND" = true ] && ([ "$NVIDIA_RUNTIME_ACTIVE" = true ] || [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]); then + echo "✅ NVIDIA Container Runtime is properly configured with hardware access." +elif [ "$NVIDIA_FOUND" = true ] && [ "$NVIDIA_RUNTIME_ACTIVE" = false ] && [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = false ]; then + echo "ℹī¸ NVIDIA devices accessible via direct passthrough instead of Container Runtime." + echo " This works but consider using the 'deploy: resources: reservations: devices:' method in docker-compose." +elif [ "$NVIDIA_FOUND" = false ] && [ "$NVIDIA_RUNTIME_ACTIVE" = true ]; then + echo "⚠ī¸ NVIDIA Container Runtime appears to be configured, but no NVIDIA devices found." + echo " Check that your host has NVIDIA drivers installed and GPUs are properly passed to the container." +elif [ "$DRI_DEVICES_FOUND" = true ] && [ "$NVIDIA_GPU_IN_LSPCI" = true ]; then + echo "ℹī¸ Limited GPU access: Only DRI devices available for NVIDIA hardware." + echo " VAAPI acceleration may work but NVENC/CUDA won't be available." + echo " For full NVIDIA capabilities, configure the NVIDIA Container Runtime." +elif [ "$DRI_DEVICES_FOUND" = true ]; then + echo "ℹī¸ Using Intel/AMD GPU hardware for acceleration via VAAPI." +else + echo "⚠ī¸ No GPU acceleration devices detected. CPU-only transcoding will be used." +fi + +# Run nvidia-smi if available +if command -v nvidia-smi >/dev/null 2>&1; then + echo "🔍 Running nvidia-smi to verify GPU visibility..." + if nvidia-smi >/dev/null 2>&1; then + echo "✅ nvidia-smi successful - GPU is accessible to container!" + echo " This confirms hardware acceleration should be available to FFmpeg." + else + echo "⚠ī¸ nvidia-smi command failed. GPU may not be properly mapped into container." + fi +else + echo "ℹī¸ nvidia-smi not installed or not in PATH." +fi + +# Show relevant environment variables with contextual suggestions +echo "🔍 Checking GPU-related environment variables..." + +# Set flags based on device detection +DRI_DEVICES_FOUND=false +for dev in /dev/dri/renderD* /dev/dri/card*; do + if [ -e "$dev" ]; then + DRI_DEVICES_FOUND=true + break + fi +done + +# Give contextual suggestions based on detected hardware +if [ "$DRI_DEVICES_FOUND" = true ]; then + # Detect Intel/AMD GPU model - skip this if we already detected GPUs earlier + if [ "$NVIDIA_GPU_IN_LSPCI" = false ] && [ "$INTEL_GPU_IN_LSPCI" = false ] && [ "$AMD_GPU_IN_LSPCI" = false ] && command -v lspci >/dev/null 2>&1; then + GPU_INFO=$(lspci -nn | grep -i "VGA\|Display" | head -1) + if [ -n "$GPU_INFO" ]; then + echo "🔍 Detected GPU: $GPU_INFO" + # Extract model for cleaner display in summary + GPU_MODEL=$(echo "$GPU_INFO" | sed -E 's/.*: (.*) \[.*/\1/' | sed 's/Corporation //' | sed 's/Technologies //') + fi + else + # Use already detected GPU model info + if [ "$NVIDIA_GPU_IN_LSPCI" = true ]; then + GPU_MODEL=$NVIDIA_MODEL + elif [ "$INTEL_GPU_IN_LSPCI" = true ]; then + GPU_MODEL=$INTEL_MODEL + elif [ "$AMD_GPU_IN_LSPCI" = true ]; then + GPU_MODEL=$AMD_MODEL + fi + fi + + if [ -n "$LIBVA_DRIVER_NAME" ]; then + echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" + else + # Check if we can detect the GPU type + if command -v lspci >/dev/null 2>&1; then + echo "ℹī¸ VAAPI driver auto-detection is usually reliable. Settings below only needed if you experience issues." + + # Intel GPU detection with more future-proof approach + if lspci | grep -q "Intel Corporation.*VGA"; then + # Check for newer Intel generations that use iHD + if lspci | grep -q "Intel Corporation.*VGA" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then + echo "💡 If needed: LIBVA_DRIVER_NAME=iHD for modern Intel GPUs (Gen12+/Arc/Xe)" + # Check for very old Intel that definitely needs i965 + elif lspci | grep -q "Intel Corporation.*VGA" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then + echo "💡 If needed: LIBVA_DRIVER_NAME=i965 for older Intel GPUs (Gen11 and below)" + else + # Generic Intel case - could be either, but iHD is more likely for newer hardware + echo "💡 If needed: Try LIBVA_DRIVER_NAME=iHD first, or LIBVA_DRIVER_NAME=i965 if that fails" + fi + elif lspci | grep -q "Advanced Micro Devices.*VGA"; then + echo "💡 If needed: LIBVA_DRIVER_NAME=radeonsi for AMD GPUs" + else + echo "ℹī¸ Common VAAPI driver options if auto-detection fails:" + echo " - For modern Intel GPUs (Gen12+/Arc/Xe): LIBVA_DRIVER_NAME=iHD" + echo " - For older Intel GPUs: LIBVA_DRIVER_NAME=i965" + echo " - For AMD GPUs: LIBVA_DRIVER_NAME=radeonsi" + fi + else + echo "ℹī¸ Intel/AMD GPU detected. Auto-detection should work in most cases." + echo " If VAAPI doesn't work, you may need to set LIBVA_DRIVER_NAME manually." + fi + fi +fi + +# Check FFmpeg hardware acceleration support +echo "🔍 Checking FFmpeg hardware acceleration capabilities..." +if command -v ffmpeg >/dev/null 2>&1; then + HWACCEL=$(ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -v "Hardware acceleration methods:" || echo "None found") + + # Define expected acceleration methods based on detected hardware + EXPECTED_METHODS="" + MISSING_METHODS="" + + if [ "$NVIDIA_FOUND" = true ]; then + EXPECTED_NVIDIA="cuda" #cuvid nvenc nvdec" keeping these in case future support is added + for method in $EXPECTED_NVIDIA; do + if echo "$HWACCEL" | grep -q "$method"; then + EXPECTED_METHODS="$EXPECTED_METHODS $method" + else + MISSING_METHODS="$MISSING_METHODS $method" + fi + done + fi + + if [ "$INTEL_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + EXPECTED_INTEL="vaapi qsv" + for method in $EXPECTED_INTEL; do + if echo "$HWACCEL" | grep -q "$method"; then + EXPECTED_METHODS="$EXPECTED_METHODS $method" + else + MISSING_METHODS="$MISSING_METHODS $method" + fi + done + fi + + if [ "$AMD_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + EXPECTED_AMD="vaapi" + for method in $EXPECTED_AMD; do + if echo "$HWACCEL" | grep -q "$method"; then + EXPECTED_METHODS="$EXPECTED_METHODS $method" + else + MISSING_METHODS="$MISSING_METHODS $method" + fi + done + fi + + # Show all available methods + echo "🔍 All available FFmpeg hardware acceleration methods:" + echo "$HWACCEL" + + # Show expected methods based on hardware + if [ -n "$EXPECTED_METHODS" ]; then + echo "✅ Hardware-appropriate acceleration methods available:$EXPECTED_METHODS" + fi + + # Show missing expected methods + if [ -n "$MISSING_METHODS" ]; then + echo "⚠ī¸ Expected acceleration methods not found:$MISSING_METHODS" + echo " This might indicate missing libraries or improper driver configuration." + fi + + # Check specific cases of interest + if [ "$NVIDIA_FOUND" = true ] && ! echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then + echo "⚠ī¸ NVIDIA GPU detected but no NVIDIA acceleration methods available." + echo " Ensure ffmpeg is built with NVIDIA support and required libraries are installed." + fi + + if ([ "$INTEL_GPU_IN_LSPCI" = true ] || [ "$AMD_GPU_IN_LSPCI" = true ]) && [ "$DRI_DEVICES_FOUND" = true ] && ! echo "$HWACCEL" | grep -q "vaapi"; then + echo "⚠ī¸ Intel/AMD GPU detected but VAAPI acceleration not available." + echo " Ensure ffmpeg is built with VAAPI support and proper drivers are installed." + fi +else + echo "⚠ī¸ FFmpeg not found in PATH." +fi + +# Provide a final summary of the hardware acceleration setup +echo "📋 ===================== SUMMARY =====================" + +# Identify which GPU type is active and working +if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VISIBLE_DEVICES" ]); then + if [ -n "$NVIDIA_MODEL" ]; then + echo "🔰 NVIDIA GPU: $NVIDIA_MODEL" + else + echo "🔰 NVIDIA GPU: ACTIVE (model detection unavailable)" + echo "ℹī¸ Note: GPU model information couldn't be retrieved, but devices are present." + echo " This may be due to missing nvidia-smi tool or container limitations." + fi + + if [ "$NVIDIA_CONTAINER_TOOLKIT_FOUND" = true ]; then + echo "✅ NVIDIA Container Toolkit: CONFIGURED CORRECTLY" + elif [ -n "$NVIDIA_VISIBLE_DEVICES" ] && [ -n "$NVIDIA_DRIVER_CAPABILITIES" ]; then + echo "✅ NVIDIA Docker configuration: USING MODERN DEPLOYMENT" + else + echo "⚠ī¸ NVIDIA setup method: DIRECT DEVICE MAPPING (functional but not optimal)" + fi + + # Add device accessibility status + if [ $NVIDIA_DEVICE_COUNT -gt 0 ]; then + if [ $NVIDIA_ACCESS_COUNT -eq $NVIDIA_DEVICE_COUNT ]; then + echo "✅ Device access: ALL NVIDIA DEVICES ACCESSIBLE ($NVIDIA_ACCESS_COUNT/$NVIDIA_DEVICE_COUNT)" + else + echo "⚠ī¸ Device access: LIMITED NVIDIA DEVICE ACCESS ($NVIDIA_ACCESS_COUNT/$NVIDIA_DEVICE_COUNT)" + echo " Some hardware acceleration features may not work properly." + fi + fi + + # Display FFmpeg NVIDIA acceleration methods + if echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then + echo "✅ FFmpeg NVIDIA acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" + fi +elif [ "$NVIDIA_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + # NVIDIA through DRI only (suboptimal but possible) + if [ -n "$NVIDIA_MODEL" ]; then + echo "🔰 NVIDIA GPU: $NVIDIA_MODEL (SUBOPTIMALLY CONFIGURED)" + else + echo "🔰 NVIDIA GPU: DETECTED BUT SUBOPTIMALLY CONFIGURED" + fi + echo "⚠ī¸ Your NVIDIA GPU is only accessible through DRI devices" + echo " - VAAPI acceleration may work for some tasks" + echo " - NVENC/CUDA acceleration is NOT available" + + # Add device accessibility status + if [ $DRI_DEVICE_COUNT -gt 0 ]; then + if [ $DRI_ACCESS_COUNT -eq $DRI_DEVICE_COUNT ]; then + echo "✅ Device access: ALL DRI DEVICES ACCESSIBLE ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI acceleration should work properly." + else + echo "⚠ī¸ Device access: LIMITED DRI DEVICE ACCESS ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI acceleration may not work properly." + fi + fi + + echo "💡 RECOMMENDATION: Use the proper NVIDIA container configuration:" + echo " deploy:" + echo " resources:" + echo " reservations:" + echo " devices:" + echo " - driver: nvidia" + echo " count: all" + echo " capabilities: [gpu]" + + if echo "$HWACCEL" | grep -q "vaapi"; then + echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" + fi +elif [ "$DRI_DEVICES_FOUND" = true ]; then + # Intel/AMD detection with model if available + if [ -n "$GPU_MODEL" ]; then + echo "🔰 GPU: $GPU_MODEL" + elif [ -n "$LIBVA_DRIVER_NAME" ]; then + echo "🔰 ${LIBVA_DRIVER_NAME^^} GPU: ACTIVE" + else + echo "🔰 INTEL/AMD GPU: ACTIVE (model detection unavailable)" + echo "ℹī¸ Note: Basic GPU drivers appear to be loaded (device nodes exist), but" + echo " couldn't identify specific model. This doesn't necessarily indicate a problem." + fi + + # Add device accessibility status + if [ $DRI_DEVICE_COUNT -gt 0 ]; then + if [ $DRI_ACCESS_COUNT -eq $DRI_DEVICE_COUNT ]; then + echo "✅ Device access: ALL DRI DEVICES ACCESSIBLE ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI hardware acceleration should work properly." + else + echo "⚠ī¸ Device access: LIMITED DRI DEVICE ACCESS ($DRI_ACCESS_COUNT/$DRI_DEVICE_COUNT)" + echo " VAAPI hardware acceleration may not work properly." + fi + fi + + # Display FFmpeg VAAPI acceleration method + if echo "$HWACCEL" | grep -q "vaapi"; then + echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + else + echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" + fi +else + echo "❌ NO GPU ACCELERATION DETECTED" + echo "⚠ī¸ Hardware acceleration is unavailable or misconfigured" +fi + +echo "📋 ==================================================" +echo "✅ GPU detection script complete." From 3098b969195e5dcbb2f81ab7896a681f7f31d87e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:03:25 -0500 Subject: [PATCH 38/55] Removed render group creation as now it's done in the base build. --- docker/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 17d995ce..ec24c818 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,10 +22,6 @@ ENV VIRTUAL_ENV=/dispatcharrpy ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /app -# Create render group for hardware acceleration support -# Remove this later as it will be included in the base image -RUN groupadd -r render || true - # Copy application code COPY . /app # Copy nginx configuration From 1ab3dcac48d09be3fd1afd1921394b380ce09369 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:04:02 -0500 Subject: [PATCH 39/55] Use base for dev. --- docker/docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 434059e2..4f42e1b0 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -3,7 +3,7 @@ services: # build: # context: .. # dockerfile: docker/Dockerfile.dev - image: ghcr.io/dispatcharr/dispatcharr:dev + image: ghcr.io/dispatcharr/dispatcharr:base container_name: dispatcharr_dev ports: - 5656:5656 From bee2226e755f7c43b82bbe455cae18d03d5dae84 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:04:32 -0500 Subject: [PATCH 40/55] Use base for debug --- docker/docker-compose.debug.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.debug.yml b/docker/docker-compose.debug.yml index df9e5017..163ebf6a 100644 --- a/docker/docker-compose.debug.yml +++ b/docker/docker-compose.debug.yml @@ -3,7 +3,7 @@ services: # build: # context: .. # dockerfile: docker/Dockerfile.dev - image: dispatcharr/dispatcharr + image: ghcr.io/dispatcharr/dispatcharr:base container_name: dispatcharr_debug ports: - 5656:5656 # API port From b43f096ea6036b1815fd895a3fe02d80144360bf Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:06:16 -0500 Subject: [PATCH 41/55] Cleaned up unneeded comment for group_add and added log level. --- docker/docker-compose.aio.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/docker-compose.aio.yml b/docker/docker-compose.aio.yml index 95834915..90cd8654 100644 --- a/docker/docker-compose.aio.yml +++ b/docker/docker-compose.aio.yml @@ -13,10 +13,8 @@ services: - DISPATCHARR_ENV=aio - REDIS_HOST=localhost - CELERY_BROKER_URL=redis://localhost:6379/0 + - DISPATCHARR_LOG_LEVEL=info # Optional for hardware acceleration - #group_add: - # - video - # #- render # Uncomment if your GPU requires it #devices: # - /dev/dri:/dev/dri # For Intel/AMD GPU acceleration (VA-API) # Uncomment the following lines for NVIDIA GPU support From 08493321dd6f80bf4e8b6b7caccd7990dff2e7d3 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:27:38 -0500 Subject: [PATCH 42/55] Fixes intel devices not matching correctly for libva_driver recommendations. --- docker/init/04-check-hwaccel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh index 04eabf1e..0e25d2cb 100644 --- a/docker/init/04-check-hwaccel.sh +++ b/docker/init/04-check-hwaccel.sh @@ -346,10 +346,10 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then # Intel GPU detection with more future-proof approach if lspci | grep -q "Intel Corporation.*VGA"; then # Check for newer Intel generations that use iHD - if lspci | grep -q "Intel Corporation.*VGA" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then + if lspci | grep -E "Intel Corporation.*VGA" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then echo "💡 If needed: LIBVA_DRIVER_NAME=iHD for modern Intel GPUs (Gen12+/Arc/Xe)" # Check for very old Intel that definitely needs i965 - elif lspci | grep -q "Intel Corporation.*VGA" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then + elif lspci | grep -E "Intel Corporation.*VGA" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then echo "💡 If needed: LIBVA_DRIVER_NAME=i965 for older Intel GPUs (Gen11 and below)" else # Generic Intel case - could be either, but iHD is more likely for newer hardware From bb73da1cb9632f1844d8a6fc138a8592aef75b30 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 20:55:35 -0500 Subject: [PATCH 43/55] Fixes invalid regex detection for intel gpu models. --- docker/init/04-check-hwaccel.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh index 0e25d2cb..fca25a5b 100644 --- a/docker/init/04-check-hwaccel.sh +++ b/docker/init/04-check-hwaccel.sh @@ -338,24 +338,26 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then if [ -n "$LIBVA_DRIVER_NAME" ]; then echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" + echo " Note: If you experience issues with hardware acceleration, try removing this" + echo " environment variable to let the system auto-detect the appropriate driver." else # Check if we can detect the GPU type if command -v lspci >/dev/null 2>&1; then echo "ℹī¸ VAAPI driver auto-detection is usually reliable. Settings below only needed if you experience issues." # Intel GPU detection with more future-proof approach - if lspci | grep -q "Intel Corporation.*VGA"; then + if lspci | grep -q "VGA compatible controller.*Intel"; then # Check for newer Intel generations that use iHD - if lspci | grep -E "Intel Corporation.*VGA" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then + if lspci | grep "VGA compatible controller.*Intel" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then echo "💡 If needed: LIBVA_DRIVER_NAME=iHD for modern Intel GPUs (Gen12+/Arc/Xe)" # Check for very old Intel that definitely needs i965 - elif lspci | grep -E "Intel Corporation.*VGA" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then + elif lspci | grep "VGA compatible controller.*Intel" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then echo "💡 If needed: LIBVA_DRIVER_NAME=i965 for older Intel GPUs (Gen11 and below)" else # Generic Intel case - could be either, but iHD is more likely for newer hardware echo "💡 If needed: Try LIBVA_DRIVER_NAME=iHD first, or LIBVA_DRIVER_NAME=i965 if that fails" fi - elif lspci | grep -q "Advanced Micro Devices.*VGA"; then + elif lspci | grep -q "VGA compatible controller.*Advanced Micro Devices"; then echo "💡 If needed: LIBVA_DRIVER_NAME=radeonsi for AMD GPUs" else echo "ℹī¸ Common VAAPI driver options if auto-detection fails:" From 40765ed46d82778421d8e004b9da8e0007d17a65 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Mon, 12 May 2025 21:19:43 -0500 Subject: [PATCH 44/55] One more attempt at proper intel detection. --- docker/init/04-check-hwaccel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh index fca25a5b..23198604 100644 --- a/docker/init/04-check-hwaccel.sh +++ b/docker/init/04-check-hwaccel.sh @@ -348,10 +348,10 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then # Intel GPU detection with more future-proof approach if lspci | grep -q "VGA compatible controller.*Intel"; then # Check for newer Intel generations that use iHD - if lspci | grep "VGA compatible controller.*Intel" | grep -E "Arc|Xe|Alchemist|Tiger Lake|Alder Lake|Raptor Lake|Meteor Lake|Gen1[2-9]"; then + if lspci | grep -q "VGA compatible controller.*Intel" | grep -q -E "Arc|Xe|Alchemist|Tiger Lake|Alder|Raptor Lake|Meteor Lake|Gen1[2-9]"; then echo "💡 If needed: LIBVA_DRIVER_NAME=iHD for modern Intel GPUs (Gen12+/Arc/Xe)" # Check for very old Intel that definitely needs i965 - elif lspci | grep "VGA compatible controller.*Intel" | grep -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then + elif lspci | grep -q "VGA compatible controller.*Intel" | grep -q -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then echo "💡 If needed: LIBVA_DRIVER_NAME=i965 for older Intel GPUs (Gen11 and below)" else # Generic Intel case - could be either, but iHD is more likely for newer hardware From dd54a13bdd4906285aa678b43577e14c2a527c42 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 00:37:07 -0500 Subject: [PATCH 45/55] Fix script exiting entrypoint if no devices were found. --- docker/init/04-check-hwaccel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh index 23198604..57d93b4a 100644 --- a/docker/init/04-check-hwaccel.sh +++ b/docker/init/04-check-hwaccel.sh @@ -124,7 +124,8 @@ else echo "📋 ==================================================" echo "✅ GPU detection script complete. No GPUs available for hardware acceleration." - exit 0 + # Don't exit the container - just return from this script + return 0 2>/dev/null || true fi # Check group membership for GPU access - context-aware based on hardware From 0a9250c3d5ee18b53223da95f371d7b6f9b581c4 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 12:32:49 -0500 Subject: [PATCH 46/55] Vastly improved logic for detecting recommended acceleration methods. --- docker/init/04-check-hwaccel.sh | 270 +++++++++++++++++++++++++------- 1 file changed, 215 insertions(+), 55 deletions(-) diff --git a/docker/init/04-check-hwaccel.sh b/docker/init/04-check-hwaccel.sh index 57d93b4a..f23cdd9a 100644 --- a/docker/init/04-check-hwaccel.sh +++ b/docker/init/04-check-hwaccel.sh @@ -139,7 +139,7 @@ NVIDIA_ENV_MISMATCH=false check_user_device_access() { local device=$1 local user=$2 - if [ -e "$device" ]; then + if [ -e "$device" ];then if su -c "test -r $device && test -w $device" - $user 2>/dev/null; then echo "✅ User $user has full access to $device" return 0 @@ -310,7 +310,7 @@ echo "🔍 Checking GPU-related environment variables..." # Set flags based on device detection DRI_DEVICES_FOUND=false for dev in /dev/dri/renderD* /dev/dri/card*; do - if [ -e "$dev" ]; then + if [ -e "$dev" ];then DRI_DEVICES_FOUND=true break fi @@ -337,6 +337,10 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then fi fi + if [ -n "$GPU_MODEL" ]; then + echo "🔍 GPU model: $GPU_MODEL" + fi + # Check for LIBVA_DRIVER_NAME environment variable if [ -n "$LIBVA_DRIVER_NAME" ]; then echo "ℹī¸ LIBVA_DRIVER_NAME is set to '$LIBVA_DRIVER_NAME'" echo " Note: If you experience issues with hardware acceleration, try removing this" @@ -346,25 +350,51 @@ if [ "$DRI_DEVICES_FOUND" = true ]; then if command -v lspci >/dev/null 2>&1; then echo "ℹī¸ VAAPI driver auto-detection is usually reliable. Settings below only needed if you experience issues." - # Intel GPU detection with more future-proof approach - if lspci | grep -q "VGA compatible controller.*Intel"; then + # Create variables to store recommended driver and supported methods + INTEL_RECOMMENDED_DRIVER="" + INTEL_SUPPORTS_QSV=false + + # Use the Intel model information we already captured + if [ "$INTEL_GPU_IN_LSPCI" = true ] && [ -n "$INTEL_MODEL" ]; then # Check for newer Intel generations that use iHD - if lspci | grep -q "VGA compatible controller.*Intel" | grep -q -E "Arc|Xe|Alchemist|Tiger Lake|Alder|Raptor Lake|Meteor Lake|Gen1[2-9]"; then - echo "💡 If needed: LIBVA_DRIVER_NAME=iHD for modern Intel GPUs (Gen12+/Arc/Xe)" - # Check for very old Intel that definitely needs i965 - elif lspci | grep -q "VGA compatible controller.*Intel" | grep -q -E "Haswell|Broadwell|Skylake|Kaby Lake|Coffee Lake|Whiskey Lake|Comet Lake"; then - echo "💡 If needed: LIBVA_DRIVER_NAME=i965 for older Intel GPUs (Gen11 and below)" + if echo "$INTEL_MODEL" | grep -q -E "Arc|Xe|Alchemist|Tiger|Alder|Raptor|Meteor|Gen1[2-9]"; then + echo "💡 Detected Intel GPU that supports iHD (e.g. Gen12+/Arc/Xe)" + echo " Recommended: LIBVA_DRIVER_NAME=iHD" + echo " Note: Only set this environment variable if hardware acceleration doesn't work by default" + INTEL_RECOMMENDED_DRIVER="iHD" + INTEL_SUPPORTS_QSV=true + elif echo "$INTEL_MODEL" | grep -q -E "Coffee|Whiskey|Comet|Gen11"; then + echo "💡 Detected Intel GPU that supports both i965 and iHD (e.g. Gen9.5/Gen11)" + echo " Preferred: LIBVA_DRIVER_NAME=iHD" + echo " Recommended: Try i965 only if iHD has compatibility issues" + echo " Note: Only set this environment variable if hardware acceleration doesn't work by default" + INTEL_RECOMMENDED_DRIVER="iHD" + INTEL_SUPPORTS_QSV=true + elif echo "$INTEL_MODEL" | grep -q -E "Haswell|Broadwell|Skylake|Kaby"; then + echo "💡 Detected Intel GPU that supports i965 (e.g. Gen9 and below)" + echo " Recommended: Set LIBVA_DRIVER_NAME=i965" + echo " Note: Only set this environment variable if hardware acceleration doesn't work by default" + INTEL_RECOMMENDED_DRIVER="i965" + # Older Intel GPUs support QSV through i965 driver but with more limitations + INTEL_SUPPORTS_QSV=false else - # Generic Intel case - could be either, but iHD is more likely for newer hardware - echo "💡 If needed: Try LIBVA_DRIVER_NAME=iHD first, or LIBVA_DRIVER_NAME=i965 if that fails" + # Generic Intel case - we're not fully confident in our recommendation + echo "💡 Unable to definitively identify Intel GPU generation" + echo " Try auto-detection first (no environment variable)" + echo " If issues occur: Try LIBVA_DRIVER_NAME=iHD first (newer GPUs)" + echo " If that fails: Try LIBVA_DRIVER_NAME=i965 (older GPUs)" + INTEL_RECOMMENDED_DRIVER="unknown" # Mark as unknown rather than assuming + INTEL_SUPPORTS_QSV="maybe" # Mark as maybe instead of assuming true fi - elif lspci | grep -q "VGA compatible controller.*Advanced Micro Devices"; then - echo "💡 If needed: LIBVA_DRIVER_NAME=radeonsi for AMD GPUs" + elif [ "$AMD_GPU_IN_LSPCI" = true ]; then + echo "💡 If auto-detection fails: Set LIBVA_DRIVER_NAME=radeonsi for AMD GPUs" + echo " Note: Only set this environment variable if hardware acceleration doesn't work by default" else echo "ℹī¸ Common VAAPI driver options if auto-detection fails:" echo " - For modern Intel GPUs (Gen12+/Arc/Xe): LIBVA_DRIVER_NAME=iHD" echo " - For older Intel GPUs: LIBVA_DRIVER_NAME=i965" echo " - For AMD GPUs: LIBVA_DRIVER_NAME=radeonsi" + echo " Note: Only set these environment variables if hardware acceleration doesn't work by default" fi else echo "ℹī¸ Intel/AMD GPU detected. Auto-detection should work in most cases." @@ -378,50 +408,140 @@ echo "🔍 Checking FFmpeg hardware acceleration capabilities..." if command -v ffmpeg >/dev/null 2>&1; then HWACCEL=$(ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -v "Hardware acceleration methods:" || echo "None found") - # Define expected acceleration methods based on detected hardware - EXPECTED_METHODS="" + # Initialize variables to store compatible and missing methods + COMPATIBLE_METHODS="" MISSING_METHODS="" - if [ "$NVIDIA_FOUND" = true ]; then - EXPECTED_NVIDIA="cuda" #cuvid nvenc nvdec" keeping these in case future support is added - for method in $EXPECTED_NVIDIA; do - if echo "$HWACCEL" | grep -q "$method"; then - EXPECTED_METHODS="$EXPECTED_METHODS $method" - else - MISSING_METHODS="$MISSING_METHODS $method" + # Format the list of hardware acceleration methods in a more readable way + echo "🔍 Available FFmpeg hardware acceleration methods:" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + # Process the list into a more readable format with relevance indicators + if [ -n "$HWACCEL" ] && [ "$HWACCEL" != "None found" ]; then + # First, show methods compatible with detected hardware + echo " 📌 Compatible with your hardware:" + COMPATIBLE_FOUND=false + + for method in $HWACCEL; do + # Skip if it's just the header line or empty + if [ "$method" = "Hardware" ] || [ -z "$method" ]; then + continue + fi + + # Check if this method is relevant to detected hardware + IS_COMPATIBLE=false + DESCRIPTION="" + + if [ "$NVIDIA_FOUND" = true ] && [[ "$method" =~ ^(cuda|cuvid|nvenc|nvdec)$ ]]; then + IS_COMPATIBLE=true + DESCRIPTION="NVIDIA GPU acceleration" + elif [ "$INTEL_GPU_IN_LSPCI" = true ] && [ "$method" = "qsv" ] && [ "$INTEL_SUPPORTS_QSV" = true ]; then + IS_COMPATIBLE=true + DESCRIPTION="Intel QuickSync acceleration" + elif [ "$method" = "vaapi" ] && (([ "$INTEL_GPU_IN_LSPCI" = true ] || [ "$AMD_GPU_IN_LSPCI" = true ]) && [ "$DRI_DEVICES_FOUND" = true ]); then + IS_COMPATIBLE=true + if [ "$INTEL_GPU_IN_LSPCI" = true ]; then + DESCRIPTION="Intel VAAPI acceleration" + else + DESCRIPTION="AMD VAAPI acceleration" + fi + fi + + # Display compatible methods and store for summary + if [ "$IS_COMPATIBLE" = true ]; then + COMPATIBLE_FOUND=true + COMPATIBLE_METHODS="$COMPATIBLE_METHODS $method" + echo " ✅ $method - $DESCRIPTION" fi done - fi - if [ "$INTEL_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then - EXPECTED_INTEL="vaapi qsv" - for method in $EXPECTED_INTEL; do - if echo "$HWACCEL" | grep -q "$method"; then - EXPECTED_METHODS="$EXPECTED_METHODS $method" - else - MISSING_METHODS="$MISSING_METHODS $method" + if [ "$COMPATIBLE_FOUND" = false ]; then + echo " ❌ No compatible acceleration methods found for your hardware" + fi + + # Then show all other available methods + echo " 📌 Other available methods (not compatible with detected hardware):" + OTHER_FOUND=false + + for method in $HWACCEL; do + # Skip if it's just the header line or empty + if [ "$method" = "Hardware" ] || [ -z "$method" ]; then + continue + fi + + # Check if this method is relevant to detected hardware + IS_COMPATIBLE=false + + if [ "$NVIDIA_FOUND" = true ] && [[ "$method" =~ ^(cuda|cuvid|nvenc|nvdec)$ ]]; then + IS_COMPATIBLE=true + elif [ "$INTEL_GPU_IN_LSPCI" = true ] && [ "$method" = "qsv" ] && [ "$INTEL_SUPPORTS_QSV" = true ]; then + IS_COMPATIBLE=true + elif [ "$method" = "vaapi" ] && (([ "$INTEL_GPU_IN_LSPCI" = true ] || [ "$AMD_GPU_IN_LSPCI" = true ]) && [ "$DRI_DEVICES_FOUND" = true ]); then + IS_COMPATIBLE=true + fi + + # Display other methods that aren't compatible + if [ "$IS_COMPATIBLE" = false ]; then + OTHER_FOUND=true + echo " ℹī¸ $method" fi done - fi - if [ "$AMD_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then - EXPECTED_AMD="vaapi" - for method in $EXPECTED_AMD; do - if echo "$HWACCEL" | grep -q "$method"; then - EXPECTED_METHODS="$EXPECTED_METHODS $method" - else - MISSING_METHODS="$MISSING_METHODS $method" + if [ "$OTHER_FOUND" = false ]; then + echo " None" + fi + + # Show expected methods that are missing + echo " 📌 Missing methods that should be available for your hardware:" + MISSING_FOUND=false + + # Check for NVIDIA methods if NVIDIA GPU is detected + if [ "$NVIDIA_FOUND" = true ]; then + EXPECTED_NVIDIA="cuda" # cuvid nvenc nvdec" keeping these in case future support is added + for method in $EXPECTED_NVIDIA; do + if ! echo "$HWACCEL" | grep -q "$method"; then + MISSING_FOUND=true + MISSING_METHODS="$MISSING_METHODS $method" + echo " ⚠ī¸ $method - NVIDIA acceleration (missing but should be available)" + fi + done + fi + + # Check for Intel methods if Intel GPU is detected + if [ "$INTEL_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + if [ "$INTEL_SUPPORTS_QSV" = true ] && ! echo "$HWACCEL" | grep -q "qsv"; then + MISSING_FOUND=true + MISSING_METHODS="$MISSING_METHODS qsv" + echo " ⚠ī¸ qsv - Intel QuickSync acceleration (missing but should be available)" fi - done + + if ! echo "$HWACCEL" | grep -q "vaapi"; then + MISSING_FOUND=true + MISSING_METHODS="$MISSING_METHODS vaapi" + echo " ⚠ī¸ vaapi - Intel VAAPI acceleration (missing but should be available)" + fi + fi + + # Check for AMD methods if AMD GPU is detected + if [ "$AMD_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then + if ! echo "$HWACCEL" | grep -q "vaapi"; then + MISSING_FOUND=true + MISSING_METHODS="$MISSING_METHODS vaapi" + echo " ⚠ī¸ vaapi - AMD VAAPI acceleration (missing but should be available)" + fi + fi + + if [ "$MISSING_FOUND" = false ]; then + echo " None - All expected methods are available" + fi + else + echo " ❌ No hardware acceleration methods found" fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - # Show all available methods - echo "🔍 All available FFmpeg hardware acceleration methods:" - echo "$HWACCEL" - - # Show expected methods based on hardware - if [ -n "$EXPECTED_METHODS" ]; then - echo "✅ Hardware-appropriate acceleration methods available:$EXPECTED_METHODS" + # Show hardware-appropriate method summary using the already gathered information + if [ -n "$COMPATIBLE_METHODS" ]; then + echo "✅ Hardware-appropriate acceleration methods available:$COMPATIBLE_METHODS" fi # Show missing expected methods @@ -430,13 +550,14 @@ if command -v ffmpeg >/dev/null 2>&1; then echo " This might indicate missing libraries or improper driver configuration." fi - # Check specific cases of interest - if [ "$NVIDIA_FOUND" = true ] && ! echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then + # Display specific cases of interest (simplify using previously captured information) + if [ "$NVIDIA_FOUND" = true ] && ! echo "$COMPATIBLE_METHODS" | grep -q "cuda\|nvenc\|cuvid"; then echo "⚠ī¸ NVIDIA GPU detected but no NVIDIA acceleration methods available." echo " Ensure ffmpeg is built with NVIDIA support and required libraries are installed." fi - if ([ "$INTEL_GPU_IN_LSPCI" = true ] || [ "$AMD_GPU_IN_LSPCI" = true ]) && [ "$DRI_DEVICES_FOUND" = true ] && ! echo "$HWACCEL" | grep -q "vaapi"; then + if (([ "$INTEL_GPU_IN_LSPCI" = true ] || [ "$AMD_GPU_IN_LSPCI" = true ]) && + [ "$DRI_DEVICES_FOUND" = true ] && ! echo "$COMPATIBLE_METHODS" | grep -q "vaapi"); then echo "⚠ī¸ Intel/AMD GPU detected but VAAPI acceleration not available." echo " Ensure ffmpeg is built with VAAPI support and proper drivers are installed." fi @@ -475,11 +596,19 @@ if [ "$NVIDIA_FOUND" = true ] && (nvidia-smi >/dev/null 2>&1 || [ -n "$NVIDIA_VI fi fi - # Display FFmpeg NVIDIA acceleration methods - if echo "$HWACCEL" | grep -q "cuda\|nvenc\|cuvid"; then + # Display FFmpeg NVIDIA acceleration methods in more detail + if echo "$COMPATIBLE_METHODS" | grep -q "cuda\|nvenc\|cuvid"; then echo "✅ FFmpeg NVIDIA acceleration: AVAILABLE" + + # Show detailed breakdown of available NVIDIA methods + NVIDIA_METHODS=$(echo "$COMPATIBLE_METHODS" | grep -o '\(cuda\|cuvid\|nvenc\|nvdec\)') + echo " Available NVIDIA methods: $NVIDIA_METHODS" + echo " Recommended for: Video transcoding with NVIDIA GPUs" else echo "⚠ī¸ FFmpeg NVIDIA acceleration: NOT DETECTED" + if [ -n "$MISSING_METHODS" ]; then + echo " Missing methods that should be available: $MISSING_METHODS" + fi fi elif [ "$NVIDIA_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then # NVIDIA through DRI only (suboptimal but possible) @@ -512,8 +641,9 @@ elif [ "$NVIDIA_GPU_IN_LSPCI" = true ] && [ "$DRI_DEVICES_FOUND" = true ]; then echo " count: all" echo " capabilities: [gpu]" - if echo "$HWACCEL" | grep -q "vaapi"; then - echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + if echo "$COMPATIBLE_METHODS" | grep -q "vaapi"; then + echo "✅ FFmpeg VAAPI acceleration: AVAILABLE (limited without NVENC)" + echo " VAAPI can be used for transcoding, but NVENC/CUDA would be more efficient" else echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" fi @@ -540,11 +670,41 @@ elif [ "$DRI_DEVICES_FOUND" = true ]; then fi fi - # Display FFmpeg VAAPI acceleration method - if echo "$HWACCEL" | grep -q "vaapi"; then + # Display FFmpeg VAAPI acceleration method with more details + if echo "$COMPATIBLE_METHODS" | grep -q "vaapi"; then echo "✅ FFmpeg VAAPI acceleration: AVAILABLE" + + # Add recommended usage information + echo " Recommended for: General video transcoding with Intel/AMD GPUs" + + # Add recommended driver information for Intel GPUs + if [ "$INTEL_GPU_IN_LSPCI" = true ] && [ -n "$INTEL_RECOMMENDED_DRIVER" ]; then + if [ "$INTEL_RECOMMENDED_DRIVER" = "unknown" ]; then + echo "ℹī¸ Uncertain about recommended VAAPI driver for this Intel GPU" + echo " Auto-detection should work, but if issues occur try iHD or i965" + else + echo "ℹī¸ Recommended VAAPI driver for this Intel GPU: $INTEL_RECOMMENDED_DRIVER" + fi + + if [ "$INTEL_SUPPORTS_QSV" = true ] && echo "$COMPATIBLE_METHODS" | grep -q "qsv"; then + echo "✅ QSV acceleration: AVAILABLE" + echo " Recommended for: Intel-specific optimized transcoding" + echo " Works best with: $INTEL_RECOMMENDED_DRIVER driver" + elif [ "$INTEL_SUPPORTS_QSV" = true ]; then + echo "ℹī¸ QSV acceleration: NOT DETECTED (may be available with proper configuration)" + echo " Your Intel GPU supports QSV but it's not available in FFmpeg" + echo " Check if FFmpeg is built with QSV support" + elif [ "$INTEL_SUPPORTS_QSV" = "maybe" ]; then + echo "ℹī¸ QSV acceleration: MAY BE AVAILABLE (depends on exact GPU model)" + fi + elif [ "$AMD_GPU_IN_LSPCI" = true ]; then + echo "ℹī¸ Recommended VAAPI driver for AMD GPUs: radeonsi" + fi else echo "⚠ī¸ FFmpeg VAAPI acceleration: NOT DETECTED" + if [ -n "$MISSING_METHODS" ]; then + echo " Missing methods that should be available: $MISSING_METHODS" + fi fi else echo "❌ NO GPU ACCELERATION DETECTED" From cff68625e0a2d9a6785fa14293c3da5931f722b2 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 12:44:32 -0500 Subject: [PATCH 47/55] Move hardware detection to end of entrypoint so it's easier to find. --- docker/entrypoint.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 23daa73a..7913ae47 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -99,7 +99,6 @@ echo "Starting init process..." . /app/docker/init/01-user-setup.sh . /app/docker/init/02-postgres.sh . /app/docker/init/03-init-dispatcharr.sh -. /app/docker/init/04-check-hwaccel.sh # Start PostgreSQL echo "Starting Postgres..." @@ -182,6 +181,15 @@ pids+=("$uwsgi_pid") # echo "✅ celery beat started with PID $beat_pid" # pids+=("$beat_pid") + +# Wait for services to fully initialize before checking hardware +echo "âŗ Waiting for services to fully initialize before hardware check..." +sleep 5 + +# Run hardware check +echo "🔍 Running hardware acceleration check..." +. /app/docker/init/04-check-hwaccel.sh + # Wait for at least one process to exit and log the process that exited first if [ ${#pids[@]} -gt 0 ]; then echo "âŗ Waiting for processes to exit..." From cbbb2a6d595c865562f8826faa8e416ee87e030e Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 16:13:44 -0500 Subject: [PATCH 48/55] Fix Profiles not loading. --- frontend/src/components/tables/ChannelsTable.jsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/tables/ChannelsTable.jsx b/frontend/src/components/tables/ChannelsTable.jsx index c8f92fc1..4a0b8d4a 100644 --- a/frontend/src/components/tables/ChannelsTable.jsx +++ b/frontend/src/components/tables/ChannelsTable.jsx @@ -203,7 +203,7 @@ const ChannelRowActions = React.memo( } ); -const ChannelsTable = ({}) => { +const ChannelsTable = ({ }) => { const theme = useMantineTheme(); /** @@ -238,6 +238,7 @@ const ChannelsTable = ({}) => { const selectedProfileChannels = useChannelsStore( (s) => s.profiles[selectedProfileId]?.channels ); + const fetchChannelProfiles = useChannelsStore((state) => state.fetchChannelProfiles); // store/settings const env_mode = useSettingsStore((s) => s.environment.env_mode); @@ -566,6 +567,11 @@ const ChannelsTable = ({}) => { setPaginationString(`${startItem} to ${endItem} of ${totalCount}`); }, [data]); + useEffect(() => { + // Fetch channel profiles when the component mounts + fetchChannelProfiles(); + }, []); + const columns = useMemo( () => [ { @@ -790,8 +796,8 @@ const ChannelsTable = ({}) => { return hasStreams ? {} // Default style for channels with streams : { - className: 'no-streams-row', // Add a class instead of background color - }; + className: 'no-streams-row', // Add a class instead of background color + }; }, }); From 0a222f27af4a56dfbf012cda3f7ac755ffa8b7c1 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 16:52:21 -0500 Subject: [PATCH 49/55] Move fetchChannelProfiles to initdata --- frontend/src/components/tables/ChannelsTable.jsx | 6 ------ frontend/src/store/auth.jsx | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/frontend/src/components/tables/ChannelsTable.jsx b/frontend/src/components/tables/ChannelsTable.jsx index 4a0b8d4a..2716e994 100644 --- a/frontend/src/components/tables/ChannelsTable.jsx +++ b/frontend/src/components/tables/ChannelsTable.jsx @@ -238,7 +238,6 @@ const ChannelsTable = ({ }) => { const selectedProfileChannels = useChannelsStore( (s) => s.profiles[selectedProfileId]?.channels ); - const fetchChannelProfiles = useChannelsStore((state) => state.fetchChannelProfiles); // store/settings const env_mode = useSettingsStore((s) => s.environment.env_mode); @@ -567,11 +566,6 @@ const ChannelsTable = ({ }) => { setPaginationString(`${startItem} to ${endItem} of ${totalCount}`); }, [data]); - useEffect(() => { - // Fetch channel profiles when the component mounts - fetchChannelProfiles(); - }, []); - const columns = useMemo( () => [ { diff --git a/frontend/src/store/auth.jsx b/frontend/src/store/auth.jsx index a55b77d4..d98183bf 100644 --- a/frontend/src/store/auth.jsx +++ b/frontend/src/store/auth.jsx @@ -39,6 +39,7 @@ const useAuthStore = create((set, get) => ({ await Promise.all([ useChannelsStore.getState().fetchChannels(), useChannelsStore.getState().fetchChannelGroups(), + useChannelsStore.getState().fetchChannelProfiles(), usePlaylistsStore.getState().fetchPlaylists(), useEPGsStore.getState().fetchEPGs(), useEPGsStore.getState().fetchEPGData(), From e7439a074fc338bf716e83eed821de9ac0b19278 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Tue, 13 May 2025 16:57:39 -0500 Subject: [PATCH 50/55] Enhance WebSocket connection handling based on authentication status --- frontend/src/WebSocket.jsx | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/frontend/src/WebSocket.jsx b/frontend/src/WebSocket.jsx index 5e80f2f9..260ec468 100644 --- a/frontend/src/WebSocket.jsx +++ b/frontend/src/WebSocket.jsx @@ -16,7 +16,7 @@ import API from './api'; import useSettingsStore from './store/settings'; import useAuthStore from './store/auth'; -export const WebsocketContext = createContext([false, () => {}, null]); +export const WebsocketContext = createContext([false, () => { }, null]); export const WebsocketProvider = ({ children }) => { const [isReady, setIsReady] = useState(false); @@ -29,6 +29,7 @@ export const WebsocketProvider = ({ children }) => { const initialBackoffDelay = 1000; // 1 second initial delay const env_mode = useSettingsStore((s) => s.environment.env_mode); const accessToken = useAuthStore((s) => s.accessToken); + const isAuthenticated = useAuthStore((s) => s.isAuthenticated); const epgs = useEPGsStore((s) => s.epgs); const updateEPG = useEPGsStore((s) => s.updateEPG); @@ -400,7 +401,18 @@ export const WebsocketProvider = ({ children }) => { // Initial connection and cleanup useEffect(() => { - connectWebSocket(); + // Only attempt to connect if the user is authenticated + if (isAuthenticated && accessToken) { + connectWebSocket(); + } else if (ws.current) { + // Close the connection if the user logs out + clearReconnectTimer(); + console.log('Closing WebSocket connection due to logout'); + ws.current.onclose = null; + ws.current.close(); + ws.current = null; + setIsReady(false); + } return () => { clearReconnectTimer(); // Clear any pending reconnect timers @@ -412,7 +424,7 @@ export const WebsocketProvider = ({ children }) => { ws.current = null; } }; - }, [connectWebSocket, clearReconnectTimer]); + }, [connectWebSocket, clearReconnectTimer, isAuthenticated, accessToken]); const setChannelStats = useChannelsStore((s) => s.setChannelStats); const fetchPlaylists = usePlaylistsStore((s) => s.fetchPlaylists); From cecc057ea4730ed3d26d2c50b44299fa71c540e5 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 14 May 2025 09:32:24 -0500 Subject: [PATCH 51/55] Make sure we can change render group gid to match host and capture errors so we don't crash server. --- docker/init/01-user-setup.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index aa478acb..e26c1038 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -48,8 +48,19 @@ if [ -e "/dev/dri/renderD128" ]; then if getent group render >/dev/null; then CURRENT_RENDER_GID=$(getent group render | cut -d: -f3) if [ "$CURRENT_RENDER_GID" != "$HOST_RENDER_GID" ]; then - echo "Changing render group GID from $CURRENT_RENDER_GID to $HOST_RENDER_GID" - groupmod -g "$HOST_RENDER_GID" render + # Check if another group already has the target GID + if getent group "$HOST_RENDER_GID" >/dev/null 2>&1; then + EXISTING_GROUP=$(getent group "$HOST_RENDER_GID" | cut -d: -f1) + echo "Warning: Cannot change render group GID to $HOST_RENDER_GID as it's already used by group '$EXISTING_GROUP'" + # Add user to the existing group with the target GID to ensure device access + if ! id -nG "$POSTGRES_USER" | grep -qw "$EXISTING_GROUP"; then + usermod -a -G "$EXISTING_GROUP" "$POSTGRES_USER" || echo "Warning: Failed to add user to $EXISTING_GROUP group" + echo "Added user $POSTGRES_USER to $EXISTING_GROUP group for GPU access" + fi + else + echo "Changing render group GID from $CURRENT_RENDER_GID to $HOST_RENDER_GID" + groupmod -g "$HOST_RENDER_GID" render || echo "Warning: Failed to change render group GID. Continuing anyway..." + fi fi else echo "Creating render group with GID $HOST_RENDER_GID" From 14c3944578bd13650f2af79d05b84160543c423c Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 14 May 2025 10:19:55 -0500 Subject: [PATCH 52/55] Don't hardcode dispatch as user for nginx. Use postgres_user. --- docker/init/01-user-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/init/01-user-setup.sh b/docker/init/01-user-setup.sh index e26c1038..d7041265 100644 --- a/docker/init/01-user-setup.sh +++ b/docker/init/01-user-setup.sh @@ -87,4 +87,4 @@ if getent group video >/dev/null 2>&1; then fi # Run nginx as specified user -sed -i 's/user www-data;/user dispatch;/g' /etc/nginx/nginx.conf +sed -i "s/user www-data;/user $POSTGRES_USER;/g" /etc/nginx/nginx.conf From 44a79d2a8a85a8b9c68f1836d9683af3ee164582 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 14 May 2025 18:49:46 -0500 Subject: [PATCH 53/55] Log UWSGI if debug is set (DISPATCHARR_DEBUG=true) --- docker/entrypoint.sh | 26 ++++++++++++++++++-------- docker/uwsgi.debug.ini | 12 ++++++++---- docker/uwsgi.dev.ini | 15 +++++++++++++-- docker/uwsgi.ini | 15 +++++++++++++-- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7913ae47..2d9f7fdc 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -53,11 +53,14 @@ if [ -n "$DISPATCHARR_TIMESTAMP" ]; then else echo "đŸ“Ļ Dispatcharr version: ${DISPATCHARR_VERSION}" fi - +export DISPATCHARR_LOG_LEVEL # Set log level with default if not provided -export DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL:-info} -echo "Environment DISPATCHARR_LOG_LEVEL detected as: '${DISPATCHARR_LOG_LEVEL}'" -echo "Setting log level to: ${DISPATCHARR_LOG_LEVEL}" +DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL:-INFO} +# Convert to uppercase +DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL^^} + + +echo "Environment DISPATCHARR_LOG_LEVEL set to: '${DISPATCHARR_LOG_LEVEL}'" # Also make the log level available in /etc/environment for all login shells #grep -q "DISPATCHARR_LOG_LEVEL" /etc/environment || echo "DISPATCHARR_LOG_LEVEL=${DISPATCHARR_LOG_LEVEL}" >> /etc/environment @@ -143,10 +146,17 @@ else uwsgi_file="/app/docker/uwsgi.ini" fi -# Pass all environment variables to the uwsgi process -# The -p/--preserve-environment flag ensures all environment variables are passed through -su -p - $POSTGRES_USER -c "cd /app && uwsgi --ini $uwsgi_file &" -uwsgi_pid=$(pgrep uwsgi | sort | head -n1) +# Set base uwsgi args +uwsgi_args="--ini $uwsgi_file" + +# Conditionally disable logging if not in debug mode +if [ "$DISPATCHARR_DEBUG" != "true" ]; then + uwsgi_args+=" --disable-logging" +fi + +# Launch uwsgi -p passes environment variables to the process +su -p - $POSTGRES_USER -c "cd /app && uwsgi $uwsgi_args &" +uwsgi_pid=$(pgrep uwsgi | sort | head -n1) echo "✅ uwsgi started with PID $uwsgi_pid" pids+=("$uwsgi_pid") diff --git a/docker/uwsgi.debug.ini b/docker/uwsgi.debug.ini index 9cd4ccdf..43ecd5ce 100644 --- a/docker/uwsgi.debug.ini +++ b/docker/uwsgi.debug.ini @@ -19,6 +19,7 @@ module = scripts.debug_wrapper:application virtualenv = /dispatcharrpy master = true env = DJANGO_SETTINGS_MODULE=dispatcharr.settings + socket = /app/uwsgi.sock chmod-socket = 777 vacuum = true @@ -58,9 +59,6 @@ ignore-sigpipe = true ignore-write-errors = true disable-write-exception = true -# Explicitly disable for-server option that confuses debugpy -for-server = false - # Debugging settings py-autoreload = 1 honour-stdin = true @@ -78,4 +76,10 @@ env = DEBUGPY_LOG_DIR=/app/debugpy_logs env = WAIT_FOR_DEBUGGER=false env = DEBUG_TIMEOUT=30 - +# Enable console logging (stdout) +log-master = true +# Enable strftime formatting for timestamps +logformat-strftime = true +log-date = %%Y-%%m-%%d %%H:%%M:%%S,000 +# Use the environment variable in log format - ensure consistent formatting with other files +log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms \ No newline at end of file diff --git a/docker/uwsgi.dev.ini b/docker/uwsgi.dev.ini index 455111d3..62a5f352 100644 --- a/docker/uwsgi.dev.ini +++ b/docker/uwsgi.dev.ini @@ -1,6 +1,8 @@ [uwsgi] -; exec-before = python manage.py collectstatic --noinput -; exec-before = python manage.py migrate --noinput +; Remove file creation commands since we're not logging to files anymore +; exec-pre = mkdir -p /data/logs +; exec-pre = touch /data/logs/uwsgi-dev.log +; exec-pre = chmod 666 /data/logs/uwsgi-dev.log ; First run Redis availability check script once exec-pre = python /app/scripts/wait_for_redis.py @@ -47,3 +49,12 @@ thunder-lock = true log-4xx = true log-5xx = true disable-logging = false + +# Logging configuration - development mode +# Enable console logging (stdout) +log-master = true +# Enable strftime formatting for timestamps +logformat-strftime = true +log-date = %%Y-%%m-%%d %%H:%%M:%%S,000 +# Use formatted time with environment variable for log level +log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms \ No newline at end of file diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 75d4de3f..5068268c 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -1,6 +1,8 @@ [uwsgi] -; exec-before = python manage.py collectstatic --noinput -; exec-before = python manage.py migrate --noinput +; Remove file creation commands since we're not logging to files anymore +; exec-pre = mkdir -p /data/logs +; exec-pre = touch /data/logs/uwsgi.log +; exec-pre = chmod 666 /data/logs/uwsgi.log ; First run Redis availability check script once exec-pre = python /app/scripts/wait_for_redis.py @@ -45,3 +47,12 @@ thunder-lock = true log-4xx = true log-5xx = true disable-logging = false + +# Logging configuration +# Enable console logging (stdout) +log-master = true +# Enable strftime formatting for timestamps +logformat-strftime = true +log-date = %%Y-%%m-%%d %%H:%%M:%%S,000 +# Use formatted time with environment variable for log level +log-format = %(ftime) $(DISPATCHARR_LOG_LEVEL) uwsgi.requests Worker ID: %(wid) %(method) %(status) %(uri) %(msecs)ms \ No newline at end of file From 2c831bd756f8e78c2309989ec0efe0ef5e6fd59a Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 14 May 2025 19:11:12 -0500 Subject: [PATCH 54/55] Fixes remote debugging. --- scripts/debug_wrapper.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/scripts/debug_wrapper.py b/scripts/debug_wrapper.py index eb71d43d..2e96e110 100644 --- a/scripts/debug_wrapper.py +++ b/scripts/debug_wrapper.py @@ -27,7 +27,7 @@ logger.info(f"Files in current directory: {os.listdir()}") logger.info(f"Python path: {sys.path}") # Default timeout in seconds -DEBUG_TIMEOUT = int(os.environ.get('DEBUG_TIMEOUT', '30')) +DEBUG_TIMEOUT = int(os.environ.get('DEBUG_TIMEOUT', '60')) # Increased default timeout # Whether to wait for debugger to attach WAIT_FOR_DEBUGGER = os.environ.get('WAIT_FOR_DEBUGGER', 'false').lower() == 'true' @@ -40,7 +40,7 @@ try: logger.info("Successfully imported debugpy") # Critical: Configure debugpy to use regular Python for the adapter, not uwsgi - python_path = '/usr/local/bin/python3' + python_path = '/dispatcharrpy/bin/python' if os.path.exists(python_path): logger.info(f"Setting debugpy adapter to use Python interpreter: {python_path}") debugpy.configure(python=python_path) @@ -50,42 +50,37 @@ try: # Don't wait for connection, just set up the debugging session logger.info("Initializing debugpy on 0.0.0.0:5678...") try: - # Set socket timeout using the existing DEBUG_TIMEOUT variable - import socket - socket.setdefaulttimeout(DEBUG_TIMEOUT) - # Use connect instead of listen to avoid the adapter process + # Configure debugpy to listen without socket timeout initially debugpy.listen(("0.0.0.0", 5678)) logger.info("debugpy now listening on 0.0.0.0:5678") - # Reset socket timeout - socket.setdefaulttimeout(None) if WAIT_FOR_DEBUGGER: logger.info(f"Waiting for debugger to attach (timeout: {DEBUG_TIMEOUT}s)...") start_time = time.time() + + # Use a more reliable approach for checking connection while not debugpy.is_client_connected() and (time.time() - start_time < DEBUG_TIMEOUT): time.sleep(1) - logger.info("Waiting for debugger connection...") + if (time.time() - start_time) % 5 == 0: # Log only every 5 seconds to reduce spam + logger.info(f"Still waiting for debugger connection... ({int(time.time() - start_time)}s)") if debugpy.is_client_connected(): - logger.info("Debugger attached!") + logger.info("Debugger attached successfully!") else: - logger.info(f"Debugger not attached after {DEBUG_TIMEOUT}s, continuing anyway...") + logger.warning(f"Debugger not attached after {DEBUG_TIMEOUT}s, continuing anyway...") except RuntimeError as re: - if "timed out waiting for adapter to connect" in str(re): + if "already in use" in str(re): + logger.warning(f"Port 5678 already in use. This might indicate another debugging session is active.") + logger.info("Continuing without debugging...") + elif "timed out waiting for adapter to connect" in str(re): logger.warning(f"debugpy.listen timed out after {DEBUG_TIMEOUT}s. This is normal in some environments.") logger.info("Continuing without debugging...") else: logger.error(f"RuntimeError with debugpy.listen: {re}", exc_info=True) logger.info("Continuing without debugging...") - except socket.timeout: - logger.warning(f"Socket timeout after {DEBUG_TIMEOUT}s while initializing debugpy.") - logger.info("Continuing without debugging...") except Exception as e: logger.error(f"Error with debugpy.listen: {e}", exc_info=True) logger.info("Continuing without debugging...") - finally: - # Ensure socket timeout is always reset - socket.setdefaulttimeout(None) except ImportError: logger.error("debugpy not installed, continuing without debugging support") From 790e3710a631fffb22a0c283af2acfc778b27af0 Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Wed, 14 May 2025 20:10:26 -0500 Subject: [PATCH 55/55] Created missing migrations. --- .../migrations/0012_alter_epgsource_status.py | 18 ++++++++++++++++++ .../migrations/0011_alter_m3uaccount_status.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 apps/epg/migrations/0012_alter_epgsource_status.py create mode 100644 apps/m3u/migrations/0011_alter_m3uaccount_status.py diff --git a/apps/epg/migrations/0012_alter_epgsource_status.py b/apps/epg/migrations/0012_alter_epgsource_status.py new file mode 100644 index 00000000..39cce295 --- /dev/null +++ b/apps/epg/migrations/0012_alter_epgsource_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-05-15 01:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epg', '0011_update_epgsource_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='epgsource', + name='status', + field=models.CharField(choices=[('idle', 'Idle'), ('fetching', 'Fetching'), ('parsing', 'Parsing'), ('error', 'Error'), ('success', 'Success'), ('disabled', 'Disabled')], default='idle', max_length=20), + ), + ] diff --git a/apps/m3u/migrations/0011_alter_m3uaccount_status.py b/apps/m3u/migrations/0011_alter_m3uaccount_status.py new file mode 100644 index 00000000..7812f317 --- /dev/null +++ b/apps/m3u/migrations/0011_alter_m3uaccount_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-05-15 01:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('m3u', '0010_add_status_fields_and_remove_auto_now'), + ] + + operations = [ + migrations.AlterField( + model_name='m3uaccount', + name='status', + field=models.CharField(choices=[('idle', 'Idle'), ('fetching', 'Fetching'), ('parsing', 'Parsing'), ('error', 'Error'), ('success', 'Success'), ('pending_setup', 'Pending Setup'), ('disabled', 'Disabled')], default='idle', max_length=20), + ), + ]