Merge 0c3c19febd into 67d8bd5598

lib/pinchflat/diagnostics/queue_diagnostics.ex

@@ -0,0 +1,203 @@
+defmodule Pinchflat.Diagnostics.QueueDiagnostics do
+  @moduledoc """
+  Provides diagnostic information about Oban job queues.
+  """
+
+  import Ecto.Query
+
+  alias Pinchflat.Repo
+
+  @queues [:default, :fast_indexing, :media_collection_indexing, :media_fetching, :remote_metadata, :local_data]
+
+  @doc """
+  Returns a list of all queue names.
+  """
+  def queue_names, do: @queues
+
+  @doc """
+  Returns health status for all queues including job counts by state.
+  """
+  def get_all_queue_stats do
+    Enum.map(@queues, fn queue_name ->
+      queue_info = Oban.check_queue(queue: queue_name)
+      job_counts = get_job_counts_for_queue(queue_name)
+
+      %{
+        name: queue_name,
+        running: length(Map.get(queue_info, :running, [])),
+        limit: Map.get(queue_info, :limit, 0),
+        paused: Map.get(queue_info, :paused, false),
+        available: Map.get(job_counts, :available, 0),
+        scheduled: Map.get(job_counts, :scheduled, 0),
+        retryable: Map.get(job_counts, :retryable, 0),
+        executing: Map.get(job_counts, :executing, 0)
+      }
+    end)
+  end
+
+  @doc """
+  Returns jobs that are in a retryable state (failed but will retry).
+  """
+  def get_retryable_jobs(limit \\ 50) do
+    from(j in Oban.Job,
+      where: j.state == "retryable",
+      order_by: [desc: j.attempted_at],
+      limit: ^limit,
+      select: %{
+        id: j.id,
+        queue: j.queue,
+        worker: j.worker,
+        state: j.state,
+        attempt: j.attempt,
+        max_attempts: j.max_attempts,
+        errors: j.errors,
+        args: j.args,
+        attempted_at: j.attempted_at,
+        scheduled_at: j.scheduled_at
+      }
+    )
+    |> Repo.all()
+  end
+
+  @doc """
+  Returns jobs that appear to be stuck (executing for too long or orphaned).
+  A job is considered stuck if it's been "executing" for more than the threshold.
+  """
+  def get_stuck_jobs(threshold_minutes \\ 30) do
+    threshold = DateTime.add(DateTime.utc_now(), -threshold_minutes * 60, :second)
+
+    from(j in Oban.Job,
+      where: j.state == "executing",
+      where: j.attempted_at < ^threshold,
+      order_by: [asc: j.attempted_at],
+      select: %{
+        id: j.id,
+        queue: j.queue,
+        worker: j.worker,
+        attempt: j.attempt,
+        attempted_at: j.attempted_at,
+        args: j.args
+      }
+    )
+    |> Repo.all()
+  end
+
+  @doc """
+  Resets stuck jobs by marking them as available for retry.
+  Returns the number of jobs reset.
+  """
+  def reset_stuck_jobs(threshold_minutes \\ 30) do
+    threshold = DateTime.add(DateTime.utc_now(), -threshold_minutes * 60, :second)
+
+    {count, _} =
+      from(j in Oban.Job,
+        where: j.state == "executing",
+        where: j.attempted_at < ^threshold
+      )
+      |> Repo.update_all(set: [state: "available", scheduled_at: DateTime.utc_now(), attempted_at: nil])
+
+    count
+  end
+
+  @doc """
+  Resets all retryable jobs by clearing their error history and marking as available.
+  Returns the number of jobs reset.
+  """
+  def reset_retryable_jobs do
+    {count, _} =
+      from(j in Oban.Job,
+        where: j.state == "retryable"
+      )
+      |> Repo.update_all(set: [state: "available", attempt: 1, errors: [], scheduled_at: DateTime.utc_now()])
+
+    count
+  end
+
+  @doc """
+  Resets a specific job by ID.
+  """
+  def reset_job(job_id) do
+    {count, _} =
+      from(j in Oban.Job,
+        where: j.id == ^job_id,
+        where: j.state in ["retryable", "executing"]
+      )
+      |> Repo.update_all(set: [state: "available", attempt: 1, errors: [], scheduled_at: DateTime.utc_now()])
+
+    count
+  end
+
+  @doc """
+  Cancels a specific job by ID.
+  """
+  def cancel_job(job_id) do
+    case Oban.cancel_job(job_id) do
+      :ok -> {:ok, :cancelled}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  @doc """
+  Returns summary statistics for the system.
+  """
+  def get_system_stats do
+    %{
+      total_pending_downloads: count_pending_downloads(),
+      total_downloaded: count_downloaded_media(),
+      total_sources: count_sources(),
+      database_size: get_database_size()
+    }
+  end
+
+  # Private functions
+
+  defp get_job_counts_for_queue(queue_name) do
+    queue_string = Atom.to_string(queue_name)
+
+    from(j in Oban.Job,
+      where: j.queue == ^queue_string,
+      where: j.state in ["available", "scheduled", "retryable", "executing"],
+      group_by: j.state,
+      select: {j.state, count(j.id)}
+    )
+    |> Repo.all()
+    |> Enum.into(%{}, fn {state, count} -> {String.to_atom(state), count} end)
+  end
+
+  defp count_pending_downloads do
+    from(m in Pinchflat.Media.MediaItem,
+      where: is_nil(m.media_filepath),
+      where: m.prevent_download == false
+    )
+    |> Repo.aggregate(:count)
+  end
+
+  defp count_downloaded_media do
+    from(m in Pinchflat.Media.MediaItem,
+      where: not is_nil(m.media_filepath)
+    )
+    |> Repo.aggregate(:count)
+  end
+
+  defp count_sources do
+    Repo.aggregate(Pinchflat.Sources.Source, :count)
+  end
+
+  defp get_database_size do
+    db_path = Application.get_env(:pinchflat, Pinchflat.Repo)[:database]
+
+    if db_path && File.exists?(db_path) do
+      case File.stat(db_path) do
+        {:ok, %{size: size}} -> format_bytes(size)
+        _ -> "Unknown"
+      end
+    else
+      "Unknown"
+    end
+  end
+
+  defp format_bytes(bytes) when bytes < 1024, do: "#{bytes} B"
+  defp format_bytes(bytes) when bytes < 1024 * 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
+  defp format_bytes(bytes) when bytes < 1024 * 1024 * 1024, do: "#{Float.round(bytes / 1024 / 1024, 1)} MB"
+  defp format_bytes(bytes), do: "#{Float.round(bytes / 1024 / 1024 / 1024, 2)} GB"
+end

lib/pinchflat_web/components/layouts/partials/sidebar.html.heex

@@ -33,6 +33,7 @@
             >
               <:submenu text="Settings" href={~p"/settings"} />
               <:submenu text="App Info" href={~p"/app_info"} />
+              <:submenu text="Diagnostics" href={~p"/diagnostics"} />
             </.sidebar_submenu>
           </ul>
         </div>

lib/pinchflat_web/controllers/settings/diagnostics_controller.ex

@@ -0,0 +1,53 @@
+defmodule PinchflatWeb.Settings.DiagnosticsController do
+  use PinchflatWeb, :controller
+
+  alias Pinchflat.Diagnostics.QueueDiagnostics
+
+  def show(conn, _params) do
+    render(conn, "show.html")
+  end
+
+  def reset_stuck_jobs(conn, _params) do
+    count = QueueDiagnostics.reset_stuck_jobs()
+
+    conn
+    |> put_flash(:info, "Reset #{count} stuck job(s). The queue will restart processing shortly.")
+    |> redirect(to: ~p"/diagnostics")
+  end
+
+  def reset_retryable_jobs(conn, _params) do
+    count = QueueDiagnostics.reset_retryable_jobs()
+
+    conn
+    |> put_flash(:info, "Reset #{count} retryable job(s). They will be retried shortly.")
+    |> redirect(to: ~p"/diagnostics")
+  end
+
+  def reset_job(conn, %{"id" => job_id}) do
+    case QueueDiagnostics.reset_job(String.to_integer(job_id)) do
+      1 ->
+        conn
+        |> put_flash(:info, "Job ##{job_id} has been reset and will retry shortly.")
+        |> redirect(to: ~p"/diagnostics")
+
+      0 ->
+        conn
+        |> put_flash(:error, "Job ##{job_id} could not be reset. It may have already completed or been cancelled.")
+        |> redirect(to: ~p"/diagnostics")
+    end
+  end
+
+  def cancel_job(conn, %{"id" => job_id}) do
+    case QueueDiagnostics.cancel_job(String.to_integer(job_id)) do
+      {:ok, :cancelled} ->
+        conn
+        |> put_flash(:info, "Job ##{job_id} has been cancelled.")
+        |> redirect(to: ~p"/diagnostics")
+
+      {:error, _reason} ->
+        conn
+        |> put_flash(:error, "Job ##{job_id} could not be cancelled.")
+        |> redirect(to: ~p"/diagnostics")
+    end
+  end
+end

lib/pinchflat_web/controllers/settings/diagnostics_html.ex

@@ -0,0 +1,79 @@
+defmodule PinchflatWeb.Settings.DiagnosticsHTML do
+  use PinchflatWeb, :html
+
+  alias Pinchflat.Diagnostics.QueueDiagnostics
+
+  embed_templates "diagnostics_html/*"
+
+  def queue_stats do
+    QueueDiagnostics.get_all_queue_stats()
+  end
+
+  def retryable_jobs do
+    QueueDiagnostics.get_retryable_jobs(20)
+  end
+
+  def stuck_jobs do
+    QueueDiagnostics.get_stuck_jobs(30)
+  end
+
+  def system_stats do
+    QueueDiagnostics.get_system_stats()
+  end
+
+  def format_worker_name(worker) do
+    worker
+    |> String.split(".")
+    |> Enum.at(-1)
+    |> format_worker_short_name()
+  end
+
+  defp format_worker_short_name("FastIndexingWorker"), do: "Fast Indexing"
+  defp format_worker_short_name("MediaDownloadWorker"), do: "Download"
+  defp format_worker_short_name("MediaCollectionIndexingWorker"), do: "Indexing"
+  defp format_worker_short_name("MediaQualityUpgradeWorker"), do: "Quality Upgrade"
+  defp format_worker_short_name("SourceMetadataStorageWorker"), do: "Metadata"
+  defp format_worker_short_name(other), do: other
+
+  def format_queue_name(queue) do
+    queue
+    |> Atom.to_string()
+    |> String.replace("_", " ")
+    |> String.split(" ")
+    |> Enum.map_join(" ", &String.capitalize/1)
+  end
+
+  def format_datetime(nil), do: "-"
+
+  def format_datetime(datetime) do
+    Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S")
+  end
+
+  def extract_last_error(errors) when is_list(errors) and length(errors) > 0 do
+    errors
+    |> List.last()
+    |> Map.get("error", "Unknown error")
+    |> String.slice(0, 200)
+  end
+
+  def extract_last_error(_), do: "No error details"
+
+  def queue_health_class(stats) do
+    cond do
+      stats.paused -> "bg-yellow-500/20 border-yellow-500"
+      stats.retryable > 0 -> "bg-red-500/20 border-red-500"
+      stats.running >= stats.limit and stats.available > 0 -> "bg-blue-500/20 border-blue-500"
+      true -> "bg-green-500/20 border-green-500"
+    end
+  end
+
+  def queue_status_text(stats) do
+    cond do
+      stats.paused -> "Paused"
+      stats.retryable > 0 -> "Has Failures"
+      stats.running >= stats.limit -> "At Capacity"
+      stats.running > 0 -> "Active"
+      true -> "Idle"
+    end
+  end
+end

lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex

@@ -0,0 +1,212 @@
+<div class="mb-6 flex gap-3 flex-row items-center justify-between">
+  <div class="flex gap-3 items-center">
+    <h2 class="text-title-md2 font-bold text-white ml-4">
+      Queue Diagnostics
+    </h2>
+  </div>
+</div>
+
+<!-- System Stats -->
+<div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 mb-6">
+  <h3 class="text-lg font-semibold text-white mb-4">System Overview</h3>
+  <% stats = system_stats() %>
+  <div class="grid grid-cols-2 md:grid-cols-4 gap-4">
+    <div class="bg-meta-4 rounded-lg p-4">
+      <p class="text-sm text-bodydark">Pending Downloads</p>
+      <p class="text-2xl font-bold text-white">{stats.total_pending_downloads}</p>
+    </div>
+    <div class="bg-meta-4 rounded-lg p-4">
+      <p class="text-sm text-bodydark">Downloaded Media</p>
+      <p class="text-2xl font-bold text-white">{stats.total_downloaded}</p>
+    </div>
+    <div class="bg-meta-4 rounded-lg p-4">
+      <p class="text-sm text-bodydark">Total Sources</p>
+      <p class="text-2xl font-bold text-white">{stats.total_sources}</p>
+    </div>
+    <div class="bg-meta-4 rounded-lg p-4">
+      <p class="text-sm text-bodydark">Database Size</p>
+      <p class="text-2xl font-bold text-white">{stats.database_size}</p>
+    </div>
+  </div>
+</div>
+
+<!-- Queue Health -->
+<div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 mb-6">
+  <div class="flex justify-between items-center mb-4">
+    <h3 class="text-lg font-semibold text-white">Queue Health</h3>
+    <.link href={~p"/diagnostics"}>
+      <.button color="bg-bodydark" rounding="rounded-lg" class="text-sm">
+        <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Refresh
+      </.button>
+    </.link>
+  </div>
+  <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+    <%= for stats <- queue_stats() do %>
+      <div class={"rounded-lg border-2 p-4 #{queue_health_class(stats)}"}>
+        <div class="flex justify-between items-start mb-2">
+          <h4 class="font-semibold text-white">{format_queue_name(stats.name)}</h4>
+          <span class={"text-xs px-2 py-1 rounded #{if stats.paused, do: "bg-yellow-500", else: "bg-meta-4"}"}>
+            {queue_status_text(stats)}
+          </span>
+        </div>
+        <div class="grid grid-cols-2 gap-2 text-sm">
+          <div>
+            <span class="text-bodydark">Running:</span>
+            <span class="text-white ml-1">{stats.running}/{stats.limit}</span>
+          </div>
+          <div>
+            <span class="text-bodydark">Available:</span>
+            <span class="text-white ml-1">{stats.available}</span>
+          </div>
+          <div>
+            <span class="text-bodydark">Scheduled:</span>
+            <span class="text-white ml-1">{stats.scheduled}</span>
+          </div>
+          <div>
+            <span class={"#{if stats.retryable > 0, do: "text-red-400", else: "text-bodydark"}"}>Retryable:</span>
+            <span class={"ml-1 #{if stats.retryable > 0, do: "text-red-400 font-bold", else: "text-white"}"}>
+              {stats.retryable}
+            </span>
+          </div>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<!-- Stuck Jobs -->
+<% stuck = stuck_jobs() %>
+<%= if length(stuck) > 0 do %>
+  <div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 mb-6">
+    <div class="flex justify-between items-center mb-4">
+      <h3 class="text-lg font-semibold text-white">
+        <.icon name="hero-exclamation-triangle" class="h-5 w-5 text-yellow-500 mr-2" /> Stuck Jobs ({length(stuck)})
+      </h3>
+      <.link
+        href={~p"/diagnostics/reset_stuck_jobs"}
+        method="post"
+        data-confirm="This will reset all stuck jobs. They will be retried. Continue?"
+      >
+        <.button color="bg-yellow-600" rounding="rounded-lg" class="text-sm">
+          <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Stuck Jobs
+        </.button>
+      </.link>
+    </div>
+    <p class="text-bodydark text-sm mb-4">
+      These jobs have been executing for more than 30 minutes and may be orphaned. This can happen after a container restart.
+    </p>
+    <div class="overflow-x-auto">
+      <table class="w-full text-sm">
+        <thead>
+          <tr class="border-b border-strokedark">
+            <th class="text-left py-2 text-bodydark">ID</th>
+            <th class="text-left py-2 text-bodydark">Queue</th>
+            <th class="text-left py-2 text-bodydark">Worker</th>
+            <th class="text-left py-2 text-bodydark">Started At</th>
+            <th class="text-left py-2 text-bodydark">Actions</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for job <- stuck do %>
+            <tr class="border-b border-strokedark/50">
+              <td class="py-2 text-white">#{job.id}</td>
+              <td class="py-2 text-white">{job.queue}</td>
+              <td class="py-2 text-white">{format_worker_name(job.worker)}</td>
+              <td class="py-2 text-white">{format_datetime(job.attempted_at)}</td>
+              <td class="py-2">
+                <.link
+                  href={~p"/diagnostics/reset_job/#{job.id}"}
+                  method="post"
+                  class="text-primary hover:underline mr-3"
+                >
+                  Reset
+                </.link>
+                <.link
+                  href={~p"/diagnostics/cancel_job/#{job.id}"}
+                  method="post"
+                  class="text-red-400 hover:underline"
+                  data-confirm="Cancel this job?"
+                >
+                  Cancel
+                </.link>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+<% end %>
+
+<!-- Retryable Jobs -->
+<% retryable = retryable_jobs() %>
+<div class="rounded-sm border border-stroke bg-white px-5 py-5 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5">
+  <div class="flex justify-between items-center mb-4">
+    <h3 class="text-lg font-semibold text-white">
+      <.icon name="hero-exclamation-circle" class="h-5 w-5 text-red-500 mr-2" /> Failed Jobs ({length(retryable)})
+    </h3>
+    <%= if length(retryable) > 0 do %>
+      <.link
+        href={~p"/diagnostics/reset_retryable_jobs"}
+        method="post"
+        data-confirm="This will reset all failed jobs and clear their error history. They will be retried from scratch. Continue?"
+      >
+        <.button color="bg-red-600" rounding="rounded-lg" class="text-sm">
+          <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Failed Jobs
+        </.button>
+      </.link>
+    <% end %>
+  </div>
+
+  <%= if length(retryable) == 0 do %>
+    <p class="text-bodydark">No failed jobs. Everything is running smoothly!</p>
+  <% else %>
+    <p class="text-bodydark text-sm mb-4">
+      These jobs have failed and are waiting to be retried. You can reset them to retry immediately or cancel them.
+    </p>
+    <div class="overflow-x-auto">
+      <table class="w-full text-sm">
+        <thead>
+          <tr class="border-b border-strokedark">
+            <th class="text-left py-2 text-bodydark">ID</th>
+            <th class="text-left py-2 text-bodydark">Queue</th>
+            <th class="text-left py-2 text-bodydark">Worker</th>
+            <th class="text-left py-2 text-bodydark">Attempts</th>
+            <th class="text-left py-2 text-bodydark">Last Error</th>
+            <th class="text-left py-2 text-bodydark">Actions</th>
+          </tr>
+        </thead>
+        <tbody>
+          <%= for job <- retryable do %>
+            <tr class="border-b border-strokedark/50">
+              <td class="py-2 text-white">#{job.id}</td>
+              <td class="py-2 text-white">{job.queue}</td>
+              <td class="py-2 text-white">{format_worker_name(job.worker)}</td>
+              <td class="py-2 text-white">{job.attempt}/{job.max_attempts}</td>
+              <td class="py-2 text-red-400 max-w-xs truncate" title={extract_last_error(job.errors)}>
+                {extract_last_error(job.errors)}
+              </td>
+              <td class="py-2">
+                <.link
+                  href={~p"/diagnostics/reset_job/#{job.id}"}
+                  method="post"
+                  class="text-primary hover:underline mr-3"
+                >
+                  Reset
+                </.link>
+                <.link
+                  href={~p"/diagnostics/cancel_job/#{job.id}"}
+                  method="post"
+                  class="text-red-400 hover:underline"
+                  data-confirm="Cancel this job?"
+                >
+                  Cancel
+                </.link>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+  <% end %>
+</div>

lib/pinchflat_web/router.ex

@@ -51,6 +51,12 @@ defmodule PinchflatWeb.Router do
     get "/app_info", Settings.SettingController, :app_info
     get "/download_logs", Settings.SettingController, :download_logs
 
+    get "/diagnostics", Settings.DiagnosticsController, :show
+    post "/diagnostics/reset_stuck_jobs", Settings.DiagnosticsController, :reset_stuck_jobs
+    post "/diagnostics/reset_retryable_jobs", Settings.DiagnosticsController, :reset_retryable_jobs
+    post "/diagnostics/reset_job/:id", Settings.DiagnosticsController, :reset_job
+    post "/diagnostics/cancel_job/:id", Settings.DiagnosticsController, :cancel_job
+
     resources "/sources", Sources.SourceController do
       post "/force_download_pending", Sources.SourceController, :force_download_pending
       post "/force_redownload", Sources.SourceController, :force_redownload