From f9f52187abe15b2bc0ffcdf62eb9d94e945be913 Mon Sep 17 00:00:00 2001 From: Daniel Da Cunha Date: Mon, 15 Dec 2025 13:23:46 +0800 Subject: [PATCH 1/3] Add queue diagnostics page Adds a new diagnostics page under Config menu to monitor and manage Oban job queues. Helps troubleshoot stuck or failed jobs without needing to query the database directly. Features: - System overview (pending downloads, total media, sources, db size) - Queue health status with running/available/scheduled/retryable counts - Stuck jobs detection (executing > 30 min) with reset/cancel actions - Failed jobs list with error details and bulk reset option --- .../diagnostics/queue_diagnostics.ex | 203 ++++++++++++++++++ .../layouts/partials/sidebar.html.heex | 1 + .../settings/diagnostics_controller.ex | 53 +++++ .../controllers/settings/diagnostics_html.ex | 80 +++++++ .../settings/diagnostics_html/show.html.heex | 186 ++++++++++++++++ lib/pinchflat_web/router.ex | 6 + 6 files changed, 529 insertions(+) create mode 100644 lib/pinchflat/diagnostics/queue_diagnostics.ex create mode 100644 lib/pinchflat_web/controllers/settings/diagnostics_controller.ex create mode 100644 lib/pinchflat_web/controllers/settings/diagnostics_html.ex create mode 100644 lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex diff --git a/lib/pinchflat/diagnostics/queue_diagnostics.ex b/lib/pinchflat/diagnostics/queue_diagnostics.ex new file mode 100644 index 0000000..4ad4fdf --- /dev/null +++ b/lib/pinchflat/diagnostics/queue_diagnostics.ex @@ -0,0 +1,203 @@ +defmodule Pinchflat.Diagnostics.QueueDiagnostics do + @moduledoc """ + Provides diagnostic information about Oban job queues. + """ + + import Ecto.Query + + alias Pinchflat.Repo + + @queues [:default, :fast_indexing, :media_collection_indexing, :media_fetching, :remote_metadata, :local_data] + + @doc """ + Returns a list of all queue names. + """ + def queue_names, do: @queues + + @doc """ + Returns health status for all queues including job counts by state. + """ + def get_all_queue_stats do + Enum.map(@queues, fn queue_name -> + queue_info = Oban.check_queue(queue: queue_name) + job_counts = get_job_counts_for_queue(queue_name) + + %{ + name: queue_name, + running: length(Map.get(queue_info, :running, [])), + limit: Map.get(queue_info, :limit, 0), + paused: Map.get(queue_info, :paused, false), + available: Map.get(job_counts, :available, 0), + scheduled: Map.get(job_counts, :scheduled, 0), + retryable: Map.get(job_counts, :retryable, 0), + executing: Map.get(job_counts, :executing, 0) + } + end) + end + + @doc """ + Returns jobs that are in a retryable state (failed but will retry). + """ + def get_retryable_jobs(limit \\ 50) do + from(j in Oban.Job, + where: j.state == "retryable", + order_by: [desc: j.attempted_at], + limit: ^limit, + select: %{ + id: j.id, + queue: j.queue, + worker: j.worker, + state: j.state, + attempt: j.attempt, + max_attempts: j.max_attempts, + errors: j.errors, + args: j.args, + attempted_at: j.attempted_at, + scheduled_at: j.scheduled_at + } + ) + |> Repo.all() + end + + @doc """ + Returns jobs that appear to be stuck (executing for too long or orphaned). + A job is considered stuck if it's been "executing" for more than the threshold. + """ + def get_stuck_jobs(threshold_minutes \\ 30) do + threshold = DateTime.add(DateTime.utc_now(), -threshold_minutes * 60, :second) + + from(j in Oban.Job, + where: j.state == "executing", + where: j.attempted_at < ^threshold, + order_by: [asc: j.attempted_at], + select: %{ + id: j.id, + queue: j.queue, + worker: j.worker, + attempt: j.attempt, + attempted_at: j.attempted_at, + args: j.args + } + ) + |> Repo.all() + end + + @doc """ + Resets stuck jobs by marking them as available for retry. + Returns the number of jobs reset. + """ + def reset_stuck_jobs(threshold_minutes \\ 30) do + threshold = DateTime.add(DateTime.utc_now(), -threshold_minutes * 60, :second) + + {count, _} = + from(j in Oban.Job, + where: j.state == "executing", + where: j.attempted_at < ^threshold + ) + |> Repo.update_all(set: [state: "available", scheduled_at: DateTime.utc_now(), attempted_at: nil]) + + count + end + + @doc """ + Resets all retryable jobs by clearing their error history and marking as available. + Returns the number of jobs reset. + """ + def reset_retryable_jobs do + {count, _} = + from(j in Oban.Job, + where: j.state == "retryable" + ) + |> Repo.update_all(set: [state: "available", attempt: 1, errors: [], scheduled_at: DateTime.utc_now()]) + + count + end + + @doc """ + Resets a specific job by ID. + """ + def reset_job(job_id) do + {count, _} = + from(j in Oban.Job, + where: j.id == ^job_id, + where: j.state in ["retryable", "executing"] + ) + |> Repo.update_all(set: [state: "available", attempt: 1, errors: [], scheduled_at: DateTime.utc_now()]) + + count + end + + @doc """ + Cancels a specific job by ID. + """ + def cancel_job(job_id) do + case Oban.cancel_job(job_id) do + :ok -> {:ok, :cancelled} + {:error, reason} -> {:error, reason} + end + end + + @doc """ + Returns summary statistics for the system. + """ + def get_system_stats do + %{ + total_pending_downloads: count_pending_downloads(), + total_downloaded: count_downloaded_media(), + total_sources: count_sources(), + database_size: get_database_size() + } + end + + # Private functions + + defp get_job_counts_for_queue(queue_name) do + queue_string = Atom.to_string(queue_name) + + from(j in Oban.Job, + where: j.queue == ^queue_string, + where: j.state in ["available", "scheduled", "retryable", "executing"], + group_by: j.state, + select: {j.state, count(j.id)} + ) + |> Repo.all() + |> Enum.into(%{}, fn {state, count} -> {String.to_atom(state), count} end) + end + + defp count_pending_downloads do + from(m in Pinchflat.Media.MediaItem, + where: is_nil(m.media_filepath), + where: m.prevent_download == false + ) + |> Repo.aggregate(:count) + end + + defp count_downloaded_media do + from(m in Pinchflat.Media.MediaItem, + where: not is_nil(m.media_filepath) + ) + |> Repo.aggregate(:count) + end + + defp count_sources do + Repo.aggregate(Pinchflat.Sources.Source, :count) + end + + defp get_database_size do + db_path = Application.get_env(:pinchflat, Pinchflat.Repo)[:database] + + if db_path && File.exists?(db_path) do + case File.stat(db_path) do + {:ok, %{size: size}} -> format_bytes(size) + _ -> "Unknown" + end + else + "Unknown" + end + end + + defp format_bytes(bytes) when bytes < 1024, do: "#{bytes} B" + defp format_bytes(bytes) when bytes < 1024 * 1024, do: "#{Float.round(bytes / 1024, 1)} KB" + defp format_bytes(bytes) when bytes < 1024 * 1024 * 1024, do: "#{Float.round(bytes / 1024 / 1024, 1)} MB" + defp format_bytes(bytes), do: "#{Float.round(bytes / 1024 / 1024 / 1024, 2)} GB" +end diff --git a/lib/pinchflat_web/components/layouts/partials/sidebar.html.heex b/lib/pinchflat_web/components/layouts/partials/sidebar.html.heex index e7f89a3..a3518bf 100644 --- a/lib/pinchflat_web/components/layouts/partials/sidebar.html.heex +++ b/lib/pinchflat_web/components/layouts/partials/sidebar.html.heex @@ -33,6 +33,7 @@ > <:submenu text="Settings" href={~p"/settings"} /> <:submenu text="App Info" href={~p"/app_info"} /> + <:submenu text="Diagnostics" href={~p"/diagnostics"} /> diff --git a/lib/pinchflat_web/controllers/settings/diagnostics_controller.ex b/lib/pinchflat_web/controllers/settings/diagnostics_controller.ex new file mode 100644 index 0000000..6b85d60 --- /dev/null +++ b/lib/pinchflat_web/controllers/settings/diagnostics_controller.ex @@ -0,0 +1,53 @@ +defmodule PinchflatWeb.Settings.DiagnosticsController do + use PinchflatWeb, :controller + + alias Pinchflat.Diagnostics.QueueDiagnostics + + def show(conn, _params) do + render(conn, "show.html") + end + + def reset_stuck_jobs(conn, _params) do + count = QueueDiagnostics.reset_stuck_jobs() + + conn + |> put_flash(:info, "Reset #{count} stuck job(s). The queue will restart processing shortly.") + |> redirect(to: ~p"/diagnostics") + end + + def reset_retryable_jobs(conn, _params) do + count = QueueDiagnostics.reset_retryable_jobs() + + conn + |> put_flash(:info, "Reset #{count} retryable job(s). They will be retried shortly.") + |> redirect(to: ~p"/diagnostics") + end + + def reset_job(conn, %{"id" => job_id}) do + case QueueDiagnostics.reset_job(String.to_integer(job_id)) do + 1 -> + conn + |> put_flash(:info, "Job ##{job_id} has been reset and will retry shortly.") + |> redirect(to: ~p"/diagnostics") + + 0 -> + conn + |> put_flash(:error, "Job ##{job_id} could not be reset. It may have already completed or been cancelled.") + |> redirect(to: ~p"/diagnostics") + end + end + + def cancel_job(conn, %{"id" => job_id}) do + case QueueDiagnostics.cancel_job(String.to_integer(job_id)) do + {:ok, :cancelled} -> + conn + |> put_flash(:info, "Job ##{job_id} has been cancelled.") + |> redirect(to: ~p"/diagnostics") + + {:error, _reason} -> + conn + |> put_flash(:error, "Job ##{job_id} could not be cancelled.") + |> redirect(to: ~p"/diagnostics") + end + end +end diff --git a/lib/pinchflat_web/controllers/settings/diagnostics_html.ex b/lib/pinchflat_web/controllers/settings/diagnostics_html.ex new file mode 100644 index 0000000..26025ef --- /dev/null +++ b/lib/pinchflat_web/controllers/settings/diagnostics_html.ex @@ -0,0 +1,80 @@ +defmodule PinchflatWeb.Settings.DiagnosticsHTML do + use PinchflatWeb, :html + + alias Pinchflat.Diagnostics.QueueDiagnostics + + embed_templates "diagnostics_html/*" + + def queue_stats do + QueueDiagnostics.get_all_queue_stats() + end + + def retryable_jobs do + QueueDiagnostics.get_retryable_jobs(20) + end + + def stuck_jobs do + QueueDiagnostics.get_stuck_jobs(30) + end + + def system_stats do + QueueDiagnostics.get_system_stats() + end + + def format_worker_name(worker) do + worker + |> String.split(".") + |> Enum.at(-1) + |> format_worker_short_name() + end + + defp format_worker_short_name("FastIndexingWorker"), do: "Fast Indexing" + defp format_worker_short_name("MediaDownloadWorker"), do: "Download" + defp format_worker_short_name("MediaCollectionIndexingWorker"), do: "Indexing" + defp format_worker_short_name("MediaQualityUpgradeWorker"), do: "Quality Upgrade" + defp format_worker_short_name("SourceMetadataStorageWorker"), do: "Metadata" + defp format_worker_short_name(other), do: other + + def format_queue_name(queue) do + queue + |> Atom.to_string() + |> String.replace("_", " ") + |> String.split(" ") + |> Enum.map(&String.capitalize/1) + |> Enum.join(" ") + end + + def format_datetime(nil), do: "-" + + def format_datetime(datetime) do + Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") + end + + def extract_last_error(errors) when is_list(errors) and length(errors) > 0 do + errors + |> List.last() + |> Map.get("error", "Unknown error") + |> String.slice(0, 200) + end + + def extract_last_error(_), do: "No error details" + + def queue_health_class(stats) do + cond do + stats.paused -> "bg-yellow-500/20 border-yellow-500" + stats.retryable > 0 -> "bg-red-500/20 border-red-500" + stats.running >= stats.limit and stats.available > 0 -> "bg-blue-500/20 border-blue-500" + true -> "bg-green-500/20 border-green-500" + end + end + + def queue_status_text(stats) do + cond do + stats.paused -> "Paused" + stats.retryable > 0 -> "Has Failures" + stats.running >= stats.limit -> "At Capacity" + stats.running > 0 -> "Active" + true -> "Idle" + end + end +end diff --git a/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex b/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex new file mode 100644 index 0000000..1c64a22 --- /dev/null +++ b/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex @@ -0,0 +1,186 @@ +
+
+

+ Queue Diagnostics +

+
+
+ + +
+

System Overview

+ <% stats = system_stats() %> +
+
+

Pending Downloads

+

<%= stats.total_pending_downloads %>

+
+
+

Downloaded Media

+

<%= stats.total_downloaded %>

+
+
+

Total Sources

+

<%= stats.total_sources %>

+
+
+

Database Size

+

<%= stats.database_size %>

+
+
+
+ + +
+
+

Queue Health

+ <.link href={~p"/diagnostics"}> + <.button color="bg-bodydark" rounding="rounded-lg" class="text-sm"> + <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Refresh + + +
+
+ <%= for stats <- queue_stats() do %> +
+
+

<%= format_queue_name(stats.name) %>

+ + <%= queue_status_text(stats) %> + +
+
+
+ Running: + <%= stats.running %>/<%= stats.limit %> +
+
+ Available: + <%= stats.available %> +
+
+ Scheduled: + <%= stats.scheduled %> +
+
+ 0, do: "text-red-400", else: "text-bodydark"}"}>Retryable: + 0, do: "text-red-400 font-bold", else: "text-white"}"}><%= stats.retryable %> +
+
+
+ <% end %> +
+
+ + +<% stuck = stuck_jobs() %> +<%= if length(stuck) > 0 do %> +
+
+

+ <.icon name="hero-exclamation-triangle" class="h-5 w-5 text-yellow-500 mr-2" /> + Stuck Jobs (<%= length(stuck) %>) +

+ <.link href={~p"/diagnostics/reset_stuck_jobs"} method="post" data-confirm="This will reset all stuck jobs. They will be retried. Continue?"> + <.button color="bg-yellow-600" rounding="rounded-lg" class="text-sm"> + <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Stuck Jobs + + +
+

+ These jobs have been executing for more than 30 minutes and may be orphaned. This can happen after a container restart. +

+
+ + + + + + + + + + + + <%= for job <- stuck do %> + + + + + + + + <% end %> + +
IDQueueWorkerStarted AtActions
#<%= job.id %><%= job.queue %><%= format_worker_name(job.worker) %><%= format_datetime(job.attempted_at) %> + <.link href={~p"/diagnostics/reset_job/#{job.id}"} method="post" class="text-primary hover:underline mr-3"> + Reset + + <.link href={~p"/diagnostics/cancel_job/#{job.id}"} method="post" class="text-red-400 hover:underline" data-confirm="Cancel this job?"> + Cancel + +
+
+
+<% end %> + + +<% retryable = retryable_jobs() %> +
+
+

+ <.icon name="hero-exclamation-circle" class="h-5 w-5 text-red-500 mr-2" /> + Failed Jobs (<%= length(retryable) %>) +

+ <%= if length(retryable) > 0 do %> + <.link href={~p"/diagnostics/reset_retryable_jobs"} method="post" data-confirm="This will reset all failed jobs and clear their error history. They will be retried from scratch. Continue?"> + <.button color="bg-red-600" rounding="rounded-lg" class="text-sm"> + <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Failed Jobs + + + <% end %> +
+ + <%= if length(retryable) == 0 do %> +

No failed jobs. Everything is running smoothly!

+ <% else %> +

+ These jobs have failed and are waiting to be retried. You can reset them to retry immediately or cancel them. +

+
+ + + + + + + + + + + + + <%= for job <- retryable do %> + + + + + + + + + <% end %> + +
IDQueueWorkerAttemptsLast ErrorActions
#<%= job.id %><%= job.queue %><%= format_worker_name(job.worker) %><%= job.attempt %>/<%= job.max_attempts %> + <%= extract_last_error(job.errors) %> + + <.link href={~p"/diagnostics/reset_job/#{job.id}"} method="post" class="text-primary hover:underline mr-3"> + Reset + + <.link href={~p"/diagnostics/cancel_job/#{job.id}"} method="post" class="text-red-400 hover:underline" data-confirm="Cancel this job?"> + Cancel + +
+
+ <% end %> +
diff --git a/lib/pinchflat_web/router.ex b/lib/pinchflat_web/router.ex index 9586f13..358d0ea 100644 --- a/lib/pinchflat_web/router.ex +++ b/lib/pinchflat_web/router.ex @@ -51,6 +51,12 @@ defmodule PinchflatWeb.Router do get "/app_info", Settings.SettingController, :app_info get "/download_logs", Settings.SettingController, :download_logs + get "/diagnostics", Settings.DiagnosticsController, :show + post "/diagnostics/reset_stuck_jobs", Settings.DiagnosticsController, :reset_stuck_jobs + post "/diagnostics/reset_retryable_jobs", Settings.DiagnosticsController, :reset_retryable_jobs + post "/diagnostics/reset_job/:id", Settings.DiagnosticsController, :reset_job + post "/diagnostics/cancel_job/:id", Settings.DiagnosticsController, :cancel_job + resources "/sources", Sources.SourceController do post "/force_download_pending", Sources.SourceController, :force_download_pending post "/force_redownload", Sources.SourceController, :force_redownload From 907766f29fb17077e17cd8d9ce15276b3b1c9a16 Mon Sep 17 00:00:00 2001 From: Daniel Da Cunha Date: Sun, 18 Jan 2026 22:21:15 +0800 Subject: [PATCH 2/3] Fix heex template formatting Use new HEEx syntax with curly braces for expressions and fix line length issues with multi-line attribute formatting. Co-Authored-By: Claude Opus 4.5 --- .../settings/diagnostics_html/show.html.heex | 84 ++++++++++++------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex b/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex index 1c64a22..a8c0195 100644 --- a/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex +++ b/lib/pinchflat_web/controllers/settings/diagnostics_html/show.html.heex @@ -13,19 +13,19 @@

Pending Downloads

-

<%= stats.total_pending_downloads %>

+

{stats.total_pending_downloads}

Downloaded Media

-

<%= stats.total_downloaded %>

+

{stats.total_downloaded}

Total Sources

-

<%= stats.total_sources %>

+

{stats.total_sources}

Database Size

-

<%= stats.database_size %>

+

{stats.database_size}

@@ -44,27 +44,29 @@ <%= for stats <- queue_stats() do %>
-

<%= format_queue_name(stats.name) %>

+

{format_queue_name(stats.name)}

- <%= queue_status_text(stats) %> + {queue_status_text(stats)}
Running: - <%= stats.running %>/<%= stats.limit %> + {stats.running}/{stats.limit}
Available: - <%= stats.available %> + {stats.available}
Scheduled: - <%= stats.scheduled %> + {stats.scheduled}
0, do: "text-red-400", else: "text-bodydark"}"}>Retryable: - 0, do: "text-red-400 font-bold", else: "text-white"}"}><%= stats.retryable %> + 0, do: "text-red-400 font-bold", else: "text-white"}"}> + {stats.retryable} +
@@ -78,10 +80,13 @@

- <.icon name="hero-exclamation-triangle" class="h-5 w-5 text-yellow-500 mr-2" /> - Stuck Jobs (<%= length(stuck) %>) + <.icon name="hero-exclamation-triangle" class="h-5 w-5 text-yellow-500 mr-2" /> Stuck Jobs ({length(stuck)})

- <.link href={~p"/diagnostics/reset_stuck_jobs"} method="post" data-confirm="This will reset all stuck jobs. They will be retried. Continue?"> + <.link + href={~p"/diagnostics/reset_stuck_jobs"} + method="post" + data-confirm="This will reset all stuck jobs. They will be retried. Continue?" + > <.button color="bg-yellow-600" rounding="rounded-lg" class="text-sm"> <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Stuck Jobs @@ -104,15 +109,24 @@ <%= for job <- stuck do %> - #<%= job.id %> - <%= job.queue %> - <%= format_worker_name(job.worker) %> - <%= format_datetime(job.attempted_at) %> + #{job.id} + {job.queue} + {format_worker_name(job.worker)} + {format_datetime(job.attempted_at)} - <.link href={~p"/diagnostics/reset_job/#{job.id}"} method="post" class="text-primary hover:underline mr-3"> + <.link + href={~p"/diagnostics/reset_job/#{job.id}"} + method="post" + class="text-primary hover:underline mr-3" + > Reset - <.link href={~p"/diagnostics/cancel_job/#{job.id}"} method="post" class="text-red-400 hover:underline" data-confirm="Cancel this job?"> + <.link + href={~p"/diagnostics/cancel_job/#{job.id}"} + method="post" + class="text-red-400 hover:underline" + data-confirm="Cancel this job?" + > Cancel @@ -129,11 +143,14 @@

- <.icon name="hero-exclamation-circle" class="h-5 w-5 text-red-500 mr-2" /> - Failed Jobs (<%= length(retryable) %>) + <.icon name="hero-exclamation-circle" class="h-5 w-5 text-red-500 mr-2" /> Failed Jobs ({length(retryable)})

<%= if length(retryable) > 0 do %> - <.link href={~p"/diagnostics/reset_retryable_jobs"} method="post" data-confirm="This will reset all failed jobs and clear their error history. They will be retried from scratch. Continue?"> + <.link + href={~p"/diagnostics/reset_retryable_jobs"} + method="post" + data-confirm="This will reset all failed jobs and clear their error history. They will be retried from scratch. Continue?" + > <.button color="bg-red-600" rounding="rounded-lg" class="text-sm"> <.icon name="hero-arrow-path" class="h-4 w-4 mr-1" /> Reset All Failed Jobs @@ -162,18 +179,27 @@ <%= for job <- retryable do %> - #<%= job.id %> - <%= job.queue %> - <%= format_worker_name(job.worker) %> - <%= job.attempt %>/<%= job.max_attempts %> + #{job.id} + {job.queue} + {format_worker_name(job.worker)} + {job.attempt}/{job.max_attempts} - <%= extract_last_error(job.errors) %> + {extract_last_error(job.errors)} - <.link href={~p"/diagnostics/reset_job/#{job.id}"} method="post" class="text-primary hover:underline mr-3"> + <.link + href={~p"/diagnostics/reset_job/#{job.id}"} + method="post" + class="text-primary hover:underline mr-3" + > Reset - <.link href={~p"/diagnostics/cancel_job/#{job.id}"} method="post" class="text-red-400 hover:underline" data-confirm="Cancel this job?"> + <.link + href={~p"/diagnostics/cancel_job/#{job.id}"} + method="post" + class="text-red-400 hover:underline" + data-confirm="Cancel this job?" + > Cancel From 0c3c19febdba528c96a815129c42b300fbae47c3 Mon Sep 17 00:00:00 2001 From: Daniel Da Cunha Date: Sun, 18 Jan 2026 22:25:01 +0800 Subject: [PATCH 3/3] Use Enum.map_join instead of Enum.map |> Enum.join Co-Authored-By: Claude Opus 4.5 --- lib/pinchflat_web/controllers/settings/diagnostics_html.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pinchflat_web/controllers/settings/diagnostics_html.ex b/lib/pinchflat_web/controllers/settings/diagnostics_html.ex index 26025ef..d40432a 100644 --- a/lib/pinchflat_web/controllers/settings/diagnostics_html.ex +++ b/lib/pinchflat_web/controllers/settings/diagnostics_html.ex @@ -40,8 +40,7 @@ defmodule PinchflatWeb.Settings.DiagnosticsHTML do |> Atom.to_string() |> String.replace("_", " ") |> String.split(" ") - |> Enum.map(&String.capitalize/1) - |> Enum.join(" ") + |> Enum.map_join(" ", &String.capitalize/1) end def format_datetime(nil), do: "-"