[Enhancement] Adds ability to enable/disable sources (#481)

* [Unrelated] updated module name for existing liveview module

* Updated toggle component and moved MP index table to a liveview

* [WIP] reverted MP index table; added source count to MP index

* Moved new live table to sources index

* Added 'enabled' boolean to sources

* Got 'enabled' logic working re: downloading pending media

* Updated sources context to do the right thing when a source is updated

* Docs and tests

* Updated slow indexing to maintain its old schedule if re-enabled

* Hooked up the enabled toggle to the sources page

* [Unrelated] added direct links to various tabs on the sources table

* More tests

* Removed unneeded guard in

* Removed outdated comment
This commit is contained in:
Kieran 2024-11-21 14:38:37 -08:00 committed by GitHub
parent 4c8c0461be
commit d9c48370df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 626 additions and 146 deletions

View file

@ -11,14 +11,27 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
alias Pinchflat.Repo
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.FastIndexing.YoutubeRss
alias Pinchflat.FastIndexing.YoutubeApi
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.Downloading.DownloadOptionBuilder
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Kicks off a new fast indexing task for a source. This will delete any existing fast indexing
tasks for the source before starting a new one.
Returns {:ok, %Task{}}
"""
def kickoff_indexing_task(%Source{} = source) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
FastIndexingWorker.kickoff_with_task(source)
end
@doc """
Fetches new media IDs for a source from YT's API or RSS, indexes them, and kicks off downloading
tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the

View file

@ -0,0 +1,29 @@
defmodule Pinchflat.Profiles.ProfilesQuery do
@moduledoc """
Query helpers for the Profiles context.
These methods are made to be one-ish liners used
to compose queries. Each method should strive to do
_one_ thing. These don't need to be tested as
they are just building blocks for other functionality
which, itself, will be tested.
"""
import Ecto.Query, warn: false
alias Pinchflat.Profiles.MediaProfile
# This allows the module to be aliased and query methods to be used
# all in one go
# usage: use Pinchflat.Profiles.ProfilesQuery
defmacro __using__(_opts) do
quote do
import Ecto.Query, warn: false
alias unquote(__MODULE__)
end
end
def new do
MediaProfile
end
end

View file

@ -25,13 +25,28 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
Starts tasks for indexing a source's media regardless of the source's indexing
frequency. It's assumed the caller will check for indexing frequency.
Returns {:ok, %Task{}}.
Returns {:ok, %Task{}}
"""
def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
job_offset_seconds = calculate_job_offset_seconds(source)
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts ++ [schedule_in: job_offset_seconds])
end
@doc """
A helper method to delete all indexing-related tasks for a source.
Optionally, you can include executing tasks in the deletion process.
Returns :ok
"""
def delete_indexing_tasks(%Source{} = source, opts \\ []) do
include_executing = Keyword.get(opts, :include_executing, false)
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: include_executing)
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: include_executing)
end
@doc """
@ -141,4 +156,14 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
changeset
end
end
# Find the difference between the current time and the last time the source was indexed
defp calculate_job_offset_seconds(%Source{last_indexed_at: nil}), do: 0
defp calculate_job_offset_seconds(source) do
offset_seconds = DateTime.diff(DateTime.utc_now(), source.last_indexed_at, :second)
index_frequency_seconds = source.index_frequency_minutes * 60
max(0, index_frequency_seconds - offset_seconds)
end
end

View file

@ -15,6 +15,7 @@ defmodule Pinchflat.Sources.Source do
alias Pinchflat.Metadata.SourceMetadata
@allowed_fields ~w(
enabled
collection_name
collection_id
collection_type
@ -64,6 +65,7 @@ defmodule Pinchflat.Sources.Source do
)a
schema "sources" do
field :enabled, :boolean, default: true
# This is _not_ used as the primary key or internally in the database
# relations. This is only used to prevent an enumeration attack on the streaming
# and RSS feed endpoints since those _must_ be public (ie: no basic auth)

View file

@ -15,8 +15,8 @@ defmodule Pinchflat.Sources do
alias Pinchflat.Metadata.SourceMetadata
alias Pinchflat.Utils.FilesystemUtils
alias Pinchflat.Downloading.DownloadingHelpers
alias Pinchflat.FastIndexing.FastIndexingWorker
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
alias Pinchflat.FastIndexing.FastIndexingHelpers
alias Pinchflat.Metadata.SourceMetadataStorageWorker
@doc """
@ -255,19 +255,40 @@ defmodule Pinchflat.Sources do
end
end
# If the source is NOT new (ie: updated) and the download_media flag has changed,
# If the source is new (ie: not persisted), do nothing
defp maybe_handle_media_tasks(%{data: %{__meta__: %{state: state}}}, _source) when state != :loaded do
:ok
end
# If the source is NOT new (ie: updated),
# enqueue or dequeue media download tasks as necessary.
defp maybe_handle_media_tasks(changeset, source) do
case {changeset.data, changeset.changes} do
{%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
current_changes = changeset.changes
applied_changes = Ecto.Changeset.apply_changes(changeset)
# We need both current_changes and applied_changes to determine
# the course of action to take. For example, we only care if a source is supposed
# to be `enabled` or not - we don't care if that information comes from the
# current changes or if that's how it already was in the database.
# Rephrased, we're essentially using it in place of `get_field/2`
case {current_changes, applied_changes} do
{%{download_media: true}, %{enabled: true}} ->
DownloadingHelpers.enqueue_pending_download_tasks(source)
{%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
{%{enabled: true}, %{download_media: true}} ->
DownloadingHelpers.enqueue_pending_download_tasks(source)
{%{download_media: false}, _} ->
DownloadingHelpers.dequeue_pending_download_tasks(source)
{%{enabled: false}, _} ->
DownloadingHelpers.dequeue_pending_download_tasks(source)
_ ->
:ok
nil
end
:ok
end
defp maybe_run_indexing_task(changeset, source) do
@ -301,13 +322,22 @@ defmodule Pinchflat.Sources do
end
defp maybe_update_slow_indexing_task(changeset, source) do
case changeset.changes do
%{index_frequency_minutes: mins} when mins > 0 ->
# See comment in `maybe_handle_media_tasks` as to why we need these
current_changes = changeset.changes
applied_changes = Ecto.Changeset.apply_changes(changeset)
case {current_changes, applied_changes} do
{%{index_frequency_minutes: mins}, %{enabled: true}} when mins > 0 ->
SlowIndexingHelpers.kickoff_indexing_task(source)
%{index_frequency_minutes: _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
{%{enabled: true}, %{index_frequency_minutes: mins}} when mins > 0 ->
SlowIndexingHelpers.kickoff_indexing_task(source)
{%{index_frequency_minutes: _}, _} ->
SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
{%{enabled: false}, _} ->
SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
_ ->
:ok
@ -315,13 +345,25 @@ defmodule Pinchflat.Sources do
end
defp maybe_update_fast_indexing_task(changeset, source) do
case changeset.changes do
%{fast_index: true} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
FastIndexingWorker.kickoff_with_task(source)
# See comment in `maybe_handle_media_tasks` as to why we need these
current_changes = changeset.changes
applied_changes = Ecto.Changeset.apply_changes(changeset)
%{fast_index: false} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
# This technically could be simplified since `maybe_update_slow_indexing_task`
# has some overlap re: deleting pending tasks, but I'm keeping it separate
# for clarity and explicitness.
case {current_changes, applied_changes} do
{%{fast_index: true}, %{enabled: true}} ->
FastIndexingHelpers.kickoff_indexing_task(source)
{%{enabled: true}, %{fast_index: true}} ->
FastIndexingHelpers.kickoff_indexing_task(source)
{%{fast_index: false}, _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
{%{enabled: false}, _} ->
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
_ ->
:ok

View file

@ -340,14 +340,15 @@ defmodule PinchflatWeb.CoreComponents do
end)
~H"""
<div x-data={"{ enabled: #{@checked}}"}>
<.label for={@id}>
<div x-data={"{ enabled: #{@checked} }"} class="" phx-update="ignore" id={"#{@id}-wrapper"}>
<.label :if={@label} for={@id}>
<%= @label %>
<span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
</.label>
<div class="relative">
<div class="relative flex flex-col">
<input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
<div class="inline-block cursor-pointer" @click="enabled = !enabled">
<%!-- This triggers a `change` event on the hidden input when the toggle is clicked --%>
<div class="inline-block cursor-pointer" @click={"enabled = !enabled; dispatchFor('#{@id}', 'change')"}>
<div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
<div
x-bind:class="enabled && '!right-1 !translate-x-full'"

View file

@ -3,7 +3,7 @@ defmodule Pinchflat.UpgradeButtonLive do
def render(assigns) do
~H"""
<form id="upgradeForm" phx-change="check_matching_text" phx-hook="supressEnterSubmission">
<form id="upgradeForm" phx-change="check_matching_text" phx-hook="supress-enter-submission">
<.input type="text" name="unlock-pro-textbox" value="" />
</form>

View file

@ -1,20 +1,31 @@
defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
use PinchflatWeb, :controller
use Pinchflat.Sources.SourcesQuery
use Pinchflat.Profiles.ProfilesQuery
alias Pinchflat.Repo
alias Pinchflat.Profiles
alias Pinchflat.Sources.Source
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.MediaProfileDeletionWorker
def index(conn, _params) do
media_profiles =
MediaProfile
|> where([mp], is_nil(mp.marked_for_deletion_at))
|> order_by(asc: :name)
|> Repo.all()
media_profiles_query =
from mp in MediaProfile,
as: :media_profile,
where: is_nil(mp.marked_for_deletion_at),
order_by: [asc: mp.name],
select: map(mp, ^MediaProfile.__schema__(:fields)),
select_merge: %{
source_count:
subquery(
from s in Source,
where: s.media_profile_id == parent_as(:media_profile).id,
select: count(s.id)
)
}
render(conn, :index, media_profiles: media_profiles)
render(conn, :index, media_profiles: Repo.all(media_profiles_query))
end
def new(conn, params) do

View file

@ -10,7 +10,6 @@
</.link>
</nav>
</div>
<div class="rounded-sm border border-stroke bg-white shadow-default dark:border-strokedark dark:bg-boxdark">
<div class="max-w-full overflow-x-auto">
<div class="flex flex-col gap-10 min-w-max">
@ -23,6 +22,11 @@
<:col :let={media_profile} label="Preferred Resolution">
<%= media_profile.preferred_resolution %>
</:col>
<:col :let={media_profile} label="Sources">
<.subtle_link href={~p"/media_profiles/#{media_profile.id}/#tab-sources"}>
<.localized_number number={media_profile.source_count} />
</.subtle_link>
</:col>
<:col :let={media_profile} label="" class="flex justify-end">
<.icon_link href={~p"/media_profiles/#{media_profile.id}/edit"} icon="hero-pencil-square" class="mr-4" />
</:col>

View file

@ -1,12 +1,11 @@
defmodule PinchflatWeb.Sources.SourceController do
use PinchflatWeb, :controller
use Pinchflat.Media.MediaQuery
use Pinchflat.Sources.SourcesQuery
alias Pinchflat.Repo
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaItem
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Media.FileSyncingWorker
alias Pinchflat.Sources.SourceDeletionWorker
@ -15,33 +14,7 @@ defmodule PinchflatWeb.Sources.SourceController do
alias Pinchflat.Metadata.SourceMetadataStorageWorker
def index(conn, _params) do
source_query =
from s in Source,
as: :source,
inner_join: mp in assoc(s, :media_profile),
where: is_nil(s.marked_for_deletion_at) and is_nil(mp.marked_for_deletion_at),
preload: [media_profile: mp],
order_by: [asc: s.custom_name],
select: map(s, ^Source.__schema__(:fields)),
select_merge: %{
downloaded_count:
subquery(
from m in MediaItem,
where: m.source_id == parent_as(:source).id,
where: ^MediaQuery.downloaded(),
select: count(m.id)
),
pending_count:
subquery(
from m in MediaItem,
join: s in assoc(m, :source),
where: m.source_id == parent_as(:source).id,
where: ^MediaQuery.pending(),
select: count(m.id)
)
}
render(conn, :index, sources: Repo.all(source_query))
render(conn, :index)
end
def new(conn, params) do

View file

@ -12,32 +12,7 @@
<div class="rounded-sm border border-stroke bg-white shadow-default dark:border-strokedark dark:bg-boxdark">
<div class="max-w-full overflow-x-auto">
<div class="flex flex-col gap-10 min-w-max">
<.table rows={@sources} table_class="text-black dark:text-white">
<:col :let={source} label="Name">
<.subtle_link href={~p"/sources/#{source.id}"}>
<%= StringUtils.truncate(source.custom_name || source.collection_name, 35) %>
</.subtle_link>
</:col>
<:col :let={source} label="Type"><%= source.collection_type %></:col>
<:col :let={source} label="Pending"><.localized_number number={source.pending_count} /></:col>
<:col :let={source} label="Downloaded"><.localized_number number={source.downloaded_count} /></:col>
<:col :let={source} label="Retention">
<%= if source.retention_period_days && source.retention_period_days > 0 do %>
<.localized_number number={source.retention_period_days} />
<.pluralize count={source.retention_period_days} word="day" />
<% else %>
<span class="text-lg">∞</span>
<% end %>
</:col>
<:col :let={source} label="Media Profile">
<.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
<%= source.media_profile.name %>
</.subtle_link>
</:col>
<:col :let={source} label="" class="flex place-content-evenly">
<.icon_link href={~p"/sources/#{source.id}/edit"} icon="hero-pencil-square" class="mx-1" />
</:col>
</.table>
<%= live_render(@conn, PinchflatWeb.Sources.IndexTableLive) %>
</div>
</div>
</div>

View file

@ -0,0 +1,103 @@
defmodule PinchflatWeb.Sources.IndexTableLive do
use PinchflatWeb, :live_view
use Pinchflat.Media.MediaQuery
use Pinchflat.Sources.SourcesQuery
alias Pinchflat.Repo
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Media.MediaItem
def render(assigns) do
~H"""
<.table rows={@sources} table_class="text-white">
<:col :let={source} label="Name">
<.subtle_link href={~p"/sources/#{source.id}"}>
<%= StringUtils.truncate(source.custom_name || source.collection_name, 35) %>
</.subtle_link>
</:col>
<:col :let={source} label="Pending">
<.subtle_link href={~p"/sources/#{source.id}/#tab-pending"}>
<.localized_number number={source.pending_count} />
</.subtle_link>
</:col>
<:col :let={source} label="Downloaded">
<.subtle_link href={~p"/sources/#{source.id}/#tab-downloaded"}>
<.localized_number number={source.downloaded_count} />
</.subtle_link>
</:col>
<:col :let={source} label="Retention">
<%= if source.retention_period_days && source.retention_period_days > 0 do %>
<.localized_number number={source.retention_period_days} />
<.pluralize count={source.retention_period_days} word="day" />
<% else %>
<span class="text-lg"></span>
<% end %>
</:col>
<:col :let={source} label="Media Profile">
<.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
<%= source.media_profile.name %>
</.subtle_link>
</:col>
<:col :let={source} label="Enabled?">
<.input
name={"source[#{source.id}][enabled]"}
value={source.enabled}
id={"source_#{source.id}_enabled"}
phx-hook="formless-input"
data-subscribe="change"
data-event-name="toggle_enabled"
data-identifier={source.id}
type="toggle"
/>
</:col>
<:col :let={source} label="" class="flex place-content-evenly">
<.icon_link href={~p"/sources/#{source.id}/edit"} icon="hero-pencil-square" class="mx-1" />
</:col>
</.table>
"""
end
def mount(_params, _session, socket) do
{:ok, assign(socket, %{sources: get_sources()})}
end
def handle_event("formless-input", %{"event" => "toggle_enabled"} = params, socket) do
source = Sources.get_source!(params["id"])
should_enable = params["value"] == "true"
{:ok, _} = Sources.update_source(source, %{enabled: should_enable})
{:noreply, assign(socket, %{sources: get_sources()})}
end
defp get_sources do
query =
from s in Source,
as: :source,
inner_join: mp in assoc(s, :media_profile),
where: is_nil(s.marked_for_deletion_at) and is_nil(mp.marked_for_deletion_at),
preload: [media_profile: mp],
order_by: [asc: s.custom_name],
select: map(s, ^Source.__schema__(:fields)),
select_merge: %{
downloaded_count:
subquery(
from m in MediaItem,
where: m.source_id == parent_as(:source).id,
where: ^MediaQuery.downloaded(),
select: count(m.id)
),
pending_count:
subquery(
from m in MediaItem,
join: s in assoc(m, :source),
where: m.source_id == parent_as(:source).id,
where: ^MediaQuery.pending(),
select: count(m.id)
)
}
Repo.all(query)
end
end

View file

@ -1,4 +1,4 @@
defmodule Pinchflat.Sources.MediaItemTableLive do
defmodule PinchflatWeb.Sources.MediaItemTableLive do
use PinchflatWeb, :live_view
use Pinchflat.Media.MediaQuery

View file

@ -39,21 +39,21 @@
<:tab title="Pending" id="pending">
<%= live_render(
@conn,
Pinchflat.Sources.MediaItemTableLive,
PinchflatWeb.Sources.MediaItemTableLive,
session: %{"source_id" => @source.id, "media_state" => "pending"}
) %>
</:tab>
<:tab title="Downloaded" id="downloaded">
<%= live_render(
@conn,
Pinchflat.Sources.MediaItemTableLive,
PinchflatWeb.Sources.MediaItemTableLive,
session: %{"source_id" => @source.id, "media_state" => "downloaded"}
) %>
</:tab>
<:tab title="Other" id="other">
<%= live_render(
@conn,
Pinchflat.Sources.MediaItemTableLive,
PinchflatWeb.Sources.MediaItemTableLive,
session: %{"source_id" => @source.id, "media_state" => "other"}
) %>
</:tab>