Misc indexing improvements (#99)

* increased file follower timeout to 10 minutes

* Removed unique index across a source's collection and its media profile

* Ensure source gets updating during slow indexing

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -9,7 +9,7 @@ defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   require Logger
 
   @poll_interval_ms Application.compile_env(:pinchflat, :file_watcher_poll_interval)
-  @activity_timeout_ms 60_000
+  @activity_timeout_ms 600_000
 
   # Client API
   @doc """

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -7,6 +7,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
 
   require Logger
 
+  alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
@@ -60,6 +61,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
+    # Reload because the source may have been updated during the (long-running) indexing process
+    # and important settings like `download_media` may have changed.
+    source = Repo.reload!(source)
 
     result =
       Enum.map(media_attributes, fn media_attrs ->
@@ -117,6 +121,10 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
+    # Reload because the source may have been updated during the (long-running) indexing process
+    # and important settings like `download_media` may have changed.
+    source = Repo.reload!(source)
+
     case Media.create_media_item_from_backend_attrs(source, media_attrs) do
       {:ok, %MediaItem{} = media_item} ->
         if source.download_media && Media.pending_download?(media_item) do

lib/pinchflat/sources/source.ex

@@ -98,7 +98,6 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> validate_required(required_fields)
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
-    |> unique_constraint([:collection_id, :media_profile_id])
   end
 
   @doc false

priv/repo/migrations/20240320181210_remove_unique_index_from_sources.exs

@@ -0,0 +1,7 @@
+defmodule Pinchflat.Repo.Migrations.RemoveUniqueIndexFromSources do
+  use Ecto.Migration
+
+  def change do
+    drop unique_index(:sources, [:collection_id, :media_profile_id])
+  end
+end

test/pinchflat/sources_test.exs

@@ -138,49 +138,6 @@ defmodule Pinchflat.SourcesTest do
       assert {:error, %Ecto.Changeset{}} = Sources.create_source(@invalid_source_attrs)
     end
 
-    test "creation enforces uniqueness of collection_id scoped to the media_profile" do
-      expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
-        {:ok,
-         Phoenix.json_library().encode!(%{
-           channel: "some channel name",
-           channel_id: "some_channel_id_12345678",
-           playlist_id: "some_channel_id_12345678",
-           playlist_title: "some channel name - videos"
-         })}
-      end)
-
-      valid_once_attrs = %{
-        media_profile_id: media_profile_fixture().id,
-        original_url: "https://www.youtube.com/channel/abc123"
-      }
-
-      assert {:ok, %Source{}} = Sources.create_source(valid_once_attrs)
-      assert {:error, %Ecto.Changeset{}} = Sources.create_source(valid_once_attrs)
-    end
-
-    test "creation lets you duplicate collection_ids as long as the media profile is different" do
-      expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
-        {:ok,
-         Phoenix.json_library().encode!(%{
-           channel: "some channel name",
-           channel_id: "some_channel_id_12345678",
-           playlist_id: "some_channel_id_12345678",
-           playlist_title: "some channel name - videos"
-         })}
-      end)
-
-      valid_attrs = %{
-        name: "some name",
-        original_url: "https://www.youtube.com/channel/abc123"
-      }
-
-      source_1_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
-      source_2_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
-
-      assert {:ok, %Source{}} = Sources.create_source(source_1_attrs)
-      assert {:ok, %Source{}} = Sources.create_source(source_2_attrs)
-    end
-
     test "creation will schedule the indexing task" do
       expect(YtDlpRunnerMock, :run, &channel_mock/3)