From 05f3deebfab7baca600da3fcacaab04e9a7654de Mon Sep 17 00:00:00 2001 From: Kieran Date: Mon, 18 Mar 2024 17:27:28 -0700 Subject: [PATCH] Source NFO downloads (#95) * Hooked up series directory finding to source metadata runner * Fixed aired NFO tag for episodes * Updated MI NFO builder to take in a filepath * Hooked up NFO generation to the source worker * Added NFO controls to form * Improved the way the source metadata worker updates the source * Consolidated NFO selection options in media profile instead of source --- .../downloading/download_option_builder.ex | 27 ++-- lib/pinchflat/downloading/media_downloader.ex | 4 +- .../metadata/metadata_file_helpers.ex | 37 +++++ lib/pinchflat/metadata/nfo_builder.ex | 34 ++++- .../source_metadata_storage_worker.ex | 69 +++++++-- lib/pinchflat/sources/source.ex | 9 ++ lib/pinchflat/sources/sources.ex | 60 +++++--- lib/pinchflat/yt_dlp/media_collection.ex | 14 +- .../media_profile_form.html.heex | 37 +++-- .../sources/source_html/source_form.html.heex | 4 +- ...20240318161405_add_nfo_path_to_sources.exs | 10 ++ .../download_option_builder_test.exs | 8 + .../metadata/metadata_file_helpers_test.exs | 48 ++++++ test/pinchflat/metadata/nfo_builder_test.exs | 48 +++--- .../source_metadata_storage_worker_test.exs | 141 +++++++++++++++--- test/pinchflat/sources_test.exs | 56 ++++++- .../yt_dlp/media_collection_test.exs | 2 +- test/support/fixtures/sources_fixtures.ex | 14 ++ 18 files changed, 513 insertions(+), 109 deletions(-) create mode 100644 priv/repo/migrations/20240318161405_add_nfo_path_to_sources.exs diff --git a/lib/pinchflat/downloading/download_option_builder.ex b/lib/pinchflat/downloading/download_option_builder.ex index 49a688c..b9233d1 100644 --- a/lib/pinchflat/downloading/download_option_builder.ex +++ b/lib/pinchflat/downloading/download_option_builder.ex @@ -3,6 +3,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do Builds the options for yt-dlp to download media based on the given media profile. """ + alias Pinchflat.Sources.Source alias Pinchflat.Media.MediaItem alias Pinchflat.Downloading.OutputPathBuilder @@ -26,6 +27,18 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do {:ok, built_options} end + @doc """ + Builds the output path for yt-dlp to download media based on the given source's + media profile. + + Returns binary() + """ + def build_output_path_for(%Source{} = source_with_preloads) do + output_path_template = source_with_preloads.media_profile.output_path_template + + build_output_path(output_path_template, source_with_preloads) + end + defp default_options do [:no_progress, :windows_filenames] end @@ -104,23 +117,19 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do end defp output_options(media_item_with_preloads) do - output_path_template = media_item_with_preloads.source.media_profile.output_path_template - [ - output: build_output_path(output_path_template, media_item_with_preloads) + output: build_output_path_for(media_item_with_preloads.source) ] end - defp build_output_path(string, media_item_with_preloads) do - additional_options_map = output_options_map(media_item_with_preloads) + defp build_output_path(string, source) do + additional_options_map = output_options_map(source) {:ok, output_path} = OutputPathBuilder.build(string, additional_options_map) Path.join(base_directory(), output_path) end - defp output_options_map(media_item_with_preloads) do - source = media_item_with_preloads.source - + defp output_options_map(source) do %{ "source_custom_name" => source.custom_name, "source_collection_type" => source.collection_type @@ -137,7 +146,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do |> String.split(~r{\.}, include_captures: true) |> List.insert_at(-3, "-thumb") |> Enum.join() - |> build_output_path(media_item_with_preloads) + |> build_output_path(media_item_with_preloads.source) end defp base_directory do diff --git a/lib/pinchflat/downloading/media_downloader.ex b/lib/pinchflat/downloading/media_downloader.ex index cf87514..1774910 100644 --- a/lib/pinchflat/downloading/media_downloader.ex +++ b/lib/pinchflat/downloading/media_downloader.ex @@ -56,7 +56,9 @@ defmodule Pinchflat.Downloading.MediaDownloader do defp determine_nfo_filepath(media_item, parsed_json) do if media_item.source.media_profile.download_nfo do - NfoBuilder.build_and_store_for_media_item(parsed_json) + filepath = Path.rootname(parsed_json["filepath"]) <> ".nfo" + + NfoBuilder.build_and_store_for_media_item(filepath, parsed_json) else nil end diff --git a/lib/pinchflat/metadata/metadata_file_helpers.ex b/lib/pinchflat/metadata/metadata_file_helpers.ex index f1e768d..dc9d95d 100644 --- a/lib/pinchflat/metadata/metadata_file_helpers.ex +++ b/lib/pinchflat/metadata/metadata_file_helpers.ex @@ -63,6 +63,43 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do Date.from_iso8601!("#{year}-#{month}-#{day}") end + @doc """ + Attempts to determine the series directory from a media filepath. + The series directory is the "root" directory for a given source + which should contain all the season-level folders of that source. + + Used for determining where to store things like NFO data and banners + for media center apps. Not useful without a media center app. + + Returns {:ok, binary()} | {:error, :indeterminable} + """ + def series_directory_from_media_filepath(media_filepath) do + # Matches "s" or "season" (case-insensitive) + # followed by an optional non-word character (. or _ or , etc) + # followed by at least one digit + # followed immediately by the end of the string + # Example matches: s1, s.1, s01 season 1, Season.01, Season_1, Season 1, Season1 + # Example non-matches: s01e01, season, series 1, + season_regex = ~r/^s(eason)?(\W|_)?\d{1,}$/i + + {series_directory, found_series_directory} = + media_filepath + |> Path.split() + |> Enum.reduce_while({[], false}, fn part, {directory_acc, _} -> + if String.match?(part, season_regex) do + {:halt, {directory_acc, true}} + else + {:cont, {directory_acc ++ [part], false}} + end + end) + + if found_series_directory do + {:ok, Path.join(series_directory)} + else + {:error, :indeterminable} + end + end + defp fetch_thumbnail_from_url(url) do http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient) {:ok, body} = http_client.get(url, [], body_format: :binary) diff --git a/lib/pinchflat/metadata/nfo_builder.ex b/lib/pinchflat/metadata/nfo_builder.ex index f328fb7..fb5ba7b 100644 --- a/lib/pinchflat/metadata/nfo_builder.ex +++ b/lib/pinchflat/metadata/nfo_builder.ex @@ -9,13 +9,11 @@ defmodule Pinchflat.Metadata.NfoBuilder do @doc """ Builds an NFO file for a media item (read: single "episode") and - stores it in the same directory as the media file. Has the same name - as the media file, but with a .nfo extension. + stores it at the specified location. Returns the filepath of the NFO file. """ - def build_and_store_for_media_item(metadata) do - filepath = Path.rootname(metadata["filepath"]) <> ".nfo" + def build_and_store_for_media_item(filepath, metadata) do nfo = build_for_media_item(metadata) FilesystemHelpers.write_p!(filepath, nfo) @@ -23,6 +21,20 @@ defmodule Pinchflat.Metadata.NfoBuilder do filepath end + @doc """ + Builds an NFO file for a souce and stores it at the specified location. + Technically works for playlists, but it's really made for channels. + + Returns the filepath of the NFO file. + """ + def build_and_store_for_source(filepath, metadata) do + nfo = build_for_source(metadata) + + FilesystemHelpers.write_p!(filepath, nfo) + + filepath + end + defp build_for_media_item(metadata) do upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"]) # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub. @@ -34,11 +46,23 @@ defmodule Pinchflat.Metadata.NfoBuilder do #{metadata["uploader"]} #{metadata["id"]} #{metadata["description"]} - #{upload_date} + #{upload_date} #{upload_date.year} #{Calendar.strftime(upload_date, "%m%d")} YouTube """ end + + defp build_for_source(metadata) do + """ + + + #{metadata["title"]} + #{metadata["description"]} + #{metadata["id"]} + YouTube + + """ + end end diff --git a/lib/pinchflat/metadata/source_metadata_storage_worker.ex b/lib/pinchflat/metadata/source_metadata_storage_worker.ex index 7d298c2..3380ea3 100644 --- a/lib/pinchflat/metadata/source_metadata_storage_worker.ex +++ b/lib/pinchflat/metadata/source_metadata_storage_worker.ex @@ -4,10 +4,7 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do use Oban.Worker, queue: :remote_metadata, tags: ["media_source", "source_metadata", "remote_metadata"], - max_attempts: 1, - # This is the only thing stopping this job from calling itself - # in an infinite loop. - unique: [period: 600] + max_attempts: 3 require Logger @@ -15,8 +12,10 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do alias Pinchflat.Repo alias Pinchflat.Tasks alias Pinchflat.Sources + alias Pinchflat.Metadata.NfoBuilder alias Pinchflat.YtDlp.MediaCollection alias Pinchflat.Metadata.MetadataFileHelpers + alias Pinchflat.Downloading.DownloadOptionBuilder @doc """ Starts the source metadata storage worker and creates a task for the source. @@ -30,27 +29,67 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do end @doc """ - Fetches and stores metadata for a source in the secret metadata location. + Fetches and stores various forms of metadata for a source: + - JSON metadata for internal use + - The series directory for the source + - The NFO file for the source (if specified) + + The worker is kicked off after a source is inserted/updated - this can + take an unknown amount of time so don't rely on this data being here + before, say, the first indexing or downloading task is complete. Returns :ok """ @impl Oban.Worker def perform(%Oban.Job{args: %{"id" => source_id}}) do - source = Repo.preload(Sources.get_source!(source_id), :metadata) - {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url) + source = Repo.preload(Sources.get_source!(source_id), [:metadata, :media_profile]) + source_metadata = fetch_source_metadata(source) + series_directory = determine_series_directory(source) - # Since updating a source kicks this job off again, we enforce job uniqueness (above) - # to once, per source, per x minutes. This is to prevent a job from calling itself - # in an infinite loop. - Sources.update_source(source, %{ - metadata: %{ - metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, metadata) - } - }) + # `run_post_commit_tasks: false` prevents this from running in an infinite loop + Sources.update_source( + source, + %{ + series_directory: series_directory, + nfo_filepath: store_source_nfo(source, series_directory, source_metadata), + metadata: %{ + metadata_filepath: store_source_metadata(source, source_metadata) + } + }, + run_post_commit_tasks: false + ) :ok rescue Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found") Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale") end + + defp fetch_source_metadata(source) do + {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url) + + metadata + end + + defp store_source_metadata(source, metadata) do + MetadataFileHelpers.compress_and_store_metadata_for(source, metadata) + end + + defp determine_series_directory(source) do + output_path = DownloadOptionBuilder.build_output_path_for(source) + {:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, output: output_path) + + case MetadataFileHelpers.series_directory_from_media_filepath(filepath) do + {:ok, series_directory} -> series_directory + {:error, _} -> nil + end + end + + defp store_source_nfo(source, series_directory, metadata) do + if source.media_profile.download_nfo && series_directory do + nfo_filepath = Path.join(series_directory, "tvshow.nfo") + + NfoBuilder.build_and_store_for_source(nfo_filepath, metadata) + end + end end diff --git a/lib/pinchflat/sources/source.ex b/lib/pinchflat/sources/source.ex index 6a39e5e..56bcb91 100644 --- a/lib/pinchflat/sources/source.ex +++ b/lib/pinchflat/sources/source.ex @@ -17,6 +17,8 @@ defmodule Pinchflat.Sources.Source do collection_id collection_type custom_name + nfo_filepath + series_directory index_frequency_minutes fast_index download_media @@ -52,6 +54,8 @@ defmodule Pinchflat.Sources.Source do field :collection_name, :string field :collection_id, :string field :collection_type, Ecto.Enum, values: [:channel, :playlist] + field :nfo_filepath, :string + field :series_directory, :string field :index_frequency_minutes, :integer, default: 60 * 24 field :fast_index, :boolean, default: false field :download_media, :boolean, default: true @@ -99,4 +103,9 @@ defmodule Pinchflat.Sources.Source do # minutes 15 end + + @doc false + def filepath_attributes do + ~w(nfo_filepath)a + end end diff --git a/lib/pinchflat/sources/sources.ex b/lib/pinchflat/sources/sources.ex index d7e387a..0920217 100644 --- a/lib/pinchflat/sources/sources.ex +++ b/lib/pinchflat/sources/sources.ex @@ -51,15 +51,19 @@ defmodule Pinchflat.Sources do though we know it's going to fail so it picks up any addl. database errors and fulfills our return contract. + You can pass options to control the behavior of the function: + - `run_post_commit_tasks` (default: true) - If false, the function will not + enqueue any tasks in `commit_and_handle_tasks`. + Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}} """ - def create_source(attrs) do + def create_source(attrs, opts \\ []) do case change_source(%Source{}, attrs, :initial) do %Ecto.Changeset{valid?: true} -> %Source{} |> maybe_change_source_from_url(attrs) |> maybe_change_indexing_frequency() - |> commit_and_handle_tasks() + |> commit_and_handle_tasks(opts) changeset -> Repo.insert(changeset) @@ -79,15 +83,19 @@ defmodule Pinchflat.Sources do though we know it's going to fail so it picks up any addl. database errors and fulfills our return contract. + You can pass options to control the behavior of the function: + - `run_post_commit_tasks` (default: true) - If false, the function will not + enqueue any tasks in `commit_and_handle_tasks`. + Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}} """ - def update_source(%Source{} = source, attrs) do + def update_source(%Source{} = source, attrs, opts \\ []) do case change_source(source, attrs, :initial) do %Ecto.Changeset{valid?: true} -> source |> maybe_change_source_from_url(attrs) |> maybe_change_indexing_frequency() - |> commit_and_handle_tasks() + |> commit_and_handle_tasks(opts) changeset -> Repo.update(changeset) @@ -102,7 +110,6 @@ defmodule Pinchflat.Sources do """ def delete_source(%Source{} = source, opts \\ []) do delete_files = Keyword.get(opts, :delete_files, false) - Tasks.delete_tasks_for(source) source @@ -111,7 +118,11 @@ defmodule Pinchflat.Sources do Media.delete_media_item(media_item, delete_files: delete_files) end) - delete_source_metadata_files(source) + if delete_files do + delete_source_files(source) + end + + delete_internal_metadata_files(source) Repo.delete(source) end @@ -134,22 +145,27 @@ defmodule Pinchflat.Sources do end end - defp delete_source_metadata_files(source) do + defp delete_source_files(source) do + mapped_struct = Map.from_struct(source) + + Source.filepath_attributes() + |> Enum.map(fn field -> mapped_struct[field] end) + |> Enum.filter(&is_binary/1) + |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1) + end + + defp delete_internal_metadata_files(source) do metadata = Repo.preload(source, :metadata).metadata || %SourceMetadata{} mapped_struct = Map.from_struct(metadata) - filepaths = - SourceMetadata.filepath_attributes() - |> Enum.map(fn field -> mapped_struct[field] end) - |> Enum.filter(&is_binary/1) - - Enum.each(filepaths, &FilesystemHelpers.delete_file_and_remove_empty_directories/1) + SourceMetadata.filepath_attributes() + |> Enum.map(fn field -> mapped_struct[field] end) + |> Enum.filter(&is_binary/1) + |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1) end defp add_source_details_to_changeset(source, changeset) do - %Ecto.Changeset{changes: changes} = changeset - - case MediaCollection.get_source_details(changes.original_url) do + case MediaCollection.get_source_details(changeset.changes.original_url) do {:ok, source_details} -> add_source_details_by_collection_type(source, changeset, source_details) @@ -198,12 +214,16 @@ defmodule Pinchflat.Sources do end end - defp commit_and_handle_tasks(changeset) do + defp commit_and_handle_tasks(changeset, opts) do + run_post_commit_tasks = Keyword.get(opts, :run_post_commit_tasks, true) + case Repo.insert_or_update(changeset) do {:ok, %Source{} = source} -> - maybe_handle_media_tasks(changeset, source) - maybe_run_indexing_task(changeset, source) - run_metadata_storage_task(source) + if run_post_commit_tasks do + maybe_handle_media_tasks(changeset, source) + maybe_run_indexing_task(changeset, source) + run_metadata_storage_task(source) + end {:ok, source} diff --git a/lib/pinchflat/yt_dlp/media_collection.ex b/lib/pinchflat/yt_dlp/media_collection.ex index 5a0912d..da86600 100644 --- a/lib/pinchflat/yt_dlp/media_collection.ex +++ b/lib/pinchflat/yt_dlp/media_collection.ex @@ -64,14 +64,14 @@ defmodule Pinchflat.YtDlp.MediaCollection do Returns {:ok, map()} | {:error, any, ...}. """ - def get_source_details(source_url) do + def get_source_details(source_url, addl_opts \\ []) do # `ignore_no_formats_error` is necessary because yt-dlp will error out if # the first video has not released yet (ie: is a premier). We don't care about # available formats since we're just getting the source details - opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1] - output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j" + command_opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1] ++ addl_opts + output_template = "%(.{channel,channel_id,playlist_id,playlist_title,filename})j" - with {:ok, output} <- backend_runner().run(source_url, opts, output_template), + with {:ok, output} <- backend_runner().run(source_url, command_opts, output_template), {:ok, parsed_json} <- Phoenix.json_library().decode(output) do {:ok, format_source_details(parsed_json)} else @@ -112,7 +112,11 @@ defmodule Pinchflat.YtDlp.MediaCollection do channel_id: response["channel_id"], channel_name: response["channel"], playlist_id: response["playlist_id"], - playlist_name: response["playlist_title"] + playlist_name: response["playlist_title"], + # It's not a name, it's a path dammit! + # This actually isn't used for the inital response - it's + # used later to update a source's metadata + filepath: response["filename"] } end diff --git a/lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex b/lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex index 3eba728..07cc07e 100644 --- a/lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex +++ b/lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex @@ -162,16 +162,6 @@ /> -
- <.input - field={f[:download_nfo]} - type="toggle" - label="Download NFO data" - help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc." - x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))" - /> -
-

Release Format Options

@@ -181,7 +171,7 @@ field={f[:shorts_behaviour]} options={friendly_format_type_options()} type="select" - label="Include Shorts?" + label="Include Shorts" help="Experimental. Please report any issues on GitHub" x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))" /> @@ -192,7 +182,7 @@ field={f[:livestream_behaviour]} options={friendly_format_type_options()} type="select" - label="Include Livestreams?" + label="Include Livestreams" help="Excludes media that comes from a past livestream" x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))" /> @@ -213,6 +203,29 @@ /> +

+ Media Center Options +

+

+ Everything in this section is experimental - please open a GitHub issue if you see something odd. + These options only work if this Media Profile's output template is set to split media into seasons. + Try the "Media Center" preset if you're not sure. +

+ +
+ <.input + field={f[:download_nfo]} + type="toggle" + label="Download NFO data" + label_suffix="(pro)" + help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc." + x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))" + /> +
+ <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile diff --git a/lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex b/lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex index a17714b..564296b 100644 --- a/lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex +++ b/lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex @@ -41,7 +41,7 @@ <.input field={f[:fast_index]} type="toggle" - label="Use Fast Indexing?" + label="Use Fast Indexing" label_suffix="(pro)" help="Experimental. Ignores 'Index Frequency'. Recommended for large channels that upload frequently. See below for more info" /> @@ -54,7 +54,7 @@ <.input field={f[:download_media]} type="toggle" - label="Download Media?" + label="Download Media" help="Unchecking still indexes media but it won't be downloaded until you enable this option" /> diff --git a/priv/repo/migrations/20240318161405_add_nfo_path_to_sources.exs b/priv/repo/migrations/20240318161405_add_nfo_path_to_sources.exs new file mode 100644 index 0000000..da1f1b9 --- /dev/null +++ b/priv/repo/migrations/20240318161405_add_nfo_path_to_sources.exs @@ -0,0 +1,10 @@ +defmodule Pinchflat.Repo.Migrations.AddNfoPathToSources do + use Ecto.Migration + + def change do + alter table(:sources) do + add :nfo_filepath, :string + add :series_directory, :string + end + end +end diff --git a/test/pinchflat/downloading/download_option_builder_test.exs b/test/pinchflat/downloading/download_option_builder_test.exs index 3f759b3..02103d6 100644 --- a/test/pinchflat/downloading/download_option_builder_test.exs +++ b/test/pinchflat/downloading/download_option_builder_test.exs @@ -222,6 +222,14 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do end end + describe "build_output_path_for/1" do + test "builds an output path for a source", %{media_item: media_item} do + path = DownloadOptionBuilder.build_output_path_for(media_item.source) + + assert path == "/tmp/test/media/%(title)S.%(ext)s" + end + end + defp update_media_profile_attribute(media_item_with_preloads, attrs) do media_item_with_preloads.source.media_profile |> Profiles.change_media_profile(attrs) diff --git a/test/pinchflat/metadata/metadata_file_helpers_test.exs b/test/pinchflat/metadata/metadata_file_helpers_test.exs index b28e497..511ec89 100644 --- a/test/pinchflat/metadata/metadata_file_helpers_test.exs +++ b/test/pinchflat/metadata/metadata_file_helpers_test.exs @@ -92,4 +92,52 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do assert Helpers.parse_upload_date(upload_date) == ~D[2021-01-01] end end + + describe "series_directory_from_media_filepath/1" do + test "returns base series directory if filepaths are setup as expected" do + good_filepaths = [ + "/media/season1/episode.mp4", + "/media/season 1/episode.mp4", + "/media/season.1/episode.mp4", + "/media/season_1/episode.mp4", + "/media/season-1/episode.mp4", + "/media/SEASON 1/episode.mp4", + "/media/SEASON.1/episode.mp4", + "/media/s1/episode.mp4", + "/media/s.1/episode.mp4", + "/media/s_1/episode.mp4", + "/media/s-1/episode.mp4", + "/media/s 1/episode.mp4", + "/media/S1/episode.mp4", + "/media/S.1/episode.mp4" + ] + + for filepath <- good_filepaths do + assert {:ok, "/media"} = Helpers.series_directory_from_media_filepath(filepath) + end + end + + test "returns an error if the season filepath can't be determined" do + bad_filepaths = [ + "/media/1/episode.mp4", + "/media/(s1)/episode.mp4", + "/media/episode.mp4", + "/media/s1e1/episode.mp4", + "/media/s1 e1/episode.mp4", + "/media/s1 (something else)/episode.mp4", + "/media/season1e1/episode.mp4", + "/media/season1 e1/episode.mp4", + "/media/seasoning1/episode.mp4", + "/media/season/episode.mp4", + "/media/series1/episode.mp4", + "/media/s/episode.mp4", + "/media/foo", + "/media/bar/" + ] + + for filepath <- bad_filepaths do + assert {:error, :indeterminable} = Helpers.series_directory_from_media_filepath(filepath) + end + end + end end diff --git a/test/pinchflat/metadata/nfo_builder_test.exs b/test/pinchflat/metadata/nfo_builder_test.exs index 696c978..c6119a4 100644 --- a/test/pinchflat/metadata/nfo_builder_test.exs +++ b/test/pinchflat/metadata/nfo_builder_test.exs @@ -2,37 +2,49 @@ defmodule Pinchflat.Metadata.NfoBuilderTest do use Pinchflat.DataCase alias Pinchflat.Metadata.NfoBuilder + alias Pinchflat.Filesystem.FilesystemHelpers setup do - {:ok, %{metadata: render_parsed_metadata(:media_metadata)}} + filepath = FilesystemHelpers.generate_metadata_tmpfile(:json) + + on_exit(fn -> File.rm!(filepath) end) + + {:ok, + %{ + metadata: render_parsed_metadata(:media_metadata), + filepath: filepath + }} end - describe "build_and_store_for_media_item/1" do - test "returns the filepath", %{metadata: metadata} do - result = NfoBuilder.build_and_store_for_media_item(metadata) + describe "build_and_store_for_media_item/2" do + test "returns the filepath", %{metadata: metadata, filepath: filepath} do + result = NfoBuilder.build_and_store_for_media_item(filepath, metadata) assert File.exists?(result) - - File.rm!(result) end - test "builds filepath based on media location", %{metadata: metadata} do - result = NfoBuilder.build_and_store_for_media_item(metadata) - - assert String.contains?(result, Path.rootname(metadata["filepath"])) - assert String.ends_with?(result, ".nfo") - - File.rm!(result) - end - - test "builds an NFO file", %{metadata: metadata} do - result = NfoBuilder.build_and_store_for_media_item(metadata) + test "builds an NFO file", %{metadata: metadata, filepath: filepath} do + result = NfoBuilder.build_and_store_for_media_item(filepath, metadata) nfo = File.read!(result) assert String.contains?(nfo, ~S()) assert String.contains?(nfo, "#{metadata["title"]}") + end + end - File.rm!(result) + describe "build_and_store_for_source/2" do + test "returns the filepath", %{metadata: metadata, filepath: filepath} do + result = NfoBuilder.build_and_store_for_source(filepath, metadata) + + assert File.exists?(result) + end + + test "builds an NFO file", %{metadata: metadata, filepath: filepath} do + result = NfoBuilder.build_and_store_for_source(filepath, metadata) + nfo = File.read!(result) + + assert String.contains?(nfo, ~S()) + assert String.contains?(nfo, "#{metadata["title"]}") end end end diff --git a/test/pinchflat/metadata/source_metadata_storage_worker_test.exs b/test/pinchflat/metadata/source_metadata_storage_worker_test.exs index 41b191e..c8a096e 100644 --- a/test/pinchflat/metadata/source_metadata_storage_worker_test.exs +++ b/test/pinchflat/metadata/source_metadata_storage_worker_test.exs @@ -2,10 +2,14 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do use Pinchflat.DataCase import Mox import Pinchflat.SourcesFixtures + import Pinchflat.ProfilesFixtures alias Pinchflat.Metadata.MetadataFileHelpers alias Pinchflat.Metadata.SourceMetadataStorageWorker + @source_details_ot "%(.{channel,channel_id,playlist_id,playlist_title,filename})j" + @metadata_ot "playlist:%()j" + setup :verify_on_exit! describe "kickoff_with_task/1" do @@ -27,8 +31,31 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do end describe "perform/1" do + test "won't call itself in an infinite loop" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()} + _url, _opts, ot when ot == @metadata_ot -> {:ok, "{}"} + end) + + source = source_fixture() + + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + + assert [] = all_enqueued(worker: SourceMetadataStorageWorker) + end + + test "does not blow up if the record doesn't exist" do + assert :ok = perform_job(SourceMetadataStorageWorker, %{id: 0}) + end + end + + describe "perform/1 when testing metadata storage" do test "sets metadata location for source" do - expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end) + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()} + _url, _opts, ot when ot == @metadata_ot -> {:ok, "{}"} + end) + source = Repo.preload(source_fixture(), :metadata) refute source.metadata @@ -43,7 +70,11 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do test "fetches and stores returned metadata for source" do source = source_fixture() file_contents = Phoenix.json_library().encode!(%{"title" => "test"}) - expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, file_contents} end) + + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()} + _url, _opts, ot when ot == @metadata_ot -> {:ok, file_contents} + end) perform_job(SourceMetadataStorageWorker, %{id: source.id}) source = Repo.preload(Repo.reload(source), :metadata) @@ -51,32 +82,106 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do assert metadata == %{"title" => "test"} end + end - test "won't call itself in an infinite loop" do - stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end) - source = source_fixture() + describe "perform/1 when determining the series_directory" do + test "sets the series directory based on the returned media filepath" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> + filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"]) + {:ok, source_details_return_fixture(%{filename: filename})} + + _url, _opts, ot when ot == @metadata_ot -> + {:ok, "{}"} + end) + + source = source_fixture(%{series_directory: nil}) perform_job(SourceMetadataStorageWorker, %{id: source.id}) - perform_job(SourceMetadataStorageWorker, %{id: source.id}) + source = Repo.reload(source) - assert [_] = all_enqueued(worker: SourceMetadataStorageWorker) + assert source.series_directory end - test "doesn't prevent over source jobs from running" do - stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end) - source_1 = source_fixture() - source_2 = source_fixture() + test "does not set the series directory if it cannot be determined" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> + filename = Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"]) - perform_job(SourceMetadataStorageWorker, %{id: source_1.id}) - perform_job(SourceMetadataStorageWorker, %{id: source_1.id}) - perform_job(SourceMetadataStorageWorker, %{id: source_2.id}) - perform_job(SourceMetadataStorageWorker, %{id: source_2.id}) + {:ok, source_details_return_fixture(%{filename: filename})} - assert [_, _] = all_enqueued(worker: SourceMetadataStorageWorker) + _url, _opts, ot when ot == @metadata_ot -> + {:ok, "{}"} + end) + + source = source_fixture(%{series_directory: nil}) + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + source = Repo.reload(source) + + refute source.series_directory + end + end + + describe "perform/1 when storing the series NFO" do + test "stores the NFO if specified" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> + filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"]) + + {:ok, source_details_return_fixture(%{filename: filename})} + + _url, _opts, ot when ot == @metadata_ot -> + {:ok, "{}"} + end) + + profile = media_profile_fixture(%{download_nfo: true}) + source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id}) + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + source = Repo.reload(source) + + assert source.nfo_filepath + assert source.nfo_filepath == Path.join([source.series_directory, "tvshow.nfo"]) + assert File.exists?(source.nfo_filepath) + + File.rm!(source.nfo_filepath) end - test "does not blow up if the record doesn't exist" do - assert :ok = perform_job(SourceMetadataStorageWorker, %{id: 0}) + test "does not store the NFO if not specified" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> + filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"]) + + {:ok, source_details_return_fixture(%{filename: filename})} + + _url, _opts, ot when ot == @metadata_ot -> + {:ok, "{}"} + end) + + profile = media_profile_fixture(%{download_nfo: false}) + source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id}) + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + source = Repo.reload(source) + + refute source.nfo_filepath + end + + test "does not store the NFO if the series directory cannot be determined" do + stub(YtDlpRunnerMock, :run, fn + _url, _opts, ot when ot == @source_details_ot -> + filename = Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"]) + + {:ok, source_details_return_fixture(%{filename: filename})} + + _url, _opts, ot when ot == @metadata_ot -> + {:ok, "{}"} + end) + + profile = media_profile_fixture(%{download_nfo: true}) + source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id}) + perform_job(SourceMetadataStorageWorker, %{id: source.id}) + source = Repo.reload(source) + + refute source.nfo_filepath end end end diff --git a/test/pinchflat/sources_test.exs b/test/pinchflat/sources_test.exs index 103a33f..7dcbe07 100644 --- a/test/pinchflat/sources_test.exs +++ b/test/pinchflat/sources_test.exs @@ -8,6 +8,7 @@ defmodule Pinchflat.SourcesTest do alias Pinchflat.Sources alias Pinchflat.Sources.Source + alias Pinchflat.Filesystem.FilesystemHelpers alias Pinchflat.Metadata.MetadataFileHelpers alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.FastIndexing.FastIndexingWorker @@ -57,7 +58,7 @@ defmodule Pinchflat.SourcesTest do end end - describe "create_source/1" do + describe "create_source/2" do test "creates a source and adds name + ID from runner response for channels" do expect(YtDlpRunnerMock, :run, &channel_mock/3) @@ -253,7 +254,23 @@ defmodule Pinchflat.SourcesTest do end end - describe "update_source/2" do + describe "create_source/2 when testing options" do + test "run_post_commit_tasks: false won't enqueue post-commit tasks" do + expect(YtDlpRunnerMock, :run, &channel_mock/3) + + valid_attrs = %{ + media_profile_id: media_profile_fixture().id, + original_url: "https://www.youtube.com/channel/abc123" + } + + assert {:ok, %Source{}} = Sources.create_source(valid_attrs, run_post_commit_tasks: false) + + refute_enqueued(worker: MediaCollectionIndexingWorker) + refute_enqueued(worker: SourceMetadataStorageWorker) + end + end + + describe "update_source/3" do test "updates with valid data updates the source" do source = source_fixture() update_attrs = %{collection_name: "some updated name"} @@ -426,6 +443,20 @@ defmodule Pinchflat.SourcesTest do end end + describe "update_source/3 when testing options" do + test "run_post_commit_tasks: false won't enqueue post-commit tasks" do + source = source_fixture(%{fast_index: false, download_media: false, index_frequency_minutes: -1}) + update_attrs = %{fast_index: true, download_media: true, index_frequency_minutes: 100} + + assert {:ok, %Source{}} = Sources.update_source(source, update_attrs, run_post_commit_tasks: false) + + refute_enqueued(worker: MediaCollectionIndexingWorker) + refute_enqueued(worker: SourceMetadataStorageWorker) + refute_enqueued(worker: MediaDownloadWorker) + refute_enqueued(worker: FastIndexingWorker) + end + end + describe "delete_source/2" do test "it deletes the source" do source = source_fixture() @@ -474,9 +505,19 @@ defmodule Pinchflat.SourcesTest do {:ok, updated_source} = Sources.update_source(source, update_attrs) - assert {:ok, _} = Sources.delete_source(updated_source, delete_files: true) + assert {:ok, _} = Sources.delete_source(updated_source) refute File.exists?(updated_source.metadata.metadata_filepath) end + + test "does not delete the source's non-metadata files" do + filepath = FilesystemHelpers.generate_metadata_tmpfile(:nfo) + source = source_fixture(%{nfo_filepath: filepath}) + + assert {:ok, _} = Sources.delete_source(source) + assert File.exists?(filepath) + + File.rm!(filepath) + end end describe "delete_source/2 when deleting files" do @@ -498,6 +539,15 @@ defmodule Pinchflat.SourcesTest do refute File.exists?(media_item.media_filepath) end + + test "deletes the source's non-metadata files" do + filepath = FilesystemHelpers.generate_metadata_tmpfile(:nfo) + source = source_fixture(%{nfo_filepath: filepath}) + + assert {:ok, _} = Sources.delete_source(source, delete_files: true) + + refute File.exists?(filepath) + end end describe "change_source/3" do diff --git a/test/pinchflat/yt_dlp/media_collection_test.exs b/test/pinchflat/yt_dlp/media_collection_test.exs index 1f88c53..8fbf6fc 100644 --- a/test/pinchflat/yt_dlp/media_collection_test.exs +++ b/test/pinchflat/yt_dlp/media_collection_test.exs @@ -97,7 +97,7 @@ defmodule Pinchflat.YtDlp.MediaCollectionTest do test "it passes the expected args to the backend runner" do expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot -> assert opts == [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1] - assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j" + assert ot == "%(.{channel,channel_id,playlist_id,playlist_title,filename})j" {:ok, "{}"} end) diff --git a/test/support/fixtures/sources_fixtures.ex b/test/support/fixtures/sources_fixtures.ex index 7966edd..0db2e78 100644 --- a/test/support/fixtures/sources_fixtures.ex +++ b/test/support/fixtures/sources_fixtures.ex @@ -85,4 +85,18 @@ defmodule Pinchflat.SourcesFixtures do source_attributes |> Enum.map_join("\n", &Phoenix.json_library().encode!(&1)) end + + def source_details_return_fixture(attrs \\ %{}) do + channel_id = Faker.String.base64(12) + + %{ + channel_id: channel_id, + channel: "Channel Name", + playlist_id: channel_id, + playlist_title: "Channel Name", + filename: Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"]) + } + |> Map.merge(attrs) + |> Phoenix.json_library().encode!() + end end