diff --git a/lib/pinchflat/downloading/media_download_worker.ex b/lib/pinchflat/downloading/media_download_worker.ex index c7ea232..0605bcd 100644 --- a/lib/pinchflat/downloading/media_download_worker.ex +++ b/lib/pinchflat/downloading/media_download_worker.ex @@ -12,6 +12,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do alias Pinchflat.Tasks alias Pinchflat.Repo alias Pinchflat.Media + alias Pinchflat.Media.FileDeletion alias Pinchflat.Downloading.MediaDownloader alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner @@ -85,6 +86,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do media_redownloaded_at: get_redownloaded_at(is_quality_upgrade) }) + :ok = FileDeletion.delete_outdated_files(media_item, updated_media_item) run_user_script(:media_downloaded, updated_media_item) :ok diff --git a/lib/pinchflat/media/file_deletion.ex b/lib/pinchflat/media/file_deletion.ex new file mode 100644 index 0000000..7ed01a8 --- /dev/null +++ b/lib/pinchflat/media/file_deletion.ex @@ -0,0 +1,54 @@ +defmodule Pinchflat.Media.FileDeletion do + @moduledoc """ + Functions for deleting files that are no longer needed by media items. + """ + + alias Pinchflat.Utils.MapUtils + alias Pinchflat.Media.MediaItem + alias Pinchflat.Utils.FilesystemUtils, as: FSUtils + + @doc """ + Deletes files that are no longer needed by a media item. + + This means that if a media item has been updated, the old and new versions + can be passed and any files that are no longer needed will be deleted. + + An example is a video that gets its quality upgraded and its name changes + between original download and re-download. The old file will exist on-disk + with the old name but the database entry will point to the new file. This + function can be used to delete the old file in this case. + + Returns :ok + """ + def delete_outdated_files(old_media_item, new_media_item) do + non_subtitle_keys = MediaItem.filepath_attributes() -- [:subtitle_filepaths] + + old_non_subtitles = Map.take(old_media_item, non_subtitle_keys) + old_subtitles = MapUtils.from_nested_list(old_media_item.subtitle_filepaths) + new_non_subtitles = Map.take(new_media_item, non_subtitle_keys) + new_subtitles = MapUtils.from_nested_list(new_media_item.subtitle_filepaths) + + handle_file_deletion(old_non_subtitles, new_non_subtitles) + handle_file_deletion(old_subtitles, new_subtitles) + + :ok + end + + defp handle_file_deletion(old_attributes, new_attributes) do + # The logic: + # - A file should only be deleted if it exists and the new file is different + # - The new attributes are the ones we're interested in keeping + # - If the old attributes have a key that doesn't exist in the new attributes, don't touch it. + # This is good for archiving but may be unpopular for other users so this may change. + + Enum.each(new_attributes, fn {key, new_filepath} -> + old_filepath = Map.get(old_attributes, key) + files_have_changed = old_filepath && new_filepath && old_filepath != new_filepath + files_exist_on_disk = files_have_changed && File.exists?(old_filepath) && File.exists?(new_filepath) + + if files_exist_on_disk && !FSUtils.filepaths_reference_same_file?(old_filepath, new_filepath) do + FSUtils.delete_file_and_remove_empty_directories(old_filepath) + end + end) + end +end diff --git a/lib/pinchflat/utils/filesystem_utils.ex b/lib/pinchflat/utils/filesystem_utils.ex index 8652192..355bb19 100644 --- a/lib/pinchflat/utils/filesystem_utils.ex +++ b/lib/pinchflat/utils/filesystem_utils.ex @@ -20,6 +20,24 @@ defmodule Pinchflat.Utils.FilesystemUtils do end end + @doc """ + Checks if two filepaths reference the same file. + + Useful if you have a relative and absolute filepath and want to be sure they're the same file. + Also works with symlinks. + + Returns boolean() + """ + def filepaths_reference_same_file?(filepath_1, filepath_2) do + {:ok, stat_1} = File.stat(filepath_1) + {:ok, stat_2} = File.stat(filepath_2) + + identifier_1 = "#{stat_1.major_device}:#{stat_1.minor_device}:#{stat_1.inode}" + identifier_2 = "#{stat_2.major_device}:#{stat_2.minor_device}:#{stat_2.inode}" + + identifier_1 == identifier_2 + end + @doc """ Generates a temporary file and returns its path. The file is empty and has the given type. Generates all the directories in the path if they don't exist. diff --git a/lib/pinchflat/utils/map_utils.ex b/lib/pinchflat/utils/map_utils.ex new file mode 100644 index 0000000..41f03a1 --- /dev/null +++ b/lib/pinchflat/utils/map_utils.ex @@ -0,0 +1,17 @@ +defmodule Pinchflat.Utils.MapUtils do + @moduledoc """ + Utility methods for working with maps + """ + + @doc """ + Converts a nested list of 2-element tuples or lists into a map. + + Returns map() + """ + def from_nested_list(list) do + Enum.reduce(list, %{}, fn + [key, value], acc -> Map.put(acc, key, value) + {key, value}, acc -> Map.put(acc, key, value) + end) + end +end diff --git a/test/pinchflat/downloading/media_download_worker_test.exs b/test/pinchflat/downloading/media_download_worker_test.exs index 315ac35..a34adfb 100644 --- a/test/pinchflat/downloading/media_download_worker_test.exs +++ b/test/pinchflat/downloading/media_download_worker_test.exs @@ -236,6 +236,26 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do perform_job(MediaDownloadWorker, %{id: media_item.id, force: true}) end + + test "deletes old files if the media item has been updated" do + expect(YtDlpRunnerMock, :run, 1, fn _url, _opts, _ot, _addl -> + tmp_media_item = media_item_with_attachments() + metadata = render_parsed_metadata(:media_metadata) + metadata = Map.put(metadata, "filepath", tmp_media_item.media_filepath) + + {:ok, Phoenix.json_library().encode!(metadata)} + end) + + expect(YtDlpRunnerMock, :run, 1, fn _url, _opts, _ot, _addl -> {:ok, ""} end) + + old_media_item = media_item_with_attachments() + perform_job(MediaDownloadWorker, %{id: old_media_item.id, force: true}) + updated_media_item = Repo.reload(old_media_item) + + assert updated_media_item.media_filepath != old_media_item.media_filepath + refute File.exists?(old_media_item.media_filepath) + assert File.exists?(updated_media_item.media_filepath) + end end describe "perform/1 when testing user script callbacks" do diff --git a/test/pinchflat/media/file_deletion_test.exs b/test/pinchflat/media/file_deletion_test.exs new file mode 100644 index 0000000..6494f21 --- /dev/null +++ b/test/pinchflat/media/file_deletion_test.exs @@ -0,0 +1,77 @@ +defmodule Pinchflat.Media.FileDeletionTest do + use Pinchflat.DataCase + + import Pinchflat.MediaFixtures + + alias Pinchflat.Media.FileDeletion + + describe "delete_outdated_files/2" do + test "deletes outdated non-subtitle files" do + new_media_item = media_item_with_attachments() + old_media_item = media_item_with_attachments() + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(new_media_item.media_filepath) + refute File.exists?(old_media_item.media_filepath) + end + + test "doesn't delete non-subtitle files if the new file is the same" do + new_media_item = media_item_with_attachments() + old_media_item = media_item_fixture(%{media_filepath: new_media_item.media_filepath}) + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(new_media_item.media_filepath) + assert File.exists?(old_media_item.media_filepath) + end + + test "doesn't delete the old file if the new file is missing that key" do + new_media_item = media_item_fixture(%{media_filepath: nil}) + old_media_item = media_item_with_attachments() + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(old_media_item.media_filepath) + end + + test "deletes outdated subtitle files" do + new_media_item = media_item_with_attachments() + old_media_item = media_item_with_attachments() + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(get_subtitle_filepath(new_media_item, "en")) + refute File.exists?(get_subtitle_filepath(old_media_item, "en")) + end + + test "keeps old subtitle files if the new file is the same" do + new_media_item = media_item_with_attachments() + old_media_item = media_item_fixture(%{subtitle_filepaths: new_media_item.subtitle_filepaths}) + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(get_subtitle_filepath(new_media_item, "en")) + assert File.exists?(get_subtitle_filepath(old_media_item, "en")) + end + + test "doesn't delete old subtitle files if the new file is missing that key" do + new_media_item = media_item_fixture(%{subtitle_filepaths: []}) + old_media_item = media_item_with_attachments() + + assert :ok = FileDeletion.delete_outdated_files(old_media_item, new_media_item) + + assert File.exists?(get_subtitle_filepath(old_media_item, "en")) + end + end + + defp get_subtitle_filepath(media_item, language) do + Enum.reduce_while(media_item.subtitle_filepaths, nil, fn [lang, filepath], acc -> + if lang == language do + {:halt, filepath} + else + {:cont, acc} + end + end) + end +end diff --git a/test/pinchflat/utils/filesystem_utils_test.exs b/test/pinchflat/utils/filesystem_utils_test.exs index a964e48..33b5c8d 100644 --- a/test/pinchflat/utils/filesystem_utils_test.exs +++ b/test/pinchflat/utils/filesystem_utils_test.exs @@ -37,6 +37,46 @@ defmodule Pinchflat.Utils.FilesystemUtilsTest do end end + describe "filepaths_reference_same_file?/2" do + setup do + filepath = FilesystemUtils.generate_metadata_tmpfile(:json) + + on_exit(fn -> File.rm!(filepath) end) + + {:ok, %{filepath: filepath}} + end + + test "returns true if the files are the same", %{filepath: filepath} do + assert FilesystemUtils.filepaths_reference_same_file?(filepath, filepath) + end + + test "returns true if different filepaths point to the same file", %{filepath: filepath} do + short_path = Path.expand(filepath) + long_path = Path.join(["/tmp", "..", filepath]) + + assert short_path != long_path + assert FilesystemUtils.filepaths_reference_same_file?(short_path, long_path) + end + + test "returns true if the files are symlinked", %{filepath: filepath} do + tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory) + other_filepath = Path.join([tmpfile_directory, "symlink.json"]) + :ok = File.ln_s!(filepath, other_filepath) + + assert FilesystemUtils.filepaths_reference_same_file?(filepath, other_filepath) + + File.rm!(other_filepath) + end + + test "returns false if the files are different", %{filepath: filepath} do + other_filepath = FilesystemUtils.generate_metadata_tmpfile(:json) + + refute FilesystemUtils.filepaths_reference_same_file?(filepath, other_filepath) + + File.rm!(other_filepath) + end + end + describe "generate_metadata_tmpfile/1" do test "creates a tmpfile and returns its path" do res = FilesystemUtils.generate_metadata_tmpfile(:json) diff --git a/test/pinchflat/utils/map_utils_test.exs b/test/pinchflat/utils/map_utils_test.exs new file mode 100644 index 0000000..c16356a --- /dev/null +++ b/test/pinchflat/utils/map_utils_test.exs @@ -0,0 +1,31 @@ +defmodule Pinchflat.Utils.MapUtilsTest do + use Pinchflat.DataCase + + alias Pinchflat.Utils.MapUtils + + describe "from_nested_list/1" do + test "creates a map from a nested 2-element tuple list" do + list = [ + {"key1", "value1"}, + {"key2", "value2"} + ] + + assert MapUtils.from_nested_list(list) == %{ + "key1" => "value1", + "key2" => "value2" + } + end + + test "creates a map from a nested 2-element list of lists" do + list = [ + ["key1", "value1"], + ["key2", "value2"] + ] + + assert MapUtils.from_nested_list(list) == %{ + "key1" => "value1", + "key2" => "value2" + } + end + end +end diff --git a/test/support/files/subtitle.srt b/test/support/files/subtitle.srt new file mode 100644 index 0000000..3251437 --- /dev/null +++ b/test/support/files/subtitle.srt @@ -0,0 +1,36 @@ +1 +00:00:00,000 --> 00:00:02,500 +Welcome to the Example Subtitle File! + +2 +00:00:03,000 --> 00:00:06,000 +This is a demonstration of SRT subtitles. + +3 +00:00:07,000 --> 00:00:10,500 +You can use SRT files to add subtitles to your videos. + +4 +00:00:12,000 --> 00:00:15,000 +Each subtitle entry consists of a number, a timecode, +and the subtitle text. + +5 +00:00:16,000 --> 00:00:20,000 +The timecode format is hours:minutes:seconds,milliseconds. + +6 +00:00:21,000 --> 00:00:25,000 +You can adjust the timing to match your video. + +7 +00:00:26,000 --> 00:00:30,000 +Make sure the subtitle text is clear and readable. + +8 +00:00:31,000 --> 00:00:35,000 +And that's how you create an SRT subtitle file! + +9 +00:00:36,000 --> 00:00:40,000 +Enjoy adding subtitles to your videos! diff --git a/test/support/fixtures/media_fixtures.ex b/test/support/fixtures/media_fixtures.ex index dacddb3..fa846a6 100644 --- a/test/support/fixtures/media_fixtures.ex +++ b/test/support/fixtures/media_fixtures.ex @@ -73,16 +73,19 @@ defmodule Pinchflat.MediaFixtures do "#{:rand.uniform(1_000_000)}" ]) - stored_media_filepath = Path.join(base_dir, "#media.mp4") + stored_media_filepath = Path.join(base_dir, "media.mp4") thumbnail_filepath = Path.join(base_dir, "thumbnail.jpg") + subtitle_filepath = Path.join(base_dir, "subtitle.en.srt") FilesystemUtils.cp_p!(media_filepath_fixture(), stored_media_filepath) FilesystemUtils.cp_p!(thumbnail_filepath_fixture(), thumbnail_filepath) + FilesystemUtils.cp_p!(subtitle_filepath_fixture(), subtitle_filepath) merged_attrs = Map.merge(attrs, %{ media_filepath: stored_media_filepath, - thumbnail_filepath: thumbnail_filepath + thumbnail_filepath: thumbnail_filepath, + subtitle_filepaths: [["en", subtitle_filepath]] }) media_item_fixture(merged_attrs) @@ -124,6 +127,16 @@ defmodule Pinchflat.MediaFixtures do ]) end + def subtitle_filepath_fixture do + Path.join([ + File.cwd!(), + "test", + "support", + "files", + "subtitle.srt" + ]) + end + def infojson_filepath_fixture do Path.join([ File.cwd!(),