mirror of
https://github.com/kieraneglin/pinchflat.git
synced 2026-01-23 10:26:07 +00:00
Redo indexing mechanism (#16)
* Bumped up the line length because I fear no man * Refactored indexing Previously, indexing worked by collecting the video IDs of only videos that matched indexing criteria. This new model instead stores ALL videos for a given source, but will only _download_ videos that meet that criteria. This lets us backfill without indexing, makes it easier to add in other backends, lets us download one-off videos for a source that don't quite meet criteria, you name it. * Updated media finders to respect format filters; Added credo file
This commit is contained in:
Kieran
GitHub
parent
e1565ad22f
commit
e0637331bb
31 changed files with 646 additions and 385 deletions
215
.credo.exs
Normal file
215
.credo.exs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
# This file contains the configuration for Credo and you are probably reading
|
||||
# this after creating it with `mix credo.gen.config`.
|
||||
#
|
||||
# If you find anything wrong or unclear in this file, please report an
|
||||
# issue on GitHub: https://github.com/rrrene/credo/issues
|
||||
#
|
||||
%{
|
||||
#
|
||||
# You can have as many configs as you like in the `configs:` field.
|
||||
configs: [
|
||||
%{
|
||||
#
|
||||
# Run any config using `mix credo -C <name>`. If no config name is given
|
||||
# "default" is used.
|
||||
#
|
||||
name: "default",
|
||||
#
|
||||
# These are the files included in the analysis:
|
||||
files: %{
|
||||
#
|
||||
# You can give explicit globs or simply directories.
|
||||
# In the latter case `**/*.{ex,exs}` will be used.
|
||||
#
|
||||
included: [
|
||||
"lib/",
|
||||
"src/",
|
||||
"test/",
|
||||
"web/",
|
||||
"apps/*/lib/",
|
||||
"apps/*/src/",
|
||||
"apps/*/test/",
|
||||
"apps/*/web/"
|
||||
],
|
||||
excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"]
|
||||
},
|
||||
#
|
||||
# Load and configure plugins here:
|
||||
#
|
||||
plugins: [],
|
||||
#
|
||||
# If you create your own checks, you must specify the source files for
|
||||
# them here, so they can be loaded by Credo before running the analysis.
|
||||
#
|
||||
requires: [],
|
||||
#
|
||||
# If you want to enforce a style guide and need a more traditional linting
|
||||
# experience, you can change `strict` to `true` below:
|
||||
#
|
||||
strict: false,
|
||||
#
|
||||
# To modify the timeout for parsing files, change this value:
|
||||
#
|
||||
parse_timeout: 5000,
|
||||
#
|
||||
# If you want to use uncolored output by default, you can change `color`
|
||||
# to `false` below:
|
||||
#
|
||||
color: true,
|
||||
#
|
||||
# You can customize the parameters of any check by adding a second element
|
||||
# to the tuple.
|
||||
#
|
||||
# To disable a check put `false` as second element:
|
||||
#
|
||||
# {Credo.Check.Design.DuplicatedCode, false}
|
||||
#
|
||||
checks: %{
|
||||
enabled: [
|
||||
#
|
||||
## Consistency Checks
|
||||
#
|
||||
{Credo.Check.Consistency.ExceptionNames, []},
|
||||
{Credo.Check.Consistency.LineEndings, []},
|
||||
{Credo.Check.Consistency.ParameterPatternMatching, []},
|
||||
{Credo.Check.Consistency.SpaceAroundOperators, []},
|
||||
{Credo.Check.Consistency.SpaceInParentheses, []},
|
||||
{Credo.Check.Consistency.TabsOrSpaces, []},
|
||||
|
||||
#
|
||||
## Design Checks
|
||||
#
|
||||
# You can customize the priority of any check
|
||||
# Priority values are: `low, normal, high, higher`
|
||||
#
|
||||
{Credo.Check.Design.AliasUsage, [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]},
|
||||
{Credo.Check.Design.TagFIXME, []},
|
||||
# You can also customize the exit_status of each check.
|
||||
# If you don't want TODO comments to cause `mix credo` to fail, just
|
||||
# set this value to 0 (zero).
|
||||
#
|
||||
{Credo.Check.Design.TagTODO, [exit_status: 2]},
|
||||
|
||||
#
|
||||
## Readability Checks
|
||||
#
|
||||
{Credo.Check.Readability.AliasOrder, []},
|
||||
{Credo.Check.Readability.FunctionNames, []},
|
||||
{Credo.Check.Readability.LargeNumbers, []},
|
||||
{Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]},
|
||||
{Credo.Check.Readability.ModuleAttributeNames, []},
|
||||
{Credo.Check.Readability.ModuleDoc, []},
|
||||
{Credo.Check.Readability.ModuleNames, []},
|
||||
{Credo.Check.Readability.ParenthesesInCondition, []},
|
||||
{Credo.Check.Readability.ParenthesesOnZeroArityDefs, []},
|
||||
{Credo.Check.Readability.PipeIntoAnonymousFunctions, []},
|
||||
{Credo.Check.Readability.PredicateFunctionNames, []},
|
||||
{Credo.Check.Readability.PreferImplicitTry, []},
|
||||
{Credo.Check.Readability.RedundantBlankLines, []},
|
||||
{Credo.Check.Readability.Semicolons, []},
|
||||
{Credo.Check.Readability.SpaceAfterCommas, []},
|
||||
{Credo.Check.Readability.StringSigils, []},
|
||||
{Credo.Check.Readability.TrailingBlankLine, []},
|
||||
{Credo.Check.Readability.TrailingWhiteSpace, []},
|
||||
{Credo.Check.Readability.UnnecessaryAliasExpansion, []},
|
||||
{Credo.Check.Readability.VariableNames, []},
|
||||
{Credo.Check.Readability.WithSingleClause, []},
|
||||
|
||||
#
|
||||
## Refactoring Opportunities
|
||||
#
|
||||
{Credo.Check.Refactor.Apply, []},
|
||||
{Credo.Check.Refactor.CondStatements, []},
|
||||
{Credo.Check.Refactor.FilterCount, []},
|
||||
{Credo.Check.Refactor.FilterFilter, []},
|
||||
{Credo.Check.Refactor.FunctionArity, []},
|
||||
{Credo.Check.Refactor.LongQuoteBlocks, []},
|
||||
{Credo.Check.Refactor.MapJoin, []},
|
||||
{Credo.Check.Refactor.MatchInCondition, []},
|
||||
{Credo.Check.Refactor.NegatedConditionsInUnless, []},
|
||||
{Credo.Check.Refactor.NegatedConditionsWithElse, []},
|
||||
{Credo.Check.Refactor.Nesting, []},
|
||||
{Credo.Check.Refactor.RedundantWithClauseResult, []},
|
||||
{Credo.Check.Refactor.RejectReject, []},
|
||||
{Credo.Check.Refactor.UnlessWithElse, []},
|
||||
{Credo.Check.Refactor.WithClauses, []},
|
||||
|
||||
#
|
||||
## Warnings
|
||||
#
|
||||
{Credo.Check.Warning.ApplicationConfigInModuleAttribute, []},
|
||||
{Credo.Check.Warning.BoolOperationOnSameValues, []},
|
||||
{Credo.Check.Warning.Dbg, []},
|
||||
{Credo.Check.Warning.ExpensiveEmptyEnumCheck, []},
|
||||
{Credo.Check.Warning.IExPry, []},
|
||||
{Credo.Check.Warning.IoInspect, []},
|
||||
{Credo.Check.Warning.MissedMetadataKeyInLoggerConfig, []},
|
||||
{Credo.Check.Warning.OperationOnSameValues, []},
|
||||
{Credo.Check.Warning.OperationWithConstantResult, []},
|
||||
{Credo.Check.Warning.RaiseInsideRescue, []},
|
||||
{Credo.Check.Warning.SpecWithStruct, []},
|
||||
{Credo.Check.Warning.UnsafeExec, []},
|
||||
{Credo.Check.Warning.UnusedEnumOperation, []},
|
||||
{Credo.Check.Warning.UnusedFileOperation, []},
|
||||
{Credo.Check.Warning.UnusedKeywordOperation, []},
|
||||
{Credo.Check.Warning.UnusedListOperation, []},
|
||||
{Credo.Check.Warning.UnusedPathOperation, []},
|
||||
{Credo.Check.Warning.UnusedRegexOperation, []},
|
||||
{Credo.Check.Warning.UnusedStringOperation, []},
|
||||
{Credo.Check.Warning.UnusedTupleOperation, []},
|
||||
{Credo.Check.Warning.WrongTestFileExtension, []}
|
||||
],
|
||||
disabled: [
|
||||
#
|
||||
# Checks scheduled for next check update (opt-in for now, just replace `false` with `[]`)
|
||||
|
||||
#
|
||||
# Controversial and experimental checks (opt-in, just move the check to `:enabled`
|
||||
# and be sure to use `mix credo --strict` to see low priority checks)
|
||||
#
|
||||
{Credo.Check.Refactor.CyclomaticComplexity, []},
|
||||
{Credo.Check.Consistency.MultiAliasImportRequireUse, []},
|
||||
{Credo.Check.Consistency.UnusedVariableNames, []},
|
||||
{Credo.Check.Design.DuplicatedCode, []},
|
||||
{Credo.Check.Design.SkipTestWithoutComment, []},
|
||||
{Credo.Check.Readability.AliasAs, []},
|
||||
{Credo.Check.Readability.BlockPipe, []},
|
||||
{Credo.Check.Readability.ImplTrue, []},
|
||||
{Credo.Check.Readability.MultiAlias, []},
|
||||
{Credo.Check.Readability.NestedFunctionCalls, []},
|
||||
{Credo.Check.Readability.OneArityFunctionInPipe, []},
|
||||
{Credo.Check.Readability.OnePipePerLine, []},
|
||||
{Credo.Check.Readability.SeparateAliasRequire, []},
|
||||
{Credo.Check.Readability.SingleFunctionToBlockPipe, []},
|
||||
{Credo.Check.Readability.SinglePipe, []},
|
||||
{Credo.Check.Readability.Specs, []},
|
||||
{Credo.Check.Readability.StrictModuleLayout, []},
|
||||
{Credo.Check.Readability.WithCustomTaggedTuple, []},
|
||||
{Credo.Check.Refactor.ABCSize, []},
|
||||
{Credo.Check.Refactor.AppendSingleItem, []},
|
||||
{Credo.Check.Refactor.DoubleBooleanNegation, []},
|
||||
{Credo.Check.Refactor.FilterReject, []},
|
||||
{Credo.Check.Refactor.IoPuts, []},
|
||||
{Credo.Check.Refactor.MapMap, []},
|
||||
{Credo.Check.Refactor.ModuleDependencies, []},
|
||||
{Credo.Check.Refactor.NegatedIsNil, []},
|
||||
{Credo.Check.Refactor.PassAsyncInTestCases, []},
|
||||
{Credo.Check.Refactor.PipeChainStart, []},
|
||||
{Credo.Check.Refactor.RejectFilter, []},
|
||||
{Credo.Check.Refactor.VariableRebinding, []},
|
||||
{Credo.Check.Warning.LazyLogging, []},
|
||||
{Credo.Check.Warning.LeakyEnvironment, []},
|
||||
{Credo.Check.Warning.MapGetUnsafePass, []},
|
||||
{Credo.Check.Warning.MixEnv, []},
|
||||
{Credo.Check.Warning.UnsafeToAtom, []}
|
||||
|
||||
# {Credo.Check.Refactor.MapInto, []},
|
||||
|
||||
#
|
||||
# Custom checks can be created using `mix credo.gen.check`.
|
||||
#
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -3,5 +3,5 @@
|
|||
subdirectories: ["priv/*/migrations"],
|
||||
plugins: [Phoenix.LiveView.HTMLFormatter],
|
||||
inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
|
||||
line_length: 100
|
||||
line_length: 120
|
||||
]
|
||||
|
|
|
|||
2
.iex.exs
2
.iex.exs
|
|
@ -44,7 +44,7 @@ defmodule IexHelpers do
|
|||
:channel -> channel_url()
|
||||
end
|
||||
|
||||
SourceDetails.get_video_ids(source)
|
||||
SourceDetails.get_media_attributes(source)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -19,15 +19,31 @@ defmodule Pinchflat.Media do
|
|||
|
||||
@doc """
|
||||
Returns a list of pending media_items for a given source, where
|
||||
pending means the `media_filepath` is `nil`.
|
||||
pending means the `media_filepath` is `nil` AND the media_item
|
||||
matches the format selection rules of the parent media_profile.
|
||||
|
||||
See `build_format_clauses` but tl;dr is it _may_ filter based
|
||||
on shorts or livestreams depending on the media_profile settings.
|
||||
|
||||
Returns [%MediaItem{}, ...].
|
||||
"""
|
||||
def list_pending_media_items_for(%Source{} = source) do
|
||||
from(
|
||||
m in MediaItem,
|
||||
where: m.source_id == ^source.id and is_nil(m.media_filepath)
|
||||
)
|
||||
media_profile = Repo.preload(source, :media_profile).media_profile
|
||||
|
||||
MediaItem
|
||||
|> where([mi], mi.source_id == ^source.id and is_nil(mi.media_filepath))
|
||||
|> where(^build_format_clauses(media_profile))
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a list of downloaded media_items for a given source.
|
||||
|
||||
Returns [%MediaItem{}, ...].
|
||||
"""
|
||||
def list_downloaded_media_items_for(%Source{} = source) do
|
||||
MediaItem
|
||||
|> where([mi], mi.source_id == ^source.id and not is_nil(mi.media_filepath))
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
|
|
@ -72,4 +88,39 @@ defmodule Pinchflat.Media do
|
|||
def change_media_item(%MediaItem{} = media_item, attrs \\ %{}) do
|
||||
MediaItem.changeset(media_item, attrs)
|
||||
end
|
||||
|
||||
defp build_format_clauses(media_profile) do
|
||||
mapped_struct = Map.from_struct(media_profile)
|
||||
|
||||
Enum.reduce(mapped_struct, dynamic(true), fn attr, dynamic ->
|
||||
case {attr, media_profile} do
|
||||
{{:shorts_behaviour, :only}, %{livestream_behaviour: :only}} ->
|
||||
dynamic([mi], ^dynamic and (mi.livestream == true or fragment("? ILIKE ?", mi.original_url, "%/shorts/%")))
|
||||
|
||||
# Technically redundant, but makes the other clauses easier to parse
|
||||
# (redundant because this condition is the same as the condition above, just flipped)
|
||||
{{:livestream_behaviour, :only}, %{shorts_behaviour: :only}} ->
|
||||
dynamic
|
||||
|
||||
{{:shorts_behaviour, :only}, _} ->
|
||||
# return records with /shorts/ in the original_url
|
||||
dynamic([mi], ^dynamic and fragment("? ILIKE ?", mi.original_url, "%/shorts/%"))
|
||||
|
||||
{{:livestream_behaviour, :only}, _} ->
|
||||
# return records with livestream: true
|
||||
dynamic([mi], ^dynamic and mi.livestream == true)
|
||||
|
||||
{{:shorts_behaviour, :exclude}, %{livestream_behaviour: lb}} when lb != :only ->
|
||||
# return records without /shorts/ in the original_url
|
||||
dynamic([mi], ^dynamic and fragment("? NOT ILIKE ?", mi.original_url, "%/shorts/%"))
|
||||
|
||||
{{:livestream_behaviour, :exclude}, %{shorts_behaviour: sb}} when sb != :only ->
|
||||
# return records with livestream: false
|
||||
dynamic([mi], ^dynamic and mi.livestream == false)
|
||||
|
||||
_ ->
|
||||
dynamic
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -13,17 +13,21 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
@allowed_fields ~w(
|
||||
title
|
||||
media_id
|
||||
original_url
|
||||
livestream
|
||||
media_filepath
|
||||
source_id
|
||||
subtitle_filepaths
|
||||
thumbnail_filepath
|
||||
metadata_filepath
|
||||
)a
|
||||
@required_fields ~w(media_id source_id)a
|
||||
@required_fields ~w(title original_url livestream media_id source_id)a
|
||||
|
||||
schema "media_items" do
|
||||
field :title, :string
|
||||
field :media_id, :string
|
||||
field :original_url, :string
|
||||
field :livestream, :boolean, default: false
|
||||
field :media_filepath, :string
|
||||
field :thumbnail_filepath, :string
|
||||
field :metadata_filepath, :string
|
||||
|
|
|
|||
|
|
@ -4,18 +4,26 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollection do
|
|||
videos (aka: a source [ie: channels, playlists]).
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Returns a list of strings representing the video ids in the collection.
|
||||
alias Pinchflat.Utils.FunctionUtils
|
||||
|
||||
Returns {:ok, [binary()]} | {:error, any, ...}.
|
||||
@doc """
|
||||
Returns a list of maps representing the videos in the collection.
|
||||
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_video_ids(url, command_opts \\ []) do
|
||||
def get_media_attributes(url, command_opts \\ []) do
|
||||
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
opts = command_opts ++ [:simulate, :skip_download]
|
||||
|
||||
case runner.run(url, opts, "%(id)s") do
|
||||
{:ok, output} -> {:ok, String.split(output, "\n", trim: true)}
|
||||
res -> res
|
||||
case runner.run(url, opts, "%(.{id,title,was_live,original_url})j") do
|
||||
{:ok, output} ->
|
||||
output
|
||||
|> String.split("\n", trim: true)
|
||||
|> Enum.map(&Phoenix.json_library().decode!/1)
|
||||
|> FunctionUtils.wrap_ok()
|
||||
|
||||
res ->
|
||||
res
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -6,11 +6,8 @@ defmodule Pinchflat.MediaClient.SourceDetails do
|
|||
it open-ish for future expansion (just in case).
|
||||
"""
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.MediaSource.Source
|
||||
|
||||
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
|
||||
alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
|
||||
|
||||
@doc """
|
||||
Gets a source's ID and name from its URL using the given backend.
|
||||
|
|
@ -22,24 +19,19 @@ defmodule Pinchflat.MediaClient.SourceDetails do
|
|||
end
|
||||
|
||||
@doc """
|
||||
Returns a list of video IDs for the given source URL OR source record using the given backend.
|
||||
Returns a list of basic video data mapsfor the given source URL OR
|
||||
source record using the given backend.
|
||||
|
||||
If passing a source record, the call to the backend may have custom options applied based on
|
||||
the `option_builder`.
|
||||
|
||||
Returns {:ok, list(binary())} | {:error, any, ...}.
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_video_ids(sourceable, backend \\ :yt_dlp)
|
||||
def get_media_attributes(sourceable, backend \\ :yt_dlp)
|
||||
|
||||
def get_video_ids(%Source{} = source, backend) do
|
||||
media_profile = Repo.preload(source, :media_profile).media_profile
|
||||
{:ok, options} = option_builder(backend).build(media_profile)
|
||||
|
||||
source_module(backend).get_video_ids(source.collection_id, options)
|
||||
def get_media_attributes(%Source{} = source, backend) do
|
||||
source_module(backend).get_media_attributes(source.collection_id)
|
||||
end
|
||||
|
||||
def get_video_ids(source_url, backend) when is_binary(source_url) do
|
||||
source_module(backend).get_video_ids(source_url)
|
||||
def get_media_attributes(source_url, backend) when is_binary(source_url) do
|
||||
source_module(backend).get_media_attributes(source_url)
|
||||
end
|
||||
|
||||
defp source_module(backend) do
|
||||
|
|
@ -47,10 +39,4 @@ defmodule Pinchflat.MediaClient.SourceDetails do
|
|||
:yt_dlp -> YtDlpSource
|
||||
end
|
||||
end
|
||||
|
||||
defp option_builder(backend) do
|
||||
case backend do
|
||||
:yt_dlp -> YtDlpIndexOptionBuilder
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ defmodule Pinchflat.MediaSource do
|
|||
alias Pinchflat.Repo
|
||||
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Tasks.SourceTasks
|
||||
alias Pinchflat.MediaSource.Source
|
||||
alias Pinchflat.MediaClient.SourceDetails
|
||||
|
|
@ -39,26 +38,6 @@ defmodule Pinchflat.MediaSource do
|
|||
|> commit_and_start_indexing()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Given a media source, creates (indexes) the media by creating media_items for each
|
||||
media ID in the source.
|
||||
|
||||
Returns [%MediaItem{}, ...] | [%Ecto.Changeset{}, ...]
|
||||
"""
|
||||
def index_media_items(%Source{} = source) do
|
||||
{:ok, media_ids} = SourceDetails.get_video_ids(source.original_url)
|
||||
|
||||
media_ids
|
||||
|> Enum.map(fn media_id ->
|
||||
attrs = %{source_id: source.id, media_id: media_id}
|
||||
|
||||
case Media.create_media_item(attrs) do
|
||||
{:ok, media_item} -> media_item
|
||||
{:error, changeset} -> changeset
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates a source. May attempt to pull additional source details from the
|
||||
original_url (if changed). May attempt to start indexing the source's
|
||||
|
|
@ -101,6 +80,9 @@ defmodule Pinchflat.MediaSource do
|
|||
This means that it'll go for it even if a changeset is otherwise invalid. This
|
||||
is pretty easy to change, but for MVP I'm not concerned.
|
||||
|
||||
NOTE: When operating in the ideal path, this effectively adds an API call
|
||||
to the source creation/update process. Should be used only when needed.
|
||||
|
||||
IDEA: Maybe I could discern `collection_type` based on the original URL?
|
||||
It also seems like it's a channel when the returned yt-dlp channel_id is the
|
||||
same as the playlist_id - maybe could use that?
|
||||
|
|
|
|||
|
|
@ -45,7 +45,9 @@ defmodule Pinchflat.Profiles.MediaProfile do
|
|||
# through the entire collection to determine if a video is a short or
|
||||
# a livestream.
|
||||
# NOTE: these can BOTH be set to :only which will download shorts and
|
||||
# livestreams _only_ and ignore regular videos.
|
||||
# livestreams _only_ and ignore regular videos. The redundant case
|
||||
# is when one is set to :only and the other is set to :exclude.
|
||||
# See `build_format_clauses` in the Media context for more.
|
||||
field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
|
||||
field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
|
||||
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
|
||||
@moduledoc """
|
||||
Builds the options for yt-dlp to index a media source based on the given media profile.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Profiles.MediaProfile
|
||||
|
||||
@doc """
|
||||
Builds the options for yt-dlp to index a media source based on the given media profile.
|
||||
"""
|
||||
def build(%MediaProfile{} = media_profile) do
|
||||
built_options = release_type_options(media_profile)
|
||||
|
||||
{:ok, built_options}
|
||||
end
|
||||
|
||||
defp release_type_options(media_profile) do
|
||||
mapped_struct = Map.from_struct(media_profile)
|
||||
|
||||
# Appending multiple match filters treats them as an OR condition,
|
||||
# so we have to be careful around combining `only` and `exclude` options.
|
||||
# eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
|
||||
# which will return all shorts AND normal videos.
|
||||
Enum.reduce(mapped_struct, [], fn attr, acc ->
|
||||
case {attr, media_profile} do
|
||||
{{:shorts_behaviour, :only}, _} ->
|
||||
acc ++ [match_filter: "original_url*=/shorts/"]
|
||||
|
||||
{{:livestream_behaviour, :only}, _} ->
|
||||
acc ++ [match_filter: "was_live"]
|
||||
|
||||
# Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
|
||||
# other type is set to `only`. There is also special behaviour if they're both excludes,
|
||||
# hence why these check against `:include` alone.
|
||||
{{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
|
||||
acc ++ [match_filter: "original_url!*=/shorts/"]
|
||||
|
||||
{{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
|
||||
acc ++ [match_filter: "!was_live"]
|
||||
|
||||
# Again, since it's an OR, there's a special syntax if they're both excluded
|
||||
# to make it an AND. Note that I'm not checking for the other permutation of
|
||||
# both excluding since this MUST get hit so adding the other version would double up.
|
||||
{{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
|
||||
acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
@ -6,6 +6,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
|
|||
alias Pinchflat.Media
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.MediaSource.Source
|
||||
alias Pinchflat.MediaClient.SourceDetails
|
||||
alias Pinchflat.Workers.MediaIndexingWorker
|
||||
alias Pinchflat.Workers.VideoDownloadWorker
|
||||
|
||||
|
|
@ -33,6 +34,32 @@ defmodule Pinchflat.Tasks.SourceTasks do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Given a media source, creates (indexes) the media by creating media_items for each
|
||||
media ID in the source.
|
||||
|
||||
Returns [%MediaItem{}, ...] | [%Ecto.Changeset{}, ...]
|
||||
"""
|
||||
def index_media_items(%Source{} = source) do
|
||||
{:ok, media_attributes} = SourceDetails.get_media_attributes(source.original_url)
|
||||
|
||||
media_attributes
|
||||
|> Enum.map(fn media_attrs ->
|
||||
attrs = %{
|
||||
source_id: source.id,
|
||||
title: media_attrs["title"],
|
||||
media_id: media_attrs["id"],
|
||||
original_url: media_attrs["original_url"],
|
||||
livestream: media_attrs["was_live"]
|
||||
}
|
||||
|
||||
case Media.create_media_item(attrs) do
|
||||
{:ok, media_item} -> media_item
|
||||
{:error, changeset} -> changeset
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Starts tasks for downloading videos for any of a sources _pending_ media items.
|
||||
Jobs are not enqueued if the source is set to not download media. This will return :ok.
|
||||
|
|
|
|||
15
lib/pinchflat/utils/function_utils.ex
Normal file
15
lib/pinchflat/utils/function_utils.ex
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
defmodule Pinchflat.Utils.FunctionUtils do
|
||||
@moduledoc """
|
||||
Utility functions for working with functions
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Wraps the provided term in an :ok tuple. Useful for fulfilling a contract, but
|
||||
other usage should be assessed to see if it's the right fit.
|
||||
|
||||
Returns {:ok, term}
|
||||
"""
|
||||
def wrap_ok(value) do
|
||||
{:ok, value}
|
||||
end
|
||||
end
|
||||
|
|
@ -47,7 +47,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
|
|||
end
|
||||
|
||||
defp index_media_and_reschedule(source) do
|
||||
MediaSource.index_media_items(source)
|
||||
SourceTasks.index_media_items(source)
|
||||
SourceTasks.enqueue_pending_media_downloads(source)
|
||||
|
||||
source
|
||||
|
|
|
|||
|
|
@ -168,8 +168,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
phx-connected={hide("#server-error")}
|
||||
hidden
|
||||
>
|
||||
Hang in there while we get back on track
|
||||
<.icon name="hero-arrow-path" class="ml-1 h-3 w-3 animate-spin" />
|
||||
Hang in there while we get back on track <.icon name="hero-arrow-path" class="ml-1 h-3 w-3 animate-spin" />
|
||||
</.flash>
|
||||
</div>
|
||||
"""
|
||||
|
|
@ -245,8 +244,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
values: ~w(checkbox color date datetime-local email file hidden month number password
|
||||
toggle range radio search select tel text textarea time url week)
|
||||
|
||||
attr :field, Phoenix.HTML.FormField,
|
||||
doc: "a form field struct retrieved from the form, for example: @form[:email]"
|
||||
attr :field, Phoenix.HTML.FormField, doc: "a form field struct retrieved from the form, for example: @form[:email]"
|
||||
|
||||
attr :errors, :list, default: []
|
||||
attr :checked, :boolean, doc: "the checked flag for checkbox inputs"
|
||||
|
|
@ -254,8 +252,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
attr :options, :list, doc: "the options to pass to Phoenix.HTML.Form.options_for_select/2"
|
||||
attr :multiple, :boolean, default: false, doc: "the multiple flag for select inputs"
|
||||
|
||||
attr :rest, :global,
|
||||
include: ~w(accept autocomplete capture cols disabled form list max maxlength min minlength
|
||||
attr :rest, :global, include: ~w(accept autocomplete capture cols disabled form list max maxlength min minlength
|
||||
multiple pattern placeholder readonly required rows size step)
|
||||
|
||||
slot :inner_block
|
||||
|
|
@ -279,15 +276,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<div phx-feedback-for={@name}>
|
||||
<label class="flex items-center gap-4 text-sm leading-6">
|
||||
<input type="hidden" name={@name} value="false" />
|
||||
<input
|
||||
type="checkbox"
|
||||
id={@id}
|
||||
name={@name}
|
||||
value="true"
|
||||
checked={@checked}
|
||||
class="rounded focus:ring-0"
|
||||
{@rest}
|
||||
/>
|
||||
<input type="checkbox" id={@id} name={@name} value="true" checked={@checked} class="rounded focus:ring-0" {@rest} />
|
||||
<%= @label %>
|
||||
</label>
|
||||
<.help :if={@help}><%= @help %></.help>
|
||||
|
|
@ -318,8 +307,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
{@rest}
|
||||
/>
|
||||
<div class="inline-block cursor-pointer" @click="enabled = !enabled">
|
||||
<div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black">
|
||||
</div>
|
||||
<div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
|
||||
<div
|
||||
x-bind:class="enabled && '!right-1 !translate-x-full'"
|
||||
class="absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full bg-white transition"
|
||||
|
|
@ -529,10 +517,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<td :if={@action != []} class="relative w-14 p-0">
|
||||
<div class="relative whitespace-nowrap py-4 text-right text-sm font-medium">
|
||||
<span class="absolute -inset-y-px -right-4 left-0 group-hover:bg-zinc-50 sm:rounded-r-xl" />
|
||||
<span
|
||||
:for={action <- @action}
|
||||
class="relative ml-4 font-semibold leading-6 text-zinc-900 hover:text-zinc-700"
|
||||
>
|
||||
<span :for={action <- @action} class="relative ml-4 font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
|
||||
<%= render_slot(action, @row_item.(row)) %>
|
||||
</span>
|
||||
</div>
|
||||
|
|
@ -607,10 +592,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
def back(assigns) do
|
||||
~H"""
|
||||
<div class="mt-16">
|
||||
<.link
|
||||
navigate={@navigate}
|
||||
class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700"
|
||||
>
|
||||
<.link navigate={@navigate} class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
|
||||
<.icon name="hero-arrow-left-solid" class="h-3 w-3" />
|
||||
<%= render_slot(@inner_block) %>
|
||||
</.link>
|
||||
|
|
@ -651,8 +633,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
JS.show(js,
|
||||
to: selector,
|
||||
transition:
|
||||
{"transition-all transform ease-out duration-300",
|
||||
"opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95",
|
||||
{"transition-all transform ease-out duration-300", "opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95",
|
||||
"opacity-100 translate-y-0 sm:scale-100"}
|
||||
)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -17,11 +17,7 @@
|
|||
<ul class="mb-6 flex flex-col gap-1.5">
|
||||
<.sidebar_item icon="hero-tv" text="Sources" navigate={~p"/sources"} />
|
||||
|
||||
<.sidebar_item
|
||||
icon="hero-adjustments-vertical"
|
||||
text="Media Profiles"
|
||||
navigate={~p"/media_profiles"}
|
||||
/>
|
||||
<.sidebar_item icon="hero-adjustments-vertical" text="Media Profiles" navigate={~p"/media_profiles"} />
|
||||
</ul>
|
||||
</div>
|
||||
</nav>
|
||||
|
|
|
|||
|
|
@ -8,11 +8,7 @@
|
|||
<div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">
|
||||
<div class="max-w-full overflow-x-auto">
|
||||
<div class="flex flex-col gap-10">
|
||||
<.source_form
|
||||
changeset={@changeset}
|
||||
media_profiles={@media_profiles}
|
||||
action={~p"/sources/#{@source}"}
|
||||
/>
|
||||
<.source_form changeset={@changeset} media_profiles={@media_profiles} action={~p"/sources/#{@source}"} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -29,16 +29,10 @@
|
|||
</.link>
|
||||
</:col>
|
||||
<:col :let={source} label="" class="flex place-content-evenly">
|
||||
<.link
|
||||
navigate={~p"/sources/#{source.id}"}
|
||||
class="hover:text-secondary duration-200 ease-in-out mx-0.5"
|
||||
>
|
||||
<.link navigate={~p"/sources/#{source.id}"} class="hover:text-secondary duration-200 ease-in-out mx-0.5">
|
||||
<.icon name="hero-eye" />
|
||||
</.link>
|
||||
<.link
|
||||
navigate={~p"/sources/#{source.id}/edit"}
|
||||
class="hover:text-secondary duration-200 ease-in-out mx-0.5"
|
||||
>
|
||||
<.link navigate={~p"/sources/#{source.id}/edit"} class="hover:text-secondary duration-200 ease-in-out mx-0.5">
|
||||
<.icon name="hero-pencil-square" />
|
||||
</.link>
|
||||
</:col>
|
||||
|
|
|
|||
|
|
@ -3,12 +3,7 @@
|
|||
Oops, something went wrong! Please check the errors below.
|
||||
</.error>
|
||||
|
||||
<.input
|
||||
field={f[:original_url]}
|
||||
type="text"
|
||||
label="Source URL"
|
||||
help="URL of a channel or playlist (required)"
|
||||
/>
|
||||
<.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
|
||||
|
||||
<.input field={f[:friendly_name]} type="text" label="Friendly Name" />
|
||||
|
||||
|
|
@ -19,12 +14,7 @@
|
|||
label="Media Profile"
|
||||
/>
|
||||
|
||||
<.input
|
||||
field={f[:collection_type]}
|
||||
options={friendly_collection_types()}
|
||||
type="select"
|
||||
label="Source Type"
|
||||
/>
|
||||
<.input field={f[:collection_type]} options={friendly_collection_types()} type="select" label="Source Type" />
|
||||
|
||||
<.input
|
||||
field={f[:index_frequency_minutes]}
|
||||
|
|
|
|||
|
|
@ -136,11 +136,7 @@
|
|||
href="https://twitter.com/elixirphoenix"
|
||||
class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 16 16"
|
||||
aria-hidden="true"
|
||||
class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
|
||||
>
|
||||
<svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
|
||||
<path d="M5.403 14c5.283 0 8.172-4.617 8.172-8.62 0-.131 0-.262-.008-.391A6.033 6.033 0 0 0 15 3.419a5.503 5.503 0 0 1-1.65.477 3.018 3.018 0 0 0 1.263-1.676 5.579 5.579 0 0 1-1.824.736 2.832 2.832 0 0 0-1.63-.916 2.746 2.746 0 0 0-1.821.319A2.973 2.973 0 0 0 8.076 3.78a3.185 3.185 0 0 0-.182 1.938 7.826 7.826 0 0 1-3.279-.918 8.253 8.253 0 0 1-2.64-2.247 3.176 3.176 0 0 0-.315 2.208 3.037 3.037 0 0 0 1.203 1.836A2.739 2.739 0 0 1 1.56 6.22v.038c0 .7.23 1.377.65 1.919.42.54 1.004.912 1.654 1.05-.423.122-.866.14-1.297.052.184.602.541 1.129 1.022 1.506a2.78 2.78 0 0 0 1.662.598 5.656 5.656 0 0 1-2.007 1.074A5.475 5.475 0 0 1 1 12.64a7.827 7.827 0 0 0 4.403 1.358" />
|
||||
</svg>
|
||||
Follow on Twitter
|
||||
|
|
@ -151,11 +147,7 @@
|
|||
href="https://elixirforum.com"
|
||||
class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 16 16"
|
||||
aria-hidden="true"
|
||||
class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
|
||||
>
|
||||
<svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
|
||||
<path d="M8 13.833c3.866 0 7-2.873 7-6.416C15 3.873 11.866 1 8 1S1 3.873 1 7.417c0 1.081.292 2.1.808 2.995.606 1.05.806 2.399.086 3.375l-.208.283c-.285.386-.01.905.465.85.852-.098 2.048-.318 3.137-.81a3.717 3.717 0 0 1 1.91-.318c.263.027.53.041.802.041Z" />
|
||||
</svg>
|
||||
Discuss on the Elixir Forum
|
||||
|
|
@ -166,11 +158,7 @@
|
|||
href="https://web.libera.chat/#elixir"
|
||||
class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 16 16"
|
||||
aria-hidden="true"
|
||||
class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
|
||||
>
|
||||
<svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
|
||||
<path
|
||||
fill-rule="evenodd"
|
||||
clip-rule="evenodd"
|
||||
|
|
@ -190,11 +178,7 @@
|
|||
href="https://discord.gg/elixir"
|
||||
class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 16 16"
|
||||
aria-hidden="true"
|
||||
class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
|
||||
>
|
||||
<svg viewBox="0 0 16 16" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
|
||||
<path d="M13.545 2.995c-1.02-.46-2.114-.8-3.257-.994a.05.05 0 0 0-.052.024c-.141.246-.297.567-.406.82a12.377 12.377 0 0 0-3.658 0 8.238 8.238 0 0 0-.412-.82.052.052 0 0 0-.052-.024 13.315 13.315 0 0 0-3.257.994.046.046 0 0 0-.021.018C.356 6.063-.213 9.036.066 11.973c.001.015.01.029.02.038a13.353 13.353 0 0 0 3.996 1.987.052.052 0 0 0 .056-.018c.308-.414.582-.85.818-1.309a.05.05 0 0 0-.028-.069 8.808 8.808 0 0 1-1.248-.585.05.05 0 0 1-.005-.084c.084-.062.168-.126.248-.191a.05.05 0 0 1 .051-.007c2.619 1.176 5.454 1.176 8.041 0a.05.05 0 0 1 .053.006c.08.065.164.13.248.192a.05.05 0 0 1-.004.084c-.399.23-.813.423-1.249.585a.05.05 0 0 0-.027.07c.24.457.514.893.817 1.307a.051.051 0 0 0 .056.019 13.31 13.31 0 0 0 4.001-1.987.05.05 0 0 0 .021-.037c.334-3.396-.559-6.345-2.365-8.96a.04.04 0 0 0-.021-.02Zm-8.198 7.19c-.789 0-1.438-.712-1.438-1.587 0-.874.637-1.586 1.438-1.586.807 0 1.45.718 1.438 1.586 0 .875-.637 1.587-1.438 1.587Zm5.316 0c-.788 0-1.438-.712-1.438-1.587 0-.874.637-1.586 1.438-1.586.807 0 1.45.718 1.438 1.586 0 .875-.63 1.587-1.438 1.587Z" />
|
||||
</svg>
|
||||
Join our Discord server
|
||||
|
|
@ -205,11 +189,7 @@
|
|||
href="https://fly.io/docs/elixir/getting-started/"
|
||||
class="group -mx-2 -my-0.5 inline-flex items-center gap-3 rounded-lg px-2 py-0.5 hover:bg-zinc-50 hover:text-zinc-900"
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 20 20"
|
||||
aria-hidden="true"
|
||||
class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600"
|
||||
>
|
||||
<svg viewBox="0 0 20 20" aria-hidden="true" class="h-4 w-4 fill-zinc-400 group-hover:fill-zinc-600">
|
||||
<path d="M1 12.5A4.5 4.5 0 005.5 17H15a4 4 0 001.866-7.539 3.504 3.504 0 00-4.504-4.272A4.5 4.5 0 004.06 8.235 4.502 4.502 0 001 12.5z" />
|
||||
</svg>
|
||||
Deploy your application
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
defmodule Pinchflat.Repo.Migrations.AddIndexingAttributesToMediaItems do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
alter table(:media_items) do
|
||||
add :livestream, :boolean, default: false, null: false
|
||||
add :original_url, :string, null: false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
|
||||
use ExUnit.Case, async: true
|
||||
import Mox
|
||||
import Pinchflat.MediaSourceFixtures
|
||||
|
||||
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
|
||||
|
||||
|
|
@ -8,22 +9,23 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
|
|||
|
||||
setup :verify_on_exit!
|
||||
|
||||
describe "get_video_ids/2" do
|
||||
test "returns a list of video ids with no blank elements" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "id1\nid2\n\nid3\n"} end)
|
||||
describe "get_media_attributes/2" do
|
||||
test "returns a list of video attributes with no blank elements" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture() <> "\n\n"} end)
|
||||
|
||||
assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
|
||||
assert {:ok, [%{"id" => "video1"}, %{"id" => "video2"}, %{"id" => "video3"}]} =
|
||||
VideoCollection.get_media_attributes(@channel_url)
|
||||
end
|
||||
|
||||
test "it passes the expected default args" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
|
||||
assert opts == [:simulate, :skip_download]
|
||||
assert ot == "%(id)s"
|
||||
assert ot == "%(.{id,title,was_live,original_url})j"
|
||||
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
|
||||
assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url)
|
||||
end
|
||||
|
||||
test "it passes the expected custom args" do
|
||||
|
|
@ -33,13 +35,13 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
|
|||
{:ok, ""}
|
||||
end)
|
||||
|
||||
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
|
||||
assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url, [:custom_arg])
|
||||
end
|
||||
|
||||
test "returns the error straight through when the command fails" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
|
||||
|
||||
assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
|
||||
assert {:error, "Big issue", 1} = VideoCollection.get_media_attributes(@channel_url)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
|
|||
assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
|
||||
end
|
||||
|
||||
test "it returns a struct composed of the returned data" do
|
||||
test "it returns a map composed of the returned data" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
|
||||
Phoenix.json_library().encode(%{
|
||||
channel: "TheUselessTrials",
|
||||
|
|
@ -43,42 +43,42 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "get_video_ids/2 when passed a string" do
|
||||
describe "get_media_attributes/2 when passed a string" do
|
||||
test "it passes the expected arguments to the backend" do
|
||||
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
|
||||
assert opts == [:simulate, :skip_download]
|
||||
assert ot == "%(id)s"
|
||||
assert ot == "%(.{id,title,was_live,original_url})j"
|
||||
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
assert {:ok, _} = SourceDetails.get_video_ids(@channel_url)
|
||||
assert {:ok, _} = SourceDetails.get_media_attributes(@channel_url)
|
||||
end
|
||||
|
||||
test "it returns a list of strings" do
|
||||
test "it returns a list of maps" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
|
||||
{:ok, "video1\nvideo2\nvideo3"}
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
assert {:ok, ["video1", "video2", "video3"]} = SourceDetails.get_video_ids(@channel_url)
|
||||
assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes(@channel_url)
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_video_ids/2 when passed a Source record" do
|
||||
describe "get_media_attributes/2 when passed a Source record" do
|
||||
test "it calls the backend with the source's collection ID" do
|
||||
source = source_fixture()
|
||||
|
||||
expect(YtDlpRunnerMock, :run, fn url, _opts, _ot ->
|
||||
assert source.collection_id == url
|
||||
{:ok, "video1\nvideo2\nvideo3"}
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
assert {:ok, _} = SourceDetails.get_video_ids(source)
|
||||
assert {:ok, _} = SourceDetails.get_media_attributes(source)
|
||||
end
|
||||
|
||||
test "it builds options based on the source's media profile" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
|
||||
assert opts == [{:match_filter, "!was_live"}, :simulate, :skip_download]
|
||||
assert opts == [:simulate, :skip_download]
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
|
|||
)
|
||||
|
||||
source = source_fixture(media_profile_id: media_profile.id)
|
||||
assert {:ok, _} = SourceDetails.get_video_ids(source)
|
||||
assert {:ok, _} = SourceDetails.get_media_attributes(source)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
|
|||
setup do
|
||||
media_item =
|
||||
Repo.preload(
|
||||
media_item_fixture(%{title: nil, media_filepath: nil}),
|
||||
media_item_fixture(%{title: "Something", media_filepath: nil}),
|
||||
[:metadata, source: :media_profile]
|
||||
)
|
||||
|
||||
|
|
@ -58,7 +58,6 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
|
|||
end
|
||||
|
||||
test "it extracts the title", %{media_item: media_item} do
|
||||
assert media_item.title == nil
|
||||
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
|
||||
assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ defmodule Pinchflat.MediaSourceTest do
|
|||
import Pinchflat.MediaSourceFixtures
|
||||
|
||||
alias Pinchflat.MediaSource
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.MediaSource.Source
|
||||
alias Pinchflat.Workers.MediaIndexingWorker
|
||||
|
||||
|
|
@ -117,59 +116,6 @@ defmodule Pinchflat.MediaSourceTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "index_media_items/1" do
|
||||
setup do
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2\nvideo3"} end)
|
||||
|
||||
{:ok, [source: source_fixture()]}
|
||||
end
|
||||
|
||||
test "it creates a media_item record for each media ID returned", %{source: source} do
|
||||
assert media_items = MediaSource.index_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
|
||||
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
|
||||
end
|
||||
|
||||
test "it attaches all media_items to the given source", %{source: source} do
|
||||
source_id = source.id
|
||||
assert media_items = MediaSource.index_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
|
||||
end
|
||||
|
||||
test "it won't duplicate media_items based on media_id and source", %{source: source} do
|
||||
_first_run = MediaSource.index_media_items(source)
|
||||
_duplicate_run = MediaSource.index_media_items(source)
|
||||
|
||||
media_items = Repo.preload(source, :media_items).media_items
|
||||
assert Enum.count(media_items) == 3
|
||||
end
|
||||
|
||||
test "it can duplicate media_ids for different sources", %{source: source} do
|
||||
other_source = source_fixture()
|
||||
|
||||
media_items = MediaSource.index_media_items(source)
|
||||
media_items_other_source = MediaSource.index_media_items(other_source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.count(media_items_other_source) == 3
|
||||
|
||||
assert Enum.map(media_items, & &1.media_id) ==
|
||||
Enum.map(media_items_other_source, & &1.media_id)
|
||||
end
|
||||
|
||||
test "it returns a list of media_items or changesets", %{source: source} do
|
||||
first_run = MediaSource.index_media_items(source)
|
||||
duplicate_run = MediaSource.index_media_items(source)
|
||||
|
||||
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
|
||||
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
|
||||
end
|
||||
end
|
||||
|
||||
describe "update_source/2" do
|
||||
test "updates with valid data updates the source" do
|
||||
source = source_fixture()
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ defmodule Pinchflat.MediaTest do
|
|||
|
||||
import Pinchflat.TasksFixtures
|
||||
import Pinchflat.MediaFixtures
|
||||
import Pinchflat.ProfilesFixtures
|
||||
import Pinchflat.MediaSourceFixtures
|
||||
|
||||
alias Pinchflat.Media
|
||||
|
|
@ -30,7 +31,7 @@ defmodule Pinchflat.MediaTest do
|
|||
end
|
||||
|
||||
describe "list_pending_media_items_for/1" do
|
||||
test "it returns pending media_items for a given source" do
|
||||
test "it returns pending without a filepath for a given source" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
|
||||
|
|
@ -50,6 +51,138 @@ defmodule Pinchflat.MediaTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "list_pending_media_items_for/1 when testing shorts" do
|
||||
test "returns shorts and normal media when shorts_behaviour is :include" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :include}).id})
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal, short]
|
||||
end
|
||||
|
||||
test "returns only shorts when shorts_behaviour is :only" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :only}).id})
|
||||
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [short]
|
||||
end
|
||||
|
||||
test "returns only normal media when shorts_behaviour is :exclude" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :exclude}).id})
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
_short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal]
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_pending_media_items_for/1 when testing livestreams" do
|
||||
test "returns livestreams and normal media when livestream_behaviour is :include" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :include}).id})
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal, livestream]
|
||||
end
|
||||
|
||||
test "returns only livestreams when livestream_behaviour is :only" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :only}).id})
|
||||
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [livestream]
|
||||
end
|
||||
|
||||
test "returns only normal media when livestream_behaviour is :exclude" do
|
||||
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :exclude}).id})
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal]
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_pending_media_items_for/1 when testing all format options" do
|
||||
test "returns livestreams, shorts, and normal media when behaviour is :include" do
|
||||
source =
|
||||
source_fixture(%{
|
||||
media_profile_id:
|
||||
media_profile_fixture(%{
|
||||
shorts_behaviour: :include,
|
||||
livestream_behaviour: :include
|
||||
}).id
|
||||
})
|
||||
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal, livestream, short]
|
||||
end
|
||||
|
||||
test "returns only livestreams and shorts when behaviour is :only" do
|
||||
source =
|
||||
source_fixture(%{
|
||||
media_profile_id:
|
||||
media_profile_fixture(%{
|
||||
shorts_behaviour: :only,
|
||||
livestream_behaviour: :only
|
||||
}).id
|
||||
})
|
||||
|
||||
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [livestream, short]
|
||||
end
|
||||
|
||||
test "returns only normal media when behaviour is :exclude" do
|
||||
source =
|
||||
source_fixture(%{
|
||||
media_profile_id:
|
||||
media_profile_fixture(%{
|
||||
shorts_behaviour: :exclude,
|
||||
livestream_behaviour: :exclude
|
||||
}).id
|
||||
})
|
||||
|
||||
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
_short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [normal]
|
||||
end
|
||||
|
||||
test ":only and :exclude return the expected results" do
|
||||
source =
|
||||
source_fixture(%{
|
||||
media_profile_id:
|
||||
media_profile_fixture(%{
|
||||
shorts_behaviour: :only,
|
||||
livestream_behaviour: :exclude
|
||||
}).id
|
||||
})
|
||||
|
||||
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
|
||||
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
|
||||
|
||||
assert Media.list_pending_media_items_for(source) == [short]
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_downloaded_media_items_for/1" do
|
||||
test "returns only media items with a media_filepath" do
|
||||
source = source_fixture()
|
||||
_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
media_item = media_item_fixture(%{source_id: source.id, media_filepath: "/video/#{Faker.File.file_name(:video)}"})
|
||||
|
||||
assert Media.list_downloaded_media_items_for(source) == [media_item]
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_media_item!/1" do
|
||||
test "it returns the media_item with given id" do
|
||||
media_item = media_item_fixture()
|
||||
|
|
@ -63,7 +196,8 @@ defmodule Pinchflat.MediaTest do
|
|||
media_id: Faker.String.base64(12),
|
||||
title: Faker.Commerce.product_name(),
|
||||
media_filepath: "/video/#{Faker.File.file_name(:video)}",
|
||||
source_id: source_fixture().id
|
||||
source_id: source_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
|
||||
}
|
||||
|
||||
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilderTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Pinchflat.Profiles.MediaProfile
|
||||
alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder
|
||||
|
||||
@media_profile %MediaProfile{
|
||||
output_path_template: "{{ title }}.%(ext)s",
|
||||
shorts_behaviour: :include,
|
||||
livestream_behaviour: :include
|
||||
}
|
||||
|
||||
describe "build/1 when testing release type options" do
|
||||
test "adds correct filter when shorts_behaviour is :only" do
|
||||
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :only}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "original_url*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
refute {:match_filter, "was_live"} in res
|
||||
end
|
||||
|
||||
test "adds correct filter when livestream_behaviour is :only" do
|
||||
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :only}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "was_live"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
refute {:match_filter, "!original_url*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url*=/shorts/"} in res
|
||||
end
|
||||
|
||||
test "adds correct filter when both livestreams and shorts are :only" do
|
||||
media_profile = %MediaProfile{
|
||||
@media_profile
|
||||
| shorts_behaviour: :only,
|
||||
livestream_behaviour: :only
|
||||
}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "original_url*=/shorts/"} in res
|
||||
assert {:match_filter, "was_live"} in res
|
||||
refute {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
end
|
||||
|
||||
test "adds correct filter when shorts_behaviour is :exclude" do
|
||||
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :exclude}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url*=/shorts/"} in res
|
||||
refute {:match_filter, "was_live"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
end
|
||||
|
||||
test "adds correct filter when livestream_behaviour is :exclude" do
|
||||
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :exclude}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "!was_live"} in res
|
||||
refute {:match_filter, "was_live"} in res
|
||||
refute {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url*=/shorts/"} in res
|
||||
end
|
||||
|
||||
test "adds correct filter when shorts and livestreams are both exclude" do
|
||||
media_profile = %MediaProfile{
|
||||
@media_profile
|
||||
| shorts_behaviour: :exclude,
|
||||
livestream_behaviour: :exclude
|
||||
}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "!was_live & original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
refute {:match_filter, "original_url*=/shorts/"} in res
|
||||
refute {:match_filter, "was_live"} in res
|
||||
end
|
||||
|
||||
test "does not add exclusion filter if one is excluded and the other is only" do
|
||||
media_profile = %MediaProfile{
|
||||
@media_profile
|
||||
| shorts_behaviour: :exclude,
|
||||
livestream_behaviour: :only
|
||||
}
|
||||
|
||||
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
|
||||
|
||||
assert {:match_filter, "was_live"} in res
|
||||
refute {:match_filter, "original_url!*=/shorts/"} in res
|
||||
refute {:match_filter, "original_url*=/shorts/"} in res
|
||||
refute {:match_filter, "!was_live"} in res
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
defmodule Pinchflat.Tasks.SourceTasksTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
import Mox
|
||||
import Pinchflat.TasksFixtures
|
||||
import Pinchflat.MediaFixtures
|
||||
import Pinchflat.MediaSourceFixtures
|
||||
|
|
@ -8,9 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
|
|||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Tasks.Task
|
||||
alias Pinchflat.Tasks.SourceTasks
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Workers.MediaIndexingWorker
|
||||
alias Pinchflat.Workers.VideoDownloadWorker
|
||||
|
||||
setup :verify_on_exit!
|
||||
|
||||
describe "kickoff_indexing_task/1" do
|
||||
test "it does not schedule a job if the interval is <= 0" do
|
||||
source = source_fixture(index_frequency_minutes: -1)
|
||||
|
|
@ -46,6 +50,61 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "index_media_items/1" do
|
||||
setup do
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
{:ok, [source: source_fixture()]}
|
||||
end
|
||||
|
||||
test "it creates a media_item record for each media ID returned", %{source: source} do
|
||||
assert media_items = SourceTasks.index_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
|
||||
assert ["Video 1", "Video 2", "Video 3"] == Enum.map(media_items, & &1.title)
|
||||
assert Enum.all?(media_items, fn mi -> mi.original_url end)
|
||||
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
|
||||
end
|
||||
|
||||
test "it attaches all media_items to the given source", %{source: source} do
|
||||
source_id = source.id
|
||||
assert media_items = SourceTasks.index_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
|
||||
end
|
||||
|
||||
test "it won't duplicate media_items based on media_id and source", %{source: source} do
|
||||
_first_run = SourceTasks.index_media_items(source)
|
||||
_duplicate_run = SourceTasks.index_media_items(source)
|
||||
|
||||
media_items = Repo.preload(source, :media_items).media_items
|
||||
assert Enum.count(media_items) == 3
|
||||
end
|
||||
|
||||
test "it can duplicate media_ids for different sources", %{source: source} do
|
||||
other_source = source_fixture()
|
||||
|
||||
media_items = SourceTasks.index_media_items(source)
|
||||
media_items_other_source = SourceTasks.index_media_items(other_source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.count(media_items_other_source) == 3
|
||||
|
||||
assert Enum.map(media_items, & &1.media_id) ==
|
||||
Enum.map(media_items_other_source, & &1.media_id)
|
||||
end
|
||||
|
||||
test "it returns a list of media_items or changesets", %{source: source} do
|
||||
first_run = SourceTasks.index_media_items(source)
|
||||
duplicate_run = SourceTasks.index_media_items(source)
|
||||
|
||||
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
|
||||
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
|
||||
end
|
||||
end
|
||||
|
||||
describe "enqueue_pending_media_downloads/1" do
|
||||
test "it enqueues a job for each pending media item" do
|
||||
source = source_fixture()
|
||||
|
|
|
|||
11
test/pinchflat/utils/function_utils_test.exs
Normal file
11
test/pinchflat/utils/function_utils_test.exs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
defmodule Pinchflat.Utils.FunctionUtilsTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Pinchflat.Utils.FunctionUtils
|
||||
|
||||
describe "wrap_ok/1" do
|
||||
test "wraps the provided term in an :ok tuple" do
|
||||
assert FunctionUtils.wrap_ok("hello") == {:ok, "hello"}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -38,32 +38,33 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
|
|||
end
|
||||
|
||||
test "it kicks off a download job for each pending media item" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
perform_job(MediaIndexingWorker, %{id: source.id})
|
||||
|
||||
assert [_] = all_enqueued(worker: VideoDownloadWorker)
|
||||
assert length(all_enqueued(worker: VideoDownloadWorker)) == 3
|
||||
end
|
||||
|
||||
test "it starts a job for any pending media item even if it's from another run" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
perform_job(MediaIndexingWorker, %{id: source.id})
|
||||
|
||||
assert [_, _] = all_enqueued(worker: VideoDownloadWorker)
|
||||
assert length(all_enqueued(worker: VideoDownloadWorker)) == 4
|
||||
end
|
||||
|
||||
test "it does not kick off a job for media items that could not be saved" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo1"} end)
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
|
||||
perform_job(MediaIndexingWorker, %{id: source.id})
|
||||
|
||||
# Only one job should be enqueued, since the second video is a duplicate
|
||||
assert [_] = all_enqueued(worker: VideoDownloadWorker)
|
||||
# Only 3 jobs should be enqueued, since the first video is a duplicate
|
||||
assert length(all_enqueued(worker: VideoDownloadWorker))
|
||||
end
|
||||
|
||||
test "it reschedules the job based on the index frequency" do
|
||||
|
|
@ -91,7 +92,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
|
|||
end
|
||||
|
||||
test "it creates the basic media_item records" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2"} end)
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
|
||||
|
|
@ -102,7 +103,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
|
|||
|> Enum.map(fn media_item -> media_item.media_id end)
|
||||
end
|
||||
|
||||
assert_changed([from: [], to: ["video1", "video2"]], media_item_fetcher, fn ->
|
||||
assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
|
||||
perform_job(MediaIndexingWorker, %{id: source.id})
|
||||
end)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ defmodule Pinchflat.MediaFixtures do
|
|||
|> Enum.into(%{
|
||||
media_id: Faker.String.base64(12),
|
||||
title: Faker.Commerce.product_name(),
|
||||
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
|
||||
livestream: false,
|
||||
media_filepath: "/video/#{Faker.File.file_name(:video)}",
|
||||
source_id: MediaSourceFixtures.source_fixture().id
|
||||
})
|
||||
|
|
|
|||
|
|
@ -29,4 +29,30 @@ defmodule Pinchflat.MediaSourceFixtures do
|
|||
|
||||
source
|
||||
end
|
||||
|
||||
def source_attributes_return_fixture do
|
||||
source_attributes = [
|
||||
%{
|
||||
id: "video1",
|
||||
title: "Video 1",
|
||||
original_url: "https://example.com/video1",
|
||||
was_live: false
|
||||
},
|
||||
%{
|
||||
id: "video2",
|
||||
title: "Video 2",
|
||||
original_url: "https://example.com/video2",
|
||||
was_live: true
|
||||
},
|
||||
%{
|
||||
id: "video3",
|
||||
title: "Video 3",
|
||||
original_url: "https://example.com/video3",
|
||||
was_live: false
|
||||
}
|
||||
]
|
||||
|
||||
source_attributes
|
||||
|> Enum.map_join("\n", &Phoenix.json_library().encode!(&1))
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue