From 8e9f02c8078bdba882830df0b54c2efee46e535a Mon Sep 17 00:00:00 2001 From: Kieran Date: Wed, 14 Aug 2024 12:37:24 -0700 Subject: [PATCH] [Enhancement] Support specifying count of `yt-dlp` workers per-queue (#355) * Updated certain Oban queues to have a configurable limit * Updated README --- .iex.exs | 8 ++++++++ README.md | 21 +++++++++++---------- config/config.exs | 10 ---------- config/runtime.exs | 14 ++++++++++++++ 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.iex.exs b/.iex.exs index ebe1a24..8533f83 100644 --- a/.iex.exs +++ b/.iex.exs @@ -23,3 +23,11 @@ alias Pinchflat.Metadata.MetadataFileHelpers alias Pinchflat.SlowIndexing.FileFollowerServer Pinchflat.Release.check_file_permissions() + +defmodule IexHelpers do + def restart do + :init.restart() + end +end + +import IexHelpers diff --git a/README.md b/README.md index 1f03132..7733563 100644 --- a/README.md +++ b/README.md @@ -145,16 +145,17 @@ If you change this setting and it works well for you, please leave a comment on ### Environment variables -| Name | Required? | Default | Notes | -| --------------------- | --------- | ------------------------- | ---------------------------------------------------------------------------------------------- | -| TZ | No | `UTC` | Must follow IANA TZ format | -| LOG_LEVEL | No | `debug` | Can be set to `info` | -| BASIC_AUTH_USERNAME | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) | -| BASIC_AUTH_PASSWORD | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) | -| EXPOSE_FEED_ENDPOINTS | No | | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) | -| JOURNAL_MODE | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) | -| TZ_DATA_DIR | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored | -| BASE_ROUTE_PATH | No | `/` | The base path for route generation. Useful when running behind certain reverse proxies | +| Name | Required? | Default | Notes | +| --------------------------- | --------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `TZ` | No | `UTC` | Must follow IANA TZ format | +| `LOG_LEVEL` | No | `debug` | Can be set to `info` but `debug` is strongly recommended | +| `BASIC_AUTH_USERNAME` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) | +| `BASIC_AUTH_PASSWORD` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) | +| `EXPOSE_FEED_ENDPOINTS` | No | | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) | +| `JOURNAL_MODE` | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) | +| `TZ_DATA_DIR` | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored | +| `BASE_ROUTE_PATH` | No | `/` | The base path for route generation. Useful when running behind certain reverse proxies | +| `YT_DLP_WORKER_CONCURRENCY` | No | `2` | The number of concurrent workers that use `yt-dlp` _per queue_. Set to 1 if you're getting IP limited, otherwise don't touch it | ## EFF donations diff --git a/config/config.exs b/config/config.exs index aab4d10..d0b5a58 100644 --- a/config/config.exs +++ b/config/config.exs @@ -57,16 +57,6 @@ config :pinchflat, Oban, {"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker}, {"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker} ]} - ], - # TODO: consider making this an env var or something? - queues: [ - default: 10, - fast_indexing: 6, - media_indexing: 2, - media_collection_indexing: 2, - media_fetching: 2, - local_data: 8, - remote_metadata: 4 ] # Configures the mailer diff --git a/config/runtime.exs b/config/runtime.exs index 06aa731..1a3c0de 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -40,6 +40,20 @@ config :pinchflat, Pinchflat.Repo, Path.join([:code.priv_dir(:pinchflat), "repo", "extensions", "sqlean-linux-#{system_arch}", "sqlean"]) ] +# Some users may want to increase the number of workers that use yt-dlp to improve speeds +# Others may want to decrease the number of these workers to lessen the chance of an IP ban +{yt_dlp_worker_count, _} = Integer.parse(System.get_env("YT_DLP_WORKER_CONCURRENCY", "2")) + +config :pinchflat, Oban, + queues: [ + default: 10, + fast_indexing: 6, + media_collection_indexing: yt_dlp_worker_count, + media_fetching: yt_dlp_worker_count, + remote_metadata: yt_dlp_worker_count, + local_data: 8 + ] + if config_env() == :prod do config_path = "/config" db_path = System.get_env("DATABASE_PATH", Path.join([config_path, "db", "pinchflat.db"]))