mirror of
https://github.com/alexta69/metube.git
synced 2026-07-18 00:58:18 +00:00
Compare commits
No commits in common. "master" and "2026.04.01" have entirely different histories.
master
...
2026.04.01
49 changed files with 3293 additions and 9163 deletions
14
.github/workflows/main.yml
vendored
14
.github/workflows/main.yml
vendored
|
|
@ -4,15 +4,13 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
|
||||
jobs:
|
||||
quality-checks:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
|
|
@ -34,7 +32,7 @@ jobs:
|
|||
env:
|
||||
CI: true
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v6
|
||||
- name: Install Python dependencies
|
||||
run: uv sync --frozen --group dev
|
||||
- name: Run backend smoke checks
|
||||
|
|
@ -42,7 +40,7 @@ jobs:
|
|||
- name: Run backend tests
|
||||
run: uv run pytest app/tests/
|
||||
- name: Run Trivy filesystem scan
|
||||
uses: aquasecurity/trivy-action@v0.36.0
|
||||
uses: aquasecurity/trivy-action@0.35.0
|
||||
with:
|
||||
scan-type: fs
|
||||
scan-ref: .
|
||||
|
|
@ -59,7 +57,7 @@ jobs:
|
|||
run: echo "date=$(date +'%Y.%m.%d')" >> "$GITHUB_OUTPUT"
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
|
@ -118,7 +116,7 @@ jobs:
|
|||
id: date
|
||||
run: echo "date=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Get commits since last release
|
||||
|
|
@ -211,7 +209,7 @@ jobs:
|
|||
git push origin ":refs/tags/$TAG_NAME" || true
|
||||
fi
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ steps.date.outputs.date }}
|
||||
name: Release ${{ steps.date.outputs.date }}
|
||||
|
|
|
|||
2
.github/workflows/update-yt-dlp.yml
vendored
2
.github/workflows/update-yt-dlp.yml
vendored
|
|
@ -10,7 +10,7 @@ jobs:
|
|||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.AUTOUPDATE_PAT }}
|
||||
-
|
||||
|
|
|
|||
4
.vscode/launch.json
vendored
4
.vscode/launch.json
vendored
|
|
@ -13,16 +13,12 @@
|
|||
"env": {
|
||||
"DOWNLOAD_DIR": "${env:USERPROFILE}/Downloads",
|
||||
"STATE_DIR": "${env:TEMP}",
|
||||
"ALLOW_YTDL_OPTIONS_OVERRIDES": "true",
|
||||
"YTDL_OPTIONS_PRESETS": "{\"sponsorblock\": {\"postprocessors\": [{\"key\": \"SponsorBlock\", \"categories\": [\"sponsor\", \"selfpromo\", \"interaction\"]}, {\"key\": \"ModifyChapters\", \"remove_sponsor_segments\": [\"sponsor\", \"selfpromo\", \"interaction\"]}]}, \"embed-subs\": {\"writesubtitles\": true, \"writeautomaticsub\": true, \"subtitleslangs\": [\"en\", \"de\"], \"postprocessors\": [{\"key\": \"FFmpegEmbedSubtitle\"}]}, \"limit-rate\": {\"ratelimit\": 5000000}}",
|
||||
}
|
||||
},
|
||||
"osx": {
|
||||
"env": {
|
||||
"DOWNLOAD_DIR": "${env:HOME}/Downloads",
|
||||
"STATE_DIR": "${env:TMPDIR}",
|
||||
"ALLOW_YTDL_OPTIONS_OVERRIDES": "true",
|
||||
"YTDL_OPTIONS_PRESETS": "{\"sponsorblock\": {\"postprocessors\": [{\"key\": \"SponsorBlock\", \"categories\": [\"sponsor\", \"selfpromo\", \"interaction\"]}, {\"key\": \"ModifyChapters\", \"remove_sponsor_segments\": [\"sponsor\", \"selfpromo\", \"interaction\"]}]}, \"embed-subs\": {\"writesubtitles\": true, \"writeautomaticsub\": true, \"subtitleslangs\": [\"en\", \"de\"], \"postprocessors\": [{\"key\": \"FFmpegEmbedSubtitle\"}]}, \"limit-rate\": {\"ratelimit\": 5000000}}",
|
||||
}
|
||||
},
|
||||
"console": "integratedTerminal"
|
||||
|
|
|
|||
157
AGENTS.md
157
AGENTS.md
|
|
@ -1,157 +0,0 @@
|
|||
# Agent Guidelines
|
||||
|
||||
## Project scope — read this before planning a feature
|
||||
|
||||
MeTube's contract is: give it a URL, it runs yt-dlp well, and correct files appear.
|
||||
The maintainer holds a deliberate line on what belongs inside that contract, and PRs
|
||||
on the wrong side of it are declined **regardless of code quality**. Check your plan
|
||||
against this line before writing any code.
|
||||
|
||||
**In scope — improving the write itself:**
|
||||
|
||||
- Features that make the file yt-dlp writes at download time come out more correct,
|
||||
using only data the extractor already provides (e.g. filling a missing album-artist
|
||||
tag from the extractor's own metadata).
|
||||
- Surfacing functionality yt-dlp itself owns and maintains as first-class UI options
|
||||
(e.g. a SponsorBlock toggle that just passes yt-dlp postprocessor params).
|
||||
- Download queue, subscriptions, output templates, and UI improvements to the
|
||||
download workflow.
|
||||
|
||||
**Out of scope — managing files after they exist:**
|
||||
|
||||
- Tag editors, metadata dialogs, or any workflow that rewrites files after the
|
||||
download has finished. This holds even for slimmed-down versions.
|
||||
- Lookups against external metadata services (iTunes, Deezer, MusicBrainz, etc.).
|
||||
More broadly: any new dependency on a third-party API, or new network egress from
|
||||
self-hosted instances, beyond what yt-dlp itself performs.
|
||||
- Library organization: moving/renaming existing files into Artist/Album layouts,
|
||||
watch-folder processing, and similar media-manager features. Dedicated tools
|
||||
(beets, MusicBrainz Picard, Lidarr) do this properly; the README points users
|
||||
to them.
|
||||
|
||||
**Corollaries that shape borderline PRs:**
|
||||
|
||||
- Site-specific intelligence (parsing playlist-ID prefixes, URL path conventions,
|
||||
and other platform internals) is extractor work and belongs upstream in yt-dlp,
|
||||
not re-implemented here — it silently breaks when the platform changes and
|
||||
MeTube would own the breakage.
|
||||
- Prefer enriching yt-dlp's info dict and letting its existing pipeline
|
||||
(FFmpegMetadata etc.) do the writing, over adding custom per-format tag-writing
|
||||
code to MeTube.
|
||||
- Supplemental processing must never fail a download that otherwise succeeded:
|
||||
warn and continue, don't raise.
|
||||
- Keep feature scope minimal on first submission. A hardcoded sensible default
|
||||
beats a configuration surface; follow-ups can add options when users actually
|
||||
ask. PRs that bundle several "reasonable next steps" invite rejection of the
|
||||
whole.
|
||||
|
||||
If a feature idea fails this test, the accepted alternative is usually a README
|
||||
section documenting how to pair MeTube with the right dedicated tool.
|
||||
|
||||
## README.md size constraint
|
||||
|
||||
The README.md is synced to Docker Hub, which has a **25,000 character limit**.
|
||||
Any change to README.md **must** keep the file under 25,000 characters (`wc -c README.md`).
|
||||
If an addition would exceed the limit, trim existing prose elsewhere — prefer tightening verbose descriptions over removing sections.
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Backend:** Python 3.13+, aiohttp, python-socketio 5.x, yt-dlp
|
||||
- **Frontend:** Angular 22, TypeScript, Bootstrap 5, SASS, ngx-socket-io
|
||||
- **Package managers:** uv (Python), pnpm (frontend)
|
||||
- **Container:** Multi-stage Docker (Node builder + Python runtime), multi-arch (amd64/arm64)
|
||||
|
||||
## Build & test commands
|
||||
|
||||
```bash
|
||||
# Frontend (run from ui/)
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm run lint
|
||||
pnpm run build
|
||||
pnpm exec ng test --watch=false
|
||||
|
||||
# Backend (run from repo root)
|
||||
uv sync --frozen --group dev
|
||||
python -m compileall app
|
||||
uv run pytest app/tests/
|
||||
```
|
||||
|
||||
All of these run in CI (`.github/workflows/main.yml`) on every push to master and must pass.
|
||||
|
||||
Gotchas:
|
||||
|
||||
- Backend tests must run **from the repo root**: `main.py` resolves the static-assets
|
||||
path relative to the cwd, and several test modules import `main`. Running from
|
||||
`app/` makes five test modules fail to import.
|
||||
- The frontend must be **built before** running backend tests (same reason — the
|
||||
assets at `ui/dist/metube/browser` must exist). The command order above is
|
||||
load-bearing.
|
||||
- `app/tests/test_ytdl_utils.py` stubs `yt_dlp` at import time. Run standalone,
|
||||
two tests fail with `AttributeError: <module 'yt_dlp'> does not have the
|
||||
attribute 'YoutubeDL'`; under the full suite the real module is imported first
|
||||
and they pass. This is a known quirk, not a bug to fix in the code under test.
|
||||
|
||||
Every non-markdown push to master builds multi-arch Docker images and cuts a dated
|
||||
release the same day. **Master is continuously released** — a PR must be
|
||||
release-ready exactly as merged; there is no stabilization window for follow-up
|
||||
fixes.
|
||||
|
||||
## Code style
|
||||
|
||||
Follow `.editorconfig`:
|
||||
- Python: 4-space indent
|
||||
- Everything else (TypeScript, YAML, JSON, HTML): 2-space indent
|
||||
- UTF-8, LF line endings, trim trailing whitespace, final newline
|
||||
|
||||
Frontend additionally uses ESLint (`ui/eslint.config.js`) and Prettier (config in `ui/package.json`: `printWidth=100`, `singleQuote=true`).
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
app/main.py — HTTP server, Socket.IO events, REST API routes, Config class
|
||||
app/ytdl.py — Download queue logic, yt-dlp integration
|
||||
app/subscriptions.py — Channel/playlist subscription manager
|
||||
app/state_store.py — JSON-based persistent storage with atomic writes
|
||||
app/dl_formats.py — Video/audio codec/quality mapping
|
||||
app/tests/ — pytest tests (asyncio_mode=auto)
|
||||
ui/src/app/ — Angular standalone components (no NgModules)
|
||||
```
|
||||
|
||||
## Key conventions
|
||||
|
||||
- Backend configuration lives in the `Config` class in `app/main.py` with env-var defaults in `_DEFAULTS`. New env vars go there.
|
||||
- Real-time communication uses Socket.IO events, not REST polling.
|
||||
- Frontend uses standalone Angular components with `inject()` for DI, RxJS Subjects for state, and `takeUntilDestroyed()` for cleanup.
|
||||
- Frontend components use OnPush change detection: subscribe callbacks must call `cdr.markForCheck()`.
|
||||
- State is persisted as JSON files via `AtomicJsonStore` in `app/state_store.py`.
|
||||
- Persisted state stays compact: the completed queue deliberately drops bulky entry data (see `_compact_persisted_entry` in `app/ytdl.py`). Don't expand what gets persisted without discussion.
|
||||
- Custom yt-dlp postprocessors added to `ytdl_params['postprocessors']` run in **list order** within a stage. When combining postprocessors, mirror the ordering the yt-dlp CLI would produce (e.g. sponsor-segment removal before chapter splitting).
|
||||
- No pre-commit hooks — linting and tests are enforced in CI only.
|
||||
|
||||
## Checklist: adding a per-download option
|
||||
|
||||
New options on the download form (the `split_by_chapters` pattern) need **all** of
|
||||
these pieces — the last three are the ones commonly missed:
|
||||
|
||||
1. `parse_download_options` in `app/main.py`.
|
||||
2. A field on `DownloadInfo` in `app/ytdl.py`.
|
||||
3. A `hasattr` backfill in `DownloadInfo.__setstate__` for old persisted records.
|
||||
4. The safe-deserialization field list in `app/ytdl.py`.
|
||||
5. UI form control + cookie persistence in `ui/src/app/app.ts` / `app.html`, and
|
||||
the payload in `downloads.service.ts` (plus its spec).
|
||||
6. The redownload path in `app.ts`, so retries carry the option.
|
||||
7. If the option makes sense for unattended downloads: threading through
|
||||
`app/subscriptions.py` (`SubscriptionInfo` field, serializer, add/update
|
||||
routes, the enqueue call) — or a note in the PR that it's deliberately
|
||||
direct-downloads-only.
|
||||
|
||||
## Security invariants
|
||||
|
||||
User input and extractor-provided metadata (titles, playlist names, URLs) are
|
||||
untrusted. Use the existing guards instead of hand-rolling:
|
||||
|
||||
- User-submitted URLs go through the SSRF guard (see `test_url_guard.py` for the
|
||||
expected behavior).
|
||||
- Anything that becomes a filesystem path goes through `_is_within_directory` and
|
||||
`_sanitize_path_component` in `app/ytdl.py` — including values that arrive via
|
||||
yt-dlp metadata, which sites can influence.
|
||||
10
Dockerfile
10
Dockerfile
|
|
@ -1,8 +1,4 @@
|
|||
# Pinned to a major version rather than the lts-alpine floating tag: that tag
|
||||
# has lagged behind and resolved to a Node patch older than the Angular CLI's
|
||||
# minimum supported version, breaking the build. node:22-alpine currently
|
||||
# satisfies @angular/cli's >=22.22.3 requirement.
|
||||
FROM node:22-alpine AS builder
|
||||
FROM node:lts-alpine AS builder
|
||||
|
||||
WORKDIR /metube
|
||||
COPY ui ./
|
||||
|
|
@ -67,11 +63,9 @@ ENV UMASK=022
|
|||
ENV DOWNLOAD_DIR=/downloads
|
||||
ENV STATE_DIR=/downloads/.metube
|
||||
ENV TEMP_DIR=/downloads
|
||||
ENV PORT=8081
|
||||
VOLUME /downloads
|
||||
EXPOSE 8081
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD case "$HTTPS" in true|True|on|1) curl -fsSk "https://localhost:${PORT}/";; *) curl -fsS "http://localhost:${PORT}/";; esac || exit 1
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD curl -fsS "http://localhost:8081/" || exit 1
|
||||
|
||||
# Add build-time argument for version
|
||||
ARG VERSION=dev
|
||||
|
|
|
|||
273
README.md
273
README.md
|
|
@ -3,12 +3,7 @@
|
|||
|
||||
|
||||
|
||||
MeTube is a self-hosted web UI for `yt-dlp`, for downloading media from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md). Docker images are multi-arch (amd64/arm64).
|
||||
|
||||
Key capabilities:
|
||||
* Download videos, audio, captions, and thumbnails from a browser UI.
|
||||
* Download playlists and channels, with configurable output and download options.
|
||||
* [Subscribe](https://github.com/alexta69/metube/wiki/Subscriptions) to channels and playlists, periodically check for new items, and queue new uploads automatically.
|
||||
Web GUI for youtube-dl (using the [yt-dlp](https://github.com/yt-dlp/yt-dlp) fork) with playlist support. Allows you to download videos from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md).
|
||||
|
||||
|
||||
|
||||
|
|
@ -18,7 +13,7 @@ Key capabilities:
|
|||
docker run -d -p 8081:8081 -v /path/to/downloads:/downloads ghcr.io/alexta69/metube
|
||||
```
|
||||
|
||||
## 🐳 Run using Docker Compose
|
||||
## 🐳 Run using docker-compose
|
||||
|
||||
```yaml
|
||||
services:
|
||||
|
|
@ -34,20 +29,11 @@ services:
|
|||
|
||||
## ⚙️ Configuration via environment variables
|
||||
|
||||
Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in Docker Compose.
|
||||
|
||||
### 🏠 Runtime & Permissions
|
||||
|
||||
* __PUID__: User under which MeTube will run. Defaults to `1000` (legacy `UID` also supported).
|
||||
* __PGID__: Group under which MeTube will run. Defaults to `1000` (legacy `GID` also supported).
|
||||
* __UMASK__: Umask value used by MeTube. Defaults to `022`.
|
||||
* __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`.
|
||||
* __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`.
|
||||
* __ENABLE_ACCESSLOG__: Whether to enable access log. Defaults to `false`.
|
||||
Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in docker-compose.
|
||||
|
||||
### ⬇️ Download Behavior
|
||||
|
||||
* __MAX_CONCURRENT_DOWNLOADS__: Maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`.
|
||||
* __MAX_CONCURRENT_DOWNLOADS__: Maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`.
|
||||
* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
|
||||
* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
|
||||
* __SUBSCRIPTION_DEFAULT_CHECK_INTERVAL__: Default minutes between automatic checks for each subscription. Defaults to `60`.
|
||||
|
|
@ -75,12 +61,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||
* __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`.
|
||||
* __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. Set to empty to use `OUTPUT_TEMPLATE` instead.
|
||||
* __OUTPUT_TEMPLATE_CHANNEL__: The template for the filenames of the downloaded videos when downloaded as a channel. Defaults to `%(channel)s/%(title)s.%(ext)s`. Set to empty to use `OUTPUT_TEMPLATE` instead.
|
||||
* __YTDL_OPTIONS__: Additional options to pass to yt-dlp, as a JSON object. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details, examples, and available options reference.
|
||||
* __YTDL_OPTIONS_FILE__: Path to a JSON file containing yt-dlp options. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
||||
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
||||
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
||||
* __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`.
|
||||
* __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected.
|
||||
|
||||
### 🌐 Web Server & URLs
|
||||
|
||||
|
|
@ -92,126 +74,16 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
||||
* __CERTFILE__: HTTPS certificate file path.
|
||||
* __KEYFILE__: HTTPS key file path.
|
||||
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube).
|
||||
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
||||
|
||||
## 🎛️ Configuring yt-dlp options
|
||||
### 🏠 Basic Setup
|
||||
|
||||
MeTube lets you customize how [yt-dlp](https://github.com/yt-dlp/yt-dlp) behaves at three levels, from broadest to most specific:
|
||||
|
||||
1. **Global options** — apply to every download by default.
|
||||
2. **Presets** — named bundles of options that users can pick per download from the UI.
|
||||
3. **Per-download overrides** — free-form options entered in the UI for a single download.
|
||||
|
||||
When a download starts, these layers are combined in order. If the same option appears in more than one layer, the more specific one wins: per-download overrides beat presets, and presets beat global options.
|
||||
|
||||
In JSON presets and overrides, setting an option to **`null`** clears that option for that download (for example, `"download_archive": null` overrides a global archive path so the archive is not used). This follows yt-dlp’s usual meaning of `None` for that option.
|
||||
|
||||
### Option format
|
||||
|
||||
yt-dlp options in MeTube are expressed as JSON objects. The keys are yt-dlp API option names, which roughly correspond to command-line flags with dashes replaced by underscores. For example, the command-line flag `--write-subs` becomes `"writesubtitles": true` in JSON.
|
||||
|
||||
> **Tip:** Some command-line flags don't have a direct single-key equivalent — for instance, `--embed-thumbnail` and `--recode-video` must be expressed via `"postprocessors"`. A full list of available API options can be found [in the yt-dlp source](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L224), and [this conversion script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) can help translate command-line flags to their API equivalents.
|
||||
|
||||
### Global options
|
||||
|
||||
Global options form the baseline for every download. There are two ways to define them, and you can use either or both:
|
||||
|
||||
**Inline via environment variable** (`YTDL_OPTIONS`) — pass a JSON object directly:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- 'YTDL_OPTIONS={"writesubtitles": true, "subtitleslangs": ["en", "de"], "updatetime": false, "writethumbnail": true}'
|
||||
```
|
||||
|
||||
**Via a JSON file** (`YTDL_OPTIONS_FILE`) — mount a file into the container and point to it:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /path/to/ytdl-options.json:/config/ytdl-options.json
|
||||
environment:
|
||||
- YTDL_OPTIONS_FILE=/config/ytdl-options.json
|
||||
```
|
||||
|
||||
where `ytdl-options.json` contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"writesubtitles": true,
|
||||
"subtitleslangs": ["en", "de"],
|
||||
"updatetime": false,
|
||||
"writethumbnail": true
|
||||
}
|
||||
```
|
||||
|
||||
The file is monitored for changes and reloaded automatically — no container restart needed. If you use both methods and they define the same key, the **file takes precedence**.
|
||||
|
||||
### Presets
|
||||
|
||||
Presets let you define named bundles of options that appear in the web UI under **Advanced Options** as "Option Presets". Users can select one or more presets per download, making it easy to apply common option combinations without editing global settings.
|
||||
|
||||
Like global options, presets can be set inline or via a file:
|
||||
|
||||
* `YTDL_OPTIONS_PRESETS` — a JSON object where each key is a preset name and its value is a set of yt-dlp options.
|
||||
* `YTDL_OPTIONS_PRESETS_FILE` — path to a JSON file containing presets, monitored and reloaded on changes.
|
||||
|
||||
If both are used and they define a preset with the same name, the **file's version takes precedence**.
|
||||
|
||||
**Example** — a presets file defining three presets:
|
||||
|
||||
```json
|
||||
{
|
||||
"sponsorblock": {
|
||||
"postprocessors": [
|
||||
{ "key": "SponsorBlock", "categories": ["sponsor", "selfpromo", "interaction"] },
|
||||
{ "key": "ModifyChapters", "remove_sponsor_segments": ["sponsor", "selfpromo", "interaction"] }
|
||||
]
|
||||
},
|
||||
"embed-subs": {
|
||||
"writesubtitles": true,
|
||||
"writeautomaticsub": true,
|
||||
"subtitleslangs": ["en", "de"],
|
||||
"postprocessors": [{ "key": "FFmpegEmbedSubtitle" }]
|
||||
},
|
||||
"limit-rate": {
|
||||
"ratelimit": 5000000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This makes three presets available in the UI:
|
||||
* **sponsorblock** — strips sponsor, self-promo, and interaction segments from videos.
|
||||
* **embed-subs** — downloads English and German subtitles and embeds them into the video file.
|
||||
* **limit-rate** — caps download speed to ~5 MB/s.
|
||||
|
||||
When multiple presets are selected for a download, they are applied in order. If two presets set the same option, the later one wins.
|
||||
|
||||
### Per-download overrides
|
||||
|
||||
For one-off tweaks, MeTube can expose a free-text JSON field in the UI ("Custom yt-dlp Options") where users type yt-dlp options that apply only to that single download. This is disabled by default:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- ALLOW_YTDL_OPTIONS_OVERRIDES=true
|
||||
```
|
||||
|
||||
Once enabled, the field appears under **Advanced Options**. Any options entered there take the highest priority, overriding both global options and selected presets.
|
||||
|
||||
> **⚠️ Security note:** Enabling this allows arbitrary yt-dlp API options to be supplied by anyone with access to the UI. Depending on the options used, this may enable arbitrary command execution inside the container. Enable only in trusted environments.
|
||||
|
||||
### How the layers combine
|
||||
|
||||
When a download starts, the final set of yt-dlp options is built in this order:
|
||||
|
||||
1. Start with **global options** (`YTDL_OPTIONS` / `YTDL_OPTIONS_FILE`).
|
||||
2. Apply each selected **preset** in order (later presets overwrite earlier ones for conflicting keys).
|
||||
3. Apply any **per-download overrides** on top (overwrite everything else for conflicting keys).
|
||||
|
||||
MeTube always forces its own flat-extract behaviour during the initial metadata fetch (`extract_flat`, `noplaylist`, etc.); presets cannot override those keys for that phase.
|
||||
|
||||
**Example:** Suppose your global options set `"writesubtitles": false`, but you select a preset that sets `"writesubtitles": true`. Subtitles will be written for that download because the preset overrides the global setting. If you additionally enter `{"writesubtitles": false}` in the per-download overrides field, that value wins and subtitles will not be written.
|
||||
|
||||
### Configuration cookbooks
|
||||
* __PUID__: User under which MeTube will run. Defaults to `1000` (legacy `UID` also supported).
|
||||
* __PGID__: Group under which MeTube will run. Defaults to `1000` (legacy `GID` also supported).
|
||||
* __UMASK__: Umask value used by MeTube. Defaults to `022`.
|
||||
* __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`.
|
||||
* __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`.
|
||||
* __ENABLE_ACCESSLOG__: Whether to enable access log. Defaults to `false`.
|
||||
|
||||
The project's Wiki contains examples of useful configurations contributed by users of MeTube:
|
||||
* [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook)
|
||||
|
|
@ -229,27 +101,62 @@ In case you need to use your browser's cookies with MeTube, for example to downl
|
|||
* After upload, the cookie indicator should show as active.
|
||||
* Use **Delete Cookies** in the same section to remove uploaded cookies.
|
||||
|
||||
## 🔗 Sending links to MeTube
|
||||
## 🔌 Browser extensions
|
||||
|
||||
Several integrations let you send URLs to MeTube from wherever you are, instead of pasting them into the UI. The browser-based ones make cross-origin requests, so they require `CORS_ALLOWED_ORIGINS` to be set; and if you're on an HTTPS page, your MeTube instance must be served over HTTPS too (with `HTTPS=true` or behind an HTTPS reverse proxy — see below).
|
||||
Browser extensions allow right-clicking videos and sending them directly to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be behind an HTTPS reverse proxy (see below) for the extensions to work.
|
||||
|
||||
__Browser extensions__ allow right-clicking videos and sending them directly to MeTube. Since extensions request from their own origin, set `CORS_ALLOWED_ORIGINS=*`.
|
||||
* __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
|
||||
* __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
|
||||
__Chrome:__ contributed by [Rpsl](https://github.com/rpsl). You can install it from [Google Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or use developer mode and install [from sources](https://github.com/Rpsl/metube-browser-extension).
|
||||
|
||||
__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
|
||||
__Firefox:__ contributed by [nanocortex](https://github.com/nanocortex). You can install it from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources from [here](https://github.com/nanocortex/metube-firefox-addon).
|
||||
|
||||
__iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
|
||||
## 📱 iOS Shortcut
|
||||
|
||||
__Raycast:__ [dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) for adding videos to MeTube directly from Raycast.
|
||||
[rithask](https://github.com/rithask) created an iOS shortcut to send URLs to MeTube from Safari. Enter the MeTube instance address when prompted which will be saved for later use. You can run the shortcut from Safari’s share menu. The shortcut can be downloaded from [this iCloud link](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6).
|
||||
|
||||
## 🎵 Pairing with a music tagger
|
||||
## 📱 iOS Compatibility
|
||||
|
||||
MeTube deliberately stops once the file is written — tagging and library organization belong to dedicated tools. Point one at your audio download folder (`AUDIO_DOWNLOAD_DIR`):
|
||||
iOS has strict requirements for video files, requiring h264 or h265 video codec and aac audio codec in MP4 container. This can sometimes be a lower quality than the best quality available. To accommodate iOS requirements, when downloading a MP4 format you can choose "Best (iOS)" to get the best quality formats as compatible as possible with iOS requirements.
|
||||
|
||||
* [beets](https://beets.io) — `beet import` matches tracks against MusicBrainz, fixes tags, and files them into an Artist/Album library; headless and scriptable.
|
||||
* [MusicBrainz Picard](https://picard.musicbrainz.org) — GUI tagger with acoustic fingerprinting.
|
||||
* [Lidarr](https://lidarr.audio) — full music library manager; add the folder as an import path.
|
||||
To force all downloads to be converted to an iOS-compatible codec, insert this as an environment variable:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- 'YTDL_OPTIONS={"format": "best", "exec": "ffmpeg -i %(filepath)q -c:v libx264 -c:a aac %(filepath)q.h264.mp4"}'
|
||||
```
|
||||
|
||||
## 🔖 Bookmarklet
|
||||
|
||||
[kushfest](https://github.com/kushfest) has created a Chrome bookmarklet for sending the currently open webpage to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be configured with `HTTPS` as `true` in the environment, or be behind an HTTPS reverse proxy (see below) for the bookmarklet to work.
|
||||
|
||||
GitHub doesn't allow embedding JavaScript as a link, so the bookmarklet has to be created manually by copying the following code to a new bookmark you create on your bookmarks bar. Change the hostname in the URL below to point to your MeTube instance.
|
||||
|
||||
```javascript
|
||||
javascript:!function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.withCredentials=true;xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}}();
|
||||
```
|
||||
|
||||
[shoonya75](https://github.com/shoonya75) has contributed a Firefox version:
|
||||
|
||||
```javascript
|
||||
javascript:(function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}})();
|
||||
```
|
||||
|
||||
The above bookmarklets use `alert()` as a success/failure notification. The following will show a toast message instead:
|
||||
|
||||
Chrome:
|
||||
|
||||
```javascript
|
||||
javascript:!function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}}();
|
||||
```
|
||||
|
||||
Firefox:
|
||||
|
||||
```javascript
|
||||
javascript:(function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}})();
|
||||
```
|
||||
|
||||
## ⚡ Raycast extension
|
||||
|
||||
[dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) that allows adding videos to MeTube directly from Raycast.
|
||||
|
||||
## 🔒 HTTPS support, and running behind a reverse proxy
|
||||
|
||||
|
|
@ -273,7 +180,13 @@ services:
|
|||
- KEYFILE=/ssl/key.pem
|
||||
```
|
||||
|
||||
MeTube can also run behind a reverse proxy for HTTPS termination or authentication. When serving under a subdirectory, set `URL_PREFIX` accordingly. MeTube uses WebSocket for real-time updates, so the proxy must pass the `Upgrade`/`Connection` headers, as in this NGINX example:
|
||||
It's also possible to run MeTube behind a reverse proxy, in order to support authentication. HTTPS support can also be added in this way.
|
||||
|
||||
When running behind a reverse proxy which remaps the URL (i.e. serves MeTube under a subdirectory and not under root), don't forget to set the URL_PREFIX environment variable to the correct value.
|
||||
|
||||
If you're using the [linuxserver/swag](https://docs.linuxserver.io/general/swag) image for your reverse proxying needs (which I can heartily recommend), it already includes ready snippets for proxying MeTube both in [subfolder](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subfolder.conf.sample) and [subdomain](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subdomain.conf.sample) modes under the `nginx/proxy-confs` directory in the configuration volume. It also includes Authelia which can be used for authentication.
|
||||
|
||||
### 🌐 NGINX
|
||||
|
||||
```nginx
|
||||
location /metube/ {
|
||||
|
|
@ -285,26 +198,66 @@ location /metube/ {
|
|||
}
|
||||
```
|
||||
|
||||
Apache, Caddy, and [linuxserver/swag](https://docs.linuxserver.io/general/swag) (with Authelia) examples are in the [Reverse proxy configurations wiki page](https://github.com/alexta69/metube/wiki/Reverse-proxy-configurations).
|
||||
Note: the extra `proxy_set_header` directives are there to make WebSocket work.
|
||||
|
||||
### 🌐 Apache
|
||||
|
||||
Contributed by [PIE-yt](https://github.com/PIE-yt). Source [here](https://gist.github.com/PIE-yt/29e7116588379032427f5bd446b2cac4).
|
||||
|
||||
```apache
|
||||
# For putting in your Apache sites site.conf
|
||||
# Serves MeTube under a /metube/ subdir (http://yourdomain.com/metube/)
|
||||
<Location /metube/>
|
||||
ProxyPass http://localhost:8081/ retry=0 timeout=30
|
||||
ProxyPassReverse http://localhost:8081/
|
||||
</Location>
|
||||
|
||||
<Location /metube/socket.io>
|
||||
RewriteEngine On
|
||||
RewriteCond %{QUERY_STRING} transport=websocket [NC]
|
||||
RewriteRule /(.*) ws://localhost:8081/socket.io/$1 [P,L]
|
||||
ProxyPass http://localhost:8081/socket.io retry=0 timeout=30
|
||||
ProxyPassReverse http://localhost:8081/socket.io
|
||||
</Location>
|
||||
```
|
||||
|
||||
### 🌐 Caddy
|
||||
|
||||
The following example Caddyfile gets a reverse proxy going behind [caddy](https://caddyserver.com).
|
||||
|
||||
```caddyfile
|
||||
example.com {
|
||||
route /metube/* {
|
||||
uri strip_prefix metube
|
||||
reverse_proxy metube:8081
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 🔄 Updating yt-dlp
|
||||
|
||||
MeTube is powered by [yt-dlp](https://github.com/yt-dlp/yt-dlp), which requires frequent updates as video sites change their layouts. A new MeTube Docker image is published automatically when a new yt-dlp stable release is available, so keep your container up to date — [watchtower](https://github.com/nicholas-fedor/watchtower) works well for this. To follow yt-dlp's nightly channel instead, set `YTDL_NIGHTLY_UPDATE_TIME`.
|
||||
The engine which powers the actual video downloads in MeTube is [yt-dlp](https://github.com/yt-dlp/yt-dlp). Since video sites regularly change their layouts, frequent updates of yt-dlp are required to keep up.
|
||||
|
||||
There's an automatic nightly build of MeTube which looks for a new version of yt-dlp, and if one exists, the build pulls it and publishes an updated docker image. Therefore, in order to keep up with the changes, it's recommended that you update your MeTube container regularly with the latest image.
|
||||
|
||||
I recommend installing and setting up [watchtower](https://github.com/nicholas-fedor/watchtower) for this purpose.
|
||||
|
||||
## 🔧 Troubleshooting and submitting issues
|
||||
|
||||
MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Issues with authentication, postprocessing, permissions, or `YTDL_OPTIONS` should be debugged with yt-dlp directly first — once working, import those options into MeTube. To test inside the container:
|
||||
Before asking a question or submitting an issue for MeTube, please remember that MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, postprocessing, permissions, other `YTDL_OPTIONS` configurations which seem not to work, or anything else that concerns the workings of the underlying yt-dlp library, need not be opened on the MeTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, and once that is working, importing the options that worked for you into `YTDL_OPTIONS`.
|
||||
|
||||
In order to test with the yt-dlp command directly, you can either download it and run it locally, or for a better simulation of its actual conditions, you can run it within the MeTube container itself. Assuming your MeTube container is called `metube`, run the following on your Docker host to get a shell inside the container:
|
||||
|
||||
```bash
|
||||
docker exec -ti metube sh
|
||||
cd /downloads
|
||||
```
|
||||
|
||||
Common issues and their fixes are collected in the [Troubleshooting FAQ](https://github.com/alexta69/metube/wiki/Troubleshooting-FAQ) on the wiki.
|
||||
Once there, you can use the yt-dlp command freely.
|
||||
|
||||
## 💡 Submitting feature requests
|
||||
|
||||
MeTube development relies on community contributions. If you need additional features, please submit a PR. Create an issue first to discuss the implementation before writing code — MeTube's scope is deliberately narrow: it downloads well and stops once the file is written. Features that improve the download itself are welcome; post-download file management (tag editing, metadata lookups, library organization) is out of scope regardless of implementation quality — see [AGENTS.md](AGENTS.md) for the full policy. Feature requests without an accompanying PR are unlikely to be fulfilled.
|
||||
MeTube development relies on code contributions by the community. The program as it currently stands fits my own use cases, and is therefore feature-complete as far as I'm concerned. If your use cases are different and require additional features, please feel free to submit PRs that implement those features. It's advisable to create an issue first to discuss the planned implementation, because in an effort to reduce bloat, some PRs may not be accepted. However, note that opening a feature request when you don't intend to implement the feature will rarely result in the request being fulfilled.
|
||||
|
||||
## 🛠️ Building and running locally
|
||||
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
import asyncio
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("bg_tasks")
|
||||
_TASKS: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def create_task(coro, *, name: str | None = None) -> asyncio.Task:
|
||||
"""create_task that keeps a strong reference and logs unexpected failures.
|
||||
|
||||
A bare ``asyncio.create_task(...)`` is only weakly referenced by the event
|
||||
loop; if nothing else holds the returned Task, it can be garbage collected
|
||||
mid-flight. Keeping a module-level strong reference (removed once the task
|
||||
finishes) avoids that, and the done-callback surfaces otherwise-silent
|
||||
failures.
|
||||
"""
|
||||
task = asyncio.get_running_loop().create_task(coro, name=name)
|
||||
_TASKS.add(task)
|
||||
|
||||
def _done(t: asyncio.Task) -> None:
|
||||
_TASKS.discard(t)
|
||||
if not t.cancelled() and t.exception() is not None:
|
||||
log.error("Background task %s failed", t.get_name(), exc_info=t.exception())
|
||||
|
||||
task.add_done_callback(_done)
|
||||
return task
|
||||
|
|
@ -3,21 +3,6 @@ import copy
|
|||
AUDIO_FORMATS = ("m4a", "mp3", "opus", "wav", "flac")
|
||||
CAPTION_MODES = ("auto_only", "manual_only", "prefer_manual", "prefer_auto")
|
||||
|
||||
|
||||
def merge_ytdl_option_layers(presets, overrides, presets_config) -> dict:
|
||||
"""Overlay named presets (in order) then per-item overrides onto a fresh dict.
|
||||
|
||||
Does NOT include any base ``YTDL_OPTIONS`` — callers layer this on top of
|
||||
their own base (a per-download build adds the global base; a subscription
|
||||
scan relies on ``**config.YTDL_OPTIONS`` already being present in its
|
||||
params). ``presets_config`` maps a preset name to its options dict.
|
||||
"""
|
||||
merged: dict = {}
|
||||
for name in presets or []:
|
||||
merged.update(presets_config.get(name, {}))
|
||||
merged.update(overrides or {})
|
||||
return merged
|
||||
|
||||
CODEC_FILTER_MAP = {
|
||||
'h264': "[vcodec~='^(h264|avc)']",
|
||||
'h265': "[vcodec~='^(h265|hevc)']",
|
||||
|
|
@ -58,10 +43,6 @@ def get_format(download_type: str, codec: str, format: str, quality: str) -> str
|
|||
quality = (quality or "best").strip().lower()
|
||||
|
||||
if format.startswith("custom:"):
|
||||
# Unreachable via the HTTP API (format is validated against a fixed
|
||||
# set in main.py), but legacy persisted downloads may carry a
|
||||
# custom: format from before that validation existed; removing this
|
||||
# would crash PersistentQueue.load() for those records.
|
||||
return format[7:]
|
||||
|
||||
if download_type == "thumbnail":
|
||||
|
|
@ -156,20 +137,7 @@ def get_opts(
|
|||
requested_subtitle_format = (format or "srt").lower()
|
||||
if requested_subtitle_format == "txt":
|
||||
requested_subtitle_format = "srt"
|
||||
opts["subtitlesformat"] = f"{requested_subtitle_format}/best"
|
||||
if requested_subtitle_format in ("srt", "vtt"):
|
||||
# subtitlesformat above is only a preference: if the extractor
|
||||
# doesn't natively offer this ext (e.g. YouTube has no native srt),
|
||||
# yt-dlp silently falls back to whatever it has. ffmpeg can only
|
||||
# convert to srt/vtt/ass/lrc, so only guarantee the requested
|
||||
# container for those; other formats stay best-effort.
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegSubtitlesConvertor",
|
||||
"format": requested_subtitle_format,
|
||||
"when": "before_dl",
|
||||
}
|
||||
)
|
||||
opts["subtitlesformat"] = requested_subtitle_format
|
||||
if mode == "manual_only":
|
||||
opts["writesubtitles"] = True
|
||||
opts["writeautomaticsub"] = False
|
||||
|
|
|
|||
485
app/main.py
485
app/main.py
|
|
@ -4,10 +4,8 @@
|
|||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from aiohttp import web
|
||||
from aiohttp.web import GracefulExit
|
||||
from aiohttp.log import access_logger
|
||||
import ssl
|
||||
import socket
|
||||
|
|
@ -16,33 +14,14 @@ import logging
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||||
from watchfiles import DefaultFilter, Change, awatch
|
||||
|
||||
import bg_tasks
|
||||
from ytdl import DownloadQueueNotifier, DownloadQueue, Download
|
||||
from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool
|
||||
from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo
|
||||
from yt_dlp.version import __version__ as yt_dlp_version
|
||||
|
||||
log = logging.getLogger('main')
|
||||
|
||||
_NIGHTLY_TIME_RE = re.compile(r'^([01]\d|2[0-3]):[0-5]\d$')
|
||||
_RESTART_FOR_UPDATE = False
|
||||
|
||||
def _request_graceful_exit() -> None:
|
||||
raise GracefulExit()
|
||||
|
||||
|
||||
def seconds_until_next_daily_time(time_hhmm: str, now: datetime | None = None) -> float:
|
||||
"""Seconds until the next occurrence of HH:MM in local time."""
|
||||
now = now or datetime.now()
|
||||
hour, minute = map(int, time_hhmm.split(':'))
|
||||
target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if target <= now:
|
||||
target += timedelta(days=1)
|
||||
return (target - now).total_seconds()
|
||||
|
||||
def parseLogLevel(logLevel):
|
||||
if not isinstance(logLevel, str):
|
||||
return None
|
||||
|
|
@ -78,10 +57,6 @@ class Config:
|
|||
'CLEAR_COMPLETED_AFTER': '0',
|
||||
'YTDL_OPTIONS': '{}',
|
||||
'YTDL_OPTIONS_FILE': '',
|
||||
'YTDL_OPTIONS_PRESETS': '{}',
|
||||
'YTDL_OPTIONS_PRESETS_FILE': '',
|
||||
'ALLOW_YTDL_OPTIONS_OVERRIDES': 'false',
|
||||
'CORS_ALLOWED_ORIGINS': '',
|
||||
'ROBOTS_TXT': '',
|
||||
'HOST': '0.0.0.0',
|
||||
'PORT': '8081',
|
||||
|
|
@ -93,10 +68,9 @@ class Config:
|
|||
'MAX_CONCURRENT_DOWNLOADS': '3',
|
||||
'LOGLEVEL': 'INFO',
|
||||
'ENABLE_ACCESSLOG': 'false',
|
||||
'YTDL_NIGHTLY_UPDATE_TIME': '',
|
||||
}
|
||||
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'ALLOW_YTDL_OPTIONS_OVERRIDES')
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
|
||||
def __init__(self):
|
||||
for k, v in self._DEFAULTS.items():
|
||||
|
|
@ -114,61 +88,15 @@ class Config:
|
|||
if not self.URL_PREFIX.endswith('/'):
|
||||
self.URL_PREFIX += '/'
|
||||
|
||||
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
||||
# default via os.environ.get, which would leave audio links root-relative and 404.
|
||||
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
||||
# PUBLIC_HOST_URL is also blank we leave it blank to preserve serving from web root.
|
||||
if not self.PUBLIC_HOST_AUDIO_URL and self.PUBLIC_HOST_URL:
|
||||
self.PUBLIC_HOST_AUDIO_URL = self._DEFAULTS['PUBLIC_HOST_AUDIO_URL']
|
||||
|
||||
for attr in ('PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL'):
|
||||
val = getattr(self, attr)
|
||||
if val and not val.endswith('/'):
|
||||
setattr(self, attr, val + '/')
|
||||
|
||||
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
||||
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
||||
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
||||
if self.YTDL_OPTIONS_PRESETS_FILE and self.YTDL_OPTIONS_PRESETS_FILE.startswith('.'):
|
||||
self.YTDL_OPTIONS_PRESETS_FILE = str(Path(self.YTDL_OPTIONS_PRESETS_FILE).resolve())
|
||||
|
||||
if self.YTDL_NIGHTLY_UPDATE_TIME and not _NIGHTLY_TIME_RE.match(self.YTDL_NIGHTLY_UPDATE_TIME):
|
||||
log.error(
|
||||
'Environment variable "YTDL_NIGHTLY_UPDATE_TIME" must be HH:MM (24-hour), got "%s"',
|
||||
self.YTDL_NIGHTLY_UPDATE_TIME,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
self._validate_int('MAX_CONCURRENT_DOWNLOADS', minimum=1)
|
||||
self._validate_int('PORT', minimum=1, maximum=65535)
|
||||
self._validate_int('CLEAR_COMPLETED_AFTER', minimum=0)
|
||||
self._validate_int('DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT', minimum=0)
|
||||
self._validate_int('SUBSCRIPTION_DEFAULT_CHECK_INTERVAL', minimum=1)
|
||||
self._validate_int('SUBSCRIPTION_SCAN_PLAYLIST_END', minimum=1)
|
||||
self._validate_int('SUBSCRIPTION_MAX_SEEN_IDS', minimum=1)
|
||||
|
||||
self._runtime_overrides = {}
|
||||
|
||||
success,_ = self.load_ytdl_options()
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
success,_ = self.load_ytdl_option_presets()
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
|
||||
def _validate_int(self, key, *, minimum=None, maximum=None):
|
||||
raw = getattr(self, key)
|
||||
try:
|
||||
value = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
log.error('Environment variable "%s" must be an integer, got "%s"', key, raw)
|
||||
sys.exit(1)
|
||||
if minimum is not None and value < minimum:
|
||||
log.error('Environment variable "%s" must be >= %d, got "%s"', key, minimum, raw)
|
||||
sys.exit(1)
|
||||
if maximum is not None and value > maximum:
|
||||
log.error('Environment variable "%s" must be <= %d, got "%s"', key, maximum, raw)
|
||||
sys.exit(1)
|
||||
|
||||
def set_runtime_override(self, key, value):
|
||||
self._runtime_overrides[key] = value
|
||||
|
|
@ -191,7 +119,6 @@ class Config:
|
|||
'PUBLIC_HOST_AUDIO_URL',
|
||||
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT',
|
||||
'SUBSCRIPTION_DEFAULT_CHECK_INTERVAL',
|
||||
'ALLOW_YTDL_OPTIONS_OVERRIDES',
|
||||
)
|
||||
|
||||
def frontend_safe(self) -> dict:
|
||||
|
|
@ -233,37 +160,6 @@ class Config:
|
|||
self._apply_runtime_overrides()
|
||||
return (True, '')
|
||||
|
||||
def load_ytdl_option_presets(self) -> tuple[bool, str]:
|
||||
try:
|
||||
self.YTDL_OPTIONS_PRESETS = json.loads(os.environ.get('YTDL_OPTIONS_PRESETS', '{}'))
|
||||
assert isinstance(self.YTDL_OPTIONS_PRESETS, dict)
|
||||
assert all(isinstance(name, str) and isinstance(options, dict) for name, options in self.YTDL_OPTIONS_PRESETS.items())
|
||||
except (json.decoder.JSONDecodeError, AssertionError):
|
||||
msg = 'Environment variable YTDL_OPTIONS_PRESETS is invalid'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
|
||||
if not self.YTDL_OPTIONS_PRESETS_FILE:
|
||||
return (True, '')
|
||||
|
||||
log.info(f'Loading yt-dlp option presets from "{self.YTDL_OPTIONS_PRESETS_FILE}"')
|
||||
if not os.path.exists(self.YTDL_OPTIONS_PRESETS_FILE):
|
||||
msg = f'File "{self.YTDL_OPTIONS_PRESETS_FILE}" not found'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
try:
|
||||
with open(self.YTDL_OPTIONS_PRESETS_FILE) as json_data:
|
||||
opts = json.load(json_data)
|
||||
assert isinstance(opts, dict)
|
||||
assert all(isinstance(name, str) and isinstance(options, dict) for name, options in opts.items())
|
||||
except (json.decoder.JSONDecodeError, AssertionError):
|
||||
msg = 'YTDL_OPTIONS_PRESETS_FILE contents is invalid'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
|
||||
self.YTDL_OPTIONS_PRESETS.update(opts)
|
||||
return (True, '')
|
||||
|
||||
config = Config()
|
||||
# Align root logger level with Config (keeps a single source of truth).
|
||||
# This re-applies the log level after Config loads, in case LOGLEVEL was
|
||||
|
|
@ -272,13 +168,7 @@ logging.getLogger().setLevel(parseLogLevel(str(config.LOGLEVEL)) or logging.INFO
|
|||
|
||||
class ObjectSerializer(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
# Prefer an explicit client-facing view when the object provides one
|
||||
# (e.g. DownloadInfo / SubscriptionInfo) so server-only or bulky fields
|
||||
# are never broadcast to browser clients.
|
||||
to_public = getattr(obj, 'to_public_dict', None)
|
||||
if callable(to_public):
|
||||
return to_public()
|
||||
# Fall back to __dict__ for other custom objects
|
||||
# First try to use __dict__ for custom objects
|
||||
if hasattr(obj, '__dict__'):
|
||||
return obj.__dict__
|
||||
# Convert iterables (generators, dict_items, etc.) to lists
|
||||
|
|
@ -292,35 +182,8 @@ class ObjectSerializer(json.JSONEncoder):
|
|||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
serializer = ObjectSerializer()
|
||||
|
||||
_STATE_DIR_REAL = os.path.realpath(config.STATE_DIR)
|
||||
|
||||
|
||||
def _is_within_state_dir(real_target: str) -> bool:
|
||||
return real_target == _STATE_DIR_REAL or real_target.startswith(_STATE_DIR_REAL + os.sep)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def state_dir_guard(request, handler):
|
||||
for prefix, base in (
|
||||
(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR),
|
||||
(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR),
|
||||
):
|
||||
if request.path.startswith(prefix):
|
||||
# request.path is already percent-decoded by aiohttp; decoding it
|
||||
# again would mangle a download whose filename contains a literal
|
||||
# '%' (e.g. "%" turning into a truncated escape) into a false 404.
|
||||
rel = request.path[len(prefix):]
|
||||
target = os.path.realpath(os.path.join(base, rel))
|
||||
if _is_within_state_dir(target):
|
||||
raise web.HTTPNotFound()
|
||||
break
|
||||
return await handler(request)
|
||||
|
||||
|
||||
app = web.Application(middlewares=[state_dir_guard])
|
||||
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
||||
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
||||
app = web.Application()
|
||||
sio = socketio.AsyncServer(cors_allowed_origins='*')
|
||||
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
||||
routes = web.RouteTableDef()
|
||||
VALID_SUBTITLE_FORMATS = {'srt', 'txt', 'vtt', 'ttml', 'sbv', 'scc', 'dfxp'}
|
||||
|
|
@ -331,157 +194,6 @@ VALID_VIDEO_CODECS = {'auto', 'h264', 'h265', 'av1', 'vp9'}
|
|||
VALID_VIDEO_FORMATS = {'any', 'mp4', 'ios'}
|
||||
VALID_AUDIO_FORMATS = {'m4a', 'mp3', 'opus', 'wav', 'flac'}
|
||||
VALID_THUMBNAIL_FORMATS = {'jpg'}
|
||||
def _parse_ytdl_options_overrides(value, *, enabled: bool) -> dict:
|
||||
if value is None or value == '':
|
||||
return {}
|
||||
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides must be valid JSON') from exc
|
||||
|
||||
if not isinstance(value, dict):
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides must be a JSON object')
|
||||
|
||||
if value and not enabled:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides are disabled')
|
||||
|
||||
return value
|
||||
|
||||
|
||||
_YOUTUBE_T_COMPACT_RE = re.compile(
|
||||
r'^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)(?:s)?)?$',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_youtube_t_compact(value: str) -> float | None:
|
||||
"""Parse YouTube-style ``t`` values: ``885``, ``885s``, ``14m45s``, ``1h2m3s``."""
|
||||
v = value.strip()
|
||||
if not v:
|
||||
return None
|
||||
if re.fullmatch(r'-?\d+(\.\d+)?', v):
|
||||
sec = float(v)
|
||||
return sec if sec >= 0 else None
|
||||
m = _YOUTUBE_T_COMPACT_RE.match(v)
|
||||
if m and any(m.groups()):
|
||||
hours = int(m.group(1) or 0)
|
||||
minutes = int(m.group(2) or 0)
|
||||
seconds = int(m.group(3) or 0)
|
||||
total = hours * 3600 + minutes * 60 + seconds
|
||||
return float(total) if total >= 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_clock_timestamp(s: str) -> float:
|
||||
"""Parse ``MM:SS``, ``H:MM:SS``, or single segment as seconds (with optional decimals)."""
|
||||
part = s.strip()
|
||||
if not part:
|
||||
raise ValueError('empty timestamp')
|
||||
segments = part.split(':')
|
||||
if len(segments) > 3:
|
||||
raise ValueError('too many segments')
|
||||
try:
|
||||
nums = [float(x) for x in segments]
|
||||
except ValueError as exc:
|
||||
raise ValueError('invalid number') from exc
|
||||
if any(x < 0 for x in nums):
|
||||
raise ValueError('negative segment')
|
||||
if len(segments) == 1:
|
||||
return nums[0]
|
||||
if len(segments) == 2:
|
||||
return nums[0] * 60 + nums[1]
|
||||
return nums[0] * 3600 + nums[1] * 60 + nums[2]
|
||||
|
||||
|
||||
def _parse_clip_timestamp_value(value) -> float:
|
||||
"""Coerce a clip boundary from JSON to seconds (non-negative)."""
|
||||
if isinstance(value, bool):
|
||||
raise web.HTTPBadRequest(reason='clip timestamp must be a number or string')
|
||||
if isinstance(value, (int, float)):
|
||||
if value < 0:
|
||||
raise web.HTTPBadRequest(reason='clip timestamp must be non-negative')
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
raise web.HTTPBadRequest(reason='clip timestamp cannot be empty')
|
||||
if ':' in s:
|
||||
try:
|
||||
return _parse_clock_timestamp(s)
|
||||
except ValueError as exc:
|
||||
raise web.HTTPBadRequest(reason='invalid clip timestamp format') from exc
|
||||
compact = _parse_youtube_t_compact(s)
|
||||
if compact is not None:
|
||||
return compact
|
||||
raise web.HTTPBadRequest(reason='invalid clip timestamp format')
|
||||
|
||||
|
||||
def _optional_clip_field(raw) -> float | None:
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, str) and not raw.strip():
|
||||
return None
|
||||
return _parse_clip_timestamp_value(raw)
|
||||
|
||||
|
||||
def _clip_field_provided_in_post(raw) -> bool:
|
||||
if raw is None:
|
||||
return False
|
||||
if isinstance(raw, str) and not raw.strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _extract_t_query_from_url(url: str) -> tuple[str, float | None]:
|
||||
"""If ``t=`` is present and parseable, return URL without ``t`` and start seconds.
|
||||
|
||||
Restricted to YouTube hosts: ``t`` is a generic query parameter name that
|
||||
other sites may use for unrelated purposes, so rewriting it there would
|
||||
silently mutate the URL and inject a bogus clip start.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
params = parse_qs(parsed.query)
|
||||
except Exception:
|
||||
return url, None
|
||||
host = (parsed.hostname or '').lower()
|
||||
if not (host in ('youtu.be', 'youtube.com') or host.endswith('.youtube.com')):
|
||||
return url, None
|
||||
t_values = params.get('t')
|
||||
if not t_values:
|
||||
return url, None
|
||||
start = _parse_youtube_t_compact(t_values[0])
|
||||
if start is None:
|
||||
return url, None
|
||||
filtered = {k: v for k, v in params.items() if k != 't'}
|
||||
new_query = urlencode(filtered, doseq=True)
|
||||
cleaned = urlunparse((
|
||||
parsed.scheme,
|
||||
parsed.netloc,
|
||||
parsed.path,
|
||||
parsed.params,
|
||||
new_query,
|
||||
parsed.fragment,
|
||||
))
|
||||
return cleaned, float(start)
|
||||
|
||||
|
||||
def _parse_ytdl_options_presets(post: dict) -> list[str]:
|
||||
"""Normalize preset names from add/subscribe body; supports list or legacy singular string."""
|
||||
raw = post.get('ytdl_options_presets')
|
||||
if raw is None:
|
||||
raw = post.get('ytdl_options_preset')
|
||||
if raw is None:
|
||||
return []
|
||||
if isinstance(raw, list):
|
||||
return [str(x).strip() for x in raw if str(x).strip()]
|
||||
if isinstance(raw, str):
|
||||
s = raw.strip()
|
||||
return [s] if s else []
|
||||
raise web.HTTPBadRequest(
|
||||
reason='ytdl_options_presets must be a JSON array of strings (or legacy ytdl_options_preset string)',
|
||||
)
|
||||
|
||||
|
||||
def _migrate_legacy_request(post: dict) -> dict:
|
||||
|
|
@ -562,19 +274,8 @@ class Notifier(DownloadQueueNotifier):
|
|||
await sio.emit('cleared', serializer.encode(id))
|
||||
|
||||
dqueue = DownloadQueue(config, Notifier())
|
||||
|
||||
|
||||
async def _download_queue_startup(app):
|
||||
await dqueue.initialize()
|
||||
|
||||
|
||||
async def _shutdown_download_manager(app):
|
||||
dqueue.close()
|
||||
Download.shutdown_manager()
|
||||
|
||||
|
||||
app.on_startup.append(_download_queue_startup)
|
||||
app.on_cleanup.append(_shutdown_download_manager)
|
||||
app.on_startup.append(lambda app: dqueue.initialize())
|
||||
app.on_cleanup.append(lambda app: Download.shutdown_manager())
|
||||
|
||||
|
||||
class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
||||
|
|
@ -594,13 +295,7 @@ class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
|||
|
||||
|
||||
submgr = SubscriptionManager(config, dqueue, MetubeSubscriptionNotifier())
|
||||
|
||||
|
||||
async def _shutdown_subscriptions(app):
|
||||
submgr.close()
|
||||
|
||||
|
||||
app.on_cleanup.append(_shutdown_subscriptions)
|
||||
app.on_cleanup.append(lambda app: submgr.close())
|
||||
|
||||
|
||||
async def _subscription_loop_startup(app):
|
||||
|
|
@ -610,26 +305,6 @@ async def _subscription_loop_startup(app):
|
|||
|
||||
app.on_startup.append(_subscription_loop_startup)
|
||||
|
||||
|
||||
async def _schedule_nightly_update() -> None:
|
||||
global _RESTART_FOR_UPDATE
|
||||
time_hhmm = config.YTDL_NIGHTLY_UPDATE_TIME
|
||||
if not time_hhmm:
|
||||
return
|
||||
delay = seconds_until_next_daily_time(time_hhmm)
|
||||
log.info('Next yt-dlp nightly update in %.0f seconds (at %s local time)', delay, time_hhmm)
|
||||
await asyncio.sleep(delay)
|
||||
log.info('Scheduled yt-dlp nightly update: requesting restart')
|
||||
_RESTART_FOR_UPDATE = True
|
||||
asyncio.get_running_loop().call_soon(_request_graceful_exit)
|
||||
|
||||
|
||||
async def _start_nightly_update_schedule(app):
|
||||
bg_tasks.create_task(_schedule_nightly_update(), name="nightly_update_schedule")
|
||||
|
||||
|
||||
app.on_startup.append(_start_nightly_update_schedule)
|
||||
|
||||
class FileOpsFilter(DefaultFilter):
|
||||
def __call__(self, change_type: int, path: str) -> bool:
|
||||
# Check if this path matches our YTDL_OPTIONS_FILE
|
||||
|
|
@ -674,14 +349,10 @@ async def watch_files():
|
|||
await sio.emit('ytdl_options_changed', serializer.encode(result))
|
||||
|
||||
log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}')
|
||||
bg_tasks.create_task(_watch_files(), name="watch_ytdl_options_file")
|
||||
|
||||
async def _watch_files_startup(app):
|
||||
await watch_files()
|
||||
|
||||
asyncio.create_task(_watch_files())
|
||||
|
||||
if config.YTDL_OPTIONS_FILE:
|
||||
app.on_startup.append(_watch_files_startup)
|
||||
app.on_startup.append(lambda app: watch_files())
|
||||
|
||||
|
||||
async def _read_json_request(request: web.Request) -> dict:
|
||||
|
|
@ -713,7 +384,6 @@ def parse_download_options(post: dict) -> dict:
|
|||
chapter_template = post.get('chapter_template')
|
||||
subtitle_language = post.get('subtitle_language')
|
||||
subtitle_mode = post.get('subtitle_mode')
|
||||
ytdl_options_overrides = post.get('ytdl_options_overrides')
|
||||
|
||||
if custom_name_prefix is None:
|
||||
custom_name_prefix = ''
|
||||
|
|
@ -737,11 +407,6 @@ def parse_download_options(post: dict) -> dict:
|
|||
quality = str(quality).strip().lower()
|
||||
subtitle_language = str(subtitle_language).strip()
|
||||
subtitle_mode = str(subtitle_mode).strip()
|
||||
ytdl_options_presets = _parse_ytdl_options_presets(post)
|
||||
ytdl_options_overrides = _parse_ytdl_options_overrides(
|
||||
ytdl_options_overrides,
|
||||
enabled=config.ALLOW_YTDL_OPTIONS_OVERRIDES,
|
||||
)
|
||||
|
||||
if chapter_template and ('..' in chapter_template or chapter_template.startswith('/') or chapter_template.startswith('\\')):
|
||||
raise web.HTTPBadRequest(reason='chapter_template must not contain ".." or start with a path separator')
|
||||
|
|
@ -749,9 +414,6 @@ def parse_download_options(post: dict) -> dict:
|
|||
raise web.HTTPBadRequest(reason='subtitle_language must match pattern [A-Za-z0-9-] and be at most 35 characters')
|
||||
if subtitle_mode not in VALID_SUBTITLE_MODES:
|
||||
raise web.HTTPBadRequest(reason=f'subtitle_mode must be one of {sorted(VALID_SUBTITLE_MODES)}')
|
||||
for preset_name in ytdl_options_presets:
|
||||
if preset_name not in config.YTDL_OPTIONS_PRESETS:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_presets must only contain configured preset names')
|
||||
|
||||
if download_type not in VALID_DOWNLOAD_TYPES:
|
||||
raise web.HTTPBadRequest(reason=f'download_type must be one of {sorted(VALID_DOWNLOAD_TYPES)}')
|
||||
|
|
@ -790,39 +452,6 @@ def parse_download_options(post: dict) -> dict:
|
|||
except (TypeError, ValueError) as exc:
|
||||
raise web.HTTPBadRequest(reason='playlist_item_limit must be an integer') from exc
|
||||
|
||||
clip_start_raw = post.get('clip_start')
|
||||
clip_end_raw = post.get('clip_end')
|
||||
clip_start: float | None
|
||||
clip_end: float | None
|
||||
if download_type in ('captions', 'thumbnail'):
|
||||
if _clip_field_provided_in_post(clip_start_raw) or _clip_field_provided_in_post(clip_end_raw):
|
||||
raise web.HTTPBadRequest(
|
||||
reason='clip_start and clip_end are only supported for video and audio downloads',
|
||||
)
|
||||
clip_start = None
|
||||
clip_end = None
|
||||
else:
|
||||
cleaned_url, url_t = _extract_t_query_from_url(url)
|
||||
if url_t is not None:
|
||||
url = cleaned_url
|
||||
explicit_start = _optional_clip_field(clip_start_raw)
|
||||
explicit_end = _optional_clip_field(clip_end_raw)
|
||||
explicit_start_provided = _clip_field_provided_in_post(clip_start_raw)
|
||||
explicit_end_provided = _clip_field_provided_in_post(clip_end_raw)
|
||||
if explicit_start_provided:
|
||||
clip_start = explicit_start
|
||||
elif explicit_end_provided:
|
||||
clip_start = 0.0
|
||||
elif url_t is not None:
|
||||
clip_start = url_t
|
||||
else:
|
||||
clip_start = None
|
||||
clip_end = explicit_end
|
||||
if clip_end is not None and clip_start is None:
|
||||
clip_start = 0.0
|
||||
if clip_start is not None and clip_end is not None and clip_end <= clip_start:
|
||||
raise web.HTTPBadRequest(reason='clip_end must be greater than clip_start')
|
||||
|
||||
return {
|
||||
'url': url,
|
||||
'download_type': download_type,
|
||||
|
|
@ -837,10 +466,6 @@ def parse_download_options(post: dict) -> dict:
|
|||
'chapter_template': chapter_template,
|
||||
'subtitle_language': subtitle_language,
|
||||
'subtitle_mode': subtitle_mode,
|
||||
'ytdl_options_presets': ytdl_options_presets,
|
||||
'ytdl_options_overrides': ytdl_options_overrides,
|
||||
'clip_start': clip_start,
|
||||
'clip_end': clip_end,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -875,21 +500,9 @@ async def add(request):
|
|||
o['chapter_template'],
|
||||
o['subtitle_language'],
|
||||
o['subtitle_mode'],
|
||||
o['ytdl_options_presets'],
|
||||
o['ytdl_options_overrides'],
|
||||
o['clip_start'],
|
||||
o['clip_end'],
|
||||
)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'presets')
|
||||
async def presets(request):
|
||||
return web.Response(
|
||||
text=serializer.encode({'presets': sorted(config.YTDL_OPTIONS_PRESETS.keys())}),
|
||||
content_type='application/json',
|
||||
)
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'cancel-add')
|
||||
async def cancel_add(request):
|
||||
dqueue.cancel_add()
|
||||
|
|
@ -899,7 +512,10 @@ async def cancel_add(request):
|
|||
@routes.post(config.URL_PREFIX + 'subscribe')
|
||||
async def subscribe(request):
|
||||
post = await _read_json_request(request)
|
||||
o = parse_download_options(post)
|
||||
try:
|
||||
o = parse_download_options(post)
|
||||
except web.HTTPBadRequest:
|
||||
raise
|
||||
cic = post.get('check_interval_minutes')
|
||||
if cic is None:
|
||||
cic = config.SUBSCRIPTION_DEFAULT_CHECK_INTERVAL
|
||||
|
|
@ -909,17 +525,6 @@ async def subscribe(request):
|
|||
raise web.HTTPBadRequest(reason='check_interval_minutes must be an integer') from exc
|
||||
if cic < 1:
|
||||
raise web.HTTPBadRequest(reason='check_interval_minutes must be at least 1')
|
||||
if o.get('clip_start') is not None or o.get('clip_end') is not None:
|
||||
raise web.HTTPBadRequest(reason='clip options are not supported for subscriptions')
|
||||
|
||||
try:
|
||||
skip_subscriber_only = coerce_optional_bool(
|
||||
post.get('skip_subscriber_only'),
|
||||
default=False,
|
||||
field_name='skip_subscriber_only',
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise web.HTTPBadRequest(reason=str(exc)) from exc
|
||||
|
||||
result = await submgr.add_subscription(
|
||||
o['url'],
|
||||
|
|
@ -936,10 +541,6 @@ async def subscribe(request):
|
|||
chapter_template=o['chapter_template'],
|
||||
subtitle_language=o['subtitle_language'],
|
||||
subtitle_mode=o['subtitle_mode'],
|
||||
ytdl_options_presets=o['ytdl_options_presets'],
|
||||
ytdl_options_overrides=o['ytdl_options_overrides'],
|
||||
title_regex=post.get('title_regex'),
|
||||
skip_subscriber_only=skip_subscriber_only,
|
||||
)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
|
|
@ -955,12 +556,7 @@ async def subscriptions_update(request):
|
|||
sub_id = post.get('id')
|
||||
if not sub_id:
|
||||
raise web.HTTPBadRequest(reason='missing subscription id')
|
||||
changes = {
|
||||
k: v
|
||||
for k, v in post.items()
|
||||
if k != 'id'
|
||||
and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex', 'skip_subscriber_only')
|
||||
}
|
||||
changes = {k: v for k, v in post.items() if k != 'id' and k in ('enabled', 'check_interval_minutes', 'name')}
|
||||
if not changes:
|
||||
raise web.HTTPBadRequest(reason='no valid fields to update')
|
||||
log.info("Subscription update requested for %s: %s", sub_id, sorted(changes.keys()))
|
||||
|
|
@ -988,20 +584,13 @@ async def subscriptions_check(request):
|
|||
result = await submgr.check_now([str(i) for i in ids] if ids else None)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
def _require_id_list(post: dict) -> list:
|
||||
ids = post.get('ids')
|
||||
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||||
raise web.HTTPBadRequest(reason="'ids' must be a non-empty list of strings")
|
||||
return ids
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'delete')
|
||||
async def delete(request):
|
||||
post = await _read_json_request(request)
|
||||
ids = _require_id_list(post)
|
||||
ids = post.get('ids')
|
||||
where = post.get('where')
|
||||
if where not in ['queue', 'done']:
|
||||
log.error("Bad request: incorrect 'where' value")
|
||||
if not ids or where not in ['queue', 'done']:
|
||||
log.error("Bad request: missing 'ids' or incorrect 'where' value")
|
||||
raise web.HTTPBadRequest()
|
||||
status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids))
|
||||
log.info(f"Download delete request processed for ids: {ids}, where: {where}")
|
||||
|
|
@ -1010,7 +599,7 @@ async def delete(request):
|
|||
@routes.post(config.URL_PREFIX + 'start')
|
||||
async def start(request):
|
||||
post = await _read_json_request(request)
|
||||
ids = _require_id_list(post)
|
||||
ids = post.get('ids')
|
||||
log.info(f"Received request to start pending downloads for ids: {ids}")
|
||||
status = await dqueue.start_pending(ids)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
|
@ -1040,12 +629,6 @@ async def upload_cookies(request):
|
|||
tmp_cookie_path = f"{COOKIES_PATH}.tmp"
|
||||
with open(tmp_cookie_path, 'wb') as f:
|
||||
f.write(content)
|
||||
# Cookies are sensitive auth material; restrict to owner read/write only
|
||||
# (the container's default umask would otherwise leave them group/world readable).
|
||||
try:
|
||||
os.chmod(tmp_cookie_path, 0o600)
|
||||
except OSError as exc:
|
||||
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
||||
os.replace(tmp_cookie_path, COOKIES_PATH)
|
||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||
log.info(f'Cookies file uploaded ({size} bytes)')
|
||||
|
|
@ -1090,15 +673,12 @@ async def cookie_status(request):
|
|||
async def history(request):
|
||||
history = { 'done': [], 'queue': [], 'pending': []}
|
||||
|
||||
# Served from the in-memory queues (like the socket 'all' event) rather
|
||||
# than saved_items(), which reloads and re-compacts the on-disk state on
|
||||
# every call.
|
||||
for _, v in dqueue.queue.items():
|
||||
history['queue'].append(v.info)
|
||||
for _, v in dqueue.done.items():
|
||||
history['done'].append(v.info)
|
||||
for _, v in dqueue.pending.items():
|
||||
history['pending'].append(v.info)
|
||||
for _, v in dqueue.queue.saved_items():
|
||||
history['queue'].append(v)
|
||||
for _, v in dqueue.done.saved_items():
|
||||
history['done'].append(v)
|
||||
for _, v in dqueue.pending.saved_items():
|
||||
history['pending'].append(v)
|
||||
|
||||
log.info("Sending download history")
|
||||
return web.Response(text=serializer.encode(history))
|
||||
|
|
@ -1110,17 +690,13 @@ async def connect(sid, environ):
|
|||
await sio.emit('subscriptions_all', serializer.encode([s.to_public_dict() for s in submgr.list_all()]), to=sid)
|
||||
await sio.emit('configuration', serializer.encode(config.frontend_safe()), to=sid)
|
||||
if config.CUSTOM_DIRS:
|
||||
# get_custom_dirs() can walk the whole download tree on a cache miss;
|
||||
# keep that off the event loop so a large library doesn't stall every
|
||||
# client's connect handshake.
|
||||
dirs = await asyncio.get_running_loop().run_in_executor(None, get_custom_dirs)
|
||||
await sio.emit('custom_dirs', serializer.encode(dirs), to=sid)
|
||||
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
|
||||
if config.YTDL_OPTIONS_FILE:
|
||||
await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid)
|
||||
|
||||
def get_custom_dirs():
|
||||
cache_ttl_seconds = 5
|
||||
now = time.monotonic()
|
||||
now = asyncio.get_running_loop().time()
|
||||
cache_key = (
|
||||
config.DOWNLOAD_DIR,
|
||||
config.AUDIO_DOWNLOAD_DIR,
|
||||
|
|
@ -1239,9 +815,8 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'upload-cookies', add_cors)
|
|||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors)
|
||||
|
||||
async def on_prepare(request, response):
|
||||
origin = request.headers.get('Origin')
|
||||
if origin and _cors_origins and ('*' in _cors_origins or origin in _cors_origins):
|
||||
response.headers['Access-Control-Allow-Origin'] = origin
|
||||
if 'Origin' in request.headers:
|
||||
response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']
|
||||
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
||||
|
||||
app.on_response_prepare.append(on_prepare)
|
||||
|
|
@ -1277,5 +852,3 @@ if __name__ == '__main__':
|
|||
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled())
|
||||
else:
|
||||
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled())
|
||||
if _RESTART_FOR_UPDATE:
|
||||
sys.exit(42)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
|||
|
||||
import base64
|
||||
import collections.abc
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -18,25 +17,6 @@ STATE_SCHEMA_VERSION = 2
|
|||
_BYTES_MARKER = "__metube_bytes__"
|
||||
_DATETIME_MARKER = "__metube_datetime__"
|
||||
|
||||
# Errnos that signal the filesystem cannot support the temp-file + rename
|
||||
# atomic-write strategy (for example an NFS-backed state dir returning EPERM on
|
||||
# mkstemp). These are safe to fall back on because they mean the atomic
|
||||
# mechanism is unavailable, not that the data write itself failed. Errors like
|
||||
# ENOSPC/EIO are deliberately excluded so a genuine storage failure surfaces
|
||||
# instead of silently truncating an existing good state file.
|
||||
_ATOMIC_UNSUPPORTED_ERRNOS = frozenset(
|
||||
e
|
||||
for e in (
|
||||
errno.EPERM,
|
||||
errno.EACCES,
|
||||
errno.ENOSYS,
|
||||
errno.EINVAL,
|
||||
getattr(errno, "EOPNOTSUPP", None),
|
||||
getattr(errno, "ENOTSUP", None),
|
||||
)
|
||||
if e is not None
|
||||
)
|
||||
|
||||
|
||||
def to_json_compatible(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
|
|
@ -82,7 +62,6 @@ class AtomicJsonStore:
|
|||
self.path = path
|
||||
self.kind = kind
|
||||
self.schema_version = schema_version
|
||||
self._direct_write_fallback_warned = False
|
||||
|
||||
def _ensure_parent(self) -> None:
|
||||
parent = os.path.dirname(self.path)
|
||||
|
|
@ -117,16 +96,6 @@ class AtomicJsonStore:
|
|||
def save(self, data: dict[str, Any]) -> None:
|
||||
self._ensure_parent()
|
||||
payload = self._build_payload(data)
|
||||
try:
|
||||
self._atomic_write(payload)
|
||||
except OSError as exc:
|
||||
if exc.errno not in _ATOMIC_UNSUPPORTED_ERRNOS:
|
||||
raise
|
||||
self._warn_direct_write_fallback(exc)
|
||||
self._direct_write(payload)
|
||||
|
||||
def _atomic_write(self, payload: dict[str, Any]) -> None:
|
||||
text = self._serialize(payload)
|
||||
parent = os.path.dirname(self.path) or "."
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
prefix=f".{os.path.basename(self.path)}.",
|
||||
|
|
@ -136,9 +105,10 @@ class AtomicJsonStore:
|
|||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
json.dump(payload, f, ensure_ascii=False, separators=(",", ":"))
|
||||
f.write("\n")
|
||||
f.flush()
|
||||
self._best_effort_fsync(f.fileno())
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, self.path)
|
||||
self._fsync_directory(parent)
|
||||
except Exception:
|
||||
|
|
@ -148,57 +118,6 @@ class AtomicJsonStore:
|
|||
pass
|
||||
raise
|
||||
|
||||
def _direct_write(self, payload: dict[str, Any]) -> None:
|
||||
# Serialize before truncating so a serialization failure never destroys
|
||||
# the existing state file (the atomic path gets this for free via its
|
||||
# temp file).
|
||||
text = self._serialize(payload)
|
||||
# Create with 0o600 so the fallback keeps the owner-only permissions the
|
||||
# atomic path gets from mkstemp; state files can contain URLs and
|
||||
# per-download option overrides that must not leak on shared mounts.
|
||||
fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
# The 0o600 mode above only applies when the file is created; force
|
||||
# it on rewrites too so an existing, broadly-permissioned state file
|
||||
# is tightened to match the atomic path. Best-effort because some
|
||||
# network filesystems reject chmod, and that must not re-crash save.
|
||||
try:
|
||||
os.fchmod(f.fileno(), 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
f.write(text)
|
||||
f.flush()
|
||||
self._best_effort_fsync(f.fileno())
|
||||
# Make the new directory entry durable too, matching the atomic path.
|
||||
self._fsync_directory(os.path.dirname(self.path) or ".")
|
||||
|
||||
@staticmethod
|
||||
def _best_effort_fsync(fileno: int) -> None:
|
||||
# Tolerate fsync being unsupported on the underlying filesystem (for
|
||||
# example a network mount that returns EINVAL/ENOSYS), but let genuine
|
||||
# storage failures such as ENOSPC/EIO surface so a non-durable write is
|
||||
# never reported as success. An unsupported fsync must not by itself
|
||||
# abandon the atomic rename path.
|
||||
try:
|
||||
os.fsync(fileno)
|
||||
except OSError as exc:
|
||||
if exc.errno not in _ATOMIC_UNSUPPORTED_ERRNOS:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _serialize(payload: dict[str, Any]) -> str:
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n"
|
||||
|
||||
def _warn_direct_write_fallback(self, exc: OSError) -> None:
|
||||
if self._direct_write_fallback_warned:
|
||||
return
|
||||
self._direct_write_fallback_warned = True
|
||||
log.warning(
|
||||
"Atomic state write failed for %s (%s); falling back to direct write",
|
||||
self.path,
|
||||
exc,
|
||||
)
|
||||
|
||||
def quarantine_invalid_file(self, exc: Exception) -> None:
|
||||
if not os.path.exists(self.path):
|
||||
return
|
||||
|
|
|
|||
|
|
@ -6,28 +6,18 @@ import asyncio
|
|||
import copy
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from dataclasses import dataclass, field, fields
|
||||
from functools import partial
|
||||
from typing import Any, Optional
|
||||
|
||||
import yt_dlp
|
||||
import yt_dlp.networking.impersonate
|
||||
import bg_tasks
|
||||
from dl_formats import merge_ytdl_option_layers
|
||||
from state_store import AtomicJsonStore, read_legacy_shelf
|
||||
from url_guard import validate_url
|
||||
|
||||
log = logging.getLogger("subscriptions")
|
||||
|
||||
# How many subscription feeds to scan at once. Bounded so one slow/hung feed
|
||||
# doesn't serialize the rest, without bursting a large subscription list at the
|
||||
# extractor (which risks rate-limiting / bot detection).
|
||||
_MAX_CONCURRENT_CHECKS = 4
|
||||
|
||||
VIDEO_ONLY_MSG = (
|
||||
"This URL points to a single video, not a channel or playlist. Use Download instead."
|
||||
)
|
||||
|
|
@ -51,9 +41,7 @@ def _impersonate_opt(ytdl_options: dict) -> dict:
|
|||
return opts
|
||||
|
||||
|
||||
def _build_ydl_params(
|
||||
config, *, playlistend: Optional[int] = None, extra_opts: Optional[dict[str, Any]] = None
|
||||
) -> dict:
|
||||
def _build_ydl_params(config, *, playlistend: Optional[int] = None) -> dict:
|
||||
params: dict[str, Any] = {
|
||||
"quiet": not logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||
"verbose": logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||
|
|
@ -63,7 +51,6 @@ def _build_ydl_params(
|
|||
"lazy_playlist": True,
|
||||
"paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR},
|
||||
**config.YTDL_OPTIONS,
|
||||
**(extra_opts or {}),
|
||||
}
|
||||
params = _impersonate_opt(params)
|
||||
if playlistend is not None and playlistend > 0:
|
||||
|
|
@ -88,11 +75,9 @@ def _is_media_entry(entry: Any) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def extract_flat_playlist(
|
||||
config, url: str, playlistend: int, *, extra_opts: Optional[dict[str, Any]] = None, _depth: int = 0
|
||||
):
|
||||
def extract_flat_playlist(config, url: str, playlistend: int, *, _depth: int = 0):
|
||||
"""Return (info_dict, entries_list) for playlist/channel URLs."""
|
||||
params = _build_ydl_params(config, playlistend=playlistend, extra_opts=extra_opts)
|
||||
params = _build_ydl_params(config, playlistend=playlistend)
|
||||
with yt_dlp.YoutubeDL(params=params) as ydl:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
if not info:
|
||||
|
|
@ -114,14 +99,10 @@ def extract_flat_playlist(
|
|||
nested_url = _entry_video_url(ent)
|
||||
if not nested_url:
|
||||
continue
|
||||
# nested_url comes from remote playlist content; guard it too.
|
||||
if validate_url(nested_url) is not None:
|
||||
continue
|
||||
nested_info, nested_entries = extract_flat_playlist(
|
||||
config,
|
||||
nested_url,
|
||||
playlistend,
|
||||
extra_opts=extra_opts,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
if nested_entries:
|
||||
|
|
@ -145,21 +126,6 @@ def _entry_id(entry: dict) -> Optional[str]:
|
|||
return url
|
||||
|
||||
|
||||
def _is_subscriber_only_entry(entry: dict) -> bool:
|
||||
"""True when yt-dlp marks the entry as channel member-only (subscriber_only availability)."""
|
||||
return str(entry.get("availability") or "") == "subscriber_only"
|
||||
|
||||
|
||||
def coerce_optional_bool(value: Any, *, default: bool = False, field_name: str = "value") -> bool:
|
||||
"""Parse optional JSON booleans for subscription settings."""
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return _coerce_bool(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{field_name} must be a boolean") from exc
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubscriptionInfo:
|
||||
id: str
|
||||
|
|
@ -179,10 +145,6 @@ class SubscriptionInfo:
|
|||
chapter_template: str = ""
|
||||
subtitle_language: str = "en"
|
||||
subtitle_mode: str = "prefer_manual"
|
||||
ytdl_options_presets: list[str] = field(default_factory=list)
|
||||
ytdl_options_overrides: dict[str, Any] = field(default_factory=dict)
|
||||
title_regex: str = ""
|
||||
skip_subscriber_only: bool = False
|
||||
last_checked: Optional[float] = None
|
||||
seen_ids: list[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
|
|
@ -203,8 +165,6 @@ class SubscriptionInfo:
|
|||
"format": self.format,
|
||||
"quality": self.quality,
|
||||
"folder": self.folder,
|
||||
"title_regex": self.title_regex,
|
||||
"skip_subscriber_only": self.skip_subscriber_only,
|
||||
"last_checked": self.last_checked,
|
||||
"seen_count": len(self.seen_ids),
|
||||
"error": self.error,
|
||||
|
|
@ -230,76 +190,24 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
|||
"chapter_template": sub.chapter_template,
|
||||
"subtitle_language": sub.subtitle_language,
|
||||
"subtitle_mode": sub.subtitle_mode,
|
||||
"ytdl_options_presets": list(sub.ytdl_options_presets),
|
||||
"ytdl_options_overrides": sub.ytdl_options_overrides,
|
||||
"title_regex": sub.title_regex,
|
||||
"skip_subscriber_only": sub.skip_subscriber_only,
|
||||
"last_checked": sub.last_checked,
|
||||
"seen_ids": list(sub.seen_ids),
|
||||
"error": sub.error,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_subscription_record(rec: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Migrate legacy ytdl_options_preset (str) to ytdl_options_presets (list)."""
|
||||
out = dict(rec)
|
||||
if "ytdl_options_presets" not in out:
|
||||
old = out.pop("ytdl_options_preset", None)
|
||||
if old is None:
|
||||
out["ytdl_options_presets"] = []
|
||||
elif isinstance(old, list):
|
||||
out["ytdl_options_presets"] = [str(x).strip() for x in old if str(x).strip()]
|
||||
elif isinstance(old, str):
|
||||
out["ytdl_options_presets"] = [old.strip()] if old.strip() else []
|
||||
else:
|
||||
out["ytdl_options_presets"] = []
|
||||
else:
|
||||
out.pop("ytdl_options_preset", None)
|
||||
return out
|
||||
|
||||
|
||||
def _subscription_from_record(record: Any) -> Optional[SubscriptionInfo]:
|
||||
field_names = {f.name for f in fields(SubscriptionInfo)}
|
||||
if isinstance(record, SubscriptionInfo):
|
||||
return record
|
||||
if isinstance(record, dict):
|
||||
try:
|
||||
normalized = _normalize_subscription_record(dict(record))
|
||||
return SubscriptionInfo(**{k: v for k, v in normalized.items() if k in field_names})
|
||||
return SubscriptionInfo(**{k: v for k, v in record.items() if k in field_names})
|
||||
except TypeError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_title_regex_value(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def validate_title_regex(value: Any) -> str:
|
||||
"""Return stored title regex string; non-empty values must compile (re.error on failure)."""
|
||||
s = _normalize_title_regex_value(value)
|
||||
if s:
|
||||
re.compile(s)
|
||||
return s
|
||||
|
||||
|
||||
def _coerce_bool(value: Any) -> bool:
|
||||
"""Accept JSON booleans and common string forms used by API clients."""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"true", "1", "on"}:
|
||||
return True
|
||||
if lowered in {"false", "0", "off"}:
|
||||
return False
|
||||
raise ValueError("enabled must be a boolean")
|
||||
|
||||
|
||||
class SubscriptionNotifier:
|
||||
"""Hook for Socket.IO / UI updates."""
|
||||
|
||||
|
|
@ -330,7 +238,6 @@ class SubscriptionManager:
|
|||
self._subs: dict[str, SubscriptionInfo] = {}
|
||||
self._url_index: dict[str, str] = {} # normalized url -> id
|
||||
self._pending_urls: set[str] = set()
|
||||
self._checks_in_flight: set[str] = set() # subscription ids being checked right now
|
||||
self._lock = asyncio.Lock()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._load_all()
|
||||
|
|
@ -388,23 +295,6 @@ class SubscriptionManager:
|
|||
def _save_locked(self) -> None:
|
||||
self._store.save({"items": [_subscription_to_record(sub) for sub in self._subs.values()]})
|
||||
|
||||
def _scan_extra_opts(
|
||||
self,
|
||||
ytdl_options_presets: Optional[list[str]],
|
||||
ytdl_options_overrides: Optional[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured presets (in order) with per-subscription overrides.
|
||||
|
||||
Applied on top of the global YTDL_OPTIONS when scanning a
|
||||
subscription's feed, so cookies/impersonation/etc. configured via a
|
||||
preset or override also take effect during the flat-playlist scan,
|
||||
not just the eventual per-video download. (The global YTDL_OPTIONS base
|
||||
is already spread into the scan params by ``_build_ydl_params``.)
|
||||
"""
|
||||
return merge_ytdl_option_layers(
|
||||
ytdl_options_presets, ytdl_options_overrides, self.config.YTDL_OPTIONS_PRESETS
|
||||
)
|
||||
|
||||
async def _queue_subscription_entries(
|
||||
self,
|
||||
entries: list[dict],
|
||||
|
|
@ -421,20 +311,15 @@ class SubscriptionManager:
|
|||
chapter_template: str,
|
||||
subtitle_language: str,
|
||||
subtitle_mode: str,
|
||||
ytdl_options_presets: Optional[list[str]] = None,
|
||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
queued_ids: list[str] = []
|
||||
queue_errors: list[str] = []
|
||||
presets = list(ytdl_options_presets or [])
|
||||
for ent in entries:
|
||||
eid = _entry_id(ent)
|
||||
vurl = _entry_video_url(ent)
|
||||
if not eid or not vurl:
|
||||
continue
|
||||
queue_entry = dict(ent)
|
||||
if "id" not in queue_entry:
|
||||
queue_entry["id"] = eid
|
||||
queue_entry["_type"] = "video"
|
||||
queue_entry["webpage_url"] = vurl
|
||||
result = await self.dqueue.add_entry(
|
||||
|
|
@ -451,8 +336,6 @@ class SubscriptionManager:
|
|||
chapter_template or None,
|
||||
subtitle_language,
|
||||
subtitle_mode,
|
||||
presets,
|
||||
ytdl_options_overrides,
|
||||
)
|
||||
if isinstance(result, dict) and result.get("status") == "error":
|
||||
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
||||
|
|
@ -471,8 +354,12 @@ class SubscriptionManager:
|
|||
def start_background_loop(self) -> None:
|
||||
if self._loop_task is not None and not self._loop_task.done():
|
||||
return
|
||||
# bg_tasks.create_task already logs unexpected task failures with the name.
|
||||
self._loop_task = bg_tasks.create_task(self._periodic_loop(), name="subscription_loop")
|
||||
self._loop_task = asyncio.create_task(self._periodic_loop())
|
||||
self._loop_task.add_done_callback(
|
||||
lambda t: log.error("Subscription loop failed: %s", t.exception())
|
||||
if not t.cancelled() and t.exception()
|
||||
else None
|
||||
)
|
||||
|
||||
async def _periodic_loop(self) -> None:
|
||||
while True:
|
||||
|
|
@ -496,30 +383,8 @@ class SubscriptionManager:
|
|||
if now - sub.last_checked < interval_sec:
|
||||
continue
|
||||
due.append(sub)
|
||||
await self._check_many(due)
|
||||
|
||||
async def _check_many(self, subs: list[SubscriptionInfo]) -> None:
|
||||
"""Check subscriptions with bounded concurrency so one slow feed does
|
||||
not serialize the rest. Failures are isolated per subscription."""
|
||||
if not subs:
|
||||
return
|
||||
sem = asyncio.Semaphore(_MAX_CONCURRENT_CHECKS)
|
||||
|
||||
async def _guarded(sub: SubscriptionInfo) -> None:
|
||||
async with sem:
|
||||
await self._check_one_unlocked(sub)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_guarded(sub) for sub in subs), return_exceptions=True
|
||||
)
|
||||
for sub, result in zip(subs, results):
|
||||
if isinstance(result, Exception):
|
||||
log.error(
|
||||
"Subscription check crashed for %s: %s",
|
||||
sub.name,
|
||||
result,
|
||||
exc_info=result,
|
||||
)
|
||||
for sub in due:
|
||||
await self._check_one_unlocked(sub)
|
||||
|
||||
async def add_subscription(
|
||||
self,
|
||||
|
|
@ -538,32 +403,10 @@ class SubscriptionManager:
|
|||
chapter_template: str,
|
||||
subtitle_language: str,
|
||||
subtitle_mode: str,
|
||||
ytdl_options_presets: Optional[list[str]] = None,
|
||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||
title_regex: Any = None,
|
||||
skip_subscriber_only: Any = None,
|
||||
) -> dict:
|
||||
url = self._normalize_url(url)
|
||||
if not url:
|
||||
return {"status": "error", "msg": "Missing URL"}
|
||||
# SSRF guard: block non-http(s) schemes and internal/metadata hosts
|
||||
# before yt-dlp fetches the feed. May do a DNS lookup, so run off-loop.
|
||||
url_error = await asyncio.get_running_loop().run_in_executor(None, validate_url, url)
|
||||
if url_error is not None:
|
||||
log.warning('Rejected subscription URL "%s": %s', url, url_error)
|
||||
return {"status": "error", "msg": url_error}
|
||||
try:
|
||||
title_regex_stored = validate_title_regex(title_regex)
|
||||
except re.error as exc:
|
||||
return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
|
||||
try:
|
||||
skip_so = coerce_optional_bool(
|
||||
skip_subscriber_only,
|
||||
default=False,
|
||||
field_name="skip_subscriber_only",
|
||||
)
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
async with self._lock:
|
||||
if url in self._url_index or url in self._pending_urls:
|
||||
|
|
@ -572,12 +415,8 @@ class SubscriptionManager:
|
|||
|
||||
try:
|
||||
scan_first = max(int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50)), 1)
|
||||
scan_extra_opts = self._scan_extra_opts(ytdl_options_presets, ytdl_options_overrides)
|
||||
try:
|
||||
info, entries = await asyncio.get_running_loop().run_in_executor(
|
||||
None,
|
||||
partial(extract_flat_playlist, self.config, url, scan_first, extra_opts=scan_extra_opts),
|
||||
)
|
||||
info, entries = extract_flat_playlist(self.config, url, scan_first)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
|
|
@ -599,8 +438,6 @@ class SubscriptionManager:
|
|||
seen_entries = [ent for ent in entries if _is_media_entry(ent)]
|
||||
all_ids: list[str] = []
|
||||
for ent in seen_entries:
|
||||
if ent.get("live_status") == "is_upcoming":
|
||||
continue # Don't mark scheduled streams as seen; queue them when they go live
|
||||
eid = _entry_id(ent)
|
||||
if eid:
|
||||
all_ids.append(eid)
|
||||
|
|
@ -623,10 +460,6 @@ class SubscriptionManager:
|
|||
chapter_template=chapter_template or "",
|
||||
subtitle_language=subtitle_language,
|
||||
subtitle_mode=subtitle_mode,
|
||||
ytdl_options_presets=list(ytdl_options_presets or []),
|
||||
ytdl_options_overrides=dict(ytdl_options_overrides or {}),
|
||||
title_regex=title_regex_stored,
|
||||
skip_subscriber_only=skip_so,
|
||||
last_checked=time.time(),
|
||||
seen_ids=list(dict.fromkeys(all_ids)),
|
||||
error=None,
|
||||
|
|
@ -673,43 +506,6 @@ class SubscriptionManager:
|
|||
return {"status": "ok"}
|
||||
|
||||
async def update_subscription(self, sub_id: str, changes: dict) -> dict:
|
||||
validated_tr: Optional[str] = None
|
||||
if "title_regex" in changes:
|
||||
try:
|
||||
validated_tr = validate_title_regex(changes["title_regex"])
|
||||
except re.error as exc:
|
||||
return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
|
||||
|
||||
skip_so_set = False
|
||||
validated_skip_so = False
|
||||
if "skip_subscriber_only" in changes:
|
||||
try:
|
||||
validated_skip_so = coerce_optional_bool(
|
||||
changes["skip_subscriber_only"],
|
||||
field_name="skip_subscriber_only",
|
||||
)
|
||||
skip_so_set = True
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
enabled_set = False
|
||||
validated_enabled = False
|
||||
if "enabled" in changes:
|
||||
try:
|
||||
validated_enabled = _coerce_bool(changes["enabled"])
|
||||
enabled_set = True
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
interval_set = False
|
||||
validated_interval = 0
|
||||
if "check_interval_minutes" in changes:
|
||||
try:
|
||||
validated_interval = max(1, int(changes["check_interval_minutes"]))
|
||||
except (TypeError, ValueError):
|
||||
return {"status": "error", "msg": "check_interval_minutes must be an integer"}
|
||||
interval_set = True
|
||||
|
||||
async with self._lock:
|
||||
sub = self._subs.get(sub_id)
|
||||
if not sub:
|
||||
|
|
@ -717,16 +513,12 @@ class SubscriptionManager:
|
|||
previous = copy.deepcopy(sub)
|
||||
old_enabled = sub.enabled
|
||||
|
||||
if enabled_set:
|
||||
sub.enabled = validated_enabled
|
||||
if interval_set:
|
||||
sub.check_interval_minutes = validated_interval
|
||||
if "enabled" in changes:
|
||||
sub.enabled = bool(changes["enabled"])
|
||||
if "check_interval_minutes" in changes:
|
||||
sub.check_interval_minutes = max(1, int(changes["check_interval_minutes"]))
|
||||
if "name" in changes and changes["name"]:
|
||||
sub.name = str(changes["name"])
|
||||
if validated_tr is not None:
|
||||
sub.title_regex = validated_tr
|
||||
if skip_so_set:
|
||||
sub.skip_subscriber_only = validated_skip_so
|
||||
|
||||
try:
|
||||
self._save_locked()
|
||||
|
|
@ -754,45 +546,22 @@ class SubscriptionManager:
|
|||
"Manual subscription check requested for %d subscription(s)",
|
||||
len(targets),
|
||||
)
|
||||
await self._check_many(targets)
|
||||
for sub in targets:
|
||||
await self._check_one_unlocked(sub)
|
||||
return {"status": "ok"}
|
||||
|
||||
async def _check_one_unlocked(self, sub: SubscriptionInfo) -> None:
|
||||
sid = sub.id
|
||||
# Prevent overlapping checks for the same subscription (e.g. the periodic
|
||||
# loop and a manual check-now firing together), which could double-queue
|
||||
# entries and drop seen_ids via a read-modify-write race.
|
||||
async with self._lock:
|
||||
if sid in self._checks_in_flight:
|
||||
log.info("Subscription check already in progress for %s, skipping", sub.name)
|
||||
return
|
||||
self._checks_in_flight.add(sid)
|
||||
try:
|
||||
await self._check_one_inner(sub)
|
||||
finally:
|
||||
async with self._lock:
|
||||
self._checks_in_flight.discard(sid)
|
||||
|
||||
async def _check_one_inner(self, sub: SubscriptionInfo) -> None:
|
||||
sid = sub.id
|
||||
scan = int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50))
|
||||
# ytdl_options_presets/overrides are set at subscription creation and
|
||||
# never mutated afterwards (update_subscription doesn't allow it), so
|
||||
# reading them off `sub` here without holding the lock is safe.
|
||||
scan_extra_opts = self._scan_extra_opts(sub.ytdl_options_presets, sub.ytdl_options_overrides)
|
||||
log.info("Checking subscription: %s", sub.name)
|
||||
try:
|
||||
info, entries = await asyncio.get_running_loop().run_in_executor(
|
||||
None,
|
||||
partial(extract_flat_playlist, self.config, sub.url, scan, extra_opts=scan_extra_opts),
|
||||
)
|
||||
info, entries = extract_flat_playlist(self.config, sub.url, scan)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if cur:
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.error = str(exc)
|
||||
cur.last_checked = time.time()
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
|
|
@ -805,13 +574,12 @@ class SubscriptionManager:
|
|||
entries = [ent for ent in entries if _is_media_entry(ent)]
|
||||
|
||||
etype = (info or {}).get("_type") or "video"
|
||||
if etype == "video":
|
||||
if etype == "video" or not entries:
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if cur:
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.error = VIDEO_ONLY_MSG
|
||||
cur.last_checked = time.time()
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
|
|
@ -821,22 +589,6 @@ class SubscriptionManager:
|
|||
log.warning("Subscription %s no longer resolves to a subscribable feed", sub.name)
|
||||
await self.notifier.subscription_updated(sub)
|
||||
return
|
||||
if not entries:
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if cur:
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.last_checked = time.time()
|
||||
cur.error = None
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs[sid] = previous
|
||||
raise
|
||||
sub = cur
|
||||
log.warning("Subscription check finished for %s: No entries found", sub.name)
|
||||
await self.notifier.subscription_updated(sub)
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
|
|
@ -856,60 +608,18 @@ class SubscriptionManager:
|
|||
dl_chapter = cur.chapter_template
|
||||
dl_sublang = cur.subtitle_language
|
||||
dl_submode = cur.subtitle_mode
|
||||
dl_ytdl_presets = list(cur.ytdl_options_presets)
|
||||
dl_ytdl_overrides = dict(cur.ytdl_options_overrides)
|
||||
dl_title_regex = cur.title_regex or ""
|
||||
dl_skip_subscriber_only = bool(cur.skip_subscriber_only)
|
||||
|
||||
new_entries: list[dict] = []
|
||||
new_ids: list[str] = []
|
||||
for ent in entries:
|
||||
eid = _entry_id(ent)
|
||||
if not eid:
|
||||
continue
|
||||
# Seen entries that are currently live are deliberately re-queued:
|
||||
# a stream first seen as 'upcoming' must still be captured once it
|
||||
# goes live. The download queue dedups by URL while a capture is
|
||||
# in flight, so this can't double-queue an active capture.
|
||||
if eid in seen and ent.get("live_status") != "is_live":
|
||||
if not eid or eid in seen:
|
||||
continue
|
||||
new_entries.append(ent)
|
||||
|
||||
pattern_re: Optional[re.Pattern[str]] = None
|
||||
if dl_title_regex:
|
||||
try:
|
||||
pattern_re = re.compile(dl_title_regex)
|
||||
except re.error:
|
||||
log.warning(
|
||||
"Invalid stored title_regex on subscription %s, ignoring filter",
|
||||
sub.name,
|
||||
)
|
||||
|
||||
queue_entries: list[dict] = []
|
||||
filtered_ids: list[str] = []
|
||||
for ent in new_entries:
|
||||
eid = _entry_id(ent)
|
||||
if pattern_re is not None:
|
||||
title = str(ent.get("title") or "")
|
||||
if not pattern_re.search(title):
|
||||
if eid:
|
||||
filtered_ids.append(eid)
|
||||
continue
|
||||
queue_entries.append(ent)
|
||||
|
||||
subscriber_filtered_ids: list[str] = []
|
||||
if dl_skip_subscriber_only:
|
||||
kept_entries: list[dict] = []
|
||||
for ent in queue_entries:
|
||||
eid = _entry_id(ent)
|
||||
if _is_subscriber_only_entry(ent):
|
||||
if eid:
|
||||
subscriber_filtered_ids.append(eid)
|
||||
continue
|
||||
kept_entries.append(ent)
|
||||
queue_entries = kept_entries
|
||||
new_ids.append(eid)
|
||||
|
||||
queued_ids, queue_errors = await self._queue_subscription_entries(
|
||||
queue_entries,
|
||||
new_entries,
|
||||
download_type=dl_type,
|
||||
codec=dl_codec,
|
||||
format=dl_format,
|
||||
|
|
@ -922,24 +632,16 @@ class SubscriptionManager:
|
|||
chapter_template=dl_chapter or "",
|
||||
subtitle_language=dl_sublang,
|
||||
subtitle_mode=dl_submode,
|
||||
ytdl_options_presets=dl_ytdl_presets,
|
||||
ytdl_options_overrides=dl_ytdl_overrides,
|
||||
)
|
||||
log.info(
|
||||
"Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed",
|
||||
"Subscription check finished for %s: %d new, %d queued, %d failed",
|
||||
sub.name,
|
||||
len(new_entries),
|
||||
len(filtered_ids),
|
||||
len(subscriber_filtered_ids),
|
||||
len(queued_ids),
|
||||
len(queue_errors),
|
||||
)
|
||||
|
||||
merged = list(
|
||||
dict.fromkeys(
|
||||
queued_ids + filtered_ids + subscriber_filtered_ids + seen_ids_snapshot
|
||||
)
|
||||
)
|
||||
merged = list(dict.fromkeys(queued_ids + seen_ids_snapshot))
|
||||
max_seen = int(getattr(self.config, "SUBSCRIPTION_MAX_SEEN_IDS", 50000))
|
||||
if len(merged) > max_seen:
|
||||
merged = merged[:max_seen]
|
||||
|
|
|
|||
|
|
@ -3,14 +3,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
import main
|
||||
|
||||
|
|
@ -21,7 +17,6 @@ def mock_dqueue(monkeypatch):
|
|||
d.initialize = AsyncMock(return_value=None)
|
||||
d.add = AsyncMock(return_value={"status": "ok"})
|
||||
d.cancel = AsyncMock(return_value={"status": "ok"})
|
||||
d.clear = AsyncMock(return_value={"status": "ok"})
|
||||
d.start_pending = AsyncMock(return_value={"status": "ok"})
|
||||
d.cancel_add = MagicMock()
|
||||
d.queue = MagicMock()
|
||||
|
|
@ -30,9 +25,6 @@ def mock_dqueue(monkeypatch):
|
|||
d.queue.saved_items = MagicMock(return_value=[])
|
||||
d.done.saved_items = MagicMock(return_value=[])
|
||||
d.pending.saved_items = MagicMock(return_value=[])
|
||||
d.queue.items = MagicMock(return_value=[])
|
||||
d.done.items = MagicMock(return_value=[])
|
||||
d.pending.items = MagicMock(return_value=[])
|
||||
d.get = MagicMock(return_value=([], []))
|
||||
monkeypatch.setattr(main, "dqueue", d)
|
||||
return d
|
||||
|
|
@ -45,8 +37,6 @@ def _valid_video_add_body(**kwargs):
|
|||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": [],
|
||||
"ytdl_options_overrides": "",
|
||||
}
|
||||
base.update(kwargs)
|
||||
return base
|
||||
|
|
@ -69,37 +59,6 @@ async def test_add_ok(mock_dqueue):
|
|||
mock_dqueue.add.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_passes_preset_and_overrides(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Preset A": {"writesubtitles": True}})
|
||||
monkeypatch.setattr(main.config, "ALLOW_YTDL_OPTIONS_OVERRIDES", True)
|
||||
req = _json_request(
|
||||
_valid_video_add_body(
|
||||
ytdl_options_presets=["Preset A"],
|
||||
ytdl_options_overrides='{"writesubtitles": true}',
|
||||
)
|
||||
)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[13] == ["Preset A"]
|
||||
assert call.args[14] == {"writesubtitles": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_legacy_string_preset_normalized(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Legacy": {}})
|
||||
body = _valid_video_add_body()
|
||||
del body["ytdl_options_presets"]
|
||||
body["ytdl_options_preset"] = "Legacy"
|
||||
req = _json_request(body)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call.args[13] == ["Legacy"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_missing_url_returns_400(mock_dqueue):
|
||||
req = _json_request({"download_type": "video", "quality": "best", "format": "any"})
|
||||
|
|
@ -165,38 +124,6 @@ async def test_add_invalid_json_body(mock_dqueue):
|
|||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_ytdl_options_override_json(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides="{bad json}"))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_rejects_ytdl_options_overrides_when_disabled(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides='{"exec": "rm -rf /"}'))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_allows_any_ytdl_options_override_key_when_enabled(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "ALLOW_YTDL_OPTIONS_OVERRIDES", True)
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides='{"exec": "echo hi"}'))
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[14] == {"exec": "echo hi"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_unknown_ytdl_preset(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_presets=["Missing"]))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_missing_ids(mock_dqueue):
|
||||
req = _json_request({"where": "queue"})
|
||||
|
|
@ -220,35 +147,11 @@ async def test_start_pending(mock_dqueue):
|
|||
mock_dqueue.start_pending.assert_awaited_once_with(["a"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("body", [{}, {"ids": "abc"}, {"ids": []}, {"ids": [1, 2]}])
|
||||
async def test_start_rejects_malformed_ids(mock_dqueue, body):
|
||||
req = _json_request(body)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.start(req)
|
||||
mock_dqueue.start_pending.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"where": "queue"},
|
||||
{"where": "queue", "ids": "abc"},
|
||||
{"where": "queue", "ids": []},
|
||||
{"where": "queue", "ids": [1, 2]},
|
||||
],
|
||||
)
|
||||
async def test_delete_rejects_malformed_ids(mock_dqueue, body):
|
||||
req = _json_request(body)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.delete(req)
|
||||
mock_dqueue.cancel.assert_not_awaited()
|
||||
mock_dqueue.clear.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_shape(mock_dqueue):
|
||||
mock_dqueue.queue.saved_items.return_value = []
|
||||
mock_dqueue.done.saved_items.return_value = []
|
||||
mock_dqueue.pending.saved_items.return_value = []
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.history(req)
|
||||
assert resp.status == 200
|
||||
|
|
@ -256,30 +159,6 @@ async def test_history_shape(mock_dqueue):
|
|||
assert set(data.keys()) == {"done", "queue", "pending"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_reads_in_memory_queues_not_disk_state(mock_dqueue):
|
||||
fake_queue_dl = MagicMock()
|
||||
fake_queue_dl.info = {"id": "q1", "title": "Queued"}
|
||||
fake_done_dl = MagicMock()
|
||||
fake_done_dl.info = {"id": "d1", "title": "Done"}
|
||||
fake_pending_dl = MagicMock()
|
||||
fake_pending_dl.info = {"id": "p1", "title": "Pending"}
|
||||
mock_dqueue.queue.items.return_value = [("q1", fake_queue_dl)]
|
||||
mock_dqueue.done.items.return_value = [("d1", fake_done_dl)]
|
||||
mock_dqueue.pending.items.return_value = [("p1", fake_pending_dl)]
|
||||
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.history(req)
|
||||
assert resp.status == 200
|
||||
data = json.loads(resp.text)
|
||||
assert [item["id"] for item in data["queue"]] == ["q1"]
|
||||
assert [item["id"] for item in data["done"]] == ["d1"]
|
||||
assert [item["id"] for item in data["pending"]] == ["p1"]
|
||||
mock_dqueue.queue.saved_items.assert_not_called()
|
||||
mock_dqueue.done.saved_items.assert_not_called()
|
||||
mock_dqueue.pending.saved_items.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_version_json(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
|
|
@ -289,15 +168,6 @@ async def test_version_json(mock_dqueue):
|
|||
assert "yt-dlp" in body and "version" in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_presets_endpoint_returns_names(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Preset B": {}, "Preset A": {}})
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.presets(req)
|
||||
assert resp.status == 200
|
||||
assert json.loads(resp.text) == {"presets": ["Preset A", "Preset B"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookie_status(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
|
|
@ -335,96 +205,3 @@ async def test_add_legacy_format_migrated(mock_dqueue):
|
|||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[1] == "audio"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_passes_clip_bounds_to_queue(mock_dqueue):
|
||||
req = _json_request(
|
||||
_valid_video_add_body(clip_start="2:26", clip_end="3:24"),
|
||||
)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[15] == pytest.approx(146.0)
|
||||
assert call.args[16] == pytest.approx(204.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscribe_rejects_clip_options(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock())
|
||||
req = _json_request(
|
||||
{
|
||||
**_valid_video_add_body(clip_start="10"),
|
||||
"check_interval_minutes": 60,
|
||||
}
|
||||
)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.subscribe(req)
|
||||
main.submgr.add_subscription.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscriptions_update_invalid_enabled_returns_error_not_500(mock_dqueue):
|
||||
req = _json_request({"id": "nonexistent", "enabled": "maybe"})
|
||||
resp = await main.subscriptions_update(req)
|
||||
assert resp.status == 200
|
||||
body = json.loads(resp.text)
|
||||
assert body["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscriptions_update_invalid_interval_returns_error_not_500(mock_dqueue):
|
||||
req = _json_request({"id": "nonexistent", "check_interval_minutes": "abc"})
|
||||
resp = await main.subscriptions_update(req)
|
||||
assert resp.status == 200
|
||||
body = json.loads(resp.text)
|
||||
assert body["status"] == "error"
|
||||
|
||||
|
||||
def test_is_within_state_dir_blocks_state_subtree():
|
||||
state_dir = main._STATE_DIR_REAL
|
||||
assert main._is_within_state_dir(state_dir)
|
||||
assert main._is_within_state_dir(os.path.join(state_dir, "cookies.txt"))
|
||||
assert main._is_within_state_dir(os.path.join(state_dir, "queue", "item.json"))
|
||||
|
||||
|
||||
def test_is_within_state_dir_allows_sibling_downloads():
|
||||
download_dir = os.path.realpath(main.config.DOWNLOAD_DIR)
|
||||
assert not main._is_within_state_dir(os.path.join(download_dir, "video.mp4"))
|
||||
assert not main._is_within_state_dir("/tmp/unrelated/video.mp4")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_blocks_state_dir_files(monkeypatch):
|
||||
download_dir = Path(main.config.DOWNLOAD_DIR)
|
||||
state_dir = download_dir / ".metube"
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
(state_dir / "cookies.txt").write_text("# Netscape HTTP Cookie File\n", encoding="utf-8")
|
||||
(download_dir / "video.mp4").write_bytes(b"video")
|
||||
# request.path is already percent-decoded by aiohttp; state_dir_guard must
|
||||
# not decode it a second time, or a filename containing a literal '%'
|
||||
# gets mangled into a false 404.
|
||||
percent_filename = "100% done.mp4"
|
||||
(download_dir / percent_filename).write_bytes(b"percent video")
|
||||
|
||||
monkeypatch.setattr(main.config, "STATE_DIR", str(state_dir))
|
||||
monkeypatch.setattr(main, "_STATE_DIR_REAL", os.path.realpath(str(state_dir)))
|
||||
|
||||
try:
|
||||
async with TestClient(TestServer(main.app)) as client:
|
||||
blocked = await client.get("/download/.metube/cookies.txt")
|
||||
assert blocked.status == 404
|
||||
|
||||
allowed = await client.get("/download/video.mp4")
|
||||
assert allowed.status == 200
|
||||
assert await allowed.read() == b"video"
|
||||
|
||||
percent_resp = await client.get("/download/" + quote(percent_filename))
|
||||
assert percent_resp.status == 200
|
||||
assert await percent_resp.read() == b"percent video"
|
||||
finally:
|
||||
(state_dir / "cookies.txt").unlink(missing_ok=True)
|
||||
(download_dir / "video.mp4").unlink(missing_ok=True)
|
||||
(download_dir / percent_filename).unlink(missing_ok=True)
|
||||
state_dir.rmdir()
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
"""Tests for the ``bg_tasks.create_task`` strong-reference/logging helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
import bg_tasks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_task_removes_itself_from_registry_on_success():
|
||||
async def _ok():
|
||||
return 42
|
||||
|
||||
task = bg_tasks.create_task(_ok(), name="ok_task")
|
||||
assert task in bg_tasks._TASKS
|
||||
result = await task
|
||||
assert result == 42
|
||||
assert task not in bg_tasks._TASKS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_task_logs_unhandled_exception(caplog):
|
||||
async def _boom():
|
||||
raise ValueError("kaboom")
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="bg_tasks"):
|
||||
task = bg_tasks.create_task(_boom(), name="boom_task")
|
||||
with pytest.raises(ValueError):
|
||||
await task
|
||||
# Let the done-callback (scheduled via call_soon) run.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert task not in bg_tasks._TASKS
|
||||
assert any("boom_task" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_task_does_not_log_on_cancellation(caplog):
|
||||
async def _sleep_forever():
|
||||
await asyncio.sleep(10)
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="bg_tasks"):
|
||||
task = bg_tasks.create_task(_sleep_forever(), name="cancel_task")
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert task not in bg_tasks._TASKS
|
||||
assert not any("cancel_task" in record.message for record in caplog.records)
|
||||
|
|
@ -23,60 +23,6 @@ class ConfigTests(unittest.TestCase):
|
|||
c = Config()
|
||||
self.assertEqual(c.URL_PREFIX, "foo/")
|
||||
|
||||
def test_public_host_url_gets_trailing_slash(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_URL="https://ytdl.example.com"),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||
|
||||
def test_public_host_audio_url_gets_trailing_slash(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_AUDIO_URL="https://audio.example.com"),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||
|
||||
def test_public_host_url_empty_stays_empty(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_URL="", PUBLIC_HOST_AUDIO_URL=""),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
||||
|
||||
def test_blank_audio_host_falls_back_to_audio_download_route(self):
|
||||
# Regression: a present-but-blank PUBLIC_HOST_AUDIO_URL must not stay empty
|
||||
# (which produced root-relative, 404ing audio links). It falls back to the
|
||||
# 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR.
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_URL="https://ytdl.example.com", PUBLIC_HOST_AUDIO_URL=""),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "audio_download/")
|
||||
|
||||
def test_public_host_url_already_slashed_unchanged(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(
|
||||
PUBLIC_HOST_URL="https://ytdl.example.com/",
|
||||
PUBLIC_HOST_AUDIO_URL="https://audio.example.com/",
|
||||
),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||
|
||||
def test_ytdl_options_json_loaded(self):
|
||||
opts = {"quiet": True, "no_warnings": True}
|
||||
with patch.dict(
|
||||
|
|
@ -87,16 +33,6 @@ class ConfigTests(unittest.TestCase):
|
|||
c = Config()
|
||||
self.assertEqual(c.YTDL_OPTIONS["quiet"], True)
|
||||
|
||||
def test_ytdl_option_presets_json_loaded(self):
|
||||
presets = {"Audio extras": {"embed_thumbnail": True}}
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS_PRESETS=json.dumps(presets)),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.YTDL_OPTIONS_PRESETS["Audio extras"]["embed_thumbnail"], True)
|
||||
|
||||
def test_invalid_ytdl_options_exits(self):
|
||||
with patch.dict(os.environ, _base_env(YTDL_OPTIONS="not-json"), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
|
|
@ -113,80 +49,6 @@ class ConfigTests(unittest.TestCase):
|
|||
safe = c.frontend_safe()
|
||||
self.assertNotIn("YTDL_OPTIONS", safe)
|
||||
self.assertNotIn("HOST", safe)
|
||||
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
||||
|
||||
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
||||
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
||||
c = Config()
|
||||
self.assertTrue(c.ALLOW_YTDL_OPTIONS_OVERRIDES)
|
||||
|
||||
def test_ytdl_nightly_update_time_empty_default(self):
|
||||
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME=""), clear=False):
|
||||
c = Config()
|
||||
self.assertEqual(c.YTDL_NIGHTLY_UPDATE_TIME, "")
|
||||
|
||||
def test_ytdl_nightly_update_time_valid(self):
|
||||
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME="04:00"), clear=False):
|
||||
c = Config()
|
||||
self.assertEqual(c.YTDL_NIGHTLY_UPDATE_TIME, "04:00")
|
||||
|
||||
def test_ytdl_nightly_update_time_invalid_exits(self):
|
||||
for bad in ("25:00", "4am", "12:60"):
|
||||
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_max_concurrent_downloads_exits(self):
|
||||
for bad in ("0", "-1", "abc"):
|
||||
with patch.dict(os.environ, _base_env(MAX_CONCURRENT_DOWNLOADS=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_port_exits(self):
|
||||
for bad in ("0", "70000", "notaport"):
|
||||
with patch.dict(os.environ, _base_env(PORT=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_clear_completed_after_exits(self):
|
||||
for bad in ("-5", "soon"):
|
||||
with patch.dict(os.environ, _base_env(CLEAR_COMPLETED_AFTER=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_clear_completed_after_zero_allowed(self):
|
||||
with patch.dict(os.environ, _base_env(CLEAR_COMPLETED_AFTER="0"), clear=False):
|
||||
c = Config()
|
||||
self.assertEqual(c.CLEAR_COMPLETED_AFTER, "0")
|
||||
|
||||
def test_invalid_default_option_playlist_item_limit_exits(self):
|
||||
for bad in ("-1", "many"):
|
||||
with patch.dict(os.environ, _base_env(DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_default_option_playlist_item_limit_zero_allowed(self):
|
||||
with patch.dict(os.environ, _base_env(DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT="0"), clear=False):
|
||||
c = Config()
|
||||
self.assertEqual(c.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT, "0")
|
||||
|
||||
def test_invalid_subscription_default_check_interval_exits(self):
|
||||
for bad in ("0", "-1", "often"):
|
||||
with patch.dict(os.environ, _base_env(SUBSCRIPTION_DEFAULT_CHECK_INTERVAL=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_subscription_scan_playlist_end_exits(self):
|
||||
for bad in ("0", "-1", "all"):
|
||||
with patch.dict(os.environ, _base_env(SUBSCRIPTION_SCAN_PLAYLIST_END=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_subscription_max_seen_ids_exits(self):
|
||||
for bad in ("0", "-1", "unlimited"):
|
||||
with patch.dict(os.environ, _base_env(SUBSCRIPTION_MAX_SEEN_IDS=bad), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_runtime_override_roundtrip(self):
|
||||
with patch.dict(os.environ, _base_env(), clear=False):
|
||||
|
|
@ -211,21 +73,6 @@ class ConfigTests(unittest.TestCase):
|
|||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_ytdl_option_presets_file_merges(self):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
||||
json.dump({"With subtitles": {"writesubtitles": True}}, f)
|
||||
path = f.name
|
||||
try:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS_PRESETS="{}", YTDL_OPTIONS_PRESETS_FILE=path),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertIn("With subtitles", c.YTDL_OPTIONS_PRESETS)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from app.dl_formats import (
|
|||
_normalize_subtitle_language,
|
||||
get_format,
|
||||
get_opts,
|
||||
merge_ytdl_option_layers,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -119,31 +118,7 @@ class DlFormatsTests(unittest.TestCase):
|
|||
|
||||
def test_get_opts_captions_txt_maps_to_srt_format(self):
|
||||
opts = get_opts("captions", "auto", "txt", "best", {})
|
||||
self.assertEqual(opts["subtitlesformat"], "srt/best")
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||
convertor = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegSubtitlesConvertor")
|
||||
self.assertEqual(convertor["format"], "srt")
|
||||
|
||||
def test_get_opts_captions_srt_guarantees_convertor(self):
|
||||
opts = get_opts("captions", "auto", "srt", "best", {})
|
||||
self.assertEqual(opts["subtitlesformat"], "srt/best")
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||
|
||||
def test_get_opts_captions_vtt_guarantees_convertor(self):
|
||||
opts = get_opts("captions", "auto", "vtt", "best", {})
|
||||
self.assertEqual(opts["subtitlesformat"], "vtt/best")
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||
convertor = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegSubtitlesConvertor")
|
||||
self.assertEqual(convertor["format"], "vtt")
|
||||
|
||||
def test_get_opts_captions_ttml_has_no_convertor(self):
|
||||
opts = get_opts("captions", "auto", "ttml", "best", {})
|
||||
self.assertEqual(opts["subtitlesformat"], "ttml/best")
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertNotIn("FFmpegSubtitlesConvertor", keys)
|
||||
self.assertEqual(opts["subtitlesformat"], "srt")
|
||||
|
||||
def test_get_opts_merges_existing_postprocessors(self):
|
||||
opts = get_opts("audio", "auto", "opus", "best", {"postprocessors": [{"key": "SponsorBlock"}]})
|
||||
|
|
@ -160,23 +135,5 @@ class DlFormatsTests(unittest.TestCase):
|
|||
self.assertEqual(_normalize_subtitle_language(" "), "en")
|
||||
|
||||
|
||||
class MergeYtdlOptionLayersTests(unittest.TestCase):
|
||||
def test_presets_applied_in_order_then_overrides(self):
|
||||
presets_config = {
|
||||
"a": {"x": 1, "y": 1},
|
||||
"b": {"y": 2, "z": 2},
|
||||
}
|
||||
merged = merge_ytdl_option_layers(["a", "b"], {"z": 3, "w": 4}, presets_config)
|
||||
# b overrides a's y; explicit overrides win over presets.
|
||||
self.assertEqual(merged, {"x": 1, "y": 2, "z": 3, "w": 4})
|
||||
|
||||
def test_no_base_options_included(self):
|
||||
# The helper only produces the preset/override layer, never base opts.
|
||||
self.assertEqual(merge_ytdl_option_layers(None, None, {}), {})
|
||||
|
||||
def test_unknown_preset_names_ignored(self):
|
||||
self.assertEqual(merge_ytdl_option_layers(["missing"], {"a": 1}, {}), {"a": 1})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -3,14 +3,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import time
|
||||
|
||||
from ytdl import Download, DownloadInfo, DownloadQueue
|
||||
from ytdl import DownloadQueue
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -27,7 +25,6 @@ def dq_env():
|
|||
cfg.TEMP_DIR = dl
|
||||
cfg.MAX_CONCURRENT_DOWNLOADS = "3"
|
||||
cfg.YTDL_OPTIONS = {}
|
||||
cfg.YTDL_OPTIONS_PRESETS = {}
|
||||
cfg.CUSTOM_DIRS = True
|
||||
cfg.CREATE_CUSTOM_DIRS = True
|
||||
cfg.CLEAR_COMPLETED_AFTER = "0"
|
||||
|
|
@ -47,37 +44,6 @@ def test_cancel_add_increments_generation(dq_env):
|
|||
assert dq._add_generation == before + 1
|
||||
|
||||
|
||||
def test_download_queue_has_dedicated_executor_sized_from_config(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
assert dq._download_executor is not None
|
||||
assert dq._download_executor._max_workers == 2 * int(dq_env.MAX_CONCURRENT_DOWNLOADS) + 2
|
||||
dq.close()
|
||||
|
||||
|
||||
def test_close_cancels_running_downloads_before_shutdown(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
|
||||
running = MagicMock()
|
||||
running.started.return_value = True
|
||||
running.running.return_value = True
|
||||
idle = MagicMock()
|
||||
idle.started.return_value = False
|
||||
idle.running.return_value = False
|
||||
|
||||
dq.queue.dict["u-running"] = running
|
||||
dq.queue.dict["u-idle"] = idle
|
||||
|
||||
dq.close()
|
||||
|
||||
# The active download's subprocess group is killed; the not-started one is
|
||||
# left alone. Executor is shut down afterwards.
|
||||
running.cancel.assert_called_once()
|
||||
idle.cancel.assert_not_called()
|
||||
assert dq._download_executor._shutdown
|
||||
|
||||
|
||||
def test_get_returns_tuple_of_lists(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
|
|
@ -89,7 +55,7 @@ def test_get_returns_tuple_of_lists(dq_env):
|
|||
async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
|
|
@ -119,7 +85,7 @@ async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
|
|||
async def test_cancel_removes_from_pending(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
|
|
@ -147,54 +113,11 @@ async def test_cancel_removes_from_pending(dq_env):
|
|||
notifier.canceled.assert_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_before_start_marks_download_canceled(dq_env):
|
||||
"""Regression test for the race condition where cancel() arrives after the
|
||||
download has been placed in the queue and ``__start_download`` has been
|
||||
scheduled via ``asyncio.create_task`` but has not yet executed. Without the
|
||||
fix, the pending task would run ``download.start()`` despite the user
|
||||
cancelling, because its ``download.canceled`` guard was never flipped."""
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/race"
|
||||
start_mock = AsyncMock()
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add(
|
||||
url,
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=True,
|
||||
)
|
||||
assert dq.queue.exists(url)
|
||||
download = dq.queue.get(url)
|
||||
assert download.canceled is False
|
||||
await dq.cancel([url])
|
||||
assert not dq.queue.exists(url)
|
||||
assert download.canceled is True
|
||||
notifier.canceled.assert_awaited_with(url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_pending_moves_to_queue(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
|
|
@ -252,697 +175,3 @@ async def test_add_entry_queues_single_video_without_reextracting(dq_env):
|
|||
|
||||
assert result["status"] == "ok"
|
||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
entry = {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Original Title",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
|
||||
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||
assert first["status"] == "ok"
|
||||
assert "msg" not in first
|
||||
|
||||
dupe_entry = {**entry, "title": "Different Title"}
|
||||
second = await dq.add_entry(dupe_entry, "audio", "auto", "mp3", "best", "", "", 0, auto_start=False)
|
||||
|
||||
assert second["status"] == "ok"
|
||||
assert "Already in queue" in second["msg"]
|
||||
# The original pending download's options must survive untouched.
|
||||
pending_dl = dq.pending.get("https://example.com/watch?v=1")
|
||||
assert pending_dl.info.download_type == "video"
|
||||
assert pending_dl.info.title == "Original Title"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_duplicate_while_queued_is_skipped(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
entry = {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||
assert first["status"] == "ok"
|
||||
assert dq.queue.exists("https://example.com/watch?v=1")
|
||||
|
||||
second = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||
|
||||
assert second["status"] == "ok"
|
||||
assert "Already in queue" in second["msg"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_download_uses_output_template_when_channel_template_empty(dq_env):
|
||||
"""Channel tabs reported as playlists must honor OUTPUT_TEMPLATE when OUTPUT_TEMPLATE_CHANNEL is empty."""
|
||||
notifier = AsyncMock()
|
||||
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = ""
|
||||
|
||||
channel_id = "UCabcd123"
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "playlist",
|
||||
"id": channel_id,
|
||||
"channel_id": channel_id,
|
||||
"channel": "Odin",
|
||||
"title": "Odin - Videos",
|
||||
"entries": [
|
||||
{
|
||||
"id": "vid1",
|
||||
"title": "Salvia Plath - Pondering",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
"channel": "Odin",
|
||||
"upload_date": "20130804",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://www.youtube.com/@odin/videos",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
url = "https://example.com/watch?v=1"
|
||||
assert dq.pending.exists(url)
|
||||
download = dq.pending.get(url)
|
||||
assert download.output_template.startswith("Odin [YT]/")
|
||||
assert "Odin - Videos" not in download.output_template
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_playlist_download_not_treated_as_channel(dq_env):
|
||||
"""Real playlists (id != channel_id) must not be promoted to channel downloads."""
|
||||
notifier = AsyncMock()
|
||||
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "playlist",
|
||||
"id": "PLxyz789",
|
||||
"channel_id": "UCabcd123",
|
||||
"channel": "Odin",
|
||||
"title": "My Playlist",
|
||||
"entries": [
|
||||
{
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://www.youtube.com/playlist?list=PLxyz789",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
url = "https://example.com/watch?v=1"
|
||||
assert dq.pending.exists(url)
|
||||
download = dq.pending.get(url)
|
||||
assert download.output_template.startswith("My Playlist/")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.YTDL_OPTIONS = {"writesubtitles": False, "cookiefile": "/tmp/global.txt"}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {
|
||||
"Preset A": {"writesubtitles": True, "proxy": "http://preset-a"},
|
||||
"Preset B": {"writesubtitles": False, "ratelimit": 1000},
|
||||
}
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid2",
|
||||
"title": "Preset Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://example.com/preset",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["Preset A", "Preset B"],
|
||||
ytdl_options_overrides={"proxy": "http://override", "embed_thumbnail": True},
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
queued = dq.pending.get("https://example.com/preset")
|
||||
assert queued.ytdl_opts["cookiefile"] == "/tmp/global.txt"
|
||||
assert queued.ytdl_opts["writesubtitles"] is False
|
||||
assert queued.ytdl_opts["ratelimit"] == 1000
|
||||
assert queued.ytdl_opts["proxy"] == "http://override"
|
||||
assert queued.ytdl_opts["embed_thumbnail"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_info_preset_null_download_archive_overrides_global(dq_env):
|
||||
"""Preset download_archive:null must apply during extract_info (global archive otherwise wins first)."""
|
||||
dq_env.YTDL_OPTIONS = {"download_archive": "/tmp/archive.txt"}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {"NoArchive": {"download_archive": None}}
|
||||
|
||||
captured_params: list = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, params=None):
|
||||
captured_params.append(params)
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid-archive",
|
||||
"title": "Archive Test",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
|
||||
result = await dq.add(
|
||||
"https://example.com/archive-test",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["NoArchive"],
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert len(captured_params) == 1
|
||||
extract_params = captured_params[0]
|
||||
assert extract_params.get("download_archive") is None
|
||||
assert extract_params["extract_flat"] is True
|
||||
assert extract_params["noplaylist"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_info_metube_extract_keys_win_over_preset(dq_env):
|
||||
"""MeTube's flat-extract settings must not be overridden by presets."""
|
||||
dq_env.YTDL_OPTIONS = {}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {
|
||||
"TryOverride": {"extract_flat": False, "noplaylist": False},
|
||||
}
|
||||
|
||||
captured_params: list = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, params=None):
|
||||
captured_params.append(params)
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid-flat",
|
||||
"title": "Flat Test",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
|
||||
result = await dq.add(
|
||||
"https://example.com/flat-test",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["TryOverride"],
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert captured_params[0]["extract_flat"] is True
|
||||
assert captured_params[0]["noplaylist"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_sets_clip_bounds_on_download_info(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://example.com/clip",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
clip_start=10.0,
|
||||
clip_end=99.5,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
download = dq.pending.get("https://example.com/clip")
|
||||
assert download.info.clip_start == 10.0
|
||||
assert download.info.clip_end == 99.5
|
||||
|
||||
|
||||
def _upcoming_entry(url: str, *, release_timestamp: float | None = None) -> dict:
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "live1",
|
||||
"title": "Upcoming Stream",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
"live_status": "is_upcoming",
|
||||
"release_timestamp": release_timestamp if release_timestamp is not None else time.time() + 3600,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_upcoming_stream_scheduled_without_starting(dq_env):
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-upcoming"
|
||||
start_mock = AsyncMock()
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
result = await dq.add_entry(
|
||||
_upcoming_entry(url),
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=True,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert dq.queue.exists(url)
|
||||
download = dq.queue.get(url)
|
||||
assert download.info.status == "scheduled"
|
||||
assert download.info.live_status == "is_upcoming"
|
||||
assert download.info.live_release_timestamp is not None
|
||||
start_mock.assert_not_called()
|
||||
assert url in dq._scheduled_probe_at
|
||||
# The "scheduled to start at ..." message must include a UTC offset
|
||||
# (a naive datetime's %z would render as an empty string here).
|
||||
assert re.search(r"[+-]\d{4}$", download.info.error)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_scheduled_starts_when_live(dq_env):
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-upcoming"
|
||||
start_mock = AsyncMock()
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add_entry(
|
||||
_upcoming_entry(url),
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=True,
|
||||
)
|
||||
|
||||
download = dq.queue.get(url)
|
||||
|
||||
def fake_probe_extract(self, probe_url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
assert probe_url == url
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "live1",
|
||||
"title": "Live Now",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
"live_status": "is_live",
|
||||
"formats": [{"format_id": "22"}],
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_probe_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq._probe_scheduled_download(download)
|
||||
|
||||
assert url not in dq._scheduled_probe_at
|
||||
assert download.info.live_status == "is_live"
|
||||
assert download.info.status == "pending"
|
||||
start_mock.assert_called_once_with(download)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_scheduled_re_registers_monitor(dq_env):
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-restart"
|
||||
release = time.time() + 7200
|
||||
|
||||
info = DownloadInfo(
|
||||
id="live1",
|
||||
title="Upcoming Stream",
|
||||
url=url,
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry=None,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
live_status="is_upcoming",
|
||||
live_release_timestamp=release,
|
||||
)
|
||||
info.status = "scheduled"
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
start_mock = AsyncMock()
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq._DownloadQueue__add_download(info, True)
|
||||
|
||||
assert dq.queue.exists(url)
|
||||
assert dq.queue.get(url).info.status == "scheduled"
|
||||
assert url in dq._scheduled_probe_at
|
||||
start_mock.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_transient_error_retries_without_failing(dq_env):
|
||||
"""A single probe failure must not abandon the scheduled stream."""
|
||||
import ytdl
|
||||
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-transient"
|
||||
start_mock = AsyncMock()
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add_entry(
|
||||
_upcoming_entry(url),
|
||||
"video", "auto", "any", "best", "", "", 0,
|
||||
auto_start=True,
|
||||
)
|
||||
download = dq.queue.get(url)
|
||||
|
||||
def boom(self, *args, **kwargs):
|
||||
raise ytdl.yt_dlp.utils.YoutubeDLError("temporary network glitch")
|
||||
|
||||
before = time.time()
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||
await dq._probe_scheduled_download(download)
|
||||
|
||||
# Still scheduled, still monitored, probe rescheduled into the future.
|
||||
assert download.info.status == "scheduled"
|
||||
assert url in dq._scheduled_probe_at
|
||||
assert dq._scheduled_probe_at[url] >= before
|
||||
assert dq._scheduled_probe_failures[url] == 1
|
||||
notifier.completed.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_gives_up_after_max_failures(dq_env):
|
||||
import ytdl
|
||||
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-dead"
|
||||
start_mock = AsyncMock()
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add_entry(
|
||||
_upcoming_entry(url),
|
||||
"video", "auto", "any", "best", "", "", 0,
|
||||
auto_start=True,
|
||||
)
|
||||
download = dq.queue.get(url)
|
||||
|
||||
def boom(self, *args, **kwargs):
|
||||
raise ytdl.yt_dlp.utils.YoutubeDLError("stream was deleted")
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||
for _ in range(ytdl._LIVE_PROBE_MAX_FAILURES):
|
||||
await dq._probe_scheduled_download(download)
|
||||
|
||||
assert url not in dq._scheduled_probe_at
|
||||
assert not dq.queue.exists(url)
|
||||
assert dq.done.exists(url)
|
||||
assert download.info.status == "error"
|
||||
notifier.completed.assert_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_recovers_after_transient_then_starts(dq_env):
|
||||
"""A transient failure followed by a successful live probe should start the download."""
|
||||
import ytdl
|
||||
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/live-recover"
|
||||
start_mock = AsyncMock()
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add_entry(
|
||||
_upcoming_entry(url),
|
||||
"video", "auto", "any", "best", "", "", 0,
|
||||
auto_start=True,
|
||||
)
|
||||
download = dq.queue.get(url)
|
||||
# The scheduling placeholder error is set on add.
|
||||
assert download.info.error
|
||||
|
||||
def boom(self, *args, **kwargs):
|
||||
raise ytdl.yt_dlp.utils.YoutubeDLError("temporary glitch")
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||
await dq._probe_scheduled_download(download)
|
||||
assert dq._scheduled_probe_failures[url] == 1
|
||||
|
||||
def live_now(self, *args, **kwargs):
|
||||
return {
|
||||
"_type": "video", "id": "live1", "title": "Live Now",
|
||||
"url": url, "webpage_url": url, "live_status": "is_live",
|
||||
"formats": [{"format_id": "22"}],
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", live_now), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq._probe_scheduled_download(download)
|
||||
|
||||
assert url not in dq._scheduled_probe_at
|
||||
assert url not in dq._scheduled_probe_failures
|
||||
assert download.info.status == "pending"
|
||||
# Placeholder error/msg cleared now that a real download is starting.
|
||||
assert download.info.error is None
|
||||
assert download.info.msg is None
|
||||
start_mock.assert_called_once_with(download)
|
||||
|
||||
|
||||
def test_seconds_until_next_probe_none_when_empty(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
assert dq._seconds_until_next_probe() is None
|
||||
|
||||
|
||||
def test_calc_download_path_allows_subfolder(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
path, err = dq._DownloadQueue__calc_download_path("video", "sub/dir")
|
||||
assert err is None
|
||||
assert os.path.realpath(path) == os.path.join(os.path.realpath(dq_env.DOWNLOAD_DIR), "sub", "dir")
|
||||
|
||||
|
||||
def test_calc_download_path_rejects_sibling_prefix_escape(dq_env):
|
||||
"""A folder resolving to a sibling sharing a name prefix must be rejected.
|
||||
|
||||
Regression test: ``startswith`` would have accepted ``../downloads-secret``
|
||||
when the base directory is ``.../downloads``.
|
||||
"""
|
||||
notifier = AsyncMock()
|
||||
base = os.path.realpath(dq_env.DOWNLOAD_DIR)
|
||||
sibling = base + "-secret"
|
||||
os.makedirs(sibling, exist_ok=True)
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
escape_folder = os.path.join("..", os.path.basename(sibling), "x")
|
||||
path, err = dq._DownloadQueue__calc_download_path("video", escape_folder)
|
||||
assert path is None
|
||||
assert err is not None and err["status"] == "error"
|
||||
|
||||
|
||||
def test_calc_download_path_rejects_parent_escape(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
path, err = dq._DownloadQueue__calc_download_path("video", "../../etc")
|
||||
assert path is None
|
||||
assert err is not None and err["status"] == "error"
|
||||
|
||||
|
||||
def test_download_info_to_public_dict_excludes_server_only_fields():
|
||||
info = DownloadInfo(
|
||||
id="vid1",
|
||||
title="Test Video",
|
||||
url="https://example.com/watch?v=1",
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry={"id": "vid1", "huge": "x" * 100000},
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
info.subtitle_files = [{"filename": "a.srt", "size": 10}]
|
||||
public = info.to_public_dict()
|
||||
assert "entry" not in public
|
||||
assert "subtitle_files" not in public
|
||||
# Client-facing fields are still present.
|
||||
assert public["url"] == "https://example.com/watch?v=1"
|
||||
assert public["title"] == "Test Video"
|
||||
assert public["status"] == "pending"
|
||||
|
||||
|
||||
def _make_download(dq_env, *, download_type="video", status="downloading", filename=None):
|
||||
info = DownloadInfo(
|
||||
id="id1",
|
||||
title="t",
|
||||
url="http://example.com/v",
|
||||
quality="best",
|
||||
download_type=download_type,
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry=None,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
info.status = status
|
||||
info.filename = filename
|
||||
info.size = 123 if filename else None
|
||||
return Download(
|
||||
dq_env.DOWNLOAD_DIR, dq_env.TEMP_DIR, "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_download_cleanup_clears_filename_on_error(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
||||
dq.queue.put(download)
|
||||
|
||||
dq._post_download_cleanup(download)
|
||||
|
||||
assert download.info.status == "error"
|
||||
assert download.info.filename is None
|
||||
assert download.info.size is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
||||
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
||||
dq.queue.put(download)
|
||||
|
||||
dq._post_download_cleanup(download)
|
||||
|
||||
assert download.info.status == "error"
|
||||
assert download.info.filename == "en.srt"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_skips_deletion_outside_download_directory(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.DELETE_FILE_ON_TRASHCAN = True
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
|
||||
outside_dir = tempfile.mkdtemp()
|
||||
outside_file = os.path.join(outside_dir, "outside.txt")
|
||||
with open(outside_file, "w") as f:
|
||||
f.write("do not delete me")
|
||||
|
||||
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
||||
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
||||
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
||||
dq.done.put(download)
|
||||
|
||||
await dq.clear([download.info.url])
|
||||
|
||||
assert os.path.exists(outside_file)
|
||||
assert not dq.done.exists(download.info.url)
|
||||
|
|
|
|||
|
|
@ -99,216 +99,6 @@ class FrontendSafeTests(unittest.TestCase):
|
|||
self.assertIn(key, safe)
|
||||
self.assertNotIn("YTDL_OPTIONS", safe)
|
||||
self.assertNotIn("DOWNLOAD_DIR", safe)
|
||||
self.assertIn("ALLOW_YTDL_OPTIONS_OVERRIDES", safe)
|
||||
|
||||
|
||||
class ParseYtdlOverridesTests(unittest.TestCase):
|
||||
def test_empty_override_string_returns_empty_dict(self):
|
||||
self.assertEqual(main._parse_ytdl_options_overrides("", enabled=False), {})
|
||||
|
||||
def test_rejects_non_object_json(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main._parse_ytdl_options_overrides('["bad"]', enabled=True)
|
||||
|
||||
def test_rejects_non_empty_overrides_when_disabled(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main._parse_ytdl_options_overrides('{"exec": "rm -rf /"}', enabled=False)
|
||||
|
||||
def test_allows_any_keys_when_enabled(self):
|
||||
self.assertEqual(
|
||||
main._parse_ytdl_options_overrides('{"exec": "rm -rf /"}', enabled=True),
|
||||
{"exec": "rm -rf /"},
|
||||
)
|
||||
|
||||
|
||||
class ParseDownloadOptionsTests(unittest.TestCase):
|
||||
def test_accepts_known_preset_and_overrides(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
previous_allow = main.config.ALLOW_YTDL_OPTIONS_OVERRIDES
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"With subtitles": {"writesubtitles": True}}
|
||||
main.config.ALLOW_YTDL_OPTIONS_OVERRIDES = True
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_preset": "With subtitles",
|
||||
"ytdl_options_overrides": '{"writesubtitles": true}',
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
main.config.ALLOW_YTDL_OPTIONS_OVERRIDES = previous_allow
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["With subtitles"])
|
||||
self.assertEqual(parsed["ytdl_options_overrides"], {"writesubtitles": True})
|
||||
|
||||
def test_accepts_multiple_presets_in_order(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {
|
||||
"A": {"writesubtitles": True},
|
||||
"B": {"writesubtitles": False},
|
||||
}
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["A", "B"],
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["A", "B"])
|
||||
|
||||
def test_legacy_singular_preset_string_normalized_to_list(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"Solo": {}}
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_preset": "Solo",
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["Solo"])
|
||||
|
||||
def test_rejects_unknown_preset(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["Missing preset"],
|
||||
})
|
||||
|
||||
def test_rejects_unknown_preset_in_list(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"Known": {}}
|
||||
try:
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["Known", "Nope"],
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
|
||||
def test_clip_start_end_seconds_and_clock(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "2:26",
|
||||
"clip_end": "3:24",
|
||||
})
|
||||
self.assertEqual(parsed["clip_start"], 146.0)
|
||||
self.assertEqual(parsed["clip_end"], 204.0)
|
||||
|
||||
def test_clip_url_t_param_strips_query_and_sets_start(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://www.youtube.com/watch?v=1&t=855s",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 855.0)
|
||||
self.assertIsNone(parsed["clip_end"])
|
||||
|
||||
def test_clip_explicit_start_wins_over_url_t(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://www.youtube.com/watch?v=1&t=100",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "50",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 50.0)
|
||||
self.assertIsNone(parsed["clip_end"])
|
||||
|
||||
def test_clip_end_only_sets_start_zero_and_strips_url_t(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://www.youtube.com/watch?v=1&t=999",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_end": "60",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 0.0)
|
||||
self.assertEqual(parsed["clip_end"], 60.0)
|
||||
|
||||
def test_clip_url_t_param_ignored_on_non_youtube_host(self):
|
||||
# 't' is a generic query param name; only rewrite it on YouTube hosts
|
||||
# so an unrelated site's URL isn't silently mutated with a bogus clip.
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1&t=855s",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1&t=855s")
|
||||
self.assertIsNone(parsed["clip_start"])
|
||||
self.assertIsNone(parsed["clip_end"])
|
||||
|
||||
def test_extract_t_query_youtu_be_short_host(self):
|
||||
cleaned, start = main._extract_t_query_from_url("https://youtu.be/abc123?t=90")
|
||||
self.assertEqual(cleaned, "https://youtu.be/abc123")
|
||||
self.assertEqual(start, 90.0)
|
||||
|
||||
def test_clip_rejects_end_before_start(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "100",
|
||||
"clip_end": "50",
|
||||
})
|
||||
|
||||
def test_clip_rejected_for_captions(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "captions",
|
||||
"codec": "auto",
|
||||
"format": "srt",
|
||||
"quality": "best",
|
||||
"clip_start": "1",
|
||||
})
|
||||
|
||||
|
||||
class GetCustomDirsTests(unittest.TestCase):
|
||||
def test_works_without_a_running_event_loop(self):
|
||||
# get_custom_dirs() used to time its cache via
|
||||
# asyncio.get_running_loop().time(), which raises RuntimeError outside
|
||||
# a running loop (e.g. when called from a plain executor thread). It
|
||||
# must work from a synchronous context too.
|
||||
result = main.get_custom_dirs()
|
||||
self.assertIn("download_dir", result)
|
||||
self.assertIn("audio_download_dir", result)
|
||||
self.assertIn("", result["download_dir"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
"""Tests for nightly yt-dlp update scheduling helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
|
||||
from main import seconds_until_next_daily_time
|
||||
|
||||
|
||||
class NightlyUpdateTests(unittest.TestCase):
|
||||
def test_seconds_until_later_today(self):
|
||||
now = datetime(2026, 6, 4, 10, 0, 0)
|
||||
delay = seconds_until_next_daily_time("15:30", now)
|
||||
self.assertEqual(delay, 5 * 3600 + 30 * 60)
|
||||
|
||||
def test_seconds_until_wraps_to_next_day(self):
|
||||
now = datetime(2026, 6, 4, 18, 0, 0)
|
||||
delay = seconds_until_next_daily_time("04:00", now)
|
||||
self.assertEqual(delay, 10 * 3600)
|
||||
|
||||
def test_seconds_until_same_minute_is_next_day(self):
|
||||
now = datetime(2026, 6, 4, 4, 0, 30)
|
||||
delay = seconds_until_next_daily_time("04:00", now)
|
||||
self.assertAlmostEqual(delay, 24 * 3600 - 30, delta=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -4,7 +4,6 @@ import os
|
|||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
from state_store import AtomicJsonStore, from_json_compatible, to_json_compatible
|
||||
|
||||
|
|
@ -22,135 +21,6 @@ class StateStoreTests(unittest.TestCase):
|
|||
self.assertEqual(payload["schema_version"], 2)
|
||||
self.assertEqual(payload["items"][0]["info"]["title"], "hello")
|
||||
|
||||
def test_save_falls_back_to_direct_write_when_mkstemp_fails(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
|
||||
with self.assertLogs("state_store", level="WARNING") as logs:
|
||||
with patch(
|
||||
"state_store.tempfile.mkstemp",
|
||||
side_effect=PermissionError(1, "Operation not permitted"),
|
||||
):
|
||||
store.save({"items": [{"key": "a"}]})
|
||||
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertTrue(any(path in message for message in logs.output))
|
||||
# Fallback keeps owner-only permissions, matching the atomic path.
|
||||
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||||
payload = store.load()
|
||||
self.assertEqual(payload["items"], [{"key": "a"}])
|
||||
|
||||
def test_fallback_tightens_permissions_on_existing_file(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("{}")
|
||||
os.chmod(path, 0o644)
|
||||
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
with patch(
|
||||
"state_store.tempfile.mkstemp",
|
||||
side_effect=PermissionError(1, "Operation not permitted"),
|
||||
):
|
||||
store.save({"items": [{"key": "a"}]})
|
||||
|
||||
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||||
self.assertEqual(store.load()["items"], [{"key": "a"}])
|
||||
|
||||
def test_save_falls_back_to_direct_write_when_replace_fails(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
|
||||
with patch(
|
||||
"state_store.os.replace",
|
||||
side_effect=PermissionError(1, "Operation not permitted"),
|
||||
):
|
||||
store.save({"items": [{"key": "a"}]})
|
||||
|
||||
self.assertTrue(os.path.exists(path))
|
||||
payload = store.load()
|
||||
self.assertEqual(payload["items"], [{"key": "a"}])
|
||||
self.assertEqual([], [name for name in os.listdir(tmp) if name.endswith(".tmp")])
|
||||
|
||||
def test_save_reraises_when_atomic_and_direct_write_fail(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
|
||||
with patch(
|
||||
"state_store.tempfile.mkstemp",
|
||||
side_effect=PermissionError(1, "Operation not permitted"),
|
||||
):
|
||||
with patch(
|
||||
"state_store.os.open",
|
||||
side_effect=PermissionError(13, "Permission denied"),
|
||||
):
|
||||
with self.assertRaises(PermissionError) as ctx:
|
||||
store.save({"items": [{"key": "a"}]})
|
||||
|
||||
self.assertEqual(ctx.exception.errno, 13)
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
def test_unsupported_fsync_keeps_atomic_path(self):
|
||||
# fsync being unsupported (EINVAL/ENOSYS) must not by itself trigger the
|
||||
# direct-write fallback; the atomic temp-file + rename path still runs.
|
||||
import errno as _errno
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
|
||||
with patch(
|
||||
"state_store.os.fsync",
|
||||
side_effect=OSError(_errno.EINVAL, "Invalid argument"),
|
||||
):
|
||||
with self.assertNoLogs("state_store", level="WARNING"):
|
||||
store.save({"items": [{"key": "a"}]})
|
||||
|
||||
self.assertEqual(store.load()["items"], [{"key": "a"}])
|
||||
self.assertEqual([], [name for name in os.listdir(tmp) if name.endswith(".tmp")])
|
||||
|
||||
def test_save_reraises_and_preserves_state_on_non_atomic_errno(self):
|
||||
# A storage failure such as ENOSPC is not an "atomic unavailable"
|
||||
# signal, so it must surface instead of falling back to a direct write
|
||||
# that would truncate the existing good state file.
|
||||
import errno as _errno
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
store.save({"items": [{"key": "good"}]})
|
||||
|
||||
with patch(
|
||||
"state_store.tempfile.mkstemp",
|
||||
side_effect=OSError(_errno.ENOSPC, "No space left on device"),
|
||||
):
|
||||
with self.assertRaises(OSError) as ctx:
|
||||
store.save({"items": [{"key": "new"}]})
|
||||
|
||||
self.assertEqual(ctx.exception.errno, _errno.ENOSPC)
|
||||
# Existing state is untouched.
|
||||
self.assertEqual(store.load()["items"], [{"key": "good"}])
|
||||
|
||||
def test_serialization_failure_preserves_existing_state(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
store.save({"items": [{"key": "good"}]})
|
||||
|
||||
# Even on the fallback path, a non-serializable payload must raise
|
||||
# before the existing good state file is touched.
|
||||
with patch(
|
||||
"state_store.tempfile.mkstemp",
|
||||
side_effect=PermissionError(1, "Operation not permitted"),
|
||||
):
|
||||
with self.assertRaises(TypeError):
|
||||
store.save({"items": object()})
|
||||
|
||||
self.assertEqual(store.load()["items"], [{"key": "good"}])
|
||||
|
||||
def test_invalid_file_is_quarantined(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shelve
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
|
@ -30,13 +28,7 @@ sys.modules.setdefault("yt_dlp", fake_yt_dlp)
|
|||
sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||
|
||||
from subscriptions import (
|
||||
SubscriptionInfo,
|
||||
SubscriptionManager,
|
||||
_is_subscriber_only_entry,
|
||||
coerce_optional_bool,
|
||||
extract_flat_playlist,
|
||||
)
|
||||
from subscriptions import SubscriptionManager, extract_flat_playlist
|
||||
|
||||
|
||||
class _Config:
|
||||
|
|
@ -47,7 +39,6 @@ class _Config:
|
|||
self.DOWNLOAD_DIR = state_dir
|
||||
self.TEMP_DIR = state_dir
|
||||
self.YTDL_OPTIONS = {}
|
||||
self.YTDL_OPTIONS_PRESETS = {}
|
||||
|
||||
|
||||
class _Queue:
|
||||
|
|
@ -84,20 +75,6 @@ def _create_legacy_shelf(path: str, record) -> None:
|
|||
shelf["sub-1"] = record
|
||||
|
||||
|
||||
class SubscriberOnlyHelperTests(unittest.TestCase):
|
||||
def test_is_subscriber_only_detects_availability(self):
|
||||
self.assertTrue(_is_subscriber_only_entry({"availability": "subscriber_only"}))
|
||||
self.assertFalse(_is_subscriber_only_entry({"availability": None}))
|
||||
self.assertFalse(_is_subscriber_only_entry({}))
|
||||
|
||||
def test_coerce_optional_bool_defaults_and_fields(self):
|
||||
self.assertFalse(coerce_optional_bool(None, default=False))
|
||||
self.assertTrue(coerce_optional_bool(True))
|
||||
self.assertFalse(coerce_optional_bool(False))
|
||||
with self.assertRaises(ValueError):
|
||||
coerce_optional_bool("maybe", field_name="skip_subscriber_only")
|
||||
|
||||
|
||||
class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
||||
def test_load_imports_legacy_subscription_shelf(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
|
@ -409,569 +386,6 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||
self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
|
||||
self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
|
||||
|
||||
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
side_effect=[
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[
|
||||
{
|
||||
"id": "v2",
|
||||
"title": "Members",
|
||||
"webpage_url": "https://example.com/v2",
|
||||
"availability": "subscriber_only",
|
||||
},
|
||||
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||
],
|
||||
),
|
||||
],
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
skip_subscriber_only=False,
|
||||
)
|
||||
self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
|
||||
await mgr.check_now([result["subscription"]["id"]])
|
||||
|
||||
sub = mgr.list_all()[0]
|
||||
self.assertIsNone(sub.error)
|
||||
self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
|
||||
self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
|
||||
|
||||
async def test_check_now_skips_subscriber_only_when_skip_enabled(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
side_effect=[
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[
|
||||
{
|
||||
"id": "v2",
|
||||
"title": "Members",
|
||||
"webpage_url": "https://example.com/v2",
|
||||
"availability": "subscriber_only",
|
||||
},
|
||||
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||
],
|
||||
),
|
||||
],
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
skip_subscriber_only=True,
|
||||
)
|
||||
self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
|
||||
await mgr.check_now([result["subscription"]["id"]])
|
||||
|
||||
sub = mgr.list_all()[0]
|
||||
self.assertIsNone(sub.error)
|
||||
self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
|
||||
self.assertEqual(queue.entries, [])
|
||||
|
||||
async def test_update_subscription_parses_string_false_enabled(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
|
||||
sub_id = result["subscription"]["id"]
|
||||
update = await mgr.update_subscription(sub_id, {"enabled": "false"})
|
||||
self.assertEqual(update["status"], "ok")
|
||||
self.assertFalse(mgr.list_all()[0].enabled)
|
||||
|
||||
async def test_update_subscription_rejects_invalid_enabled_value(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
|
||||
sub_id = result["subscription"]["id"]
|
||||
update_result = await mgr.update_subscription(sub_id, {"enabled": "maybe"})
|
||||
self.assertEqual(update_result["status"], "error")
|
||||
stored = mgr.get(sub_id)
|
||||
self.assertTrue(stored.enabled)
|
||||
|
||||
update_result = await mgr.update_subscription(
|
||||
sub_id, {"check_interval_minutes": "abc"}
|
||||
)
|
||||
self.assertEqual(update_result["status"], "error")
|
||||
self.assertEqual(mgr.get(sub_id).check_interval_minutes, 60)
|
||||
|
||||
async def test_add_subscription_rejects_invalid_title_regex(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
title_regex="[",
|
||||
)
|
||||
self.assertEqual(result["status"], "error")
|
||||
self.assertIn("title_regex", result["msg"].lower())
|
||||
self.assertEqual(mgr.list_all(), [])
|
||||
|
||||
async def test_add_subscription_stores_and_exposes_title_regex(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
title_regex="EPISODE",
|
||||
)
|
||||
self.assertEqual(result["status"], "ok")
|
||||
self.assertEqual(result["subscription"]["title_regex"], "EPISODE")
|
||||
self.assertEqual(mgr.list_all()[0].title_regex, "EPISODE")
|
||||
|
||||
async def test_check_now_title_regex_queues_only_matches_and_marks_unmatched_seen(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
side_effect=[
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "Old", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[
|
||||
{
|
||||
"id": "v2",
|
||||
"title": "Minecraft | EPISODE 1",
|
||||
"webpage_url": "https://example.com/v2",
|
||||
},
|
||||
{
|
||||
"id": "v3",
|
||||
"title": "Unrelated IRL",
|
||||
"webpage_url": "https://example.com/v3",
|
||||
},
|
||||
{
|
||||
"id": "v1",
|
||||
"title": "Old",
|
||||
"webpage_url": "https://example.com/v1",
|
||||
},
|
||||
],
|
||||
),
|
||||
],
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
title_regex="EPISODE",
|
||||
)
|
||||
await mgr.check_now([result["subscription"]["id"]])
|
||||
self.assertEqual([e["webpage_url"] for e, _, _ in queue.entries], ["https://example.com/v2"])
|
||||
sub = mgr.list_all()[0]
|
||||
self.assertEqual(sub.seen_ids[:3], ["v2", "v3", "v1"])
|
||||
|
||||
async def test_check_now_title_regex_queue_failure_keeps_matched_id_unseen(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
side_effect=[
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "Old", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[
|
||||
{
|
||||
"id": "v2",
|
||||
"title": "Show | EPISODE 1",
|
||||
"webpage_url": "https://example.com/v2",
|
||||
},
|
||||
{
|
||||
"id": "v3",
|
||||
"title": "Other",
|
||||
"webpage_url": "https://example.com/v3",
|
||||
},
|
||||
],
|
||||
),
|
||||
],
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
title_regex="EPISODE",
|
||||
)
|
||||
queue.fail = True
|
||||
await mgr.check_now([result["subscription"]["id"]])
|
||||
sub = mgr.list_all()[0]
|
||||
self.assertEqual(sub.error, "queue failed")
|
||||
self.assertEqual(set(sub.seen_ids), {"v1", "v3"})
|
||||
self.assertNotIn("v2", sub.seen_ids)
|
||||
|
||||
async def test_update_subscription_rejects_invalid_title_regex(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
upd = await mgr.update_subscription(sub_id, {"title_regex": "("})
|
||||
self.assertEqual(upd["status"], "error")
|
||||
self.assertEqual(mgr.list_all()[0].title_regex, "")
|
||||
|
||||
async def test_update_subscription_persists_valid_title_regex(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
upd = await mgr.update_subscription(sub_id, {"title_regex": "foo|bar"})
|
||||
self.assertEqual(upd["status"], "ok")
|
||||
self.assertEqual(upd["subscription"]["title_regex"], "foo|bar")
|
||||
self.assertEqual(mgr.list_all()[0].title_regex, "foo|bar")
|
||||
|
||||
async def test_update_subscription_skip_subscriber_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
|
||||
upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": True})
|
||||
self.assertEqual(upd["status"], "ok")
|
||||
self.assertTrue(upd["subscription"]["skip_subscriber_only"])
|
||||
self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
|
||||
|
||||
async def test_update_subscription_rejects_invalid_skip_subscriber_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": "maybe"})
|
||||
self.assertEqual(upd["status"], "error")
|
||||
self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
|
||||
|
||||
def test_persistence_includes_title_regex(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
json_path = os.path.join(tmp, "subscriptions.json")
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"kind": "subscriptions",
|
||||
"items": [
|
||||
{
|
||||
"id": "sub-1",
|
||||
"name": "Channel",
|
||||
"url": "https://example.com/channel",
|
||||
"enabled": True,
|
||||
"check_interval_minutes": 60,
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"folder": "",
|
||||
"custom_name_prefix": "",
|
||||
"auto_start": True,
|
||||
"playlist_item_limit": 0,
|
||||
"split_by_chapters": False,
|
||||
"chapter_template": "",
|
||||
"subtitle_language": "en",
|
||||
"subtitle_mode": "prefer_manual",
|
||||
"ytdl_options_presets": [],
|
||||
"ytdl_options_overrides": {},
|
||||
"title_regex": "EPISODE",
|
||||
"last_checked": None,
|
||||
"seen_ids": [],
|
||||
"error": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
f,
|
||||
)
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
self.assertEqual(mgr.list_all()[0].title_regex, "EPISODE")
|
||||
self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
|
||||
|
||||
def test_persistence_includes_skip_subscriber_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
json_path = os.path.join(tmp, "subscriptions.json")
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"kind": "subscriptions",
|
||||
"items": [
|
||||
{
|
||||
"id": "sub-1",
|
||||
"name": "Channel",
|
||||
"url": "https://example.com/channel",
|
||||
"enabled": True,
|
||||
"check_interval_minutes": 60,
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"folder": "",
|
||||
"custom_name_prefix": "",
|
||||
"auto_start": True,
|
||||
"playlist_item_limit": 0,
|
||||
"split_by_chapters": False,
|
||||
"chapter_template": "",
|
||||
"subtitle_language": "en",
|
||||
"subtitle_mode": "prefer_manual",
|
||||
"ytdl_options_presets": [],
|
||||
"ytdl_options_overrides": {},
|
||||
"title_regex": "",
|
||||
"skip_subscriber_only": True,
|
||||
"last_checked": None,
|
||||
"seen_ids": [],
|
||||
"error": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
f,
|
||||
)
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
|
||||
|
||||
|
||||
class ExtractFlatPlaylistTests(unittest.TestCase):
|
||||
def test_descends_one_level_when_root_entries_are_nested_collections(self):
|
||||
responses = iter(
|
||||
|
|
@ -1024,223 +438,6 @@ class ExtractFlatPlaylistTests(unittest.TestCase):
|
|||
self.assertEqual(info.get("_type"), "playlist")
|
||||
self.assertEqual([entry["webpage_url"] for entry in entries], ["https://example.com/v1"])
|
||||
|
||||
def test_extra_opts_applied_on_top_of_config_options(self):
|
||||
captured: dict = {}
|
||||
|
||||
class _FakeYDL:
|
||||
def __init__(self, params):
|
||||
captured.update(params)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {"_type": "video"}
|
||||
|
||||
cfg = _Config(tempfile.mkdtemp())
|
||||
with patch("subscriptions.yt_dlp.YoutubeDL", _FakeYDL, create=True):
|
||||
extract_flat_playlist(cfg, "https://example.com/v1", 50, extra_opts={"cookiefile": "x"})
|
||||
|
||||
self.assertEqual(captured.get("cookiefile"), "x")
|
||||
|
||||
|
||||
def _make_scan_capturing_fake_ydl(captured_params: list, entries: list[dict]):
|
||||
class _FakeYDL:
|
||||
def __init__(self, params):
|
||||
captured_params.append(params)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {"_type": "channel", "title": "Channel", "entries": entries}
|
||||
|
||||
return _FakeYDL
|
||||
|
||||
|
||||
class SubscriptionScanExtraOptsTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_add_subscription_scan_applies_presets_and_overrides(self):
|
||||
captured_params: list = []
|
||||
fake_ydl = _make_scan_capturing_fake_ydl(
|
||||
captured_params,
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = _Config(tmp)
|
||||
cfg.YTDL_OPTIONS_PRESETS = {"mypreset": {"cookiefile": "preset.txt"}}
|
||||
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||
|
||||
with patch("subscriptions.yt_dlp.YoutubeDL", fake_ydl, create=True):
|
||||
await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
ytdl_options_presets=["mypreset"],
|
||||
ytdl_options_overrides={"extra": "override"},
|
||||
)
|
||||
|
||||
self.assertTrue(captured_params)
|
||||
self.assertEqual(captured_params[0].get("cookiefile"), "preset.txt")
|
||||
self.assertEqual(captured_params[0].get("extra"), "override")
|
||||
|
||||
async def test_check_now_scan_applies_stored_subscription_presets(self):
|
||||
entries = [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = _Config(tmp)
|
||||
cfg.YTDL_OPTIONS_PRESETS = {"mypreset": {"cookiefile": "preset.txt"}}
|
||||
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||
|
||||
add_captured: list = []
|
||||
with patch(
|
||||
"subscriptions.yt_dlp.YoutubeDL",
|
||||
_make_scan_capturing_fake_ydl(add_captured, entries),
|
||||
create=True,
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
ytdl_options_presets=["mypreset"],
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
|
||||
check_captured: list = []
|
||||
with patch(
|
||||
"subscriptions.yt_dlp.YoutubeDL",
|
||||
_make_scan_capturing_fake_ydl(check_captured, entries),
|
||||
create=True,
|
||||
):
|
||||
await mgr.check_now([sub_id])
|
||||
|
||||
self.assertTrue(check_captured)
|
||||
self.assertEqual(check_captured[0].get("cookiefile"), "preset.txt")
|
||||
|
||||
|
||||
class SubscriptionEventLoopTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_check_now_does_not_block_event_loop(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
sub_id = result["subscription"]["id"]
|
||||
|
||||
def _slow_extract(config, url, playlistend, **kwargs):
|
||||
time.sleep(0.3)
|
||||
return (
|
||||
{"_type": "channel", "title": "Channel"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
)
|
||||
|
||||
with patch("subscriptions.extract_flat_playlist", side_effect=_slow_extract):
|
||||
check_task = asyncio.ensure_future(mgr.check_now([sub_id]))
|
||||
# If check_now() blocked the event loop, this would not complete
|
||||
# until after the slow extraction finishes.
|
||||
await asyncio.wait_for(asyncio.sleep(0.05), timeout=0.2)
|
||||
self.assertFalse(check_task.done())
|
||||
await check_task
|
||||
|
||||
async def test_check_many_isolates_a_crashing_subscription(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
|
||||
good = SubscriptionInfo(id="good", name="Good", url="https://example.com/good")
|
||||
bad = SubscriptionInfo(id="bad", name="Bad", url="https://example.com/bad")
|
||||
other = SubscriptionInfo(id="other", name="Other", url="https://example.com/other")
|
||||
|
||||
checked: list[str] = []
|
||||
|
||||
async def fake_check(sub):
|
||||
if sub.id == "bad":
|
||||
raise RuntimeError("boom")
|
||||
checked.append(sub.id)
|
||||
|
||||
with patch.object(mgr, "_check_one_unlocked", side_effect=fake_check):
|
||||
# The crashing subscription must not prevent the others running.
|
||||
await mgr._check_many([good, bad, other])
|
||||
|
||||
self.assertIn("good", checked)
|
||||
self.assertIn("other", checked)
|
||||
|
||||
async def test_check_many_bounded_concurrency(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
subs = [
|
||||
SubscriptionInfo(id=str(i), name=str(i), url=f"https://example.com/{i}")
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
import subscriptions as subs_mod
|
||||
|
||||
concurrent = 0
|
||||
peak = 0
|
||||
|
||||
async def fake_check(sub):
|
||||
nonlocal concurrent, peak
|
||||
concurrent += 1
|
||||
peak = max(peak, concurrent)
|
||||
await asyncio.sleep(0.02)
|
||||
concurrent -= 1
|
||||
|
||||
with patch.object(mgr, "_check_one_unlocked", side_effect=fake_check):
|
||||
await mgr._check_many(subs)
|
||||
|
||||
# Never exceed the configured bound, but do run more than one at once.
|
||||
self.assertLessEqual(peak, subs_mod._MAX_CONCURRENT_CHECKS)
|
||||
self.assertGreater(peak, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
"""Tests for the SSRF URL guard (``url_guard.validate_url``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from url_guard import validate_url
|
||||
|
||||
|
||||
def _addrinfo(*addrs, family=socket.AF_INET):
|
||||
return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (addr, 0)) for addr in addrs]
|
||||
|
||||
|
||||
class NonUrlInputTests(unittest.TestCase):
|
||||
"""Bare IDs and yt-dlp search/extractor prefixes must pass untouched."""
|
||||
|
||||
def test_bare_video_id_allowed(self):
|
||||
self.assertIsNone(validate_url("dQw4w9WgXcQ"))
|
||||
|
||||
def test_ytsearch_prefix_allowed(self):
|
||||
self.assertIsNone(validate_url("ytsearch:some song"))
|
||||
|
||||
def test_empty_string_allowed(self):
|
||||
self.assertIsNone(validate_url(""))
|
||||
|
||||
def test_non_string_rejected(self):
|
||||
self.assertIsNotNone(validate_url(None))
|
||||
|
||||
|
||||
class SchemeTests(unittest.TestCase):
|
||||
def test_file_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("file:///etc/passwd"))
|
||||
|
||||
def test_ftp_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("ftp://example.com/x"))
|
||||
|
||||
def test_data_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("data://text/plain;base64,AAAA"))
|
||||
|
||||
|
||||
class HostnameBlocklistTests(unittest.TestCase):
|
||||
def test_localhost_blocked_without_lookup(self):
|
||||
with mock.patch("url_guard.socket.getaddrinfo") as gai:
|
||||
self.assertIsNotNone(validate_url("http://localhost:8080/x"))
|
||||
gai.assert_not_called()
|
||||
|
||||
def test_localhost_subdomain_blocked(self):
|
||||
self.assertIsNotNone(validate_url("http://foo.localhost/x"))
|
||||
|
||||
def test_gcp_metadata_name_blocked(self):
|
||||
self.assertIsNotNone(validate_url("http://metadata.google.internal/x"))
|
||||
|
||||
|
||||
class AddressResolutionTests(unittest.TestCase):
|
||||
def _validate_with_addrs(self, url, *addrs, family=socket.AF_INET):
|
||||
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo(*addrs, family=family)):
|
||||
return validate_url(url)
|
||||
|
||||
def test_public_https_allowed(self):
|
||||
self.assertIsNone(self._validate_with_addrs("https://youtube.com/watch?v=x", "142.250.1.1"))
|
||||
|
||||
def test_public_http_allowed(self):
|
||||
self.assertIsNone(self._validate_with_addrs("http://example.com/x", "93.184.216.34"))
|
||||
|
||||
def test_link_local_metadata_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://metadata/x", "169.254.169.254"))
|
||||
|
||||
def test_loopback_ipv4_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://127.0.0.1/x", "127.0.0.1"))
|
||||
|
||||
def test_private_rfc1918_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://intranet/x", "10.0.0.5"))
|
||||
|
||||
def test_decimal_ip_form_blocked(self):
|
||||
# 2852039166 == 169.254.169.254; the OS resolver normalizes it.
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://2852039166/x", "169.254.169.254"))
|
||||
|
||||
def test_ipv6_loopback_blocked(self):
|
||||
self.assertIsNotNone(
|
||||
self._validate_with_addrs("http://[::1]/x", "::1", family=socket.AF_INET6)
|
||||
)
|
||||
|
||||
def test_ipv4_mapped_ipv6_metadata_blocked(self):
|
||||
self.assertIsNotNone(
|
||||
self._validate_with_addrs(
|
||||
"http://evil/x", "::ffff:169.254.169.254", family=socket.AF_INET6
|
||||
)
|
||||
)
|
||||
|
||||
def test_mixed_public_and_private_blocked(self):
|
||||
# If any resolved address is internal, reject the whole URL.
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://mixed/x", "142.250.1.1", "127.0.0.1"))
|
||||
|
||||
def test_resolution_failure_defers_to_ytdlp(self):
|
||||
with mock.patch("url_guard.socket.getaddrinfo", side_effect=socket.gaierror):
|
||||
self.assertIsNone(validate_url("http://does-not-resolve.example/x"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -3,20 +3,16 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||
fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
|
||||
fake_postprocessor = types.ModuleType("yt_dlp.postprocessor")
|
||||
fake_postprocessor_common = types.ModuleType("yt_dlp.postprocessor.common")
|
||||
fake_utils = types.ModuleType("yt_dlp.utils")
|
||||
|
||||
|
||||
|
|
@ -26,171 +22,26 @@ class _ImpersonateTarget:
|
|||
return value
|
||||
|
||||
|
||||
class _PostProcessor:
|
||||
def __init__(self, downloader=None):
|
||||
self._downloader = downloader
|
||||
|
||||
|
||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||
fake_networking.impersonate = fake_impersonate
|
||||
fake_postprocessor_common.PostProcessor = _PostProcessor
|
||||
# The inner ``key`` group mirrors the real ``STR_FORMAT_RE_TMPL`` so that
|
||||
# ``_OUTTMPL_FIELD_RE`` (compiled at import time) has the named group that
|
||||
# ``_resolve_outtmpl_fields`` reads via ``match.group('key')``.
|
||||
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>(?P<key>{}))\)(?P<format>[-0-9.]*{})"
|
||||
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>{})\)(?P<format>[-0-9.]*{})"
|
||||
fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
|
||||
fake_yt_dlp.networking = fake_networking
|
||||
fake_yt_dlp.postprocessor = fake_postprocessor
|
||||
fake_yt_dlp.utils = fake_utils
|
||||
sys.modules.setdefault("yt_dlp", fake_yt_dlp)
|
||||
sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||
sys.modules.setdefault("yt_dlp.postprocessor", fake_postprocessor)
|
||||
sys.modules.setdefault("yt_dlp.postprocessor.common", fake_postprocessor_common)
|
||||
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
||||
|
||||
from ytdl import (
|
||||
Download,
|
||||
DownloadInfo,
|
||||
_compact_persisted_entry,
|
||||
_convert_srt_to_txt_file,
|
||||
_AlbumArtistPostProcessor,
|
||||
_output_dir_escapes,
|
||||
_resolve_outtmpl_fields,
|
||||
_outtmpl_substitute_field,
|
||||
_sanitize_entry_for_pickle,
|
||||
_sanitize_path_component,
|
||||
)
|
||||
|
||||
# Detect whether the real yt-dlp is loaded (as opposed to the minimal fake
|
||||
# shim above). _resolve_outtmpl_fields needs YoutubeDL at runtime.
|
||||
_has_real_ytdlp = hasattr(sys.modules.get("yt_dlp"), "YoutubeDL")
|
||||
|
||||
|
||||
class AlbumArtistPostProcessorTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.postprocessor = _AlbumArtistPostProcessor()
|
||||
|
||||
def test_fills_album_artist_from_artist(self):
|
||||
info = {'album': 'CrasH Talk', 'artist': 'ScHoolboy Q'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
|
||||
|
||||
def test_uses_main_artist_for_featured_track(self):
|
||||
info = {
|
||||
'album': 'CrasH Talk',
|
||||
'artists': ['ScHoolboy Q · Travis Scott'],
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
|
||||
|
||||
def test_uses_topic_channel_artist_for_joint_album(self):
|
||||
info = {
|
||||
'album': 'Watch the Throne',
|
||||
'artists': ['JAY-Z', 'Kanye West'],
|
||||
'channel': 'JAY-Z & Kanye West - Topic',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'JAY-Z & Kanye West')
|
||||
|
||||
def test_uses_topic_uploader_and_strips_suffix_for_compilation(self):
|
||||
info = {
|
||||
'album': 'Compilation',
|
||||
'artist': 'Track Artist',
|
||||
'channel': 'Regular Channel',
|
||||
'uploader': 'Various Artists - Topic',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Various Artists')
|
||||
|
||||
def test_regular_channel_falls_back_to_main_artist(self):
|
||||
info = {
|
||||
'album': 'Album',
|
||||
'artist': 'Track Artist',
|
||||
'channel': 'Label Channel',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Track Artist')
|
||||
|
||||
def test_preserves_explicit_various_artists(self):
|
||||
info = {
|
||||
'album': 'Revenge of the Dreamers III',
|
||||
'artist': 'J. Cole',
|
||||
'album_artist': 'Various Artists',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Various Artists')
|
||||
|
||||
def test_preserves_existing_album_artists_list(self):
|
||||
info = {
|
||||
'album': 'Album',
|
||||
'artist': 'Track Artist',
|
||||
'album_artists': ['Album Artist'],
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artists'], ['Album Artist'])
|
||||
self.assertNotIn('album_artist', result)
|
||||
|
||||
def test_uses_first_artist_when_artist_list_has_multiple_entries(self):
|
||||
info = {'album': 'Album', 'artists': ['Main Artist', 'Featured Artist']}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Main Artist')
|
||||
|
||||
def test_does_not_fill_without_album(self):
|
||||
info = {'artist': 'Standalone Artist'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertNotIn('album_artist', result)
|
||||
self.assertNotIn('album_artists', result)
|
||||
|
||||
def test_does_not_fill_without_artist(self):
|
||||
info = {'album': 'Instrumental Album'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertNotIn('album_artist', result)
|
||||
self.assertNotIn('album_artists', result)
|
||||
|
||||
|
||||
class AlbumArtistRegistrationTests(unittest.TestCase):
|
||||
def test_audio_download_registers_pre_process_postprocessor(self):
|
||||
download = _make_test_download()
|
||||
download.info.download_type = 'audio'
|
||||
fake_ydl = MagicMock()
|
||||
|
||||
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
|
||||
result = download._make_youtube_dl({'quiet': True})
|
||||
|
||||
self.assertIs(result, fake_ydl)
|
||||
postprocessor, = fake_ydl.add_post_processor.call_args.args
|
||||
self.assertIsInstance(postprocessor, _AlbumArtistPostProcessor)
|
||||
self.assertEqual(fake_ydl.add_post_processor.call_args.kwargs, {'when': 'pre_process'})
|
||||
|
||||
def test_video_download_does_not_register_postprocessor(self):
|
||||
download = _make_test_download()
|
||||
fake_ydl = MagicMock()
|
||||
|
||||
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
|
||||
download._make_youtube_dl({'quiet': True})
|
||||
|
||||
fake_ydl.add_post_processor.assert_not_called()
|
||||
|
||||
|
||||
class SanitizePathComponentTests(unittest.TestCase):
|
||||
def test_replaces_windows_invalid_chars(self):
|
||||
|
|
@ -200,118 +51,16 @@ class SanitizePathComponentTests(unittest.TestCase):
|
|||
self.assertIs(_sanitize_path_component(None), None)
|
||||
self.assertEqual(_sanitize_path_component(42), 42)
|
||||
|
||||
def test_strips_path_separators_and_traversal(self):
|
||||
result = _sanitize_path_component('../../../../etc/x')
|
||||
self.assertNotIn('..', result)
|
||||
self.assertNotIn('/', result)
|
||||
self.assertNotIn('\\', result)
|
||||
|
||||
def test_strips_leading_absolute_path_separator(self):
|
||||
result = _sanitize_path_component('/tmp/x')
|
||||
self.assertFalse(result.startswith('/'))
|
||||
self.assertFalse(result.startswith('\\'))
|
||||
self.assertEqual(result, '_tmp_x')
|
||||
|
||||
def test_collapses_slashes_in_legitimate_titles(self):
|
||||
self.assertEqual(_sanitize_path_component('AC/DC'), 'AC_DC')
|
||||
|
||||
def test_empty_after_strip_becomes_underscore(self):
|
||||
self.assertEqual(_sanitize_path_component(' '), '_')
|
||||
|
||||
|
||||
@unittest.skipUnless(_has_real_ytdlp, "requires real yt-dlp")
|
||||
class ResolveOuttmplFieldsTests(unittest.TestCase):
|
||||
"""Tests for _resolve_outtmpl_fields (delegates to yt-dlp's template engine)."""
|
||||
|
||||
def test_simple_playlist_substitution(self):
|
||||
info = {"playlist_title": "My PL", "playlist_index": "03"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_title)s/%(title)s.%(ext)s", info, ("playlist",))
|
||||
self.assertEqual(result, "My PL/%(title)s.%(ext)s")
|
||||
class OuttmplSubstituteFieldTests(unittest.TestCase):
|
||||
def test_simple_substitution(self):
|
||||
self.assertEqual(_outtmpl_substitute_field("%(title)s", "title", "Hello"), "Hello")
|
||||
|
||||
def test_format_spec_int(self):
|
||||
info = {"playlist_index": "3"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_index)02d-%(title)s", info, ("playlist",))
|
||||
self.assertEqual(result, "03-%(title)s")
|
||||
self.assertEqual(_outtmpl_substitute_field("%(idx)02d", "idx", 3), "03")
|
||||
|
||||
def test_non_targeted_fields_unchanged(self):
|
||||
info = {"playlist_title": "PL"}
|
||||
result = _resolve_outtmpl_fields("%(title)s/%(ext)s", info, ("playlist",))
|
||||
self.assertEqual(result, "%(title)s/%(ext)s")
|
||||
|
||||
def test_default_value(self):
|
||||
info = {"playlist_index": "1"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_title|Unknown)s/%(playlist_index)s", info, ("playlist",))
|
||||
self.assertEqual(result, "Unknown/1")
|
||||
|
||||
def test_channel_prefix(self):
|
||||
info = {"channel": "MyChan", "channel_index": "05"}
|
||||
result = _resolve_outtmpl_fields("%(channel)s/%(channel_index)02d-%(title)s", info, ("channel",))
|
||||
self.assertEqual(result, "MyChan/05-%(title)s")
|
||||
|
||||
def test_math_operation(self):
|
||||
info = {"playlist_index": "3"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_index+100)d", info, ("playlist",))
|
||||
self.assertEqual(result, "103")
|
||||
|
||||
def test_playlist_count_and_autonumber(self):
|
||||
info = {
|
||||
"playlist_title": "My PL",
|
||||
"playlist_index": "03",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 3,
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
}
|
||||
result = _resolve_outtmpl_fields(
|
||||
"%(playlist_title)s/%(playlist_autonumber)s of %(playlist_count)s - %(title)s.%(ext)s",
|
||||
info,
|
||||
("playlist",),
|
||||
)
|
||||
# playlist_autonumber is auto-padded by yt-dlp using __last_playlist_index
|
||||
self.assertEqual(result, "My PL/03 of 10 - %(title)s.%(ext)s")
|
||||
|
||||
def test_conditional_playlist_index(self):
|
||||
info = {
|
||||
"playlist_index": "5",
|
||||
"playlist_count": 10,
|
||||
}
|
||||
result = _resolve_outtmpl_fields(
|
||||
"%(playlist_index&{} - |)s%(title)s.%(ext)s",
|
||||
info,
|
||||
("playlist",),
|
||||
)
|
||||
self.assertEqual(result, "5 - %(title)s.%(ext)s")
|
||||
|
||||
def test_malicious_playlist_title_cannot_escape_via_template(self):
|
||||
malicious_title = '/tmp/METUBE_ARBITRARY_WRITE_POC'
|
||||
entry = {
|
||||
'playlist_title': malicious_title,
|
||||
'playlist_index': '1',
|
||||
'title': 'video',
|
||||
'ext': 'mp4',
|
||||
}
|
||||
sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()}
|
||||
template = '%(playlist_title)s/%(title)s.%(ext)s'
|
||||
result = _resolve_outtmpl_fields(template, sanitized, ('playlist',))
|
||||
marker = result.find('%(')
|
||||
literal_prefix = result[:marker] if marker != -1 else result
|
||||
self.assertNotIn('..', literal_prefix)
|
||||
self.assertFalse(literal_prefix.startswith('/'))
|
||||
self.assertFalse(literal_prefix.startswith('\\'))
|
||||
|
||||
|
||||
class OutputDirEscapesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.base_dir = tempfile.mkdtemp()
|
||||
|
||||
def test_relative_traversal_escapes(self):
|
||||
self.assertTrue(_output_dir_escapes(self.base_dir, '../../tmp/x/%(title)s.%(ext)s'))
|
||||
|
||||
def test_absolute_path_escapes(self):
|
||||
self.assertTrue(_output_dir_escapes(self.base_dir, '/tmp/x/%(title)s.%(ext)s'))
|
||||
|
||||
def test_normal_playlist_dir_stays_inside(self):
|
||||
self.assertFalse(_output_dir_escapes(self.base_dir, 'Playlist/%(title)s.%(ext)s'))
|
||||
def test_missing_field_unchanged(self):
|
||||
self.assertEqual(_outtmpl_substitute_field("%(other)s", "title", "x"), "%(other)s")
|
||||
|
||||
|
||||
class SanitizeEntryForPickleTests(unittest.TestCase):
|
||||
|
|
@ -351,105 +100,6 @@ class SanitizeEntryForPickleTests(unittest.TestCase):
|
|||
self.assertEqual(out, {"z": 1, "a": 2})
|
||||
|
||||
|
||||
def _make_test_download() -> Download:
|
||||
info = DownloadInfo(
|
||||
id="id1",
|
||||
title="t",
|
||||
url="http://example.com/v",
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry=None,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
||||
|
||||
|
||||
class ProgressThrottleTests(unittest.TestCase):
|
||||
def test_downloading_ticks_are_throttled(self):
|
||||
dl = _make_test_download()
|
||||
forwarded = []
|
||||
dl.status_queue = types.SimpleNamespace(put=forwarded.append)
|
||||
hook = dl._make_progress_hook()
|
||||
|
||||
with patch("ytdl.time.monotonic", side_effect=[100.0, 100.1, 100.6, 100.7]):
|
||||
hook({"status": "downloading", "downloaded_bytes": 1})
|
||||
hook({"status": "downloading", "downloaded_bytes": 2})
|
||||
hook({"status": "downloading", "downloaded_bytes": 3})
|
||||
hook({"status": "downloading", "downloaded_bytes": 4})
|
||||
|
||||
# Only the 1st and 3rd ticks are >= 0.5s apart from the last forwarded one.
|
||||
self.assertEqual(len(forwarded), 2)
|
||||
|
||||
def test_finished_and_error_statuses_always_forwarded(self):
|
||||
dl = _make_test_download()
|
||||
forwarded = []
|
||||
dl.status_queue = types.SimpleNamespace(put=forwarded.append)
|
||||
hook = dl._make_progress_hook()
|
||||
|
||||
with patch("ytdl.time.monotonic", side_effect=[200.0, 200.1]):
|
||||
hook({"status": "downloading"})
|
||||
hook({"status": "finished"})
|
||||
hook({"status": "downloading"})
|
||||
hook({"status": "error", "msg": "boom"})
|
||||
|
||||
statuses = [item.get("status") for item in forwarded]
|
||||
self.assertIn("finished", statuses)
|
||||
self.assertIn("error", statuses)
|
||||
|
||||
|
||||
class CancelProcessGroupTests(unittest.TestCase):
|
||||
def test_cancel_kills_group_when_child_is_group_leader(self):
|
||||
# Child successfully ran os.setpgrp(): its pgid equals its own pid.
|
||||
dl = _make_test_download()
|
||||
dl.proc = types.SimpleNamespace(pid=4321)
|
||||
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||
|
||||
with patch.object(Download, "running", return_value=True), \
|
||||
patch("ytdl.os.getpgid", return_value=4321) as mock_getpgid, \
|
||||
patch("ytdl.os.killpg") as mock_killpg:
|
||||
dl.cancel()
|
||||
|
||||
mock_getpgid.assert_called_once_with(4321)
|
||||
mock_killpg.assert_called_once_with(4321, signal.SIGKILL)
|
||||
self.assertTrue(dl.canceled)
|
||||
|
||||
def test_cancel_does_not_killpg_parent_group_kills_child_only(self):
|
||||
# Child has NOT become its own group leader yet (pgid != pid, e.g. it is
|
||||
# still in the server's process group). killpg must NOT be called — that
|
||||
# would SIGKILL the whole server — and we fall back to proc.kill().
|
||||
dl = _make_test_download()
|
||||
dl.proc = types.SimpleNamespace(pid=4321, kill=MagicMock())
|
||||
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||
|
||||
with patch.object(Download, "running", return_value=True), \
|
||||
patch("ytdl.os.getpgid", return_value=999), \
|
||||
patch("ytdl.os.killpg") as mock_killpg:
|
||||
dl.cancel()
|
||||
|
||||
mock_killpg.assert_not_called()
|
||||
dl.proc.kill.assert_called_once()
|
||||
self.assertTrue(dl.canceled)
|
||||
|
||||
def test_cancel_falls_back_to_proc_kill_when_getpgid_unavailable(self):
|
||||
dl = _make_test_download()
|
||||
dl.proc = types.SimpleNamespace(pid=4321, kill=MagicMock())
|
||||
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||
|
||||
with patch.object(Download, "running", return_value=True), \
|
||||
patch("ytdl.os.getpgid", side_effect=OSError("no such process")):
|
||||
dl.cancel()
|
||||
|
||||
dl.proc.kill.assert_called_once()
|
||||
self.assertTrue(dl.canceled)
|
||||
|
||||
|
||||
class ConvertSrtToTxtTests(unittest.TestCase):
|
||||
def test_basic_conversion(self):
|
||||
srt = """1
|
||||
|
|
@ -470,77 +120,6 @@ Second line
|
|||
self.assertIn("Hello world", content)
|
||||
self.assertIn("Second line", content)
|
||||
|
||||
def test_vtt_input_strips_header_and_metadata(self):
|
||||
# yt-dlp can fall back to VTT even when srt/txt was requested (the
|
||||
# extractor may not offer a native srt track); the converter must not
|
||||
# leak VTT-only header/metadata lines into the plain-text output.
|
||||
vtt = """WEBVTT
|
||||
Kind: captions
|
||||
Language: en
|
||||
|
||||
NOTE
|
||||
This is a note block
|
||||
|
||||
1
|
||||
00:00:01.000 --> 00:00:02.000
|
||||
Hello <b>world</b>
|
||||
|
||||
2
|
||||
00:00:03.000 --> 00:00:04.000
|
||||
Second line
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "sub.vtt"
|
||||
path.write_text(vtt, encoding="utf-8")
|
||||
txt_path = _convert_srt_to_txt_file(str(path))
|
||||
self.assertIsNotNone(txt_path)
|
||||
content = Path(txt_path).read_text(encoding="utf-8")
|
||||
self.assertIn("Hello world", content)
|
||||
self.assertIn("Second line", content)
|
||||
self.assertNotIn("WEBVTT", content)
|
||||
self.assertNotIn("Kind:", content)
|
||||
self.assertNotIn("Language:", content)
|
||||
self.assertNotIn("This is a note block", content)
|
||||
|
||||
def test_vtt_standalone_header_block_is_stripped(self):
|
||||
# Some VTT files put a blank line after WEBVTT, so Kind:/Language: form
|
||||
# their own block. That header block (before the first timed cue) must
|
||||
# still be stripped.
|
||||
vtt = """WEBVTT
|
||||
|
||||
Kind: captions
|
||||
Language: en
|
||||
|
||||
1
|
||||
00:00:01.000 --> 00:00:02.000
|
||||
Hello world
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "sub.vtt"
|
||||
path.write_text(vtt, encoding="utf-8")
|
||||
content = Path(_convert_srt_to_txt_file(str(path))).read_text(encoding="utf-8")
|
||||
self.assertIn("Hello world", content)
|
||||
self.assertNotIn("Kind:", content)
|
||||
self.assertNotIn("Language:", content)
|
||||
|
||||
def test_cue_text_starting_with_metadata_keyword_is_kept(self):
|
||||
# A real caption line beginning with "Kind:"/"Language:" must NOT be
|
||||
# dropped as if it were VTT header metadata.
|
||||
srt = """1
|
||||
00:00:01,000 --> 00:00:02,000
|
||||
Kind: regards, everyone
|
||||
|
||||
2
|
||||
00:00:03,000 --> 00:00:04,000
|
||||
Language: they spoke French
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "sub.srt"
|
||||
path.write_text(srt, encoding="utf-8")
|
||||
content = Path(_convert_srt_to_txt_file(str(path))).read_text(encoding="utf-8")
|
||||
self.assertIn("Kind: regards, everyone", content)
|
||||
self.assertIn("Language: they spoke French", content)
|
||||
|
||||
|
||||
class DownloadInfoSetstateTests(unittest.TestCase):
|
||||
def _base_state(self, **kwargs):
|
||||
|
|
@ -640,12 +219,8 @@ class CompactPersistedEntryTests(unittest.TestCase):
|
|||
entry = {
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 1,
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
"formats": [{"id": "huge"}],
|
||||
"description": "big blob",
|
||||
}
|
||||
|
|
@ -657,12 +232,8 @@ class CompactPersistedEntryTests(unittest.TestCase):
|
|||
{
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 1,
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,89 +0,0 @@
|
|||
"""Lightweight SSRF guard for user-submitted URLs.
|
||||
|
||||
MeTube hands user-submitted URLs to yt-dlp, whose generic extractor will fetch
|
||||
any ``http(s)`` URL. Without a guard, an attacker can make the server fetch
|
||||
internal endpoints (cloud metadata services, loopback, RFC1918 hosts, etc.) and
|
||||
have the response saved to the download directory and served back.
|
||||
|
||||
This module provides a single cheap validator applied at every URL ingress. It
|
||||
intentionally does NOT attempt DNS-rebinding pinning, redirect-chain
|
||||
re-validation, or validation of every media URL yt-dlp derives from remote
|
||||
metadata — network isolation (e.g. Docker) remains the backstop for those.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
log = logging.getLogger('url_guard')
|
||||
|
||||
_ALLOWED_SCHEMES = ('http', 'https')
|
||||
|
||||
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
|
||||
# subdomain of it are conventionally loopback, and the GCP metadata name is a
|
||||
# well-known SSRF target that may resolve via a resolver we don't control.
|
||||
_BLOCKED_HOSTNAMES = ('localhost', 'metadata.google.internal')
|
||||
|
||||
|
||||
def _hostname_is_blocked(hostname: str) -> bool:
|
||||
host = hostname.rstrip('.').lower()
|
||||
for blocked in _BLOCKED_HOSTNAMES:
|
||||
if host == blocked or host.endswith('.' + blocked):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _address_is_global(addr: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
return False
|
||||
# Unwrap IPv4-mapped/compatible IPv6 (e.g. ::ffff:169.254.169.254) so the
|
||||
# embedded IPv4 address is judged on its own merits.
|
||||
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
||||
ip = ip.ipv4_mapped
|
||||
return ip.is_global
|
||||
|
||||
|
||||
def validate_url(url: str) -> str | None:
|
||||
"""Return an error message if the URL is disallowed, else ``None``.
|
||||
|
||||
Inputs without a ``://`` scheme separator (bare video IDs, ``ytsearch:``
|
||||
and other yt-dlp search/extractor prefixes) are allowed unchanged so that
|
||||
non-URL entries keep working.
|
||||
"""
|
||||
if not isinstance(url, str):
|
||||
return 'Invalid URL'
|
||||
|
||||
candidate = url.strip()
|
||||
if '://' not in candidate:
|
||||
# Not an absolute URL: bare video IDs, ytsearch: prefixes, etc.
|
||||
return None
|
||||
|
||||
parts = urlsplit(candidate)
|
||||
scheme = parts.scheme.lower()
|
||||
if scheme not in _ALLOWED_SCHEMES:
|
||||
return f'URL scheme "{parts.scheme}" is not allowed (only http and https)'
|
||||
|
||||
hostname = parts.hostname
|
||||
if not hostname:
|
||||
return 'URL is missing a host'
|
||||
|
||||
if _hostname_is_blocked(hostname):
|
||||
return f'Refusing to fetch internal host "{hostname}"'
|
||||
|
||||
try:
|
||||
addrinfo = socket.getaddrinfo(hostname, parts.port, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror:
|
||||
# Let yt-dlp surface a normal resolution error rather than masking it.
|
||||
return None
|
||||
except (UnicodeError, ValueError):
|
||||
return f'Invalid host "{hostname}"'
|
||||
|
||||
for family, _type, _proto, _canonname, sockaddr in addrinfo:
|
||||
addr = sockaddr[0]
|
||||
if not _address_is_global(addr):
|
||||
return f'Refusing to fetch internal address "{addr}" for host "{hostname}"'
|
||||
|
||||
return None
|
||||
821
app/ytdl.py
821
app/ytdl.py
File diff suppressed because it is too large
Load diff
|
|
@ -2,54 +2,11 @@
|
|||
|
||||
PUID="${UID:-$PUID}"
|
||||
PGID="${GID:-$PGID}"
|
||||
AUDIO_DOWNLOAD_DIR="${AUDIO_DOWNLOAD_DIR:-$DOWNLOAD_DIR}"
|
||||
|
||||
echo "Setting umask to ${UMASK}"
|
||||
umask ${UMASK}
|
||||
echo "Creating download directory (${DOWNLOAD_DIR}), audio download directory (${AUDIO_DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
|
||||
mkdir -p "${DOWNLOAD_DIR}" "${AUDIO_DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
|
||||
do_upgrade() {
|
||||
echo "Upgrading yt-dlp to nightly channel..."
|
||||
if ! python3 -m pip --version >/dev/null 2>&1; then
|
||||
echo "pip not found; attempting ensurepip"
|
||||
python3 -m ensurepip --upgrade >/dev/null 2>&1 || true
|
||||
fi
|
||||
if ! python3 -m pip install -U --pre "yt-dlp[default,curl-cffi,deno]"; then
|
||||
echo "Warning: yt-dlp nightly upgrade failed; continuing with existing installation"
|
||||
return 1
|
||||
fi
|
||||
echo "yt-dlp nightly upgrade complete"
|
||||
return 0
|
||||
}
|
||||
|
||||
run_supervised() {
|
||||
while true; do
|
||||
"$@" &
|
||||
child_pid=$!
|
||||
trap 'kill -TERM "$child_pid" 2>/dev/null; wait "$child_pid" 2>/dev/null' TERM INT
|
||||
wait "$child_pid"
|
||||
exit_code=$?
|
||||
trap - TERM INT
|
||||
if [ "$exit_code" -eq 42 ]; then
|
||||
echo "MeTube requested yt-dlp update restart (exit 42)"
|
||||
do_upgrade || true
|
||||
continue
|
||||
fi
|
||||
return "$exit_code"
|
||||
done
|
||||
}
|
||||
|
||||
nightly_enabled() {
|
||||
[ -n "${YTDL_NIGHTLY_UPDATE_TIME}" ]
|
||||
}
|
||||
|
||||
disable_nightly_for_non_root() {
|
||||
if nightly_enabled; then
|
||||
echo "YTDL_NIGHTLY_UPDATE_TIME is set but this container runs as a non-root user; nightly yt-dlp updates are not supported. Ignoring YTDL_NIGHTLY_UPDATE_TIME."
|
||||
unset YTDL_NIGHTLY_UPDATE_TIME
|
||||
fi
|
||||
}
|
||||
echo "Creating download directory (${DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
|
||||
mkdir -p "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
|
||||
if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
||||
if [ "${PUID}" -eq 0 ]; then
|
||||
|
|
@ -57,22 +14,15 @@ if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
|||
fi
|
||||
if [ "${CHOWN_DIRS:-true}" != "false" ]; then
|
||||
echo "Changing ownership of download and state directories to ${PUID}:${PGID}"
|
||||
chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${AUDIO_DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
fi
|
||||
if nightly_enabled; then
|
||||
echo "YTDL_NIGHTLY_UPDATE_TIME is set to ${YTDL_NIGHTLY_UPDATE_TIME}; upgrading yt-dlp on startup"
|
||||
do_upgrade || true
|
||||
chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
fi
|
||||
echo "Starting BgUtils POT Provider"
|
||||
gosu "${PUID}":"${PGID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||
echo "Running MeTube as user ${PUID}:${PGID}"
|
||||
run_supervised gosu "${PUID}":"${PGID}" python3 app/main.py
|
||||
exit $?
|
||||
exec gosu "${PUID}":"${PGID}" python3 app/main.py
|
||||
else
|
||||
echo "User set by docker; running MeTube as `id -u`:`id -g`"
|
||||
disable_nightly_for_non_root
|
||||
echo "Starting BgUtils POT Provider"
|
||||
bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||
run_supervised python3 app/main.py
|
||||
exit $?
|
||||
exec python3 app/main.py
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
"build": "ng build",
|
||||
"build:watch": "ng build --watch --configuration development",
|
||||
"build:watch": "ng build --watch",
|
||||
"test": "ng test",
|
||||
"lint": "ng lint"
|
||||
},
|
||||
|
|
@ -21,44 +21,43 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"packageManager": "pnpm@11.5.2",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/animations": "^22.0.6",
|
||||
"@angular/common": "^22.0.6",
|
||||
"@angular/compiler": "^22.0.6",
|
||||
"@angular/core": "^22.0.6",
|
||||
"@angular/forms": "^22.0.6",
|
||||
"@angular/platform-browser": "^22.0.6",
|
||||
"@angular/platform-browser-dynamic": "^22.0.6",
|
||||
"@angular/service-worker": "^22.0.6",
|
||||
"@angular/animations": "^21.2.6",
|
||||
"@angular/common": "^21.2.6",
|
||||
"@angular/compiler": "^21.2.6",
|
||||
"@angular/core": "^21.2.6",
|
||||
"@angular/forms": "^21.2.6",
|
||||
"@angular/platform-browser": "^21.2.6",
|
||||
"@angular/platform-browser-dynamic": "^21.2.6",
|
||||
"@angular/service-worker": "^21.2.6",
|
||||
"@fortawesome/angular-fontawesome": "~4.0.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.3.1",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.3.1",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.3.1",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.3.1",
|
||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||
"@ng-select/ng-select": "^23.2.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.2.0",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.2.0",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.2.0",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.2.0",
|
||||
"@ng-bootstrap/ng-bootstrap": "^20.0.0",
|
||||
"@ng-select/ng-select": "^21.7.0",
|
||||
"@popperjs/core": "^2.11.8",
|
||||
"bootstrap": "^5.3.8",
|
||||
"ngx-cookie-service": "^22.0.0",
|
||||
"ngx-cookie-service": "^21.3.1",
|
||||
"ngx-socket-io": "~4.10.0",
|
||||
"rxjs": "~7.8.2",
|
||||
"tslib": "^2.8.1",
|
||||
"zone.js": "0.15.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-eslint/builder": "22.0.0",
|
||||
"@angular/build": "^22.0.7",
|
||||
"@angular/cli": "^22.0.7",
|
||||
"@angular/compiler-cli": "^22.0.6",
|
||||
"@angular/localize": "^22.0.6",
|
||||
"@eslint/js": "^9.39.5",
|
||||
"angular-eslint": "22.0.0",
|
||||
"eslint": "^9.39.5",
|
||||
"@angular-eslint/builder": "21.1.0",
|
||||
"@angular/build": "^21.2.5",
|
||||
"@angular/cli": "^21.2.5",
|
||||
"@angular/compiler-cli": "^21.2.6",
|
||||
"@angular/localize": "^21.2.6",
|
||||
"@eslint/js": "^9.39.4",
|
||||
"angular-eslint": "21.1.0",
|
||||
"eslint": "^9.39.4",
|
||||
"jsdom": "^27.4.0",
|
||||
"typescript": "~6.0.3",
|
||||
"typescript-eslint": "8.62.0",
|
||||
"vitest": "^4.1.10"
|
||||
"typescript": "~5.9.3",
|
||||
"typescript-eslint": "8.47.0",
|
||||
"vitest": "^4.1.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
4276
ui/pnpm-lock.yaml
generated
4276
ui/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +0,0 @@
|
|||
allowBuilds:
|
||||
'@parcel/watcher': true
|
||||
core-js: true
|
||||
esbuild: true
|
||||
lmdb: true
|
||||
msgpackr-extract: true
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { ApplicationConfig, provideBrowserGlobalErrorListeners, isDevMode, provideZoneChangeDetection } from '@angular/core';
|
||||
import { provideServiceWorker } from '@angular/service-worker';
|
||||
import { provideHttpClient, withInterceptorsFromDi, withXhr } from '@angular/common/http';
|
||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http';
|
||||
|
||||
export const appConfig: ApplicationConfig = {
|
||||
providers: [
|
||||
|
|
@ -12,6 +12,6 @@ export const appConfig: ApplicationConfig = {
|
|||
// or after 30 seconds (whichever comes first).
|
||||
registrationStrategy: 'registerWhenStable:30000'
|
||||
}),
|
||||
provideHttpClient(withXhr(), withInterceptorsFromDi()),
|
||||
provideHttpClient(withInterceptorsFromDi()),
|
||||
]
|
||||
};
|
||||
|
|
|
|||
|
|
@ -279,12 +279,13 @@
|
|||
</div>
|
||||
<div class="col-12 col-md-6 col-lg-3">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Subtitle output format for captions mode." triggers="click" autoClose="outside" container="body">Format</span>
|
||||
<span class="input-group-text">Format</span>
|
||||
<select class="form-select"
|
||||
name="format"
|
||||
[(ngModel)]="format"
|
||||
(change)="formatChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Subtitle output format for captions mode">
|
||||
@for (f of formatOptions; track f.id) {
|
||||
<option [ngValue]="f.id">{{ f.text }}</option>
|
||||
}
|
||||
|
|
@ -293,7 +294,7 @@
|
|||
</div>
|
||||
<div class="col-12 col-md-6 col-lg-3">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Subtitle language (you can type any language code)." triggers="click" autoClose="outside" container="body">Language</span>
|
||||
<span class="input-group-text">Language</span>
|
||||
<input class="form-control"
|
||||
type="text"
|
||||
list="subtitleLanguageOptions"
|
||||
|
|
@ -301,7 +302,8 @@
|
|||
[(ngModel)]="subtitleLanguage"
|
||||
(change)="subtitleLanguageChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
placeholder="e.g. en, es, zh-Hans">
|
||||
placeholder="e.g. en, es, zh-Hans"
|
||||
ngbTooltip="Subtitle language (you can type any language code)">
|
||||
<datalist id="subtitleLanguageOptions">
|
||||
@for (lang of subtitleLanguages; track lang.id) {
|
||||
<option [value]="lang.id">{{ lang.text }}</option>
|
||||
|
|
@ -311,12 +313,13 @@
|
|||
</div>
|
||||
<div class="col-12 col-md-6 col-lg-3">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Choose manual, auto, or fallback preference for captions mode." triggers="click" autoClose="outside" container="body">Subtitle Source</span>
|
||||
<span class="input-group-text">Subtitle Source</span>
|
||||
<select class="form-select"
|
||||
name="subtitleMode"
|
||||
[(ngModel)]="subtitleMode"
|
||||
(change)="subtitleModeChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Choose manual, auto, or fallback preference for captions mode">
|
||||
@for (mode of subtitleModes; track mode.id) {
|
||||
<option [ngValue]="mode.id">{{ mode.text }}</option>
|
||||
}
|
||||
|
|
@ -366,35 +369,82 @@
|
|||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="collapse show" id="advancedOptions" [ngbCollapse]="!isAdvancedOpen">
|
||||
<div class="pt-1 pb-2">
|
||||
<!-- Output -->
|
||||
<div class="settings-section-label">Output</div>
|
||||
<div class="row g-3">
|
||||
<div class="py-2">
|
||||
<!-- Advanced Settings -->
|
||||
<div class="row g-3 mb-2">
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Type to filter existing folders, or enter a new folder name." triggers="click" autoClose="outside" container="body">Download Folder</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
placeholder="Default"
|
||||
name="folder"
|
||||
[(ngModel)]="folder"
|
||||
[ngbTypeahead]="searchFolder"
|
||||
[editable]="!!downloads.configuration['CREATE_CUSTOM_DIRS']"
|
||||
(focus)="folderFocus$.next($any($event.target).value)"
|
||||
(click)="folderClick$.next($any($event.target).value)"
|
||||
#folderTypeahead="ngbTypeahead"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
<span class="input-group-text">Auto Start</span>
|
||||
<select class="form-select"
|
||||
name="autoStart"
|
||||
[(ngModel)]="autoStart"
|
||||
(change)="autoStartChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Automatically start downloads when added">
|
||||
<option [ngValue]="true">Yes</option>
|
||||
<option [ngValue]="false">No</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Add a prefix to downloaded filenames." triggers="click" autoClose="outside" container="body">Custom Name Prefix</span>
|
||||
<span class="input-group-text">Download Folder</span>
|
||||
@if (customDirs$ | async; as customDirs) {
|
||||
<ng-select [items]="customDirs"
|
||||
placeholder="Default"
|
||||
[addTag]="allowCustomDir.bind(this)"
|
||||
addTagText="Create directory"
|
||||
bindLabel="folder"
|
||||
[(ngModel)]="folder"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
[virtualScroll]="true"
|
||||
[clearable]="true"
|
||||
[loading]="downloads.loading"
|
||||
[searchable]="true"
|
||||
[closeOnSelect]="true"
|
||||
ngbTooltip="Choose where to save downloads. Type to create a new folder." />
|
||||
}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text">Custom Name Prefix</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
placeholder="Default"
|
||||
name="customNamePrefix"
|
||||
[(ngModel)]="customNamePrefix"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Add a prefix to downloaded filenames">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text">Items Limit</span>
|
||||
<input type="number"
|
||||
min="0"
|
||||
class="form-control"
|
||||
placeholder="Default"
|
||||
name="playlistItemLimit"
|
||||
(keydown)="isNumber($event)"
|
||||
[(ngModel)]="playlistItemLimit"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Maximum number of items to download from a playlist or channel (0 = no limit)">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text">Subscription Check (min)</span>
|
||||
<input type="number"
|
||||
min="1"
|
||||
class="form-control"
|
||||
name="checkIntervalMinutes"
|
||||
(keydown)="isNumber($event)"
|
||||
[(ngModel)]="checkIntervalMinutes"
|
||||
(ngModelChange)="checkIntervalChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="How often to poll subscriptions for new videos">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
|
|
@ -403,217 +453,94 @@
|
|||
<div class="form-check form-switch">
|
||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||
name="splitByChapters" [(ngModel)]="splitByChapters" (change)="splitByChaptersChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Split video into separate files by chapters">
|
||||
<label class="form-check-label" for="checkbox-split-chapters">Split by chapters</label>
|
||||
</div>
|
||||
</div>
|
||||
@if (splitByChapters) {
|
||||
<div class="col">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Output template for chapter files." triggers="click" autoClose="outside" container="body">Template</span>
|
||||
<span class="input-group-text">Template</span>
|
||||
<input type="text" class="form-control" name="chapterTemplate" [(ngModel)]="chapterTemplate"
|
||||
(change)="chapterTemplateChanged()" [disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
(change)="chapterTemplateChanged()" [disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
ngbTooltip="Output template for chapter files">
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
@if (downloadType === 'video' || downloadType === 'audio') {
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Optional start time (seconds, M:SS, or H:MM:SS). Blank = from start or YouTube &t= in URL." triggers="click" autoClose="outside" container="body">Clip start</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
name="clipStart"
|
||||
[(ngModel)]="clipStart"
|
||||
(change)="clipStartChanged()"
|
||||
placeholder="e.g. 2:26"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Optional end time. Blank = until end of media." triggers="click" autoClose="outside" container="body">Clip end</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
name="clipEnd"
|
||||
[(ngModel)]="clipEnd"
|
||||
(change)="clipEndChanged()"
|
||||
placeholder="e.g. 3:24"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
|
||||
<!-- Behavior -->
|
||||
<div class="settings-section-label">Behavior</div>
|
||||
<div class="row g-3">
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Automatically start downloads when added." triggers="click" autoClose="outside" container="body">Auto Start</span>
|
||||
<select class="form-select"
|
||||
name="autoStart"
|
||||
[(ngModel)]="autoStart"
|
||||
(change)="autoStartChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
<option [ngValue]="true">Yes</option>
|
||||
<option [ngValue]="false">No</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Maximum number of items to download from a playlist or channel (0 = no limit)." triggers="click" autoClose="outside" container="body">Items Limit</span>
|
||||
<input type="number"
|
||||
min="0"
|
||||
class="form-control"
|
||||
placeholder="Default"
|
||||
name="playlistItemLimit"
|
||||
(keydown)="isNumber($event)"
|
||||
[(ngModel)]="playlistItemLimit"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="How often to poll subscriptions for new videos." triggers="click" autoClose="outside" container="body">Subscription Check (min)</span>
|
||||
<input type="number"
|
||||
min="1"
|
||||
class="form-control"
|
||||
name="checkIntervalMinutes"
|
||||
(keydown)="isNumber($event)"
|
||||
[(ngModel)]="checkIntervalMinutes"
|
||||
(ngModelChange)="checkIntervalChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="In subscriptions, only titles matching this Python-style regex are queued. Empty = all. Case-sensitive; use (?i) in the pattern for case-insensitive." triggers="click" autoClose="outside" container="body">Subscription Title Filter</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
name="titleRegex"
|
||||
[(ngModel)]="titleRegex"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||
placeholder="Optional regex">
|
||||
</div>
|
||||
</div>
|
||||
<!-- Advanced Actions -->
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="form-check form-switch">
|
||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-skip-subscriber-only"
|
||||
name="skipSubscriberOnly" [(ngModel)]="skipSubscriberOnly"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading" />
|
||||
<label class="form-check-label" for="checkbox-skip-subscriber-only">
|
||||
<span class="help-title" tabindex="0" role="button"
|
||||
ngbPopover="When enabled, subscription checks skip videos marked members-only by yt-dlp (channel Join). Ignored for one-off downloads."
|
||||
triggers="click" autoClose="outside" container="body"
|
||||
(click)="$event.preventDefault(); $event.stopPropagation()"
|
||||
(keydown.enter)="$event.preventDefault(); $event.stopPropagation()"
|
||||
(keydown.space)="$event.preventDefault(); $event.stopPropagation()">Skip members-only subscription videos</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- yt-dlp -->
|
||||
<div class="settings-section-label">yt-dlp</div>
|
||||
<div class="row g-3">
|
||||
<div class="col-12" [class.col-md-6]="allowYtdlOptionsOverrides()">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Choose one or more yt-dlp option presets configured on the server (applied in order)." triggers="click" autoClose="outside" container="body">Option Presets</span>
|
||||
<ng-select
|
||||
class="flex-grow-1"
|
||||
name="ytdlOptionsPresets"
|
||||
[items]="ytdlOptionPresetNames"
|
||||
[multiple]="true"
|
||||
[closeOnSelect]="false"
|
||||
placeholder="Default"
|
||||
[(ngModel)]="ytdlOptionsPresets"
|
||||
(ngModelChange)="ytdlOptionsPresetsChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading" />
|
||||
</div>
|
||||
</div>
|
||||
@if (allowYtdlOptionsOverrides()) {
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text help-title" ngbPopover="Optional per-download yt-dlp overrides as a JSON object." triggers="click" autoClose="outside" container="body">Custom yt-dlp Options</span>
|
||||
<input type="text"
|
||||
class="form-control"
|
||||
placeholder='e.g. {"writesubtitles": true}'
|
||||
name="ytdlOptionsOverrides"
|
||||
[(ngModel)]="ytdlOptionsOverrides"
|
||||
(change)="ytdlOptionsOverridesChanged()"
|
||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
|
||||
<!-- Tools -->
|
||||
<div class="settings-section-label">Tools</div>
|
||||
<div class="row g-3">
|
||||
<div class="col-md-4">
|
||||
<div class="action-group-label help-title" ngbPopover="Upload a cookies.txt file from your browser to authenticate restricted or private downloads." triggers="click" autoClose="outside" container="body">Cookies</div>
|
||||
<input type="file" id="cookie-upload" class="d-none" accept=".txt"
|
||||
(change)="onCookieFileSelect($event)"
|
||||
[disabled]="cookieUploadInProgress || addInProgress">
|
||||
<div class="btn-group w-100" role="group">
|
||||
<label class="btn mb-0"
|
||||
[class]="hasCookies ? 'btn cookie-active-btn mb-0' : 'btn cookie-btn mb-0'"
|
||||
[class.disabled]="cookieUploadInProgress || addInProgress"
|
||||
for="cookie-upload">
|
||||
@if (cookieUploadInProgress) {
|
||||
<span class="spinner-border spinner-border-sm me-2" role="status"></span>
|
||||
} @else {
|
||||
<fa-icon [icon]="faUpload" class="me-2" />
|
||||
}
|
||||
{{ hasCookies ? 'Replace Cookies' : 'Upload Cookies' }}
|
||||
</label>
|
||||
@if (hasCookies) {
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
(click)="deleteCookies()"
|
||||
[disabled]="cookieUploadInProgress || addInProgress"
|
||||
ngbTooltip="Remove uploaded cookies">
|
||||
<fa-icon [icon]="faTrashAlt" />
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
<div class="cookie-status" [class.active]="hasCookies">
|
||||
@if (hasCookies) {
|
||||
<fa-icon [icon]="faCheckCircle" class="me-1" />
|
||||
Cookies active
|
||||
} @else {
|
||||
No cookies configured
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-8">
|
||||
<div class="action-group-label">Bulk Actions</div>
|
||||
<div class="row g-2">
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="openBatchImportModal()">
|
||||
<fa-icon [icon]="faFileImport" class="me-2" />
|
||||
Import URLs
|
||||
</button>
|
||||
<hr class="my-3">
|
||||
<div class="row g-3">
|
||||
<div class="col-md-4">
|
||||
<div class="action-group-label">Cookies</div>
|
||||
<input type="file" id="cookie-upload" class="d-none" accept=".txt"
|
||||
(change)="onCookieFileSelect($event)"
|
||||
[disabled]="cookieUploadInProgress || addInProgress">
|
||||
<div class="btn-group w-100" role="group">
|
||||
<label class="btn mb-0"
|
||||
[class]="hasCookies ? 'btn cookie-active-btn mb-0' : 'btn cookie-btn mb-0'"
|
||||
[class.disabled]="cookieUploadInProgress || addInProgress"
|
||||
for="cookie-upload"
|
||||
ngbTooltip="Upload a cookies.txt file for authenticated downloads">
|
||||
@if (cookieUploadInProgress) {
|
||||
<span class="spinner-border spinner-border-sm me-2" role="status"></span>
|
||||
} @else {
|
||||
<fa-icon [icon]="faUpload" class="me-2" />
|
||||
}
|
||||
{{ hasCookies ? 'Replace Cookies' : 'Upload Cookies' }}
|
||||
</label>
|
||||
@if (hasCookies) {
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
(click)="deleteCookies()"
|
||||
[disabled]="cookieUploadInProgress || addInProgress"
|
||||
ngbTooltip="Remove uploaded cookies">
|
||||
<fa-icon [icon]="faTrashAlt" />
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
<div class="cookie-status" [class.active]="hasCookies">
|
||||
@if (hasCookies) {
|
||||
<fa-icon [icon]="faCheckCircle" class="me-1" />
|
||||
Cookies active
|
||||
} @else {
|
||||
No cookies configured
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="exportBatchUrls('all')">
|
||||
<fa-icon [icon]="faFileExport" class="me-2" />
|
||||
Export URLs
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="copyBatchUrls('all')">
|
||||
<fa-icon [icon]="faCopy" class="me-2" />
|
||||
Copy URLs
|
||||
</button>
|
||||
<div class="col-md-8">
|
||||
<div class="action-group-label">Bulk Actions</div>
|
||||
<div class="row g-2">
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="openBatchImportModal()">
|
||||
<fa-icon [icon]="faFileImport" class="me-2" />
|
||||
Import URLs
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="exportBatchUrls('all')">
|
||||
<fa-icon [icon]="faFileExport" class="me-2" />
|
||||
Export URLs
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-4">
|
||||
<button type="button"
|
||||
class="btn btn-secondary w-100"
|
||||
(click)="copyBatchUrls('all')">
|
||||
<fa-icon [icon]="faCopy" class="me-2" />
|
||||
Copy URLs
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -644,19 +571,6 @@
|
|||
@if (batchImportStatus) {
|
||||
<small>{{ batchImportStatus }}</small>
|
||||
}
|
||||
@if (batchImportTotal > 0) {
|
||||
<div class="progress mt-2" style="height: 20px;">
|
||||
<div class="progress-bar"
|
||||
[class.bg-danger]="batchImportFailures > 0"
|
||||
role="progressbar"
|
||||
[style.width.%]="(batchImportCount / batchImportTotal) * 100"
|
||||
[attr.aria-valuenow]="batchImportCount"
|
||||
[attr.aria-valuemin]="0"
|
||||
[attr.aria-valuemax]="batchImportTotal">
|
||||
{{ batchImportCount }} / {{ batchImportTotal }}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
|
|
@ -699,38 +613,23 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@for (download of downloads.queue | keyvalue: asIsOrder; track download.key) {
|
||||
@for (download of downloads.queue | keyvalue: asIsOrder; track download.value.id) {
|
||||
<tr [class.disabled]='download.value.deleting'>
|
||||
<td>
|
||||
<app-item-checkbox [id]="download.key" [master]="queueMasterCheckboxRef" [checkable]="download.value" />
|
||||
</td>
|
||||
<td title="{{ download.value.filename }}">
|
||||
<div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
|
||||
<div class="d-flex align-items-center flex-wrap gap-2">
|
||||
<span>{{ download.value.title }}</span>
|
||||
@if (download.value.live_status === 'is_live' && download.value.status !== 'scheduled') {
|
||||
<span class="badge bg-danger">LIVE</span>
|
||||
}
|
||||
</div>
|
||||
@if (download.value.status === 'scheduled') {
|
||||
<span class="badge bg-warning text-dark">
|
||||
<fa-icon [icon]="faClock" />
|
||||
Waiting for stream
|
||||
@if (liveCountdownSeconds(download.value); as secs) {
|
||||
- starts in {{ secs | eta }}
|
||||
}
|
||||
</span>
|
||||
} @else {
|
||||
<ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
|
||||
[value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
|
||||
}
|
||||
<div>{{ download.value.title }} </div>
|
||||
<ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
|
||||
[value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ download.value.speed | speed }}</td>
|
||||
<td>{{ download.value.eta | eta }}</td>
|
||||
<td>
|
||||
<div class="d-flex">
|
||||
@if (download.value.status === 'pending' || download.value.status === 'scheduled') {
|
||||
@if (download.value.status === 'pending') {
|
||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Start download for ' + download.value.title" (click)="downloadItemByKey(download.key)"><fa-icon [icon]="faDownload" /></button>
|
||||
}
|
||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Remove ' + download.value.title + ' from queue'" (click)="delDownload('queue', download.key)"><fa-icon [icon]="faTrashAlt" /></button>
|
||||
|
|
@ -769,7 +668,7 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@for (entry of cachedSortedDone; track entry[0]) {
|
||||
@for (entry of cachedSortedDone; track entry[1].id) {
|
||||
<tr [class.disabled]='entry[1].deleting'>
|
||||
<td>
|
||||
<app-item-checkbox [id]="entry[0]" [master]="doneMasterCheckboxRef" [checkable]="entry[1]" />
|
||||
|
|
@ -856,9 +755,6 @@
|
|||
@if (entry[1].filename) {
|
||||
<a href="{{buildDownloadLink(entry[1])}}" download class="btn btn-link" [attr.aria-label]="'Download result file for ' + entry[1].title"><fa-icon [icon]="faDownload" /></a>
|
||||
}
|
||||
@if (entry[1].filename && canShareDownloads()) {
|
||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Share result file for ' + entry[1].title" (click)="shareDownload(entry[1])"><fa-icon [icon]="faShareNodes" /></button>
|
||||
}
|
||||
<a href="{{entry[1].url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + entry[1].title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Delete completed item ' + entry[1].title" (click)="delDownload('done', entry[0])"><fa-icon [icon]="faTrashAlt" /></button>
|
||||
</div>
|
||||
|
|
@ -941,7 +837,6 @@
|
|||
</th>
|
||||
<th scope="col">Name</th>
|
||||
<th scope="col">URL</th>
|
||||
<th scope="col" class="text-nowrap"><span class="help-title" ngbPopover="Subscriptions only — which new video titles to queue when this feed is checked. Does not affect manual downloads." triggers="click" autoClose="outside" container="body">Filter</span></th>
|
||||
<th scope="col" class="text-nowrap">Interval (min)</th>
|
||||
<th scope="col" class="text-nowrap">Last checked</th>
|
||||
<th scope="col">Status</th>
|
||||
|
|
@ -960,32 +855,6 @@
|
|||
</td>
|
||||
<td>{{ entry[1].name }}</td>
|
||||
<td class="text-break"><a [href]="entry[1].url" target="_blank" rel="noopener">{{ entry[1].url }}</a></td>
|
||||
<td>
|
||||
@if (editingTitleRegexId === entry[0]) {
|
||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||
<input type="text"
|
||||
class="form-control form-control-sm flex-grow-1"
|
||||
[name]="'subTitleRegex' + entry[0]"
|
||||
[(ngModel)]="titleRegexEditDraft"
|
||||
[disabled]="downloads.loading" />
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||
(click)="saveTitleRegex(entry[0])"
|
||||
[disabled]="downloads.loading">Save</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||
(click)="cancelEditTitleRegex()"
|
||||
[disabled]="downloads.loading">Cancel</button>
|
||||
</div>
|
||||
} @else {
|
||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||
<span class="text-muted small text-break"
|
||||
[class.text-secondary]="!entry[1].title_regex">{{ entry[1].title_regex || '—' }}</span>
|
||||
<button type="button" class="btn btn-link btn-sm p-0"
|
||||
(click)="beginEditTitleRegex(entry[0], entry[1].title_regex)"
|
||||
[disabled]="downloads.loading"
|
||||
ngbTooltip="Edit subscription title filter (subscriptions only; not for one-off downloads)">Edit</button>
|
||||
</div>
|
||||
}
|
||||
</td>
|
||||
<td>{{ entry[1].check_interval_minutes }}</td>
|
||||
<td class="text-nowrap">
|
||||
@if (entry[1].last_checked !== null) {
|
||||
|
|
@ -1078,5 +947,3 @@
|
|||
}
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<app-toast-container />
|
||||
|
|
|
|||
|
|
@ -182,18 +182,6 @@ main
|
|||
opacity: 0.65
|
||||
pointer-events: none
|
||||
|
||||
.settings-section-label
|
||||
font-size: 0.8rem
|
||||
text-transform: uppercase
|
||||
letter-spacing: 0.1em
|
||||
font-weight: 600
|
||||
color: var(--bs-body-color)
|
||||
margin-top: 1.75rem
|
||||
margin-bottom: 0.75rem
|
||||
|
||||
&:first-child
|
||||
margin-top: 0
|
||||
|
||||
.action-group-label
|
||||
font-size: 0.7rem
|
||||
text-transform: uppercase
|
||||
|
|
@ -201,12 +189,6 @@ main
|
|||
color: var(--bs-secondary-color)
|
||||
margin-bottom: 0.4rem
|
||||
|
||||
.help-title
|
||||
cursor: help
|
||||
|
||||
&:focus
|
||||
outline: none
|
||||
|
||||
.cookie-status
|
||||
font-size: 0.8rem
|
||||
margin-top: 0.35rem
|
||||
|
|
|
|||
|
|
@ -1,108 +1,7 @@
|
|||
import { TestBed } from '@angular/core/testing';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { Subject, of } from 'rxjs';
|
||||
import { App } from './app';
|
||||
import { DownloadsService } from './services/downloads.service';
|
||||
import { SubscriptionsService } from './services/subscriptions.service';
|
||||
import { ToastService } from './services/toast.service';
|
||||
import { CookieService } from 'ngx-cookie-service';
|
||||
|
||||
class DownloadsServiceStub {
|
||||
loading = false;
|
||||
queue = new Map();
|
||||
done = new Map();
|
||||
configuration: Record<string, unknown> = { CUSTOM_DIRS: true, CREATE_CUSTOM_DIRS: true, ALLOW_YTDL_OPTIONS_OVERRIDES: false };
|
||||
customDirs = { download_dir: [], audio_download_dir: [] };
|
||||
queueChanged = new Subject<void>();
|
||||
doneChanged = new Subject<void>();
|
||||
configurationChanged = new Subject<Record<string, unknown>>();
|
||||
customDirsChanged = new Subject<Record<string, string[]>>();
|
||||
ytdlOptionsChanged = new Subject<Record<string, unknown>>();
|
||||
updated = new Subject<void>();
|
||||
|
||||
getCookieStatus() {
|
||||
return of({ status: 'ok', has_cookies: false });
|
||||
}
|
||||
|
||||
getPresets() {
|
||||
return of({ presets: ['Preset A'] });
|
||||
}
|
||||
|
||||
add() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
cancelAdd() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
startById() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
delById() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
delByFilter() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
startByFilter() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
uploadCookies() {
|
||||
return of({ status: 'ok' });
|
||||
}
|
||||
|
||||
deleteCookies() {
|
||||
return of({ status: 'ok' });
|
||||
}
|
||||
}
|
||||
|
||||
class SubscriptionsServiceStub {
|
||||
subscriptions = new Map();
|
||||
subscriptionsChanged = new Subject<void>();
|
||||
subscribeCalls: unknown[] = [];
|
||||
|
||||
subscribe(payload: unknown) {
|
||||
this.subscribeCalls.push(payload);
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
delete() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
update() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
refreshList() {
|
||||
return of([]);
|
||||
}
|
||||
}
|
||||
|
||||
class CookieServiceStub {
|
||||
private cookies = new Map<string, string>();
|
||||
|
||||
get(name: string) {
|
||||
return this.cookies.get(name) ?? '';
|
||||
}
|
||||
|
||||
set(name: string, value: string) {
|
||||
this.cookies.set(name, value);
|
||||
}
|
||||
|
||||
check(name: string) {
|
||||
return this.cookies.has(name);
|
||||
}
|
||||
}
|
||||
|
||||
describe('App', () => {
|
||||
let downloads: DownloadsServiceStub;
|
||||
|
||||
beforeEach(async () => {
|
||||
Object.defineProperty(window, 'matchMedia', {
|
||||
writable: true,
|
||||
|
|
@ -116,20 +15,8 @@ describe('App', () => {
|
|||
dispatchEvent: vi.fn(),
|
||||
})),
|
||||
});
|
||||
downloads = new DownloadsServiceStub();
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [App],
|
||||
providers: [
|
||||
{ provide: DownloadsService, useValue: downloads },
|
||||
{ provide: SubscriptionsService, useClass: SubscriptionsServiceStub },
|
||||
{ provide: CookieService, useClass: CookieServiceStub },
|
||||
{
|
||||
provide: HttpClient,
|
||||
useValue: {
|
||||
get: vi.fn().mockReturnValue(of({ 'yt-dlp': 'test', version: 'test' })),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
|
|
@ -138,148 +25,4 @@ describe('App', () => {
|
|||
const app = fixture.componentInstance;
|
||||
expect(app).toBeTruthy();
|
||||
});
|
||||
|
||||
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
expect(app.asIsOrder()).toBe(0);
|
||||
});
|
||||
|
||||
it('hides manual override input when disabled', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
fixture.componentInstance.isAdvancedOpen = true;
|
||||
fixture.detectChanges();
|
||||
|
||||
const root = fixture.nativeElement as HTMLElement;
|
||||
expect(root.querySelector('input[name="ytdlOptionsOverrides"]')).toBeNull();
|
||||
|
||||
const presetWrapper = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.col-12');
|
||||
expect(presetWrapper?.classList.contains('col-md-6')).toBe(false);
|
||||
|
||||
const presetRow = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.row');
|
||||
expect(presetRow?.querySelector('input[name="checkIntervalMinutes"]')).toBeNull();
|
||||
});
|
||||
|
||||
it('shows manual override input when enabled', () => {
|
||||
downloads.configuration['ALLOW_YTDL_OPTIONS_OVERRIDES'] = true;
|
||||
|
||||
const fixture = TestBed.createComponent(App);
|
||||
fixture.componentInstance.isAdvancedOpen = true;
|
||||
fixture.detectChanges();
|
||||
|
||||
const root = fixture.nativeElement as HTMLElement;
|
||||
expect(root.querySelector('input[name="ytdlOptionsOverrides"]')).not.toBeNull();
|
||||
|
||||
const presetWrapper = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.col-12');
|
||||
expect(presetWrapper?.classList.contains('col-md-6')).toBe(true);
|
||||
|
||||
const presetRow = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.row');
|
||||
expect(presetRow?.querySelector('input[name="checkIntervalMinutes"]')).toBeNull();
|
||||
expect(presetRow?.querySelector('input[name="ytdlOptionsOverrides"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('does not submit manual overrides when disabled', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
|
||||
app.ytdlOptionsOverrides = '{"exec":"echo hi"}';
|
||||
|
||||
const payload = app['buildAddPayload']();
|
||||
|
||||
expect(payload.ytdlOptionsOverrides).toBe('');
|
||||
});
|
||||
|
||||
it('shows waiting badge for scheduled live stream', () => {
|
||||
downloads.queue.set('https://example.com/live', {
|
||||
id: 'live1',
|
||||
title: 'Upcoming Stream',
|
||||
url: 'https://example.com/live',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'scheduled',
|
||||
live_status: 'is_upcoming',
|
||||
live_release_timestamp: Date.now() / 1000 + 3600,
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
});
|
||||
downloads.queueChanged.next();
|
||||
|
||||
const fixture = TestBed.createComponent(App);
|
||||
fixture.detectChanges();
|
||||
|
||||
const root = fixture.nativeElement as HTMLElement;
|
||||
expect(root.textContent).toContain('Waiting for stream');
|
||||
expect(root.textContent).toContain('starts in');
|
||||
});
|
||||
|
||||
it('includes titleRegex in subscribe payload', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.titleRegex = 'EPISODE';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as { titleRegex: string; skipSubscriberOnly: boolean };
|
||||
expect(payload.titleRegex).toBe('EPISODE');
|
||||
expect(payload.skipSubscriberOnly).toBe(false);
|
||||
});
|
||||
|
||||
it('includes skipSubscriberOnly true when checked', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.skipSubscriberOnly = true;
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as { skipSubscriberOnly: boolean };
|
||||
expect(payload.skipSubscriberOnly).toBe(true);
|
||||
});
|
||||
|
||||
it('omits clip fields from subscribe payload', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.clipStart = '1:00';
|
||||
app.clipEnd = '2:00';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as Record<string, unknown>;
|
||||
expect('clipStart' in payload).toBe(false);
|
||||
expect('clipEnd' in payload).toBe(false);
|
||||
});
|
||||
|
||||
it('buildAddPayload includes clip times', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
app.clipStart = '0:10';
|
||||
app.clipEnd = '1:20';
|
||||
const payload = app['buildAddPayload']();
|
||||
expect(payload.clipStart).toBe('0:10');
|
||||
expect(payload.clipEnd).toBe('1:20');
|
||||
});
|
||||
|
||||
it('blocks subscribe with invalid title regex', () => {
|
||||
const toasts = TestBed.inject(ToastService);
|
||||
const errorSpy = vi.spyOn(toasts, 'error').mockImplementation(() => undefined);
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.titleRegex = '[';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(0);
|
||||
expect(errorSpy).toHaveBeenCalledWith('Invalid subscription title filter (regex)');
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,20 +1,17 @@
|
|||
import { DatePipe, KeyValuePipe, NgTemplateOutlet } from '@angular/common';
|
||||
import { AsyncPipe, DatePipe, KeyValuePipe, NgTemplateOutlet } from '@angular/common';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { AfterViewInit, ChangeDetectionStrategy, ChangeDetectorRef, Component, DestroyRef, ElementRef, viewChild, inject, OnDestroy, OnInit } from '@angular/core';
|
||||
import { Observable, OperatorFunction, Subject, Subscription, from, map, merge, debounceTime, distinctUntilChanged, filter, finalize, mergeMap, takeUntil, tap } from 'rxjs';
|
||||
import { Observable, Subscription, map, distinctUntilChanged, finalize } from 'rxjs';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
|
||||
import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
|
||||
import { NgbModule, NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
|
||||
import { NgSelectModule } from '@ng-select/ng-select';
|
||||
import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload, faPause, faPlay, faShareNodes } from '@fortawesome/free-solid-svg-icons';
|
||||
import { NgbModule } from '@ng-bootstrap/ng-bootstrap';
|
||||
import { NgSelectModule } from '@ng-select/ng-select';
|
||||
import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload, faPause, faPlay } from '@fortawesome/free-solid-svg-icons';
|
||||
import { faGithub } from '@fortawesome/free-brands-svg-icons';
|
||||
import { CookieService } from 'ngx-cookie-service';
|
||||
import { AddDownloadPayload, DownloadsService } from './services/downloads.service';
|
||||
import { MeTubeSocket } from './services/metube-socket.service';
|
||||
import { SubscriptionsService } from './services/subscriptions.service';
|
||||
import { ToastService } from './services/toast.service';
|
||||
import { BatchUrlsService, BatchUrlFilter } from './services/batch-urls.service';
|
||||
import { SubscriptionRow } from './interfaces/subscription';
|
||||
import { Themes } from './theme';
|
||||
import {
|
||||
|
|
@ -34,7 +31,7 @@ import {
|
|||
State,
|
||||
} from './interfaces';
|
||||
import { EtaPipe, SpeedPipe, FileSizePipe } from './pipes';
|
||||
import { SelectAllCheckboxComponent, ItemCheckboxComponent, ToastContainerComponent } from './components/';
|
||||
import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/';
|
||||
|
||||
@Component({
|
||||
selector: 'app-root',
|
||||
|
|
@ -43,6 +40,7 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent, ToastContainerCompon
|
|||
FormsModule,
|
||||
NgTemplateOutlet,
|
||||
KeyValuePipe,
|
||||
AsyncPipe,
|
||||
DatePipe,
|
||||
FontAwesomeModule,
|
||||
NgbModule,
|
||||
|
|
@ -52,7 +50,6 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent, ToastContainerCompon
|
|||
FileSizePipe,
|
||||
SelectAllCheckboxComponent,
|
||||
ItemCheckboxComponent,
|
||||
ToastContainerComponent,
|
||||
],
|
||||
templateUrl: './app.html',
|
||||
styleUrl: './app.sass',
|
||||
|
|
@ -60,9 +57,6 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent, ToastContainerCompon
|
|||
export class App implements AfterViewInit, OnInit, OnDestroy {
|
||||
downloads = inject(DownloadsService);
|
||||
subscriptionsSvc = inject(SubscriptionsService);
|
||||
private toasts = inject(ToastService);
|
||||
private batchUrls = inject(BatchUrlsService);
|
||||
private socket = inject(MeTubeSocket);
|
||||
private cookieService = inject(CookieService);
|
||||
private http = inject(HttpClient);
|
||||
private cdr = inject(ChangeDetectorRef);
|
||||
|
|
@ -87,21 +81,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
playlistItemLimit!: number;
|
||||
splitByChapters: boolean;
|
||||
chapterTemplate: string;
|
||||
clipStart = '';
|
||||
clipEnd = '';
|
||||
subtitleLanguage: string;
|
||||
subtitleMode: string;
|
||||
ytdlOptionsPresets: string[] = [];
|
||||
ytdlOptionsOverrides: string;
|
||||
ytdlOptionPresetNames: string[] = [];
|
||||
addInProgress = false;
|
||||
cancelRequested = false;
|
||||
subscribeInProgress = false;
|
||||
checkIntervalMinutes = 60;
|
||||
titleRegex = '';
|
||||
skipSubscriberOnly = false;
|
||||
editingTitleRegexId: string | null = null;
|
||||
titleRegexEditDraft = '';
|
||||
cachedSubs: [string, SubscriptionRow][] = [];
|
||||
selectedSubscriptionIds = new Set<string>();
|
||||
checkingSubscriptionIds = new Set<string>();
|
||||
|
|
@ -111,22 +96,13 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
cookieUploadInProgress = false;
|
||||
themes: Theme[] = Themes;
|
||||
activeTheme: Theme | undefined;
|
||||
readonly folderTypeahead = viewChild<NgbTypeahead>('folderTypeahead');
|
||||
folderFocus$ = new Subject<string>();
|
||||
folderClick$ = new Subject<string>();
|
||||
showBatchPanel = false;
|
||||
customDirs$!: Observable<string[]>;
|
||||
showBatchPanel = false;
|
||||
batchImportModalOpen = false;
|
||||
batchImportText = '';
|
||||
batchImportStatus = '';
|
||||
batchImportCount = 0;
|
||||
batchImportTotal = 0;
|
||||
batchImportFailures = 0;
|
||||
importInProgress = false;
|
||||
private batchImportCancel$ = new Subject<void>();
|
||||
// Maximum number of /add requests to have in-flight at once during a batch
|
||||
// import. Keeps the server from being hit with hundreds of simultaneous
|
||||
// yt-dlp metadata extractions when a user pastes a huge URL list.
|
||||
private static readonly BATCH_IMPORT_CONCURRENCY = 4;
|
||||
cancelImportFlag = false;
|
||||
ytDlpOptionsUpdateTime: string | null = null;
|
||||
ytDlpVersion: string | null = null;
|
||||
metubeVersion: string | null = null;
|
||||
|
|
@ -137,7 +113,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
lastCopiedErrorId: string | null = null;
|
||||
private previousDownloadType = 'video';
|
||||
private addRequestSub?: Subscription;
|
||||
private liveCountdownTimer?: ReturnType<typeof setInterval>;
|
||||
private selectionsByType: Record<string, {
|
||||
codec: string;
|
||||
format: string;
|
||||
|
|
@ -194,7 +169,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
faUpload = faUpload;
|
||||
faPause = faPause;
|
||||
faPlay = faPlay;
|
||||
faShareNodes = faShareNodes;
|
||||
subtitleLanguages = [
|
||||
{ id: 'en', text: 'English' },
|
||||
{ id: 'ar', text: 'Arabic' },
|
||||
|
|
@ -255,12 +229,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||
// Will be set from backend configuration, use empty string as placeholder
|
||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
||||
this.clipEnd = this.cookieService.get('metube_clip_end') || '';
|
||||
this.subtitleLanguage = this.cookieService.get('metube_subtitle_language') || 'en';
|
||||
this.subtitleMode = this.cookieService.get('metube_subtitle_mode') || 'prefer_manual';
|
||||
this.ytdlOptionsPresets = this.loadYtdlOptionsPresetsFromCookie();
|
||||
this.ytdlOptionsOverrides = this.cookieService.get('metube_ytdl_options_overrides') || '';
|
||||
const allowedDownloadTypes = new Set(this.downloadTypes.map(t => t.id));
|
||||
const allowedVideoCodecs = new Set(this.videoCodecs.map(c => c.id));
|
||||
if (!allowedDownloadTypes.has(this.downloadType)) {
|
||||
|
|
@ -291,7 +261,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
// Subscribe to download updates
|
||||
this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||
this.updateMetrics();
|
||||
this.syncLiveCountdownTimer();
|
||||
this.cdr.markForCheck();
|
||||
});
|
||||
this.downloads.doneChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||
|
|
@ -302,7 +271,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
// Subscribe to real-time updates
|
||||
this.downloads.updated.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||
this.updateMetrics();
|
||||
this.syncLiveCountdownTimer();
|
||||
this.cdr.markForCheck();
|
||||
});
|
||||
|
||||
|
|
@ -319,7 +287,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
});
|
||||
this.getConfiguration();
|
||||
this.getYtdlOptionsUpdateTime();
|
||||
this.getYtdlOptionPresets();
|
||||
this.customDirs$ = this.getMatchingCustomDir();
|
||||
this.setTheme(this.activeTheme!);
|
||||
|
||||
this.colorSchemeMediaQuery.addEventListener('change', this.onColorSchemeChanged);
|
||||
|
|
@ -338,23 +306,20 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
// Initialize action button states for already-loaded entries.
|
||||
this.updateDoneActionButtons();
|
||||
this.fetchVersionInfo();
|
||||
this.socket.fromEvent('connect')
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe(() => this.fetchVersionInfo());
|
||||
}
|
||||
|
||||
ngOnDestroy() {
|
||||
this.addRequestSub?.unsubscribe();
|
||||
if (this.liveCountdownTimer) {
|
||||
clearInterval(this.liveCountdownTimer);
|
||||
}
|
||||
this.colorSchemeMediaQuery.removeEventListener('change', this.onColorSchemeChanged);
|
||||
}
|
||||
|
||||
// keyvalue comparator that preserves insertion order (Angular's keyvalue
|
||||
// pipe sorts by key by default): https://github.com/angular/angular/issues/31420
|
||||
// workaround to allow fetching of Map values in the order they were inserted
|
||||
// https://github.com/angular/angular/issues/31420
|
||||
|
||||
|
||||
|
||||
asIsOrder() {
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
qualityChanged() {
|
||||
|
|
@ -385,30 +350,34 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
return this.downloads.configuration['CUSTOM_DIRS'];
|
||||
}
|
||||
|
||||
allowYtdlOptionsOverrides() {
|
||||
return this.downloads.configuration['ALLOW_YTDL_OPTIONS_OVERRIDES'] === true;
|
||||
allowCustomDir(tag: string) {
|
||||
if (this.downloads.configuration['CREATE_CUSTOM_DIRS']) {
|
||||
return tag;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
searchFolder: OperatorFunction<string, readonly string[]> = (text$: Observable<string>) => {
|
||||
const debouncedText$ = text$.pipe(debounceTime(150), distinctUntilChanged());
|
||||
const clicksWithClosedPopup$ = this.folderClick$.pipe(
|
||||
filter(() => !this.folderTypeahead()?.isPopupOpen()),
|
||||
);
|
||||
return merge(debouncedText$, this.folderFocus$, clicksWithClosedPopup$).pipe(
|
||||
map(term => {
|
||||
const dirs = this.isAudioType()
|
||||
? (this.downloads.customDirs?.['audio_download_dir'] ?? [])
|
||||
: (this.downloads.customDirs?.['download_dir'] ?? []);
|
||||
const t = (term ?? '').toLowerCase();
|
||||
return (t === '' ? dirs : dirs.filter(d => d.toLowerCase().includes(t))).slice(0, 10);
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
isAudioType() {
|
||||
return this.downloadType === 'audio';
|
||||
}
|
||||
|
||||
getMatchingCustomDir() : Observable<string[]> {
|
||||
return this.downloads.customDirsChanged.asObservable().pipe(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
map((output: any) => {
|
||||
// Keep logic consistent with app/ytdl.py
|
||||
if (this.isAudioType()) {
|
||||
console.debug("Showing audio-specific download directories");
|
||||
return output["audio_download_dir"];
|
||||
} else {
|
||||
console.debug("Showing default download directories");
|
||||
return output["download_dir"];
|
||||
}
|
||||
}),
|
||||
distinctUntilChanged((prev, curr) => JSON.stringify(prev) === JSON.stringify(curr))
|
||||
);
|
||||
}
|
||||
|
||||
getYtdlOptionsUpdateTime() {
|
||||
this.downloads.ytdlOptionsChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
|
|
@ -417,7 +386,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
const date = new Date(data['update_time'] * 1000);
|
||||
this.ytDlpOptionsUpdateTime=date.toLocaleString();
|
||||
}else{
|
||||
this.toasts.error("Error reloading yt-dlp options: " + data['msg']);
|
||||
alert("Error reload yt-dlp options: "+data['msg']);
|
||||
}
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
|
@ -427,8 +396,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.downloads.configurationChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
next: (config: any) => {
|
||||
const playlistItemLimit = parseInt(String(config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'] ?? '0'), 10);
|
||||
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
||||
const playlistItemLimit = config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'];
|
||||
if (playlistItemLimit !== '0') {
|
||||
this.playlistItemLimit = playlistItemLimit;
|
||||
}
|
||||
// Set chapter template from backend config if not already set by cookie
|
||||
|
|
@ -446,62 +415,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
});
|
||||
}
|
||||
|
||||
getYtdlOptionPresets() {
|
||||
this.downloads.getPresets().pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
||||
next: (data) => {
|
||||
this.ytdlOptionPresetNames = Array.isArray(data?.presets)
|
||||
? data.presets.filter((preset): preset is string => typeof preset === 'string')
|
||||
: [];
|
||||
if (this.ytdlOptionsPresets?.length) {
|
||||
const valid = new Set(this.ytdlOptionPresetNames);
|
||||
const filtered = this.ytdlOptionsPresets.filter((p) => valid.has(p));
|
||||
if (filtered.length !== this.ytdlOptionsPresets.length) {
|
||||
this.ytdlOptionsPresets = filtered;
|
||||
this.ytdlOptionsPresetsChanged();
|
||||
}
|
||||
}
|
||||
this.cdr.markForCheck();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private loadYtdlOptionsPresetsFromCookie(): string[] {
|
||||
const jsonCookie = this.cookieService.get('metube_ytdl_options_presets');
|
||||
if (jsonCookie) {
|
||||
try {
|
||||
const parsed = JSON.parse(jsonCookie) as unknown;
|
||||
if (Array.isArray(parsed)) {
|
||||
return parsed.filter((p): p is string => typeof p === 'string' && p.length > 0);
|
||||
}
|
||||
} catch {
|
||||
// fall through to legacy cookie
|
||||
}
|
||||
}
|
||||
const legacy = this.cookieService.get('metube_ytdl_options_preset')?.trim();
|
||||
return legacy ? [legacy] : [];
|
||||
}
|
||||
|
||||
private validateYtdlOptionsOverrides(value: string): boolean {
|
||||
if (!this.allowYtdlOptionsOverrides()) {
|
||||
return true;
|
||||
}
|
||||
const trimmed = value?.trim() || '';
|
||||
if (!trimmed) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
|
||||
this.toasts.error('Custom yt-dlp options must be a JSON object');
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
this.toasts.error('Custom yt-dlp options must be valid JSON');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private rebuildCachedSubs() {
|
||||
this.cachedSubs = Array.from(this.subscriptionsSvc.subscriptions.entries());
|
||||
const validIds = new Set(this.cachedSubs.map(([id]) => id));
|
||||
|
|
@ -523,19 +436,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
return status?.status === 'error' ? status.msg || null : null;
|
||||
}
|
||||
|
||||
private handleActionResult(res: unknown, fallbackMsg: string) {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || fallbackMsg);
|
||||
}
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
||||
private refreshSubscriptionsWithAlert() {
|
||||
this.subscriptionsSvc.refreshList().pipe(takeUntilDestroyed(this.destroyRef)).subscribe((refreshRes) => {
|
||||
const error = this.getStatusError(refreshRes);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Refresh subscriptions failed');
|
||||
alert(error || 'Refresh subscriptions failed');
|
||||
return;
|
||||
}
|
||||
this.cdr.markForCheck();
|
||||
|
|
@ -579,36 +484,18 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
}
|
||||
const payload = this.buildAddPayload();
|
||||
if (!payload.url?.trim()) {
|
||||
this.toasts.error('Please enter a URL');
|
||||
alert('Please enter a URL');
|
||||
return;
|
||||
}
|
||||
const tr = (this.titleRegex || '').trim();
|
||||
if (tr) {
|
||||
try {
|
||||
void RegExp(tr);
|
||||
} catch {
|
||||
this.toasts.error('Invalid subscription title filter (regex)');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
||||
this.toasts.error('Chapter template must include %(section_number)');
|
||||
alert('Chapter template must include %(section_number)');
|
||||
return;
|
||||
}
|
||||
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
||||
return;
|
||||
}
|
||||
// Subscriptions do not support clip ranges (backend rejects clip fields).
|
||||
const { clipStart: _clipStart, clipEnd: _clipEnd, ...subscribeBase } = payload;
|
||||
void _clipStart;
|
||||
void _clipEnd;
|
||||
this.subscribeInProgress = true;
|
||||
this.subscriptionsSvc
|
||||
.subscribe({
|
||||
...subscribeBase,
|
||||
...payload,
|
||||
checkIntervalMinutes: this.checkIntervalMinutes,
|
||||
titleRegex: tr,
|
||||
skipSubscriberOnly: this.skipSubscriberOnly,
|
||||
})
|
||||
.pipe(
|
||||
takeUntilDestroyed(this.destroyRef),
|
||||
|
|
@ -621,53 +508,19 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
next: (res) => {
|
||||
const r = res as { status?: string; msg?: string };
|
||||
if (r.status === 'error') {
|
||||
this.toasts.error(r.msg || 'Subscribe failed');
|
||||
alert(r.msg || 'Subscribe failed');
|
||||
} else {
|
||||
this.addUrl = '';
|
||||
this.titleRegex = '';
|
||||
this.skipSubscriberOnly = false;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
beginEditTitleRegex(id: string, current: string | undefined) {
|
||||
this.editingTitleRegexId = id;
|
||||
this.titleRegexEditDraft = current ?? '';
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
||||
cancelEditTitleRegex() {
|
||||
this.editingTitleRegexId = null;
|
||||
this.titleRegexEditDraft = '';
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
||||
saveTitleRegex(id: string) {
|
||||
const raw = (this.titleRegexEditDraft || '').trim();
|
||||
if (raw) {
|
||||
try {
|
||||
void RegExp(raw);
|
||||
} catch {
|
||||
this.toasts.error('Invalid subscription title filter (regex)');
|
||||
return;
|
||||
}
|
||||
}
|
||||
this.subscriptionsSvc.update(id, { title_regex: raw }).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Update subscription failed');
|
||||
return;
|
||||
}
|
||||
this.cancelEditTitleRegex();
|
||||
});
|
||||
}
|
||||
|
||||
deleteSubscription(id: string) {
|
||||
this.subscriptionsSvc.delete([id]).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Delete subscription failed');
|
||||
alert(error || 'Delete subscription failed');
|
||||
return;
|
||||
}
|
||||
this.selectedSubscriptionIds.delete(id);
|
||||
|
|
@ -683,7 +536,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.subscriptionsSvc.delete(ids).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Delete subscriptions failed');
|
||||
alert(error || 'Delete subscriptions failed');
|
||||
return;
|
||||
}
|
||||
this.selectedSubscriptionIds.clear();
|
||||
|
|
@ -709,7 +562,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
.subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Subscription check failed');
|
||||
alert(error || 'Subscription check failed');
|
||||
return;
|
||||
}
|
||||
this.refreshSubscriptionsWithAlert();
|
||||
|
|
@ -756,7 +609,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
.subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Subscription check failed');
|
||||
alert(error || 'Subscription check failed');
|
||||
return;
|
||||
}
|
||||
this.refreshSubscriptionsWithAlert();
|
||||
|
|
@ -779,7 +632,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.subscriptionsSvc.update(row.id, { enabled: !row.enabled }).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Update subscription failed');
|
||||
alert(error || 'Update subscription failed');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -832,14 +685,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: this.settingsCookieExpiryDays });
|
||||
}
|
||||
|
||||
clipStartChanged() {
|
||||
this.cookieService.set('metube_clip_start', this.clipStart, { expires: this.settingsCookieExpiryDays });
|
||||
}
|
||||
|
||||
clipEndChanged() {
|
||||
this.cookieService.set('metube_clip_end', this.clipEnd, { expires: this.settingsCookieExpiryDays });
|
||||
}
|
||||
|
||||
subtitleLanguageChanged() {
|
||||
this.cookieService.set('metube_subtitle_language', this.subtitleLanguage, { expires: this.settingsCookieExpiryDays });
|
||||
this.saveSelection(this.downloadType);
|
||||
|
|
@ -850,18 +695,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.saveSelection(this.downloadType);
|
||||
}
|
||||
|
||||
ytdlOptionsPresetsChanged() {
|
||||
this.cookieService.set(
|
||||
'metube_ytdl_options_presets',
|
||||
JSON.stringify(this.ytdlOptionsPresets ?? []),
|
||||
{ expires: this.settingsCookieExpiryDays },
|
||||
);
|
||||
}
|
||||
|
||||
ytdlOptionsOverridesChanged() {
|
||||
this.cookieService.set('metube_ytdl_options_overrides', this.ytdlOptionsOverrides, { expires: this.settingsCookieExpiryDays });
|
||||
}
|
||||
|
||||
isVideoType() {
|
||||
return this.downloadType === 'video';
|
||||
}
|
||||
|
|
@ -1047,7 +880,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
}
|
||||
|
||||
private buildAddPayload(overrides: Partial<AddDownloadPayload> = {}): AddDownloadPayload {
|
||||
const allowYtdlOptionsOverrides = this.allowYtdlOptionsOverrides();
|
||||
return {
|
||||
url: overrides.url ?? this.addUrl,
|
||||
downloadType: overrides.downloadType ?? this.downloadType,
|
||||
|
|
@ -1062,12 +894,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
||||
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
||||
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
||||
ytdlOptionsPresets: overrides.ytdlOptionsPresets ?? [...this.ytdlOptionsPresets],
|
||||
ytdlOptionsOverrides: allowYtdlOptionsOverrides
|
||||
? (overrides.ytdlOptionsOverrides ?? this.ytdlOptionsOverrides)
|
||||
: '',
|
||||
clipStart: overrides.clipStart ?? this.clipStart,
|
||||
clipEnd: overrides.clipEnd ?? this.clipEnd,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1076,24 +902,18 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
|
||||
// Validate chapter template if chapter splitting is enabled
|
||||
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
||||
this.toasts.error('Chapter template must include %(section_number)');
|
||||
return;
|
||||
}
|
||||
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
||||
alert('Chapter template must include %(section_number)');
|
||||
return;
|
||||
}
|
||||
|
||||
console.debug('Downloading:', payload);
|
||||
this.addInProgress = true;
|
||||
this.cancelRequested = false;
|
||||
this.addRequestSub?.unsubscribe();
|
||||
this.addRequestSub = this.downloads.add(payload).subscribe((status: Status) => {
|
||||
if (status.status === 'error' && !this.cancelRequested) {
|
||||
this.toasts.error(`Error adding URL: ${status.msg}`);
|
||||
alert(`Error adding URL: ${status.msg}`);
|
||||
} else if (status.status !== 'error') {
|
||||
// e.g. "Already in queue: ..." when the backend skipped a duplicate.
|
||||
if (status.msg) {
|
||||
this.toasts.info(status.msg);
|
||||
}
|
||||
this.addUrl = '';
|
||||
}
|
||||
this.resetAddState();
|
||||
|
|
@ -1122,31 +942,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
}
|
||||
|
||||
downloadItemByKey(id: string) {
|
||||
this.downloads.startById([id]).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
||||
}
|
||||
|
||||
liveCountdownSeconds(download: Download): number | null {
|
||||
const ts = download.live_release_timestamp;
|
||||
if (ts == null || download.status !== 'scheduled') {
|
||||
return null;
|
||||
}
|
||||
return Math.max(0, ts - Date.now() / 1000);
|
||||
}
|
||||
|
||||
private syncLiveCountdownTimer() {
|
||||
const hasScheduled = Array.from(this.downloads.queue.values()).some(
|
||||
(download) => download.status === 'scheduled',
|
||||
);
|
||||
if (hasScheduled && !this.liveCountdownTimer) {
|
||||
this.liveCountdownTimer = setInterval(() => this.cdr.markForCheck(), 1000);
|
||||
} else if (!hasScheduled && this.liveCountdownTimer) {
|
||||
clearInterval(this.liveCountdownTimer);
|
||||
this.liveCountdownTimer = undefined;
|
||||
}
|
||||
this.downloads.startById([id]).subscribe();
|
||||
}
|
||||
|
||||
retryDownload(key: string, download: Download) {
|
||||
const payload = this.buildAddPayload({
|
||||
this.addDownload({
|
||||
url: download.url,
|
||||
downloadType: download.download_type,
|
||||
codec: download.codec,
|
||||
|
|
@ -1160,45 +960,28 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
chapterTemplate: download.chapter_template,
|
||||
subtitleLanguage: download.subtitle_language,
|
||||
subtitleMode: download.subtitle_mode,
|
||||
ytdlOptionsPresets: download.ytdl_options_presets?.length
|
||||
? [...download.ytdl_options_presets]
|
||||
: [],
|
||||
ytdlOptionsOverrides: download.ytdl_options_overrides ? JSON.stringify(download.ytdl_options_overrides) : '',
|
||||
clipStart: download.clip_start != null ? String(download.clip_start) : '',
|
||||
clipEnd: download.clip_end != null ? String(download.clip_end) : '',
|
||||
});
|
||||
// Only remove the done-list record once the retry is confirmed queued —
|
||||
// deleting it eagerly would silently lose history if the re-add fails.
|
||||
this.downloads.add(payload)
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
this.toasts.error(`Error retrying ${download.title}: ${status.msg}`);
|
||||
this.cdr.markForCheck();
|
||||
return;
|
||||
}
|
||||
this.downloads.delById('done', [key]).subscribe();
|
||||
});
|
||||
this.downloads.delById('done', [key]).subscribe();
|
||||
}
|
||||
|
||||
delDownload(where: State, id: string) {
|
||||
this.downloads.delById(where, [id]).subscribe((res) => this.handleActionResult(res, 'Delete failed'));
|
||||
this.downloads.delById(where, [id]).subscribe();
|
||||
}
|
||||
|
||||
startSelectedDownloads(where: State){
|
||||
this.downloads.startByFilter(where, dl => !!dl.checked).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
||||
this.downloads.startByFilter(where, dl => !!dl.checked).subscribe();
|
||||
}
|
||||
|
||||
delSelectedDownloads(where: State) {
|
||||
this.downloads.delByFilter(where, dl => !!dl.checked).subscribe((res) => this.handleActionResult(res, 'Delete failed'));
|
||||
this.downloads.delByFilter(where, dl => !!dl.checked).subscribe();
|
||||
}
|
||||
|
||||
clearCompletedDownloads() {
|
||||
this.downloads.delByFilter('done', dl => dl.status === 'finished').subscribe((res) => this.handleActionResult(res, 'Clear completed failed'));
|
||||
this.downloads.delByFilter('done', dl => dl.status === 'finished').subscribe();
|
||||
}
|
||||
|
||||
clearFailedDownloads() {
|
||||
this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe((res) => this.handleActionResult(res, 'Clear failed downloads failed'));
|
||||
this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe();
|
||||
}
|
||||
|
||||
retryFailedDownloads() {
|
||||
|
|
@ -1209,39 +992,19 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
});
|
||||
}
|
||||
|
||||
// Chromium-based browsers silently drop programmatic downloads beyond ~10 when
|
||||
// triggered in a tight loop. Trigger in batches with a short pause in between so
|
||||
// large selections download cleanly. See issue #1008.
|
||||
private static readonly DOWNLOAD_BATCH_SIZE = 10;
|
||||
private static readonly DOWNLOAD_BATCH_DELAY_MS = 1000;
|
||||
|
||||
async downloadSelectedFiles() {
|
||||
const selected: Download[] = [];
|
||||
this.downloads.done.forEach((dl) => {
|
||||
downloadSelectedFiles() {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
this.downloads.done.forEach((dl, _) => {
|
||||
if (dl.status === 'finished' && dl.checked) {
|
||||
selected.push(dl);
|
||||
const link = document.createElement('a');
|
||||
link.href = this.buildDownloadLink(dl);
|
||||
link.setAttribute('download', dl.filename);
|
||||
link.setAttribute('target', '_self');
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
}
|
||||
});
|
||||
|
||||
for (let i = 0; i < selected.length; i++) {
|
||||
const dl = selected[i];
|
||||
const link = document.createElement('a');
|
||||
link.href = this.buildDownloadLink(dl);
|
||||
link.setAttribute('download', dl.filename);
|
||||
link.setAttribute('target', '_self');
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
|
||||
if (
|
||||
(i + 1) % App.DOWNLOAD_BATCH_SIZE === 0 &&
|
||||
i + 1 < selected.length
|
||||
) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, App.DOWNLOAD_BATCH_DELAY_MS),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buildDownloadLink(download: Download) {
|
||||
|
|
@ -1257,80 +1020,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
return baseDir + encodeURIComponent(download.filename);
|
||||
}
|
||||
|
||||
// Web Share API support — primarily for iOS Safari / Chrome, lets the user
|
||||
// hand the downloaded file off to the platform share sheet (Photos.app,
|
||||
// Files, third-party apps, AirDrop). Falls back silently to the standard
|
||||
// download flow on platforms without navigator.share / canShare.
|
||||
canShareDownloads(): boolean {
|
||||
// navigator.share alone is not enough — Desktop Safari implements
|
||||
// navigator.share but not canShare with files. We explicitly require
|
||||
// both, since we always intend to share a file (not a URL).
|
||||
return typeof navigator !== 'undefined'
|
||||
&& typeof navigator.share === 'function'
|
||||
&& typeof navigator.canShare === 'function';
|
||||
}
|
||||
|
||||
// Conservative warning threshold for the share sheet — iOS' actual
|
||||
// refusal limit varies between ~50 MB (older versions) and ~150 MB
|
||||
// (recent ones). 80 MB warns the user before the time-wasting fetch+
|
||||
// copy of a too-large file that the platform will then reject.
|
||||
private static readonly SHARE_SIZE_WARN_BYTES = 80 * 1024 * 1024;
|
||||
|
||||
async shareDownload(download: Download): Promise<void> {
|
||||
if (!this.canShareDownloads()) {
|
||||
return;
|
||||
}
|
||||
// Pre-flight size check: warn the user about the iOS share-sheet
|
||||
// soft-fail on large files, before we spend time fetching the whole
|
||||
// file into memory only to have navigator.canShare reject it.
|
||||
if (download.size && download.size > App.SHARE_SIZE_WARN_BYTES) {
|
||||
const sizeMb = Math.round(download.size / 1024 / 1024);
|
||||
const proceed = await this.toasts.confirm(
|
||||
`This file is ${sizeMb} MB. iOS' share sheet often refuses files ` +
|
||||
`larger than ~100 MB and the share will silently fail. ` +
|
||||
`Try anyway? (Use the download button instead if it fails.)`,
|
||||
'Try anyway',
|
||||
'Cancel',
|
||||
);
|
||||
if (!proceed) return;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(this.buildDownloadLink(download));
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} fetching file for share`);
|
||||
}
|
||||
const blob = await response.blob();
|
||||
const file = new File([blob], download.filename, {
|
||||
type: blob.type || 'application/octet-stream',
|
||||
});
|
||||
const payload: ShareData = { files: [file], title: download.title };
|
||||
if (!navigator.canShare(payload)) {
|
||||
// The platform refused the payload — most commonly because the
|
||||
// file is too large for the iOS share sheet, or the MIME type
|
||||
// isn't accepted. Tell the user so they can fall back to the
|
||||
// download button right next to this one instead of staring at
|
||||
// a button that quietly did nothing.
|
||||
console.warn('navigator.canShare rejected payload for', download.filename);
|
||||
this.toasts.error(
|
||||
`Your device's share sheet doesn't accept this file ` +
|
||||
`(most likely because it's too large). ` +
|
||||
`Please use the download button instead.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
await navigator.share(payload);
|
||||
} catch (err) {
|
||||
const e = err as { name?: string; message?: string };
|
||||
// AbortError = user dismissed the share sheet → silent no-op.
|
||||
if (e.name === 'AbortError') return;
|
||||
console.error('Share failed:', err);
|
||||
this.toasts.error(
|
||||
`Share failed: ${e.message || 'unknown error'}. ` +
|
||||
`Please use the download button instead.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
buildResultItemTooltip(download: Download) {
|
||||
const parts = [];
|
||||
if (download.msg) {
|
||||
|
|
@ -1391,10 +1080,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.batchImportModalOpen = true;
|
||||
this.batchImportText = '';
|
||||
this.batchImportStatus = '';
|
||||
this.batchImportCount = 0;
|
||||
this.batchImportTotal = 0;
|
||||
this.batchImportFailures = 0;
|
||||
this.importInProgress = false;
|
||||
this.cancelImportFlag = false;
|
||||
setTimeout(() => {
|
||||
const textarea = document.getElementById('batch-import-textarea');
|
||||
if (textarea instanceof HTMLTextAreaElement) {
|
||||
|
|
@ -1416,82 +1103,116 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
.map(url => url.trim())
|
||||
.filter(url => url.length > 0);
|
||||
if (urls.length === 0) {
|
||||
this.toasts.error('No valid URLs found.');
|
||||
alert('No valid URLs found.');
|
||||
return;
|
||||
}
|
||||
this.importInProgress = true;
|
||||
this.batchImportCount = 0;
|
||||
this.batchImportFailures = 0;
|
||||
this.batchImportTotal = urls.length;
|
||||
this.updateBatchImportStatus();
|
||||
|
||||
from(urls).pipe(
|
||||
mergeMap(
|
||||
url => this.downloads.add(this.buildAddPayload({ url })).pipe(
|
||||
// downloads.add() already catches HTTP errors and emits a single
|
||||
// Status value, so `tap` (not `finalize`) is the right place to
|
||||
// count. This avoids incrementing the counter when an in-flight
|
||||
// request is aborted by cancellation.
|
||||
tap((status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
this.batchImportFailures++;
|
||||
console.error(`Error adding URL ${url}: ${status.msg}`);
|
||||
}
|
||||
this.batchImportCount++;
|
||||
this.updateBatchImportStatus();
|
||||
this.cdr.markForCheck();
|
||||
}),
|
||||
),
|
||||
App.BATCH_IMPORT_CONCURRENCY,
|
||||
),
|
||||
takeUntil(this.batchImportCancel$),
|
||||
takeUntilDestroyed(this.destroyRef),
|
||||
finalize(() => {
|
||||
this.cancelImportFlag = false;
|
||||
this.batchImportStatus = `Starting to import ${urls.length} URLs...`;
|
||||
let index = 0;
|
||||
const delayBetween = 1000;
|
||||
const processNext = () => {
|
||||
if (this.cancelImportFlag) {
|
||||
this.batchImportStatus = `Import cancelled after ${index} of ${urls.length} URLs.`;
|
||||
this.importInProgress = false;
|
||||
this.updateBatchImportStatus(true);
|
||||
this.cdr.markForCheck();
|
||||
}),
|
||||
).subscribe();
|
||||
}
|
||||
|
||||
private updateBatchImportStatus(done = false): void {
|
||||
const parts: string[] = [];
|
||||
if (done) {
|
||||
const processed = this.batchImportCount;
|
||||
if (processed < this.batchImportTotal) {
|
||||
parts.push(`Import cancelled after ${processed} of ${this.batchImportTotal} URLs.`);
|
||||
} else {
|
||||
parts.push(`Finished importing ${this.batchImportTotal} URLs.`);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
parts.push(`Importing ${this.batchImportCount} of ${this.batchImportTotal} URLs...`);
|
||||
}
|
||||
if (this.batchImportFailures > 0) {
|
||||
parts.push(`${this.batchImportFailures} failed.`);
|
||||
}
|
||||
this.batchImportStatus = parts.join(' ');
|
||||
if (index >= urls.length) {
|
||||
this.batchImportStatus = `Finished importing ${urls.length} URLs.`;
|
||||
this.importInProgress = false;
|
||||
return;
|
||||
}
|
||||
const url = urls[index];
|
||||
this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`;
|
||||
// Pass current selection options to backend
|
||||
this.downloads.add(this.buildAddPayload({ url }))
|
||||
.subscribe({
|
||||
next: (status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
alert(`Error adding URL ${url}: ${status.msg}`);
|
||||
}
|
||||
index++;
|
||||
setTimeout(processNext, delayBetween);
|
||||
},
|
||||
error: (err) => {
|
||||
console.error(`Error importing URL ${url}:`, err);
|
||||
index++;
|
||||
setTimeout(processNext, delayBetween);
|
||||
}
|
||||
});
|
||||
};
|
||||
processNext();
|
||||
}
|
||||
|
||||
// Cancel the batch import process: aborts in-flight and pending requests
|
||||
// immediately via the cancellation Subject wired into the pipeline.
|
||||
// Cancel the batch import process
|
||||
cancelBatchImport(): void {
|
||||
if (this.importInProgress) {
|
||||
this.batchImportCancel$.next();
|
||||
this.cancelImportFlag = true;
|
||||
this.batchImportStatus += ' Cancelling...';
|
||||
}
|
||||
}
|
||||
|
||||
// Export URLs based on filter: 'pending', 'completed', 'failed', or 'all'
|
||||
exportBatchUrls(filter: BatchUrlFilter): void {
|
||||
this.batchUrls.export(filter);
|
||||
exportBatchUrls(filter: 'pending' | 'completed' | 'failed' | 'all'): void {
|
||||
let urls: string[];
|
||||
if (filter === 'pending') {
|
||||
urls = Array.from(this.downloads.queue.values()).map(dl => dl.url);
|
||||
} else if (filter === 'completed') {
|
||||
// Only finished downloads in the "done" Map
|
||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'finished').map(dl => dl.url);
|
||||
} else if (filter === 'failed') {
|
||||
// Only error downloads from the "done" Map
|
||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'error').map(dl => dl.url);
|
||||
} else {
|
||||
// All: pending + both finished and error in done
|
||||
urls = [
|
||||
...Array.from(this.downloads.queue.values()).map(dl => dl.url),
|
||||
...Array.from(this.downloads.done.values()).map(dl => dl.url)
|
||||
];
|
||||
}
|
||||
if (!urls.length) {
|
||||
alert('No URLs found for the selected filter.');
|
||||
return;
|
||||
}
|
||||
const content = urls.join('\n');
|
||||
const blob = new Blob([content], { type: 'text/plain' });
|
||||
const downloadUrl = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = downloadUrl;
|
||||
a.download = 'metube_urls.txt';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
window.URL.revokeObjectURL(downloadUrl);
|
||||
}
|
||||
|
||||
// Copy URLs to clipboard based on filter: 'pending', 'completed', 'failed', or 'all'
|
||||
copyBatchUrls(filter: BatchUrlFilter): void {
|
||||
this.batchUrls.copy(filter);
|
||||
copyBatchUrls(filter: 'pending' | 'completed' | 'failed' | 'all'): void {
|
||||
let urls: string[];
|
||||
if (filter === 'pending') {
|
||||
urls = Array.from(this.downloads.queue.values()).map(dl => dl.url);
|
||||
} else if (filter === 'completed') {
|
||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'finished').map(dl => dl.url);
|
||||
} else if (filter === 'failed') {
|
||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'error').map(dl => dl.url);
|
||||
} else {
|
||||
urls = [
|
||||
...Array.from(this.downloads.queue.values()).map(dl => dl.url),
|
||||
...Array.from(this.downloads.done.values()).map(dl => dl.url)
|
||||
];
|
||||
}
|
||||
if (!urls.length) {
|
||||
alert('No URLs found for the selected filter.');
|
||||
return;
|
||||
}
|
||||
const content = urls.join('\n');
|
||||
navigator.clipboard.writeText(content)
|
||||
.then(() => alert('URLs copied to clipboard.'))
|
||||
.catch(() => alert('Failed to copy URLs.'));
|
||||
}
|
||||
|
||||
fetchVersionInfo(): void {
|
||||
// eslint-disable-next-line no-useless-escape
|
||||
// eslint-disable-next-line no-useless-escape
|
||||
const baseUrl = `${window.location.origin}${window.location.pathname.replace(/\/[^\/]*$/, '/')}`;
|
||||
const versionUrl = `${baseUrl}version`;
|
||||
this.http.get<{ 'yt-dlp': string, version: string }>(versionUrl)
|
||||
|
|
@ -1547,7 +1268,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
};
|
||||
const fail = (err?: unknown) => {
|
||||
console.error('Clipboard write failed:', err);
|
||||
this.toasts.error('Failed to copy to clipboard. Your browser may require HTTPS for clipboard access.');
|
||||
alert('Failed to copy to clipboard. Your browser may require HTTPS for clipboard access.');
|
||||
};
|
||||
if (navigator.clipboard?.writeText) {
|
||||
navigator.clipboard.writeText(text).then(done).catch(fail);
|
||||
|
|
@ -1583,7 +1304,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.hasCookies = true;
|
||||
} else {
|
||||
this.refreshCookieStatus();
|
||||
this.toasts.error(`Error uploading cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||
alert(`Error uploading cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||
}
|
||||
this.cookieUploadInProgress = false;
|
||||
input.value = '';
|
||||
|
|
@ -1592,7 +1313,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
this.refreshCookieStatus();
|
||||
this.cookieUploadInProgress = false;
|
||||
input.value = '';
|
||||
this.toasts.error('Error uploading cookies.');
|
||||
alert('Error uploading cookies.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -1626,11 +1347,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
return;
|
||||
}
|
||||
this.refreshCookieStatus();
|
||||
this.toasts.error(`Error deleting cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||
alert(`Error deleting cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||
},
|
||||
error: () => {
|
||||
this.refreshCookieStatus();
|
||||
this.toasts.error('Error deleting cookies.');
|
||||
alert('Error deleting cookies.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -1654,7 +1375,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||
speed += download.speed || 0;
|
||||
} else if (download.status === 'preparing') {
|
||||
active++;
|
||||
} else if (download.status === 'pending' || download.status === 'scheduled') {
|
||||
} else if (download.status === 'pending') {
|
||||
queued++;
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,2 @@
|
|||
export { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
export { ItemCheckboxComponent } from './slave-checkbox.component';
|
||||
export { ToastContainerComponent } from './toast-container.component';
|
||||
export { ItemCheckboxComponent } from './slave-checkbox.component';
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { Component, ElementRef, viewChild, output, input, ChangeDetectionStrategy } from "@angular/core";
|
||||
import { Component, ElementRef, viewChild, output, input } from "@angular/core";
|
||||
import { Checkable } from "../interfaces";
|
||||
import { FormsModule } from "@angular/forms";
|
||||
|
||||
|
|
@ -10,10 +10,7 @@ import { FormsModule } from "@angular/forms";
|
|||
<label class="form-check-label visually-hidden" for="{{id()}}-select-all">Select all</label>
|
||||
</div>
|
||||
`,
|
||||
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
||||
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
||||
changeDetection: ChangeDetectionStrategy.Eager,
|
||||
imports: [
|
||||
imports: [
|
||||
FormsModule
|
||||
]
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { Component, input, ChangeDetectionStrategy } from '@angular/core';
|
||||
import { Component, input } from '@angular/core';
|
||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
import { Checkable } from '../interfaces';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
|
|
@ -11,10 +11,7 @@ import { FormsModule } from '@angular/forms';
|
|||
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
||||
</div>
|
||||
`,
|
||||
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
||||
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
||||
changeDetection: ChangeDetectionStrategy.Eager,
|
||||
imports: [
|
||||
imports: [
|
||||
FormsModule
|
||||
]
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
import { ChangeDetectionStrategy, Component, inject } from '@angular/core';
|
||||
import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
|
||||
import { faCheckCircle, faTimesCircle, faInfoCircle, faXmark } from '@fortawesome/free-solid-svg-icons';
|
||||
import { ToastService } from '../services/toast.service';
|
||||
|
||||
@Component({
|
||||
selector: 'app-toast-container',
|
||||
changeDetection: ChangeDetectionStrategy.OnPush,
|
||||
imports: [FontAwesomeModule],
|
||||
template: `
|
||||
<div class="toast-container position-fixed top-0 end-0 p-3" style="z-index: 1100;" aria-live="polite" aria-atomic="true">
|
||||
@for (toast of toasts.toasts(); track toast.id) {
|
||||
<div class="toast show align-items-center border-0 mb-2"
|
||||
[class.text-bg-danger]="toast.level === 'error'"
|
||||
[class.text-bg-success]="toast.level === 'success'"
|
||||
[class.text-bg-primary]="toast.level === 'info'"
|
||||
role="alert" aria-live="assertive" aria-atomic="true">
|
||||
<div class="d-flex">
|
||||
<div class="toast-body d-flex align-items-start gap-2">
|
||||
@if (toast.level === 'error') {
|
||||
<fa-icon [icon]="faTimesCircle" class="mt-1" />
|
||||
} @else if (toast.level === 'success') {
|
||||
<fa-icon [icon]="faCheckCircle" class="mt-1" />
|
||||
} @else {
|
||||
<fa-icon [icon]="faInfoCircle" class="mt-1" />
|
||||
}
|
||||
<span style="white-space: pre-line;">{{ toast.message }}</span>
|
||||
</div>
|
||||
@if (!toast.actions) {
|
||||
<button type="button" class="btn-close btn-close-white me-2 m-auto"
|
||||
aria-label="Close" (click)="toasts.dismiss(toast.id)"></button>
|
||||
}
|
||||
</div>
|
||||
@if (toast.actions) {
|
||||
<div class="d-flex justify-content-end gap-2 px-3 pb-2">
|
||||
@for (action of toast.actions; track action.label) {
|
||||
<button type="button"
|
||||
class="btn btn-sm"
|
||||
[class.btn-light]="!action.primary"
|
||||
[class.btn-outline-light]="action.primary"
|
||||
(click)="toasts.respond(toast.id, action.value)">
|
||||
{{ action.label }}
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
`,
|
||||
})
|
||||
export class ToastContainerComponent {
|
||||
protected readonly toasts = inject(ToastService);
|
||||
protected readonly faCheckCircle = faCheckCircle;
|
||||
protected readonly faTimesCircle = faTimesCircle;
|
||||
protected readonly faInfoCircle = faInfoCircle;
|
||||
protected readonly faXmark = faXmark;
|
||||
}
|
||||
|
|
@ -14,12 +14,6 @@ export interface Download {
|
|||
chapter_template?: string;
|
||||
subtitle_language?: string;
|
||||
subtitle_mode?: string;
|
||||
ytdl_options_presets?: string[];
|
||||
ytdl_options_overrides?: Record<string, unknown>;
|
||||
clip_start?: number;
|
||||
clip_end?: number;
|
||||
live_status?: string;
|
||||
live_release_timestamp?: number;
|
||||
status: string;
|
||||
msg: string;
|
||||
percent: number;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ export interface SubscriptionRow {
|
|||
format: string;
|
||||
quality: string;
|
||||
folder: string;
|
||||
title_regex?: string;
|
||||
skip_subscriber_only?: boolean;
|
||||
last_checked: number | null;
|
||||
seen_count: number;
|
||||
error: string | null;
|
||||
|
|
|
|||
|
|
@ -10,16 +10,15 @@ describe('FileSizePipe', () => {
|
|||
it('formats bytes and larger units', () => {
|
||||
const pipe = new FileSizePipe();
|
||||
expect(pipe.transform(500)).toContain('Bytes');
|
||||
expect(pipe.transform(1000)).toContain('Bytes');
|
||||
expect(pipe.transform(1024)).toContain('KB');
|
||||
expect(pipe.transform(1024 ** 2)).toContain('MB');
|
||||
expect(pipe.transform(1024 ** 3)).toContain('GB');
|
||||
expect(pipe.transform(1000)).toContain('KB');
|
||||
expect(pipe.transform(1000 * 1000)).toContain('MB');
|
||||
expect(pipe.transform(1000 ** 3)).toContain('GB');
|
||||
});
|
||||
|
||||
it('handles boundaries between units', () => {
|
||||
const pipe = new FileSizePipe();
|
||||
expect(pipe.transform(1023)).toContain('Bytes');
|
||||
expect(pipe.transform(1024)).toContain('KB');
|
||||
expect(pipe.transform(1025)).toContain('KB');
|
||||
expect(pipe.transform(999)).toContain('Bytes');
|
||||
expect(pipe.transform(1000)).toContain('KB');
|
||||
expect(pipe.transform(1001)).toContain('KB');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,10 +8,9 @@ export class FileSizePipe implements PipeTransform {
|
|||
if (isNaN(value) || value === 0) return '0 Bytes';
|
||||
|
||||
const units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||
const k = 1024; // Matches SpeedPipe's base so file sizes and transfer speeds agree.
|
||||
const unitIndex = Math.floor(Math.log(value) / Math.log(k));
|
||||
const unitIndex = Math.floor(Math.log(value) / Math.log(1000)); // Use 1000 for common units
|
||||
|
||||
const unitValue = value / Math.pow(k, unitIndex);
|
||||
const unitValue = value / Math.pow(1000, unitIndex);
|
||||
return `${unitValue.toFixed(2)} ${units[unitIndex]}`;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
import { inject, Injectable } from '@angular/core';
|
||||
import { DownloadsService } from './downloads.service';
|
||||
import { ToastService } from './toast.service';
|
||||
|
||||
export type BatchUrlFilter = 'pending' | 'completed' | 'failed' | 'all';
|
||||
|
||||
/**
|
||||
* Encapsulates collecting download URLs by status and exporting/copying them.
|
||||
* Extracted from the main app component to keep it focused on view concerns.
|
||||
*/
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class BatchUrlsService {
|
||||
private downloads = inject(DownloadsService);
|
||||
private toasts = inject(ToastService);
|
||||
|
||||
collect(filter: BatchUrlFilter): string[] {
|
||||
const queueUrls = () => Array.from(this.downloads.queue.values()).map((dl) => dl.url);
|
||||
const doneUrls = (status?: string) =>
|
||||
Array.from(this.downloads.done.values())
|
||||
.filter((dl) => status === undefined || dl.status === status)
|
||||
.map((dl) => dl.url);
|
||||
switch (filter) {
|
||||
case 'pending':
|
||||
return queueUrls();
|
||||
case 'completed':
|
||||
return doneUrls('finished');
|
||||
case 'failed':
|
||||
return doneUrls('error');
|
||||
default:
|
||||
return [...queueUrls(), ...doneUrls()];
|
||||
}
|
||||
}
|
||||
|
||||
export(filter: BatchUrlFilter): void {
|
||||
const urls = this.collect(filter);
|
||||
if (!urls.length) {
|
||||
this.toasts.info('No URLs found for the selected filter.');
|
||||
return;
|
||||
}
|
||||
const blob = new Blob([urls.join('\n')], { type: 'text/plain' });
|
||||
const downloadUrl = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = downloadUrl;
|
||||
a.download = 'metube_urls.txt';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
window.URL.revokeObjectURL(downloadUrl);
|
||||
}
|
||||
|
||||
copy(filter: BatchUrlFilter): void {
|
||||
const urls = this.collect(filter);
|
||||
if (!urls.length) {
|
||||
this.toasts.info('No URLs found for the selected filter.');
|
||||
return;
|
||||
}
|
||||
navigator.clipboard
|
||||
.writeText(urls.join('\n'))
|
||||
.then(() => this.toasts.success('URLs copied to clipboard.'))
|
||||
.catch(() => this.toasts.error('Failed to copy URLs.'));
|
||||
}
|
||||
}
|
||||
|
|
@ -39,10 +39,6 @@ function basePayload(): AddDownloadPayload {
|
|||
chapterTemplate: '',
|
||||
subtitleLanguage: 'en',
|
||||
subtitleMode: 'prefer_manual',
|
||||
ytdlOptionsPresets: [],
|
||||
ytdlOptionsOverrides: '',
|
||||
clipStart: '',
|
||||
clipEnd: '',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -83,40 +79,11 @@ describe('DownloadsService', () => {
|
|||
chapter_template: '',
|
||||
subtitle_language: 'en',
|
||||
subtitle_mode: 'prefer_manual',
|
||||
ytdl_options_presets: [],
|
||||
ytdl_options_overrides: '',
|
||||
}),
|
||||
);
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('add() sends clip_start and clip_end when set', () => {
|
||||
service
|
||||
.add({
|
||||
...basePayload(),
|
||||
clipStart: '1:00',
|
||||
clipEnd: '2:00',
|
||||
})
|
||||
.subscribe();
|
||||
const req = httpMock.expectOne('add');
|
||||
expect(req.request.body).toEqual(
|
||||
expect.objectContaining({
|
||||
clip_start: '1:00',
|
||||
clip_end: '2:00',
|
||||
}),
|
||||
);
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('getPresets() fetches configured preset names', () => {
|
||||
service.getPresets().subscribe((result) => {
|
||||
expect(result).toEqual({ presets: ['Preset A'] });
|
||||
});
|
||||
const req = httpMock.expectOne('presets');
|
||||
expect(req.request.method).toBe('GET');
|
||||
req.flush({ presets: ['Preset A'] });
|
||||
});
|
||||
|
||||
it('cancelAdd posts to cancel-add', () => {
|
||||
service.cancelAdd().subscribe();
|
||||
const req = httpMock.expectOne('cancel-add');
|
||||
|
|
@ -159,61 +126,6 @@ describe('DownloadsService', () => {
|
|||
req.flush({});
|
||||
});
|
||||
|
||||
it('delById resets deleting flag and emits error status on HTTP failure', () => {
|
||||
const dl: Download = {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'finished',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
deleting: false,
|
||||
};
|
||||
service.queue.set('u1', dl);
|
||||
let queueChangedCount = 0;
|
||||
service.queueChanged.subscribe(() => queueChangedCount++);
|
||||
let result: unknown;
|
||||
let threw = false;
|
||||
|
||||
service.delById('queue', ['u1']).subscribe({
|
||||
next: (res) => { result = res; },
|
||||
error: () => { threw = true; },
|
||||
});
|
||||
expect(dl.deleting).toBe(true);
|
||||
const req = httpMock.expectOne('delete');
|
||||
req.flush({ msg: 'boom' }, { status: 500, statusText: 'Server Error' });
|
||||
|
||||
expect(threw).toBe(false);
|
||||
expect(dl.deleting).toBe(false);
|
||||
expect(queueChangedCount).toBeGreaterThan(0);
|
||||
expect((result as { status: string }).status).toBe('error');
|
||||
});
|
||||
|
||||
it('startById surfaces HTTP errors as a status object instead of throwing', () => {
|
||||
let result: unknown;
|
||||
let threw = false;
|
||||
|
||||
service.startById(['a']).subscribe({
|
||||
next: (res) => { result = res; },
|
||||
error: () => { threw = true; },
|
||||
});
|
||||
const req = httpMock.expectOne('start');
|
||||
req.flush({ msg: 'nope' }, { status: 500, statusText: 'Server Error' });
|
||||
|
||||
expect(threw).toBe(false);
|
||||
expect((result as { status: string }).status).toBe('error');
|
||||
});
|
||||
|
||||
it('handleHTTPError extracts msg from object body', async () => {
|
||||
const err = new HttpErrorResponse({
|
||||
error: { msg: 'bad' },
|
||||
|
|
@ -281,15 +193,6 @@ describe('DownloadsService', () => {
|
|||
expect(updated?.deleting).toBe(true);
|
||||
});
|
||||
|
||||
it('socket updated ignores events for urls not already in the queue', () => {
|
||||
expect(service.queue.has('unknown-url')).toBe(false);
|
||||
socket.emit(
|
||||
'updated',
|
||||
JSON.stringify({ url: 'unknown-url', title: 't', status: 'downloading' }),
|
||||
);
|
||||
expect(service.queue.has('unknown-url')).toBe(false);
|
||||
});
|
||||
|
||||
it('socket completed moves entry to done', () => {
|
||||
service.queue.set('u1', {
|
||||
id: '1',
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@ export interface AddDownloadPayload {
|
|||
chapterTemplate: string;
|
||||
subtitleLanguage: string;
|
||||
subtitleMode: string;
|
||||
ytdlOptionsPresets: string[];
|
||||
ytdlOptionsOverrides: string;
|
||||
clipStart?: string;
|
||||
clipEnd?: string;
|
||||
}
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
|
|
@ -69,14 +65,8 @@ export class DownloadsService {
|
|||
.subscribe((strdata: string) => {
|
||||
const data: Download = JSON.parse(strdata);
|
||||
const dl: Download | undefined = this.queue.get(data.url);
|
||||
// An 'added' event always precedes legitimate updates. If the row is
|
||||
// gone (canceled/completed already processed), this update is stale —
|
||||
// applying it would resurrect a ghost row until the next full refresh.
|
||||
if (!dl) {
|
||||
return;
|
||||
}
|
||||
data.checked = !!dl.checked;
|
||||
data.deleting = !!dl.deleting;
|
||||
data.checked = !!dl?.checked;
|
||||
data.deleting = !!dl?.deleting;
|
||||
this.queue.set(data.url, data);
|
||||
this.updated.next();
|
||||
});
|
||||
|
|
@ -137,7 +127,7 @@ export class DownloadsService {
|
|||
}
|
||||
|
||||
public add(payload: AddDownloadPayload) {
|
||||
const body: Record<string, unknown> = {
|
||||
return this.http.post<Status>('add', {
|
||||
url: payload.url,
|
||||
download_type: payload.downloadType,
|
||||
codec: payload.codec,
|
||||
|
|
@ -151,28 +141,13 @@ export class DownloadsService {
|
|||
chapter_template: payload.chapterTemplate,
|
||||
subtitle_language: payload.subtitleLanguage,
|
||||
subtitle_mode: payload.subtitleMode,
|
||||
ytdl_options_presets: payload.ytdlOptionsPresets,
|
||||
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
||||
};
|
||||
const cs = payload.clipStart?.trim();
|
||||
const ce = payload.clipEnd?.trim();
|
||||
if (cs) body['clip_start'] = cs;
|
||||
if (ce) body['clip_end'] = ce;
|
||||
return this.http.post<Status>('add', body).pipe(
|
||||
}).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
public getPresets() {
|
||||
return this.http.get<{ presets: string[] }>('presets').pipe(
|
||||
catchError(() => of({ presets: [] }))
|
||||
);
|
||||
}
|
||||
|
||||
public startById(ids: string[]) {
|
||||
return this.http.post<Status>('start', {ids: ids}).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
return this.http.post('start', {ids: ids});
|
||||
}
|
||||
|
||||
public delById(where: State, ids: string[]) {
|
||||
|
|
@ -185,22 +160,7 @@ export class DownloadsService {
|
|||
}
|
||||
}
|
||||
}
|
||||
return this.http.post<Status>('delete', {where: where, ids: ids}).pipe(
|
||||
catchError((err: HttpErrorResponse) => {
|
||||
// Request failed — the rows would otherwise stay disabled forever
|
||||
// with no way to retry, since nothing ever clears `deleting`.
|
||||
if (map) {
|
||||
for (const id of ids) {
|
||||
const obj = map.get(id);
|
||||
if (obj) {
|
||||
obj.deleting = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
(where === 'queue' ? this.queueChanged : this.doneChanged).next();
|
||||
return this.handleHTTPError(err);
|
||||
})
|
||||
);
|
||||
return this.http.post('delete', {where: where, ids: ids});
|
||||
}
|
||||
|
||||
public startByFilter(where: State, filter: (dl: Download) => boolean) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,2 @@
|
|||
export { DownloadsService } from './downloads.service';
|
||||
export { MeTubeSocket } from './metube-socket.service';
|
||||
export { ToastService } from './toast.service';
|
||||
export { BatchUrlsService } from './batch-urls.service';
|
||||
export { MeTubeSocket } from './metube-socket.service';
|
||||
|
|
@ -10,8 +10,6 @@ import { AddDownloadPayload } from './downloads.service';
|
|||
|
||||
export interface SubscribePayload extends AddDownloadPayload {
|
||||
checkIntervalMinutes: number;
|
||||
titleRegex: string;
|
||||
skipSubscriberOnly: boolean;
|
||||
}
|
||||
|
||||
@Injectable({
|
||||
|
|
@ -96,11 +94,7 @@ export class SubscriptionsService {
|
|||
chapter_template: payload.chapterTemplate,
|
||||
subtitle_language: payload.subtitleLanguage,
|
||||
subtitle_mode: payload.subtitleMode,
|
||||
ytdl_options_presets: payload.ytdlOptionsPresets,
|
||||
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
||||
check_interval_minutes: payload.checkIntervalMinutes,
|
||||
title_regex: payload.titleRegex,
|
||||
skip_subscriber_only: payload.skipSubscriberOnly,
|
||||
})
|
||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
|
@ -109,15 +103,7 @@ export class SubscriptionsService {
|
|||
return this.http.post('subscriptions/delete', { ids }).pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
||||
update(
|
||||
id: string,
|
||||
changes: Partial<
|
||||
Pick<
|
||||
SubscriptionRow,
|
||||
'enabled' | 'check_interval_minutes' | 'name' | 'title_regex' | 'skip_subscriber_only'
|
||||
>
|
||||
>,
|
||||
) {
|
||||
update(id: string, changes: Partial<Pick<SubscriptionRow, 'enabled' | 'check_interval_minutes' | 'name'>>) {
|
||||
return this.http
|
||||
.post('subscriptions/update', { id, ...changes })
|
||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
import { Injectable, signal } from '@angular/core';
|
||||
|
||||
export type ToastLevel = 'info' | 'success' | 'error';
|
||||
|
||||
export interface ToastAction {
|
||||
label: string;
|
||||
value: boolean;
|
||||
primary?: boolean;
|
||||
}
|
||||
|
||||
export interface Toast {
|
||||
id: number;
|
||||
level: ToastLevel;
|
||||
message: string;
|
||||
actions?: ToastAction[];
|
||||
/** Resolver for confirm() toasts; resolved when the user picks an action or dismisses. */
|
||||
_resolve?: (value: boolean) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight non-blocking notification service. Replaces the blocking
|
||||
* window.alert()/confirm() dialogs that previously littered the app component.
|
||||
*/
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class ToastService {
|
||||
private counter = 0;
|
||||
readonly toasts = signal<Toast[]>([]);
|
||||
|
||||
info(message: string): void {
|
||||
this.show('info', message, 4000);
|
||||
}
|
||||
|
||||
success(message: string): void {
|
||||
this.show('success', message, 4000);
|
||||
}
|
||||
|
||||
error(message: string): void {
|
||||
this.show('error', message, 8000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Show a confirmation toast with confirm/cancel actions. Resolves true when
|
||||
* confirmed, false when cancelled or auto-dismissed.
|
||||
*/
|
||||
confirm(message: string, confirmLabel = 'OK', cancelLabel = 'Cancel'): Promise<boolean> {
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const id = ++this.counter;
|
||||
this.toasts.update((list) => [
|
||||
...list,
|
||||
{
|
||||
id,
|
||||
level: 'info',
|
||||
message,
|
||||
actions: [
|
||||
{ label: cancelLabel, value: false },
|
||||
{ label: confirmLabel, value: true, primary: true },
|
||||
],
|
||||
_resolve: resolve,
|
||||
},
|
||||
]);
|
||||
});
|
||||
}
|
||||
|
||||
respond(id: number, value: boolean): void {
|
||||
const toast = this.toasts().find((t) => t.id === id);
|
||||
toast?._resolve?.(value);
|
||||
this.remove(id);
|
||||
}
|
||||
|
||||
dismiss(id: number): void {
|
||||
const toast = this.toasts().find((t) => t.id === id);
|
||||
// A confirm toast dismissed without an explicit choice resolves to false.
|
||||
toast?._resolve?.(false);
|
||||
this.remove(id);
|
||||
}
|
||||
|
||||
private remove(id: number): void {
|
||||
this.toasts.update((list) => list.filter((t) => t.id !== id));
|
||||
}
|
||||
|
||||
private show(level: ToastLevel, message: string, autoDismissMs: number): void {
|
||||
const id = ++this.counter;
|
||||
this.toasts.update((list) => [...list, { id, level, message }]);
|
||||
setTimeout(() => this.remove(id), autoDismissMs);
|
||||
}
|
||||
}
|
||||
|
|
@ -12,13 +12,5 @@
|
|||
],
|
||||
"exclude": [
|
||||
"src/**/*.spec.ts"
|
||||
],
|
||||
"angularCompilerOptions": {
|
||||
"extendedDiagnostics": {
|
||||
"checks": {
|
||||
"nullishCoalescingNotNullable": "suppress",
|
||||
"optionalChainNotNullable": "suppress"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue