Videos: Auto-install/update yt-dlp in the dev environment #4982

Signed-off-by: Michael Mayer <michael@photoprism.app>

Dockerfile

@@ -20,3 +20,5 @@ WORKDIR "/go/src/github.com/photoprism/photoprism"
 # Copy source to image.
 COPY . .
 COPY --chown=root:root /scripts/dist/ /scripts/
+
+RUN sudo /scripts/install-yt-dlp.sh

compose.intel.yaml

@@ -115,8 +115,8 @@ services:
       PHOTOPRISM_FFMPEG_ENCODER: "intel"             # H.264/AVC encoder (software, intel, nvidia, apple, raspberry, or vaapi)
       PHOTOPRISM_FFMPEG_SIZE: "1920"                 # video size limit in pixels (720-7680) (default: 3840)
       # PHOTOPRISM_FFMPEG_BITRATE: "60"              # video bitrate limit in Mbps (default: 60)
-      ## Run/install on first startup (options: update tensorflow https intel gpu davfs):
-      PHOTOPRISM_INIT: "https intel tensorflow"
+      ## Run/install on first startup (options: update tensorflow https intel gpu davfs yt-dlp):
+      PHOTOPRISM_INIT: "https intel tensorflow yt-dlp"
     ## Share hardware devices with FFmpeg for hardware video transcoding:
     devices:
       - "/dev/dri:/dev/dri"

compose.nvidia.yaml

@@ -114,14 +114,14 @@ services:
       PHOTOPRISM_THUMB_FILTER: "auto"                # downscaling filter (imaging best to worst: blackman, lanczos, cubic, linear, nearest)
       PHOTOPRISM_THUMB_UNCACHED: "true"              # enables on-demand thumbnail rendering (high memory and cpu usage)
       TF_CPP_MIN_LOG_LEVEL: 1                        # show TensorFlow log messages for development
-      ## Run/install on first startup, see https://github.com/photoprism/photoprism/blob/develop/scripts/dist/Makefile:
-      PHOTOPRISM_INIT: "https tensorflow-gpu"        # common options: update https tensorflow tensorflow-gpu intel gpu davfs
       ## Nvidia Video Transcoding (https://docs.photoprism.app/getting-started/advanced/transcoding/#nvidia-container-toolkit):
       NVIDIA_VISIBLE_DEVICES: "all"
       NVIDIA_DRIVER_CAPABILITIES: "all"
       PHOTOPRISM_FFMPEG_ENCODER: "nvidia"            # H.264/AVC encoder (software, intel, nvidia, apple, raspberry, or vaapi)
       PHOTOPRISM_FFMPEG_SIZE: "1920"                 # video size limit in pixels (720-7680) (default: 3840)
       # PHOTOPRISM_FFMPEG_BITRATE: "60"              # video bitrate limit in Mbps (default: 60)
+      ## Run/install on first startup (options: update tensorflow https intel gpu davfs yt-dlp):
+      PHOTOPRISM_INIT: "https tensorflow-gpu yt-dlp"
     ## Share hardware devices with FFmpeg and TensorFlow (optional):
     # devices:
     #  - "/dev/dri:/dev/dri"                         # Intel QSV (Broadwell and later) or VAAPI (Haswell and earlier)

compose.postgres.yaml

@@ -24,7 +24,6 @@ services:
       - "go-mod:/go/pkg/mod"
     shm_size: "2gb"
     environment:
-      PHOTOPRISM_INIT: "https"
       PHOTOPRISM_ADMIN_USER: "admin"                       # admin login username
       PHOTOPRISM_ADMIN_PASSWORD: "photoprism"              # initial admin password (8-72 characters)
       PHOTOPRISM_AUTH_MODE: "password"                     # authentication mode (public, password)
@@ -66,7 +65,8 @@ services:
       PHOTOPRISM_THUMB_SIZE_UNCACHED: 7680    # on-demand rendering size limit (default 7680, min 720, max 7680)
       PHOTOPRISM_JPEG_SIZE: 7680              # size limit for converted image files in pixels (720-30000)
       TF_CPP_MIN_LOG_LEVEL: 1                 # show TensorFlow log messages for development
-
+      ## Run/install on first startup (options: update tensorflow https intel gpu davfs yt-dlp):
+      PHOTOPRISM_INIT: "https yt-dlp"
   ## PostgreSQL Database Server
   ## Docs: https://www.postgresql.org/docs/
   postgres:

compose.yaml

@@ -124,8 +124,8 @@ services:
       # LIBVA_DRIVER_NAME: "i965"                    # For Intel architectures Haswell and older which do not support QSV yet but use VAAPI instead
       PHOTOPRISM_FFMPEG_SIZE: "1920"                 # video size limit in pixels (720-7680) (default: 3840)
       # PHOTOPRISM_FFMPEG_BITRATE: "60"              # video bitrate limit in Mbps (default: 60)
-      ## Run/install on first startup (options: update tensorflow https intel gpu davfs):
-      PHOTOPRISM_INIT: "https"
+      ## Run/install on first startup (options: update tensorflow https intel gpu davfs yt-dlp):
+      PHOTOPRISM_INIT: "https yt-dlp"
     ## Share hardware devices with FFmpeg and TensorFlow (optional):
     # devices:
     #  - "/dev/dri:/dev/dri"                         # Intel QSV (Broadwell and later) or VAAPI (Haswell and earlier)

internal/photoprism/dl/dl_test.go

@@ -221,6 +221,10 @@ func TestDownloadSections(t *testing.T) {
 	fileName := fs.Abs("./testdata/duration_test_file")
 	duration := 5
 
+	defer func() {
+		_ = os.Remove(fileName)
+	}()
+
 	cmd := exec.Command(FindFFmpegBin(), "-version")
 	_, err := cmd.Output()
 
@@ -292,7 +296,6 @@ func TestDownloadSections(t *testing.T) {
 	}
 
 	_ = dr.Close()
-	_ = os.Remove(fileName)
 }
 
 func TestErrorNotAPlaylist(t *testing.T) {

internal/photoprism/dl/download_test.go

@@ -4,9 +4,14 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"os"
 	"os/exec"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/photoprism/photoprism/pkg/fs"
 )
 
 func TestDownload(t *testing.T) {
@@ -14,6 +19,7 @@ func TestDownload(t *testing.T) {
 		t.Skip("skipping test in short mode.")
 	}
 
+	// Fetch metadata.
 	stderrBuf := &bytes.Buffer{}
 	r, err := NewMetadata(context.Background(), testVideoRawURL, Options{
 		StderrFn: func(cmd *exec.Cmd) io.Writer {
@@ -23,6 +29,22 @@ func TestDownload(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// Write metadata to "./testdata/download-test.json".
+	jsonName := fs.Abs("./testdata/download-test.json")
+	if json := r.Info.JSON(); json != nil {
+		if err = fs.WriteFile(jsonName, json, fs.ModeFile); err != nil {
+			t.Errorf("%s could not be created: %s", jsonName, err)
+		}
+	} else {
+		t.Errorf("%s could not be created because json data is nil", jsonName)
+	}
+
+	// Remove metadata file.
+	assert.FileExists(t, jsonName)
+	_ = os.Remove(jsonName)
+
+	// Download video based on metadata.
 	dr, err := r.Download(context.Background(), r.Info.Formats[0].FormatID)
 	if err != nil {
 		t.Fatal(err)

internal/photoprism/dl/info.go

@@ -100,6 +100,12 @@ type Info struct {
 	Format
 }
 
+// JSON returns the information as JSON string.
+func (i *Info) JSON() []byte {
+	b, _ := json.Marshal(i)
+	return b
+}
+
 func infoFromURL(
 	ctx context.Context,
 	rawURL string,
@@ -116,7 +122,7 @@ func infoFromURL(
 		// TODO: needed?
 		"--restrict-filenames",
 		// use .netrc authentication data
-		"--netrc",
+		// "--netrc",
 		// provide url via stdin for security, youtube-dl has some run command args
 		"--batch-file", "-",
 		// dump info json

internal/photoprism/dl/options.go

@@ -101,7 +101,7 @@ func (result Metadata) DownloadWithOptions(
 		// TODO: needed?
 		"--restrict-filenames",
 		// use .netrc authentication data
-		"--netrc",
+		// "--netrc",
 	)
 
 	if options.Output != "" {