Prod packaging v1 (#23)

* Sets media directory for prod

* Improved logging; Added docker files for prod

* Improved fetching SECRET_KEY_BASE for selfhosted instances

.dockerignore

@@ -0,0 +1,45 @@
+# This file excludes paths from the Docker build context.
+#
+# By default, Docker's build context includes all files (and folders) in the
+# current directory. Even if a file isn't copied into the container it is still sent to
+# the Docker daemon.
+#
+# There are multiple reasons to exclude files from the build context:
+#
+# 1. Prevent nested folders from being copied into the container (ex: exclude
+#    /assets/node_modules when copying /assets)
+# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc)
+# 3. Avoid sending files containing sensitive information
+#
+# More information on using .dockerignore is available here:
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+.dockerignore
+
+# Ignore git, but keep git HEAD and refs to access current commit hash if needed:
+#
+# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat
+# d0b8727759e1e0e7aa3d41707d12376e373d5ecc
+.git
+!.git/HEAD
+!.git/refs
+
+# Common development/test artifacts
+/cover/
+/doc/
+/test/
+/tmp/
+.elixir_ls
+
+# Mix artifacts
+/_build/
+/deps/
+*.ez
+
+# Generated on crash by the VM
+erl_crash.dump
+
+# Static artifacts - These should be fetched and built inside the Docker image
+/assets/node_modules/
+/priv/static/assets/
+/priv/static/cache_manifest.json

config/config.exs

@@ -13,13 +13,12 @@ config :pinchflat,
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
   yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner,
-  # TODO: figure this out
-  media_directory: :not_implemented,
+  media_directory: "/downloads",
   metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"])
 
 # Configures the endpoint
 config :pinchflat, PinchflatWeb.Endpoint,
-  url: [host: "localhost"],
+  url: [host: "localhost", port: 8945],
   adapter: Phoenix.Endpoint.Cowboy2Adapter,
   render_errors: [
     formats: [html: PinchflatWeb.ErrorHTML, json: PinchflatWeb.ErrorJSON],

config/runtime.exs

@@ -1,4 +1,5 @@
 import Config
+require Logger
 
 # config/runtime.exs is executed for all environments, including
 # during releases. It is executed after compilation and before the
@@ -28,6 +29,8 @@ if config_env() == :prod do
       For example: /etc/pinchflat/pinchflat.db
       """
 
+  config :pinchflat, yt_dlp_executable: System.find_executable("yt-dlp")
+
   config :pinchflat, Pinchflat.Repo,
     database: database_path,
     pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5")
@@ -38,26 +41,36 @@ if config_env() == :prod do
   # to check this value into version control, so we use an environment
   # variable instead.
   secret_key_base =
-    System.get_env("SECRET_KEY_BASE") ||
-      raise """
-      environment variable SECRET_KEY_BASE is missing.
-      You can generate one by calling: mix phx.gen.secret
-      """
+    if System.get_env("SECRET_KEY_BASE") do
+      System.get_env("SECRET_KEY_BASE")
+    else
+      if System.get_env("RUN_CONTEXT") == "selfhosted" do
+        # Using the default SECRET_KEY_BASE in a conventional production environment
+        # is dangerous. Please set the SECRET_KEY_BASE environment variable if you're
+        # deploying this to an internet-facing server. If you're running this in a
+        # private network, it's likely safe to use the default value. If you want
+        # to be extra safe, run `mix phx.gen.secret` and set the SECRET_KEY_BASE
+        # environment variable to the output of that command.
 
-  host = System.get_env("PHX_HOST") || "example.com"
-  port = String.to_integer(System.get_env("PORT") || "4000")
+        "ZkuQMStdmUzBv+gO3m3XZrtQW76e+AX3QIgTLajw3b/HkTLMEx+DOXr2WZsSS+n8"
+      else
+        raise """
+        environment variable SECRET_KEY_BASE is missing.
+        You can generate one by calling: mix phx.gen.secret
+        """
+      end
+    end
 
   config :pinchflat, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
 
   config :pinchflat, PinchflatWeb.Endpoint,
-    url: [host: host, port: 443, scheme: "https"],
     http: [
       # Enable IPv6 and bind on all interfaces.
       # Set it to  {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
       # See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html
       # for details about using IPv6 vs IPv4 and loopback vs public addresses.
       ip: {0, 0, 0, 0, 0, 0, 0, 0},
-      port: port
+      port: String.to_integer(System.get_env("PORT") || "4000")
     ],
     secret_key_base: secret_key_base
 

docker-compose.ci.yml

@@ -1,7 +1,9 @@
 version: '3'
 services:
   phx:
-    build: .
+    build:
+      context: .
+      dockerfile: dev.Dockerfile
     volumes:
       - '.:/app'
     ports:

docker-compose.yml

@@ -1,12 +1,14 @@
 version: '3'
 services:
   phx:
-    build: .
+    build:
+      context: .
+      dockerfile: dev.Dockerfile
     volumes:
       - '.:/app'
     ports:
       - '4008:4008'
     command:
-      - ./docker-run.sh
+      - ./docker-run.dev.sh
     stdin_open: true
     tty: true

lib/pinchflat/application.ex

@@ -21,6 +21,8 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Endpoint
     ]
 
+    :ok = Oban.Telemetry.attach_default_logger()
+
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pinchflat.Supervisor]

lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex

@@ -30,7 +30,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
     print_to_file_opts = [{:print_to_file, output_template}, json_output_path]
     formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts)
 
-    Logger.debug("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
+    Logger.info("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
 
     case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->

lib/pinchflat/release.ex

@@ -0,0 +1,28 @@
+defmodule Pinchflat.Release do
+  @moduledoc """
+  Used for executing DB release tasks when run in production without Mix
+  installed.
+  """
+  @app :pinchflat
+
+  def migrate do
+    load_app()
+
+    for repo <- repos() do
+      {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true))
+    end
+  end
+
+  def rollback(repo, version) do
+    load_app()
+    {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
+  end
+
+  defp repos do
+    Application.fetch_env!(@app, :ecto_repos)
+  end
+
+  defp load_app do
+    Application.load(@app)
+  end
+end

rel/overlays/bin/docker_start

@@ -0,0 +1,5 @@
+#!/bin/sh
+/app/bin/migrate
+
+cd -P -- "$(dirname -- "$0")"
+PHX_SERVER=true exec ./pinchflat start

rel/overlays/bin/migrate

@@ -0,0 +1,3 @@
+#!/bin/sh
+cd -P -- "$(dirname -- "$0")"
+exec ./pinchflat eval Pinchflat.Release.migrate

selfhosted.Dockerfile

@@ -0,0 +1,119 @@
+# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian
+# instead of Alpine to avoid DNS resolution issues in production.
+#
+# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu
+# https://hub.docker.com/_/ubuntu?tab=tags
+#
+# This file is based on these images:
+#
+#   - https://hub.docker.com/r/hexpm/elixir/tags - for the build image
+#   - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20231009-slim - for the release image
+#   - https://pkgs.org/ - resource for finding needed packages
+#   - Ex: hexpm/elixir:1.16.0-erlang-26.2.1-debian-bullseye-20231009-slim
+#
+ARG ELIXIR_VERSION=1.16.0
+ARG OTP_VERSION=26.2.1
+ARG DEBIAN_VERSION=bullseye-20231009-slim
+
+ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
+ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
+
+FROM ${BUILDER_IMAGE} as builder
+
+# install build dependencies
+RUN apt-get update -y && apt-get install -y build-essential git curl \
+    && apt-get clean && rm -f /var/lib/apt/lists/*_*
+
+# prepare build dir
+WORKDIR /app
+
+# Install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
+RUN bash nodesource_setup.sh
+RUN apt-get install -y nodejs
+RUN npm install -g yarn
+
+# install hex + rebar
+RUN mix local.hex --force && \
+    mix local.rebar --force
+
+# set build ENV
+ENV MIX_ENV="prod"
+
+# install mix dependencies
+COPY mix.exs mix.lock ./
+RUN mix deps.get --only $MIX_ENV
+RUN mkdir config
+
+# copy compile-time config files before we compile dependencies
+# to ensure any relevant config change will trigger the dependencies
+# to be re-compiled.
+COPY config/config.exs config/${MIX_ENV}.exs config/
+RUN mix deps.compile
+
+COPY priv priv
+
+COPY lib lib
+
+COPY assets assets
+
+# compile assets
+RUN yarn --cwd assets install
+RUN mix assets.deploy
+
+# Compile the release
+RUN mix compile
+
+# Changes to config/runtime.exs don't require recompiling the code
+COPY config/runtime.exs config/
+
+COPY rel rel
+RUN mix release
+
+# start a new build stage so that the final image will only contain
+# the compiled release and other runtime necessities
+FROM ${RUNNER_IMAGE}
+
+RUN apt-get update -y
+RUN apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \
+  python3 python3-pip ffmpeg
+RUN apt-get clean && rm -f /var/lib/apt/lists/*_*
+
+# Download YT-DLP
+# NOTE: If you're seeing weird issues, consider using the FFMPEG released by yt-dlp
+RUN python3 -m pip install -U --pre yt-dlp
+
+# Set the locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+WORKDIR "/app"
+RUN chown nobody /app
+
+# Set up data volumes
+RUN mkdir /config /downloads
+RUN chown nobody /config /downloads
+VOLUME /config
+VOLUME /downloads
+
+# set runner ENV
+ENV MIX_ENV="prod"
+ENV DATABASE_PATH="/config/pinchflat.db"
+ENV PORT=8945
+ENV RUN_CONTEXT="selfhosted"
+
+# Only copy the final release from the build stage
+COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./
+
+USER nobody
+
+# If using an environment that doesn't automatically reap zombie processes, it is
+# advised to add an init process such as tini via `apt-get install`
+# above and adding an entrypoint. See https://github.com/krallin/tini for details
+# ENTRYPOINT ["/tini", "--"]
+
+# Start the app
+CMD ["/app/bin/docker_start"]