mirror of
https://github.com/johannesjo/super-productivity.git
synced 2026-07-17 16:37:43 +00:00
f15a20ba8d
A bare CREATE INDEX CONCURRENTLY aborted mid-build (the in-image step
timeout firing at its 1800s default when a raised MIGRATION_TIMEOUT was
not forwarded, an external stop, or OOM) leaves the migration failed and
an INVALID index of the target name, wedging the deploy. Deployed images
reported only a bare "prisma migrate deploy failed (exit 143)" with no
recovery steps, so operators had no way forward.
- deploy.sh: forward MIGRATE_STEP_TIMEOUT into the migrator so a large
MIGRATION_TIMEOUT is not silently capped at the image default (1800s)
and kill a slow CREATE INDEX CONCURRENTLY early (the root cause).
- migrate-deploy.sh: normalize BusyBox `timeout` 143 -> 124 so a step
timeout hits the timeout branch, not the generic one.
- migrate-deploy.sh: emit_interrupted_recovery_hint() prints copy-paste
recovery (drop the INVALID index, roll the record back, re-run) for an
interrupted CONCURRENTLY build, from BOTH the 124 timeout branch (where
a normalized 143 -- the incident's own signal -- lands) and the generic
non-gate branch (OOM/137), plus the existing P3009 re-run path. Every
path prints guidance only and NEVER auto-resolves a bare CREATE.
- env.example: document the now-forwarded MIGRATE_STEP_TIMEOUT knob.
- tests: cover 143 normalization, the incident's 143->124 timeout-branch
recovery, the P3009 and non-gate bare-create recovery, and the
forwarded step timeout.
The bare CREATE stays bare and fail-loud on purpose: 20260613000001
ships in v18.11.0-v18.14.0 and is applied natively on healthy deploys,
so converting it to a drop-then-create shape would change its checksum
and break every DB that already applied it.
Recovery for the current wedged deploy (20260613000001):
DROP INDEX CONCURRENTLY IF EXISTS "operations_entity_ids_gin";
prisma migrate resolve --rolled-back \
20260613000001_add_operation_entity_ids_gin_index
then re-run the deploy with the step timeout raised enough for the GIN
build (and clear any idle-in-transaction blocker first).
331 lines
13 KiB
Bash
331 lines
13 KiB
Bash
#!/bin/sh
|
|
set -eu
|
|
|
|
# Generic, name-agnostic Prisma migration deploy + recovery.
|
|
#
|
|
# Prisma 5.x wraps every migration in a transaction. PostgreSQL forbids
|
|
# CREATE/DROP INDEX CONCURRENTLY inside a transaction block, so such a
|
|
# migration fails with P3018 / SQLSTATE 25001 ("cannot run inside a
|
|
# transaction block"), and a later deploy then refuses with P3009 (the
|
|
# migration is stuck in a failed state).
|
|
#
|
|
# This script applies migrations and, ONLY for that specific failure mode,
|
|
# recovers by running the failing migration's own SQL out-of-band (no
|
|
# transaction), then marks it applied and retries. It hardcodes no migration
|
|
# names: the failing migration is read from Prisma's own output and its SQL is
|
|
# read from prisma/migrations/<name>/migration.sql.
|
|
#
|
|
# This script is COPYed into the image next to prisma/migrations in the same
|
|
# build, so it is always version-locked to the migrations it must handle. All
|
|
# three call sites (host deploy.sh, image startup CMD, helm initContainer)
|
|
# invoke it, so it carries its own step timeout as defense-in-depth (deploy.sh
|
|
# also wraps it; the CMD/initContainer paths have no outer timeout).
|
|
#
|
|
# RECOVERABLE shape (only this is auto-recovered): a migration whose SQL
|
|
# contains BOTH a DROP INDEX CONCURRENTLY and a CREATE INDEX CONCURRENTLY, so
|
|
# re-running it out-of-band is idempotent and clears a half-built INVALID index
|
|
# first:
|
|
#
|
|
# DROP INDEX CONCURRENTLY IF EXISTS "x";
|
|
# CREATE INDEX CONCURRENTLY "x" ON ...;
|
|
#
|
|
# A bare `CREATE INDEX CONCURRENTLY` (no DROP) is INTENTIONALLY not recovered:
|
|
# such migrations (e.g. 20260511000000) are written to fail loudly rather than
|
|
# be marked applied with a possibly-INVALID index. They fall through to a loud
|
|
# failure here by gate, deterministically.
|
|
#
|
|
# Statements must end with `;` at end of line, comments must be full-line `--`,
|
|
# and `;` must not appear inside string literals (true for all index DDL).
|
|
|
|
SCHEMA="prisma/schema.prisma"
|
|
MIGRATIONS_DIR="prisma/migrations"
|
|
# 3 recoverable CONCURRENTLY migrations today + a final clean pass + slack.
|
|
# The real infinite-loop backstop is the LAST_RECOVERED guard below; this is
|
|
# just a tight upper bound. Overridable for emergencies.
|
|
MAX_ATTEMPTS="${MIGRATE_MAX_ATTEMPTS:-6}"
|
|
# Per-step timeout. A CONCURRENTLY build blocked by a long-running transaction
|
|
# can hang forever; without this the CMD/initContainer paths never fail.
|
|
STEP_TIMEOUT="${MIGRATE_STEP_TIMEOUT:-1800}"
|
|
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
with_timeout() {
|
|
wt_rc=0
|
|
timeout "$STEP_TIMEOUT" "$@" || wt_rc=$?
|
|
# GNU coreutils `timeout` exits 124 on expiry; BusyBox `timeout` (shipped by
|
|
# the node:*-alpine runtime image) instead lets the child die from the
|
|
# default SIGTERM and returns 128+15=143. Normalize so the single 124
|
|
# timeout branch is reached on both. Under this wrapper a 143 is timeout's
|
|
# own SIGTERM, not an unrelated external kill.
|
|
if [ "$wt_rc" -eq 143 ]; then
|
|
wt_rc=124
|
|
fi
|
|
return "$wt_rc"
|
|
}
|
|
else
|
|
with_timeout() { "$@"; }
|
|
fi
|
|
|
|
MIGRATE_LOG=""
|
|
MIGRATE_STATUS=0
|
|
LAST_RECOVERED=""
|
|
|
|
STMT_FILE="$(mktemp "${TMPDIR:-/tmp}/supersync-stmts.XXXXXX")"
|
|
cleanup() {
|
|
rm -f "$STMT_FILE" "$MIGRATE_LOG"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
run_migrate_deploy() {
|
|
# Drop the previous attempt's log so retries don't leak temp files (the
|
|
# trap only ever sees the last value).
|
|
[ -n "$MIGRATE_LOG" ] && rm -f "$MIGRATE_LOG"
|
|
MIGRATE_LOG="$(mktemp "${TMPDIR:-/tmp}/supersync-migrate.XXXXXX")"
|
|
set +e
|
|
with_timeout npx prisma migrate deploy >"$MIGRATE_LOG" 2>&1
|
|
MIGRATE_STATUS=$?
|
|
set -e
|
|
cat "$MIGRATE_LOG"
|
|
}
|
|
|
|
# Failing migration name, from Prisma's own output, validated to the migration
|
|
# directory charset (rejects path traversal / metacharacters). Empty = unknown.
|
|
# Always exits 0 (used in `name=$(...)` under `set -e`).
|
|
parse_failing_migration() {
|
|
# P3018: precise "Migration name:" line.
|
|
name=$(sed -n 's/^Migration name: *\([^ ].*[^ ]\) *$/\1/p' "$MIGRATE_LOG" | tail -n1)
|
|
if [ -z "$name" ]; then
|
|
# P3009: the specific failed-migration sentence.
|
|
name=$(sed -n 's/.*`\([0-9]\{14\}_[A-Za-z0-9_]*\)` migration started at.*failed.*/\1/p' \
|
|
"$MIGRATE_LOG" | tail -n1)
|
|
fi
|
|
if [ -z "$name" ]; then
|
|
# Last resort: a backticked migration-shaped token ("Applying migration").
|
|
name=$(grep -oE '`[0-9]{14}_[A-Za-z0-9_]+`' "$MIGRATE_LOG" | tr -d '`' | tail -n1 || true)
|
|
fi
|
|
case "$name" in
|
|
[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*)
|
|
# Reject anything outside the migration-name charset (defence in depth).
|
|
case "$name" in
|
|
*[!A-Za-z0-9_]*) name="" ;;
|
|
esac
|
|
;;
|
|
*) name="" ;;
|
|
esac
|
|
printf '%s' "$name"
|
|
}
|
|
|
|
log_has() {
|
|
grep -q "$1" "$MIGRATE_LOG"
|
|
}
|
|
|
|
is_transaction_block_failure() {
|
|
log_has 'P3018' &&
|
|
{ log_has 'cannot run inside a transaction block' || log_has '25001'; }
|
|
}
|
|
|
|
is_stuck_failed_migration() {
|
|
log_has 'P3009'
|
|
}
|
|
|
|
migration_sql_path() {
|
|
printf '%s/%s/migration.sql' "$MIGRATIONS_DIR" "$1"
|
|
}
|
|
|
|
# Guard: only auto-recover the idempotent drop-then-create CONCURRENTLY shape.
|
|
# A bare CREATE (no DROP) or any non-CONCURRENTLY migration fails this and is
|
|
# never auto-resolved.
|
|
is_recoverable_concurrently_migration() {
|
|
sql="$1"
|
|
[ -f "$sql" ] &&
|
|
grep -Eqi 'DROP[[:space:]]+INDEX[[:space:]]+CONCURRENTLY' "$sql" &&
|
|
grep -Eqi 'CREATE[[:space:]]+INDEX[[:space:]]+CONCURRENTLY' "$sql"
|
|
}
|
|
|
|
# The intentionally-fail-loud shape: a bare CREATE INDEX CONCURRENTLY with no
|
|
# DROP. Not auto-recovered (an interrupted build leaves an INVALID index that
|
|
# must be handled deliberately), but distinguished from a plain non-index
|
|
# migration so the loud failure can print the correct manual steps.
|
|
is_bare_create_concurrently() {
|
|
sql="$1"
|
|
[ -f "$sql" ] &&
|
|
grep -Eqi 'CREATE[[:space:]]+(UNIQUE[[:space:]]+)?INDEX[[:space:]]+CONCURRENTLY' "$sql" &&
|
|
! grep -Eqi 'DROP[[:space:]]+INDEX[[:space:]]+CONCURRENTLY' "$sql"
|
|
}
|
|
|
|
# One statement per line; multi-line statements collapsed to a single line
|
|
# (index DDL is whitespace-insensitive and has no line-spanning literals).
|
|
split_statements() {
|
|
awk '
|
|
/^[[:space:]]*--/ { next }
|
|
{
|
|
sub(/^[[:space:]]+/, ""); sub(/[[:space:]]+$/, "")
|
|
if ($0 == "") next
|
|
stmt = (stmt == "" ? $0 : stmt " " $0)
|
|
if ($0 ~ /;$/) { print stmt; stmt = "" }
|
|
}
|
|
END { if (stmt != "") print stmt }
|
|
' "$1"
|
|
}
|
|
|
|
# Single-quote a value for safe shell paste (a'b -> 'a'\''b').
|
|
shell_quote() {
|
|
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")"
|
|
}
|
|
|
|
print_manual_recovery() {
|
|
name="$1"
|
|
sql="$2"
|
|
echo ""
|
|
echo "Manual recovery for $name (copy-paste):"
|
|
echo " npx prisma migrate resolve --rolled-back $(shell_quote "$name")"
|
|
split_statements "$sql" > "$STMT_FILE"
|
|
while IFS= read -r stmt; do
|
|
[ -n "$stmt" ] || continue
|
|
echo " printf '%s\\n' $(shell_quote "$stmt") | npx prisma db execute --schema $SCHEMA --stdin"
|
|
done < "$STMT_FILE"
|
|
echo " npx prisma migrate resolve --applied $(shell_quote "$name") # only after every statement above succeeds"
|
|
}
|
|
|
|
# Copy-paste recovery for an interrupted bare CREATE INDEX CONCURRENTLY. An
|
|
# aborted concurrent build leaves an INVALID index of the same name, so a plain
|
|
# re-run of the migration fails with "already exists"; the INVALID index must be
|
|
# dropped first. Then clear the failed record so the next deploy re-applies the
|
|
# migration natively (single-statement CONCURRENTLY needs no out-of-band run).
|
|
print_bare_create_recovery() {
|
|
name="$1"
|
|
sql="$2"
|
|
idx="$(grep -Ei 'CREATE[[:space:]]+(UNIQUE[[:space:]]+)?INDEX[[:space:]]+CONCURRENTLY' "$sql" |
|
|
grep -oE '"[^"]+"' | head -n1 | tr -d '"')"
|
|
echo ""
|
|
echo "Manual recovery for $name (interrupted bare CREATE INDEX CONCURRENTLY, copy-paste):"
|
|
if [ -n "$idx" ]; then
|
|
echo " printf '%s\\n' $(shell_quote "DROP INDEX CONCURRENTLY IF EXISTS \"$idx\";") | npx prisma db execute --schema $SCHEMA --stdin"
|
|
else
|
|
echo " # Drop any INVALID index left by the interrupted build (see $sql), e.g.:"
|
|
echo " # DROP INDEX CONCURRENTLY IF EXISTS \"<index_name>\";"
|
|
fi
|
|
echo " npx prisma migrate resolve --rolled-back $(shell_quote "$name")"
|
|
echo " # Then re-run the deploy; $name re-applies natively."
|
|
}
|
|
|
|
# Print copy-paste recovery for an INTERRUPTED CONCURRENTLY index build (a
|
|
# migrate step aborted by a timeout SIGTERM, OOM, or external stop), if — and
|
|
# only if — the failing migration is one. An aborted CONCURRENTLY build leaves
|
|
# an INVALID index of the target name, so a plain re-run cannot rebuild it. This
|
|
# only ever prints guidance; it never resolves a migration, and a non-index or
|
|
# unidentifiable failure prints nothing. Safe to call from any failure branch.
|
|
emit_interrupted_recovery_hint() {
|
|
hint_name="$(parse_failing_migration)"
|
|
[ -n "$hint_name" ] || return 0
|
|
hint_sql="$(migration_sql_path "$hint_name")"
|
|
if is_bare_create_concurrently "$hint_sql"; then
|
|
print_bare_create_recovery "$hint_name" "$hint_sql"
|
|
elif is_recoverable_concurrently_migration "$hint_sql"; then
|
|
echo ""
|
|
echo "$hint_name is an auto-recoverable CONCURRENTLY migration; re-run the deploy to finish it (the re-run drops any INVALID index and rebuilds)."
|
|
fi
|
|
}
|
|
|
|
fail_loudly() {
|
|
echo ""
|
|
echo "ERROR: $1"
|
|
echo " Not auto-recovered. Investigate the migration; do not blindly"
|
|
echo " mark it applied. See https://pris.ly/d/migrate-resolve"
|
|
exit "${2:-$MIGRATE_STATUS}"
|
|
}
|
|
|
|
recover_migration() {
|
|
name="$1"
|
|
sql="$2"
|
|
|
|
echo ""
|
|
echo "==> Recovering $name outside Prisma migrate (CONCURRENTLY cannot run in a transaction)..."
|
|
|
|
set +e
|
|
with_timeout npx prisma migrate resolve --rolled-back "$name"
|
|
resolve_status=$?
|
|
set -e
|
|
if [ "$resolve_status" -ne 0 ]; then
|
|
echo " Migration $name was not in a failed state; continuing with out-of-band SQL."
|
|
fi
|
|
|
|
split_statements "$sql" > "$STMT_FILE"
|
|
exec_rc=0
|
|
while IFS= read -r stmt; do
|
|
[ -n "$stmt" ] || continue
|
|
echo " -> $stmt"
|
|
if ! printf '%s\n' "$stmt" | with_timeout npx prisma db execute --schema "$SCHEMA" --stdin; then
|
|
exec_rc=1
|
|
break
|
|
fi
|
|
done < "$STMT_FILE"
|
|
if [ "$exec_rc" -ne 0 ]; then
|
|
echo ""
|
|
echo "ERROR: an out-of-band statement for $name failed."
|
|
echo " $name was NOT marked applied (schema may be incomplete)."
|
|
print_manual_recovery "$name" "$sql"
|
|
exit 1
|
|
fi
|
|
|
|
with_timeout npx prisma migrate resolve --applied "$name"
|
|
echo " $name applied out-of-band and marked applied."
|
|
}
|
|
|
|
attempt=0
|
|
while :; do
|
|
run_migrate_deploy
|
|
if [ "$MIGRATE_STATUS" -eq 0 ]; then
|
|
exit 0
|
|
fi
|
|
if [ "$MIGRATE_STATUS" -eq 124 ]; then
|
|
# This branch also catches a normalized 143 (with_timeout maps BusyBox's
|
|
# SIGTERM exit to 124), i.e. the incident's own signal. A timed-out/aborted
|
|
# CONCURRENTLY build leaves an INVALID index + a failed record, so raising
|
|
# the timeout alone will not let a plain re-run rebuild it — surface the
|
|
# drop-index recovery here so the FIRST failure is actionable.
|
|
emit_interrupted_recovery_hint
|
|
fail_loudly "prisma migrate deploy timed out after ${STEP_TIMEOUT}s (a long-running transaction may be blocking CREATE/DROP INDEX CONCURRENTLY). Clear the blocker, then raise MIGRATION_TIMEOUT (it forwards to MIGRATE_STEP_TIMEOUT) and re-run." 1
|
|
fi
|
|
|
|
attempt=$((attempt + 1))
|
|
if [ "$attempt" -ge "$MAX_ATTEMPTS" ]; then
|
|
fail_loudly "prisma migrate deploy still failing after $attempt attempts."
|
|
fi
|
|
|
|
if ! is_transaction_block_failure && ! is_stuck_failed_migration; then
|
|
# A non-P3018/P3009 exit is usually a genuine error (bad SQL, unreachable
|
|
# DB), but OOM (137) or another non-timeout kill can also abort an in-flight
|
|
# CONCURRENTLY build before Prisma records the failure. (A timeout SIGTERM is
|
|
# normalized to 124 above and handled there — it never reaches here.) Surface
|
|
# the drop-index recovery when the in-flight migration is a CONCURRENTLY
|
|
# build so the FIRST failure is actionable (deploy.sh promises "recovery
|
|
# steps above"); guidance only, never auto-resolves.
|
|
emit_interrupted_recovery_hint
|
|
fail_loudly "prisma migrate deploy failed (exit $MIGRATE_STATUS)."
|
|
fi
|
|
|
|
name="$(parse_failing_migration)"
|
|
if [ -z "$name" ]; then
|
|
fail_loudly "could not determine the failing migration from Prisma output."
|
|
fi
|
|
|
|
sql="$(migration_sql_path "$name")"
|
|
if ! is_recoverable_concurrently_migration "$sql"; then
|
|
if is_bare_create_concurrently "$sql"; then
|
|
print_bare_create_recovery "$name" "$sql"
|
|
fi
|
|
fail_loudly "$name is not a recoverable drop-then-create CONCURRENTLY index migration (a bare CREATE is intentionally fail-loud); refusing to auto-resolve."
|
|
fi
|
|
|
|
if [ "$name" = "$LAST_RECOVERED" ]; then
|
|
echo ""
|
|
echo "ERROR: $name failed again after out-of-band recovery."
|
|
print_manual_recovery "$name" "$sql"
|
|
exit 1
|
|
fi
|
|
|
|
recover_migration "$name" "$sql"
|
|
LAST_RECOVERED="$name"
|
|
echo ""
|
|
echo "==> Retrying prisma migrate deploy after recovering $name..."
|
|
done
|