super-productivity/packages/super-sync-server/tests/migration-sql.spec.ts
Johannes Millan 5ed1f2c5d0
fix(supersync): prevent migrator advisory-lock leak on deploy timeout (#9126)
A timed-out `docker compose run` in deploy.sh SIGTERMs the compose CLI but
can leave the migrator container (and its DB connection) running detached,
which keeps Prisma's session-level migration advisory lock held and wedges
the next deploy with P1002.

- deploy.sh: run_migrator() names each one-off migrator container from a
  single MIGRATOR_NAME_PREFIX and force-removes it (inline + $$-scoped EXIT
  sweep) so a timed-out run can't orphan it and leak the lock; add timeout -k.
- migrate-deploy.sh: recognize P1002 advisory-lock timeouts and print
  copy-paste diagnosis + cleanup guidance instead of a generic exit 1; never
  auto-terminate a backend (an active CONCURRENTLY build legitimately holds it).
- tests: P1002 fake-npx case + non-vacuous recovery test; assert the
  name/sweep-filter prefix single-source-of-truth invariant.
- env.example: document the recommended higher MIGRATION_TIMEOUT and the new
  force-remove-on-timeout behavior.
2026-07-17 20:18:54 +02:00

394 lines
19 KiB
TypeScript

import { readdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
const currentDir = dirname(fileURLToPath(import.meta.url));
const migrationsDir = join(currentDir, '../prisma/migrations');
const readMigration = (name: string): string =>
readFileSync(join(migrationsDir, name, 'migration.sql'), 'utf8');
const allMigrationSql = (): string =>
readdirSync(migrationsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => readMigration(entry.name))
.join('\n');
describe('performance migrations', () => {
it('adds the entity sequence index without a blocking or destructive migration', () => {
const migrationSql = readFileSync(
join(
currentDir,
'../prisma/migrations/20260511000000_add_entity_sequence_index/migration.sql',
),
'utf8',
);
expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY');
expect(migrationSql).not.toMatch(/\bIF\s+NOT\s+EXISTS\b/i);
expect(migrationSql).toContain(
'"operations_user_id_entity_type_entity_id_server_seq_idx"',
);
expect(migrationSql).toContain(
'ON "operations"("user_id", "entity_type", "entity_id", "server_seq")',
);
expect(migrationSql).not.toMatch(/\bDROP\s+INDEX\b/i);
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds partial full-state sequence index and drops redundant indexes', () => {
const migrationSql = readFileSync(
join(
currentDir,
'../prisma/migrations/20260512000000_add_full_state_sequence_index_drop_redundant_indexes/migration.sql',
),
'utf8',
);
expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY');
expect(migrationSql).toContain('"operations_user_id_full_state_server_seq_idx"');
expect(migrationSql).toContain('ON "operations"("user_id", "server_seq")');
expect(migrationSql).toContain(
`WHERE "op_type" IN ('SYNC_IMPORT', 'BACKUP_IMPORT', 'REPAIR')`,
);
expect(migrationSql).toContain(
'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_op_type_idx"',
);
expect(migrationSql).toContain(
'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_entity_type_entity_id_idx"',
);
expect(migrationSql).toContain(
'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_server_seq_idx"',
);
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bALTER\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds partial encrypted-op sequence index concurrently', () => {
const migrationSql = readFileSync(
join(
currentDir,
'../prisma/migrations/20260514000000_add_encrypted_ops_partial_index/migration.sql',
),
'utf8',
);
expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY');
expect(migrationSql).toContain(
'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_server_seq_encrypted_idx"',
);
expect(migrationSql).toContain('"operations_user_id_server_seq_encrypted_idx"');
expect(migrationSql).toContain('ON "operations"("user_id", "server_seq")');
expect(migrationSql).toContain('WHERE "is_payload_encrypted" = true');
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds operation payload_bytes as a metadata-only column (no table rewrite)', () => {
const migrationSql = readFileSync(
join(
currentDir,
'../prisma/migrations/20260514000001_add_operation_payload_bytes/migration.sql',
),
'utf8',
);
// ADD COLUMN ... NOT NULL DEFAULT <constant> is a metadata-only operation on
// PostgreSQL 11+ (the default is stored in pg_attribute, no table rewrite).
// These guards lock in the fast path: a future edit to a volatile/expression
// default or a separate UPDATE backfill would rewrite/lock a 100M-row table.
expect(migrationSql).toMatch(
/ALTER TABLE "operations"\s+ADD COLUMN "payload_bytes" BIGINT NOT NULL DEFAULT 0/i,
);
expect(migrationSql).not.toMatch(/\bUPDATE\b/i);
expect(migrationSql).not.toMatch(/\bUSING\b/i);
expect(migrationSql).not.toMatch(/DEFAULT\s+(?!0\b)/i);
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds the payload_bytes unbackfilled partial index concurrently', () => {
const migrationSql = readFileSync(
join(
currentDir,
'../prisma/migrations/20260514000002_add_payload_bytes_unbackfilled_index/migration.sql',
),
'utf8',
);
expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY');
expect(migrationSql).toContain(
'DROP INDEX CONCURRENTLY IF EXISTS "operations_payload_bytes_unbackfilled_idx"',
);
expect(migrationSql).toContain('"operations_payload_bytes_unbackfilled_idx"');
expect(migrationSql).toContain('ON "operations"("user_id", "id")');
// Partial predicate must match the boot self-check / quota probe
// (payload_bytes = 0) so the index drains to empty post-backfill.
expect(migrationSql).toContain('WHERE "payload_bytes" = 0');
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bALTER\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds the operation entity_ids column as a metadata-only column (no table rewrite)', () => {
const migrationSql = readMigration('20260613000000_add_operation_entity_ids');
// Same fast-path guards as payload_bytes: ADD COLUMN with a constant default is
// metadata-only on PG 11+. A future edit to an expression default or a separate
// UPDATE backfill would rewrite/lock a 100M-row table — #8334 is forward-only by
// design (pre-migration rows fall back to the scalar entity_id), so no backfill.
expect(migrationSql).toMatch(
/ALTER TABLE "operations"\s+ADD COLUMN "entity_ids" TEXT\[\] NOT NULL DEFAULT '\{\}'/i,
);
expect(migrationSql).not.toMatch(/\bUPDATE\b/i);
expect(migrationSql).not.toMatch(/\bUSING\b/i);
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('adds the entity_ids GIN index concurrently as a single native-apply statement', () => {
const migrationSql = readMigration(
'20260613000001_add_operation_entity_ids_gin_index',
);
expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY');
expect(migrationSql).toContain('"operations_entity_ids_gin"');
expect(migrationSql).toContain('USING GIN ("entity_ids")');
// Bare CREATE (no IF NOT EXISTS / no drop-then-create): an interrupted concurrent
// build must fail loudly, matching the 20260511000000 precedent.
expect(migrationSql).not.toMatch(/\bIF\s+NOT\s+EXISTS\b/i);
expect(migrationSql).not.toMatch(/\bALTER\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i);
expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i);
});
it('runs migrations before replacing the app during compose deploys', () => {
const deployScript = readFileSync(join(currentDir, '../scripts/deploy.sh'), 'utf8');
const runtimeMigrateScript = readFileSync(
join(currentDir, '../scripts/migrate-deploy.sh'),
'utf8',
);
const buildAndPushScript = readFileSync(
join(currentDir, '../scripts/build-and-push.sh'),
'utf8',
);
const dockerfile = readFileSync(join(currentDir, '../Dockerfile'), 'utf8');
const composeFile = readFileSync(join(currentDir, '../docker-compose.yml'), 'utf8');
const composeBuildFile = readFileSync(
join(currentDir, '../docker-compose.build.yml'),
'utf8',
);
const helmDeployment = readFileSync(
join(currentDir, '../helm/supersync/templates/deployment.yaml'),
'utf8',
);
const dockerWorkflow = readFileSync(
join(currentDir, '../../../.github/workflows/supersync-docker.yml'),
'utf8',
);
const migrationCommand = 'sh scripts/migrate-deploy.sh';
const startCommand = 'up -d --wait --wait-timeout "$WAIT_TIMEOUT"';
const externalDbStartCommand =
'up -d --wait --wait-timeout "$WAIT_TIMEOUT" --no-deps supersync caddy';
expect(deployScript).toContain('POSTGRES_WAIT_TIMEOUT');
expect(deployScript).toContain('load_env_value()');
expect(deployScript).toContain('POSTGRES_SERVICE="${POSTGRES_SERVICE-postgres}"');
expect(deployScript).toContain('@db:5432');
expect(deployScript).toContain('@postgres:5432');
expect(deployScript).toContain('SUPER_SYNC_DEPLOY_REEXECED');
expect(deployScript).toMatch(/git hash-object/);
expect(deployScript).toMatch(/exec\s+"\$DEPLOY_SCRIPT_FILE"/);
expect(deployScript).toContain('verify_supersync_image_revision()');
expect(deployScript).toContain('supersync_image_source_revision()');
expect(deployScript).toContain('assert_clean_supersync_image_inputs()');
expect(deployScript).toContain('git log -1 --format=%H');
expect(deployScript).toContain('../../.dockerignore');
expect(deployScript).toContain('git ls-files --others --exclude-standard');
expect(deployScript).toContain('packages/shared-schema');
expect(deployScript).toContain('Refusing to build a labeled supersync image');
expect(deployScript).toContain('SUPERSYNC_SKIP_IMAGE_REVISION_CHECK');
expect(deployScript).toContain('org.opencontainers.image.revision');
expect(deployScript).toContain('config --format json');
expect(deployScript).toContain('.services.supersync.image // empty');
expect(deployScript).toContain('jq is required');
expect(deployScript).toContain('docker compose config --format json failed');
expect(deployScript).toContain('docker image inspect');
expect(deployScript).toContain('run --rm --no-deps --interactive=false -T');
expect(deployScript).toContain('-e "MIGRATE_STEP_TIMEOUT=$MIGRATE_STEP_TIMEOUT"');
// One-off migrator containers are named per-deploy and force-removed (inline
// + an EXIT trap) so a timed-out `docker compose run` cannot orphan the
// container and leak Prisma's advisory lock into the next deploy (P1002).
expect(deployScript).toContain('run_migrator()');
expect(deployScript).toContain('--name "$name"');
// Container name and EXIT-sweep filter must derive from ONE prefix, else a
// rename in one spot silently breaks the sweep backstop this PR relies on.
expect(deployScript).toContain('MIGRATOR_NAME_PREFIX="supersync-migrator-$$"');
expect(deployScript).toContain('local name="${MIGRATOR_NAME_PREFIX}-$RANDOM"');
expect(deployScript).toContain('--filter "name=${MIGRATOR_NAME_PREFIX}-"');
expect(deployScript).toContain('docker rm -f "$name"');
expect(deployScript).toContain('trap cleanup_migrator_containers EXIT');
// The host forwards its migration budget into the image so the in-image
// per-step timeout can't silently cap a large MIGRATION_TIMEOUT at the
// image default (1800s) and kill a slow CREATE INDEX CONCURRENTLY early.
expect(deployScript).toContain(
'MIGRATE_STEP_TIMEOUT="${MIGRATE_STEP_TIMEOUT:-$((MIGRATION_TIMEOUT',
);
expect(deployScript).toContain('prisma db execute');
expect(deployScript).toContain(migrationCommand);
expect(deployScript).toContain('Migrator container started');
expect(deployScript).toContain('prisma db execute --schema prisma/schema.prisma');
// Recovery now lives in the in-image scripts/migrate-deploy.sh. The host
// must NOT re-hardcode migration names or index DDL: that lockstep
// host/image coupling is exactly what caused the production skew bug.
expect(deployScript).not.toMatch(/_INDEX_MIGRATION=/);
expect(deployScript).not.toContain('run_concurrent_index_sql');
expect(deployScript).not.toContain('CREATE INDEX CONCURRENTLY "operations');
// Host still owns the timeout + exit-code policy around the migrator: it
// passes MIGRATION_TIMEOUT into run_migrator, which wraps the container run
// in `timeout` and force-removes the container afterward.
expect(deployScript).toContain('run_migrator "$MIGRATION_TIMEOUT"');
expect(deployScript).toContain('timeout -k 30 "$run_timeout"');
expect(deployScript).toContain('prisma migrate deploy timed out');
expect(deployScript).toContain('database migrations failed (exit $MIGRATE_STATUS)');
expect(deployScript).toContain(externalDbStartCommand);
expect(deployScript).toContain('RUN_MIGRATIONS_ON_STARTUP');
expect(deployScript.indexOf(migrationCommand)).toBeLessThan(
deployScript.indexOf(startCommand),
);
expect(dockerfile).toContain('ARG VCS_REF=unknown');
expect(dockerfile).toContain('LABEL org.opencontainers.image.revision=$VCS_REF');
expect(dockerfile).toContain('RUN_MIGRATIONS_ON_STARTUP');
expect(dockerfile).toContain('sh scripts/migrate-deploy.sh');
expect(dockerfile).toContain('NODE_OPTIONS=--max-old-space-size=576');
expect(composeBuildFile).toContain('VCS_REF: ${SUPERSYNC_BUILD_SHA:-local}');
expect(buildAndPushScript).toContain('supersync_image_source_revision()');
expect(buildAndPushScript).toContain('assert_clean_supersync_image_inputs');
expect(buildAndPushScript).toContain('git -C "$REPO_ROOT" log -1 --format=%H');
expect(buildAndPushScript).toContain('.dockerignore');
expect(buildAndPushScript).toContain('git -C "$REPO_ROOT" ls-files --others');
expect(buildAndPushScript).toContain('--build-arg "VCS_REF=$VCS_REF"');
expect(dockerWorkflow).toContain('push:');
expect(dockerWorkflow).toContain('branches:');
expect(dockerWorkflow).toContain('- master');
expect(dockerWorkflow).toContain('fetch-depth: 0');
expect(dockerWorkflow).toContain('.dockerignore');
expect(dockerWorkflow).toContain('packages/super-sync-server/**');
expect(dockerWorkflow).toContain('Resolve image source revision');
expect(dockerWorkflow).toContain('Could not resolve SuperSync image source revision');
expect(dockerWorkflow).toContain('revision=$revision');
expect(dockerWorkflow).toContain('VCS_REF=${{ steps.source-ref.outputs.revision }}');
expect(dockerWorkflow).not.toContain('labels: ${{ steps.meta.outputs.labels }}');
expect(helmDeployment).toContain('sh scripts/migrate-deploy.sh');
// Architectural invariant (the actual bug class): the generic runtime
// script must NOT hardcode any migration name or index DDL — that lockstep
// coupling is what went stale and broke the production deploy. Behavioral
// coverage of the recovery logic lives in migrate-deploy-script.spec.ts.
expect(runtimeMigrateScript).toContain('npx prisma migrate deploy');
expect(runtimeMigrateScript).not.toMatch(/_INDEX_MIGRATION=/);
expect(runtimeMigrateScript).not.toContain(
'operations_user_id_server_seq_encrypted_idx',
);
expect(runtimeMigrateScript).not.toContain(
'operations_payload_bytes_unbackfilled_idx',
);
expect(runtimeMigrateScript).not.toContain(
'operations_user_id_full_state_server_seq_idx',
);
expect(composeFile).toContain(
'RUN_MIGRATIONS_ON_STARTUP=${RUN_MIGRATIONS_ON_STARTUP:-false}',
);
expect(composeFile).toContain(
'SUPERSYNC_PAYLOAD_BYTES_BACKFILL_COMPLETE=${SUPERSYNC_PAYLOAD_BYTES_BACKFILL_COMPLETE:-false}',
);
expect(composeFile).toContain(
'psql -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" -c "SELECT 1"',
);
expect(composeFile).toContain('aliases:');
expect(composeFile).toContain('- db');
});
it('backfills operation payload bytes with per-user batched updates', () => {
const script = readFileSync(
join(currentDir, '../scripts/migrate-payload-bytes.ts'),
'utf8',
);
const packageJson = readFileSync(join(currentDir, '../package.json'), 'utf8');
expect(script).toContain('SELECT DISTINCT user_id');
// Batch size sized for throughput: a tiny batch made a 100M-row backfill take
// tens of hours, prolonging the slow octet_length() quota fallback window.
expect(script).toContain('const DEFAULT_BATCH_SIZE = 500');
expect(script).toContain('const MAX_BATCH_SIZE = 1000');
// The override is still clamped so a fat-fingered value cannot OOM the
// Node process building the VALUES string.
expect(script).toContain('Math.min(parsed, MAX_BATCH_SIZE)');
expect(script).toContain('userId,');
expect(script).toContain('FROM (VALUES ${values}) AS v(id, bytes)');
expect(script).toContain('SET payload_bytes = v.bytes');
expect(script).toContain('storage_used_bytes = usage.total_bytes');
expect(packageJson).toContain(
'"migrate-payload-bytes": "node dist/scripts/migrate-payload-bytes.js"',
);
expect(packageJson).toContain(
'"migrate-payload-bytes:dev": "ts-node scripts/migrate-payload-bytes.ts"',
);
expect(script).not.toContain('prisma.operation.update({');
});
});
// Regression coverage for issue #8187: the migration chain must be able to
// create a fresh database on its own, and must stay in sync with schema.prisma.
describe('schema bootstrap and drift (#8187)', () => {
const migrationNames = (): string[] =>
readdirSync(migrationsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
it('starts with a baseline that creates the base tables (no ALTER on a missing table)', () => {
const sql = readMigration('0_init');
// migrate deploy applies migrations in lexicographic order; the baseline
// must sort before the first incremental (ALTER-only) migration so the
// tables those migrations ALTER actually exist on a fresh database.
expect(migrationNames()[0]).toBe('0_init');
for (const table of ['users', 'operations', 'user_sync_state', 'sync_devices']) {
expect(sql).toContain(`CREATE TABLE "${table}"`);
}
});
it('adds the magic-link login_token columns and index that schema.prisma requires', () => {
const sql = readMigration('20260601000000_add_login_token');
expect(sql).toMatch(/ADD COLUMN IF NOT EXISTS "login_token" TEXT/i);
expect(sql).toMatch(/ADD COLUMN IF NOT EXISTS "login_token_expires_at" BIGINT/i);
expect(sql).toMatch(
/CREATE INDEX IF NOT EXISTS "users_login_token_idx" ON "users"\("login_token"\)/i,
);
});
it('keeps every @map column in schema.prisma backed by a migration', () => {
const schema = readFileSync(join(currentDir, '../prisma/schema.prisma'), 'utf8');
const migrations = allMigrationSql();
// The #8187 root cause was a column declared in schema.prisma (login_token)
// with no migration creating it, so a migrate-only database crashed at
// runtime with `column users.login_token does not exist`. Guard the whole
// bug class: every `@map("col")` (single @, so model `@@map` table names are
// excluded by the lookbehind) must appear as a quoted identifier in some
// migration. Quoting avoids substring matches (e.g. "login_token" must not
// be satisfied by "login_token_expires_at").
const mappedColumns = [...schema.matchAll(/(?<!@)@map\("([^"]+)"\)/g)].map(
(match) => match[1],
);
expect(mappedColumns.length).toBeGreaterThan(0);
const missing = mappedColumns.filter((column) => !migrations.includes(`"${column}"`));
expect(missing).toEqual([]);
});
});