diff --git a/packages/super-sync-server/docs/testing-guide.md b/packages/super-sync-server/docs/testing-guide.md new file mode 100644 index 0000000000..800fa2fc29 --- /dev/null +++ b/packages/super-sync-server/docs/testing-guide.md @@ -0,0 +1,441 @@ +# SuperSync Encryption-at-Rest Testing Guide + +This guide covers Phase 2 testing and validation procedures for the LUKS encryption implementation. + +## Overview + +**Purpose**: Validate encryption setup before production migration +**Environment**: Test VM or local development environment +**Duration**: 1-2 days for complete testing +**Prerequisites**: Phase 1 tooling installed and verified + +## Pre-Test Checklist + +- [ ] Test environment available (VM or local) +- [ ] All Phase 1 scripts created and executable +- [ ] Prerequisites verified (`./tools/verify-prerequisites.sh`) +- [ ] AES-NI support confirmed (`grep aes /proc/cpuinfo`) +- [ ] Docker and Docker Compose installed +- [ ] Sufficient disk space (3x database size minimum) + +## Test 1: Setup Validation + +**Objective**: Verify LUKS volume creation and unlock procedures + +### Steps + +```bash +# 1. Create test encrypted volume (10GB for testing) +cd /opt/supersync/packages/super-sync-server +sudo ./tools/setup-encrypted-volume.sh --size 10G --name pg-data-encrypted-test + +# 2. Verify volume is created and mounted +sudo cryptsetup status pg-data-encrypted-test +mountpoint /mnt/pg-data-encrypted +ls -la /mnt/pg-data-encrypted + +# 3. Test unlock with operational passphrase +sudo umount /mnt/pg-data-encrypted +sudo cryptsetup luksClose pg-data-encrypted-test +sudo ./tools/unlock-encrypted-volume.sh pg-data-encrypted-test + +# 4. Test unlock with recovery passphrase +sudo umount /mnt/pg-data-encrypted +sudo cryptsetup luksClose pg-data-encrypted-test +sudo ./tools/unlock-encrypted-volume.sh pg-data-encrypted-test +# Enter recovery passphrase + +# 5. Test with WRONG passphrase (should fail) +sudo umount /mnt/pg-data-encrypted +sudo cryptsetup luksClose pg-data-encrypted-test +sudo cryptsetup luksOpen /var/lib/supersync-encrypted.img pg-data-encrypted-test +# Enter incorrect passphrase - should fail + +# 6. Unlock again for next tests +sudo ./tools/unlock-encrypted-volume.sh pg-data-encrypted-test +``` + +### Success Criteria + +- ✅ Volume creates without errors +- ✅ Both passphrases unlock successfully +- ✅ Wrong passphrase fails cleanly +- ✅ Audit log created: `/var/log/luks-audit.log` +- ✅ LUKS header backup created: `/var/backups/luks-header-*.img.enc` + +## Test 2: Migration Dry Run + +**Objective**: Test full migration procedure with sample data + +### Setup Test Database + +```bash +# 1. Start SuperSync with unencrypted volume +cd /opt/supersync/packages/super-sync-server +docker compose up -d + +# 2. Wait for services to be healthy +docker compose ps +curl http://localhost:1900/health + +# 3. Create test data +# Option A: Manual via API +curl -X POST http://localhost:1900/auth/register \ + -H "Content-Type: application/json" \ + -d '{"email":"test@example.com","password":"test123"}' + +# Option B: Use existing E2E test fixtures +# (if available) + +# 4. Verify data exists +docker exec postgres psql -U supersync supersync -c "\dt" +docker exec postgres psql -U supersync supersync -c "SELECT COUNT(*) FROM users;" +``` + +### Execute Migration + +```bash +# 1. Run migration script +sudo ./tools/migrate-to-encrypted-volume.sh + +# 2. Update Docker Compose configuration +docker compose -f docker-compose.yml -f docker-compose.encrypted.yaml config + +# 3. Start with encrypted volume +docker compose -f docker-compose.yml -f docker-compose.encrypted.yaml up -d + +# 4. Verify migration +sudo ./tools/verify-migration.sh +``` + +### Success Criteria + +- ✅ Pre-migration backup created +- ✅ PostgreSQL stops cleanly +- ✅ All files copied (rsync completes without errors) +- ✅ File counts match (verify-migration.sh passes) +- ✅ Checksums match (no corruption) +- ✅ PostgreSQL starts on encrypted volume +- ✅ Data accessible (same row counts) +- ✅ SuperSync health check passes + +## Test 3: Performance Benchmarking + +**Objective**: Measure encryption overhead + +### Baseline (Unencrypted) + +```bash +# Start with unencrypted volume +docker compose -f docker-compose.yml up -d + +# Wait for healthy +until curl -sf http://localhost:1900/health > /dev/null; do sleep 1; done + +# Measure database operations +echo "=== Baseline Performance ===" > performance-results.txt +date >> performance-results.txt + +# Database benchmark +docker exec postgres pgbench -i -s 10 supersync +docker exec postgres pgbench -c 10 -j 2 -t 1000 supersync | tee -a performance-results.txt + +# I/O metrics +iostat -x 5 12 >> performance-results.txt +``` + +### Encrypted Performance + +```bash +# Start with encrypted volume +docker compose -f docker-compose.yml -f docker-compose.encrypted.yaml up -d + +# Wait for healthy +until curl -sf http://localhost:1900/health > /dev/null; do sleep 1; done + +# Same benchmarks +echo "=== Encrypted Performance ===" >> performance-results.txt +date >> performance-results.txt + +docker exec postgres pgbench -c 10 -j 2 -t 1000 supersync | tee -a performance-results.txt +iostat -x 5 12 >> performance-results.txt +``` + +### Analyze Results + +```bash +# Calculate overhead percentage +cat performance-results.txt + +# Expected with AES-NI: +# - TPS (transactions/sec): within 10% of baseline +# - Latency: +3-10% +# - Disk utilization: +5-10% +``` + +### Success Criteria + +- ✅ Overhead < 10% for database operations +- ✅ Overhead < 15% for bulk operations +- ✅ No errors during benchmark +- ✅ AES-NI acceleration confirmed (check `/proc/crypto`) + +## Test 4: Backup and Restore + +**Objective**: Validate encrypted backup procedures + +### Create Backup Passphrase + +```bash +# Generate backup passphrase +diceware -n 8 | sudo tee /run/secrets/backup_passphrase > /dev/null +sudo chmod 600 /run/secrets/backup_passphrase +``` + +### Test Backup Creation + +```bash +# 1. Create encrypted backup +sudo POSTGRES_CONTAINER=postgres ./tools/backup-encrypted.sh + +# 2. Verify backup file +BACKUP=$(ls -t /var/backups/supersync/*.enc | head -1) +ls -lh "$BACKUP" + +# 3. Verify file is encrypted (not plaintext) +file "$BACKUP" # Should NOT say "text" or "SQL" + +# 4. Check audit log +tail /var/log/backup-audit.log +``` + +### Test Restore + +```bash +# 1. Create test database +docker exec postgres createdb -U supersync backup_test + +# 2. Decrypt and restore +BACKUP=$(ls -t /var/backups/supersync/*.enc | head -1) + +openssl enc -d -aes-256-cbc -pbkdf2 -iter 1000000 \ + -pass file:/run/secrets/backup_passphrase \ + -in "$BACKUP" | \ + gunzip | \ + docker exec -i postgres psql -U supersync backup_test + +# 3. Verify row counts match +PROD_COUNT=$(docker exec postgres psql -U supersync supersync -t -c "SELECT COUNT(*) FROM users;") +TEST_COUNT=$(docker exec postgres psql -U supersync backup_test -t -c "SELECT COUNT(*) FROM users;") + +echo "Production: $PROD_COUNT" +echo "Backup: $TEST_COUNT" + +# 4. Cleanup +docker exec postgres dropdb -U supersync backup_test +``` + +### Test Wrong Passphrase + +```bash +# Should fail cleanly +echo "wrongpassphrase" > /tmp/wrong_pass + +openssl enc -d -aes-256-cbc -pbkdf2 -iter 1000000 \ + -pass file:/tmp/wrong_pass \ + -in "$BACKUP" 2>&1 | grep -i "bad decrypt" + +rm /tmp/wrong_pass +``` + +### Success Criteria + +- ✅ Backup creates without errors +- ✅ Backup file is encrypted +- ✅ Restore succeeds with correct passphrase +- ✅ Row counts match production +- ✅ Restore fails with wrong passphrase +- ✅ Audit log updated + +## Test 5: Rollback Procedure + +**Objective**: Practice emergency rollback + +### Execute Rollback + +```bash +# 1. Document current state +docker exec postgres psql -U supersync supersync -c "SELECT COUNT(*) FROM users;" > pre-rollback-counts.txt + +# 2. Stop services +docker compose down + +# 3. Unmount and close encrypted volume +sudo umount /mnt/pg-data-encrypted +sudo cryptsetup luksClose pg-data-encrypted-test + +# 4. Restore to unencrypted (using pre-migration backup) +BACKUP="/var/backups/supersync/pre-migration-*.sql.gz" + +# Start unencrypted postgres +docker compose -f docker-compose.yml up -d postgres + +# Wait for ready +until docker exec postgres pg_isready -U supersync; do sleep 1; done + +# Drop and recreate database +docker exec postgres psql -U supersync -c "DROP DATABASE IF EXISTS supersync;" +docker exec postgres psql -U supersync -c "CREATE DATABASE supersync;" + +# Restore +gunzip < $BACKUP | docker exec -i postgres psql -U supersync supersync + +# 5. Start all services +docker compose -f docker-compose.yml up -d + +# 6. Verify +docker exec postgres psql -U supersync supersync -c "SELECT COUNT(*) FROM users;" > post-rollback-counts.txt +diff pre-rollback-counts.txt post-rollback-counts.txt + +curl http://localhost:1900/health +``` + +### Success Criteria + +- ✅ Rollback completes without errors +- ✅ Row counts match pre-migration +- ✅ SuperSync health check passes +- ✅ No data loss +- ✅ Rollback time < 30 minutes (documented) + +## Test 6: Backup Rotation + +**Objective**: Validate backup retention policy + +### Setup Test Backups + +```bash +# Create fake backups with different dates +sudo mkdir -p /var/backups/supersync + +# Daily (recent) +for i in {0..10}; do + DATE=$(date -d "$i days ago" +%Y%m%d-%H%M%S) + sudo touch -d "$i days ago" "/var/backups/supersync/supersync-$DATE.sql.gz.enc" +done + +# Weekly (older) +for i in {1..6}; do + DATE=$(date -d "$((i * 7)) days ago" +%Y%m%d-%H%M%S) + sudo touch -d "$((i * 7)) days ago" "/var/backups/supersync/supersync-$DATE.sql.gz.enc" +done + +# Count before rotation +echo "Before: $(find /var/backups/supersync -name '*.enc' | wc -l) backups" +``` + +### Run Rotation + +```bash +# Execute rotation +sudo ./tools/backup-rotate.sh + +# Check results +echo "After: $(find /var/backups/supersync -maxdepth 1 -name '*.enc' | wc -l) daily" +echo "Weekly: $(find /var/backups/supersync/weekly -name '*.enc' | wc -l) weekly" +echo "Monthly: $(find /var/backups/supersync/monthly -name '*.enc' | wc -l) monthly" + +# Verify logs +tail -20 /var/log/backup-rotation.log +``` + +### Success Criteria + +- ✅ Backups > 7 days deleted +- ✅ Weekly backups preserved +- ✅ Monthly backups preserved +- ✅ At least one backup remains +- ✅ Rotation log created + +## Results Template + +Document test results in: `/packages/super-sync-server/docs/phase2-test-results.md` + +```markdown +# Phase 2 Test Results + +**Date**: YYYY-MM-DD +**Tester**: [Name] +**Environment**: [VM/Local/Cloud] + +## Test Results Summary + +| Test | Status | Notes | +| ------------------------ | ------- | ---------------- | +| 1. Setup Validation | ✅ PASS | | +| 2. Migration Dry Run | ✅ PASS | | +| 3. Performance Benchmark | ✅ PASS | Overhead: X% | +| 4. Backup/Restore | ✅ PASS | | +| 5. Rollback | ✅ PASS | Duration: XX min | +| 6. Backup Rotation | ✅ PASS | | + +## Performance Results + +**Baseline (Unencrypted)**: + +- TPS: XXX transactions/sec +- Latency avg: XX ms +- Latency 95th: XX ms + +**Encrypted**: + +- TPS: XXX transactions/sec (-X%) +- Latency avg: XX ms (+X%) +- Latency 95th: XX ms (+X%) + +**Overhead**: X% (within acceptable < 10%) + +## Issues Encountered + +None / [List any issues] + +## Recommendations + +- [ ] Ready for production migration +- [ ] Issues to address: [List] +- [ ] Performance acceptable: YES/NO + +## Sign-Off + +Tested by: [Name] +Date: [Date] +Approved for production: YES/NO +``` + +## Cleanup After Testing + +```bash +# Stop services +docker compose down + +# Remove test volume +sudo umount /mnt/pg-data-encrypted +sudo cryptsetup luksClose pg-data-encrypted-test +sudo rm /var/lib/supersync-encrypted.img + +# Remove test backups +sudo rm -rf /var/backups/supersync/* + +# Clear logs +sudo rm /var/log/luks-audit.log +sudo rm /var/log/backup-audit.log +``` + +## Next Steps + +After successful Phase 2 testing: + +1. Document all test results +2. Review with team +3. Proceed to Phase 3: Migration Planning +4. Schedule production migration window diff --git a/packages/super-sync-server/tools/backup-rotate.sh b/packages/super-sync-server/tools/backup-rotate.sh new file mode 100755 index 0000000000..a20ce61f17 --- /dev/null +++ b/packages/super-sync-server/tools/backup-rotate.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Backup Rotation Script for SuperSync +# Implements retention policy: 7 daily, 4 weekly, 12 monthly + +set -e + +BACKUP_DIR="/var/backups/supersync" +LOG_FILE="/var/log/backup-rotation.log" + +log() { + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE" +} + +# Create log directory if needed +mkdir -p "$(dirname "$LOG_FILE")" + +log "=== Starting backup rotation ===" + +# Create backup directory if it doesn't exist +mkdir -p "$BACKUP_DIR" + +# Count backups before rotation +BEFORE_COUNT=$(find "$BACKUP_DIR" -maxdepth 1 -name "supersync-*.enc" -type f 2>/dev/null | wc -l) +if [ -d "$BACKUP_DIR" ]; then + BEFORE_SIZE=$(du -sh "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo "0") +else + BEFORE_SIZE="0" +fi +log "Before rotation: $BEFORE_COUNT backups, $BEFORE_SIZE total" + +# 1. Daily backups: Keep last 7 days +log "Rotating daily backups (keep 7 days)..." +DELETED_COUNT=0 + +# Find and delete daily backups older than 7 days +while IFS= read -r -d '' file; do + rm -f "$file" + DELETED_COUNT=$((DELETED_COUNT + 1)) +done < <(find "$BACKUP_DIR" -maxdepth 1 -name "supersync-*.enc" -type f -mtime +7 -print0 2>/dev/null) + +log "Deleted $DELETED_COUNT old daily backups" + +# 2. Weekly backups: Keep first backup of each week for 4 weeks +log "Managing weekly backups (keep 4 weeks)..." +WEEKLY_DIR="$BACKUP_DIR/weekly" +mkdir -p "$WEEKLY_DIR" + +# Find first backup from each of the last 4 weeks (Sunday) +for week in $(seq 1 4); do + START_DATE=$(date -d "$week weeks ago sunday" +%Y%m%d 2>/dev/null || date -v-${week}w -v+0d +%Y%m%d 2>/dev/null || echo "") + + if [ -z "$START_DATE" ]; then + log "Warning: Could not calculate date for week $week" + continue + fi + + # Find first backup from that week + WEEKLY_BACKUP=$(find "$BACKUP_DIR" -maxdepth 1 -name "supersync-${START_DATE}*.enc" \ + -type f 2>/dev/null | head -1) + + if [ -n "$WEEKLY_BACKUP" ]; then + BACKUP_NAME=$(basename "$WEEKLY_BACKUP") + if [ ! -f "$WEEKLY_DIR/$BACKUP_NAME" ]; then + cp "$WEEKLY_BACKUP" "$WEEKLY_DIR/" 2>/dev/null || log "Warning: Could not copy weekly backup" + log "Preserved weekly backup: $BACKUP_NAME" + fi + fi +done + +# Remove weekly backups older than 4 weeks (28 days) +find "$WEEKLY_DIR" -name "*.enc" -type f -mtime +28 -delete 2>/dev/null || true + +# 3. Monthly backups: Keep first backup of each month for 12 months +log "Managing monthly backups (keep 12 months)..." +MONTHLY_DIR="$BACKUP_DIR/monthly" +mkdir -p "$MONTHLY_DIR" + +# Find first backup from each of the last 12 months +for month in $(seq 1 12); do + MONTH_DATE=$(date -d "$month months ago" +%Y%m01 2>/dev/null || date -v-${month}m -v1d +%Y%m01 2>/dev/null || echo "") + + if [ -z "$MONTH_DATE" ]; then + log "Warning: Could not calculate date for month $month" + continue + fi + + # Find first backup from that month + MONTHLY_BACKUP=$(find "$BACKUP_DIR" -maxdepth 1 -name "supersync-${MONTH_DATE}*.enc" \ + -type f 2>/dev/null | head -1) + + if [ -n "$MONTHLY_BACKUP" ]; then + BACKUP_NAME=$(basename "$MONTHLY_BACKUP") + if [ ! -f "$MONTHLY_DIR/$BACKUP_NAME" ]; then + cp "$MONTHLY_BACKUP" "$MONTHLY_DIR/" 2>/dev/null || log "Warning: Could not copy monthly backup" + log "Preserved monthly backup: $BACKUP_NAME" + fi + fi +done + +# Remove monthly backups older than 12 months (365 days) +find "$MONTHLY_DIR" -name "*.enc" -type f -mtime +365 -delete 2>/dev/null || true + +# 4. Summary +AFTER_COUNT=$(find "$BACKUP_DIR" -maxdepth 1 -name "supersync-*.enc" -type f 2>/dev/null | wc -l) +AFTER_SIZE=$(du -sh "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo "0") +WEEKLY_COUNT=$(find "$WEEKLY_DIR" -name "*.enc" -type f 2>/dev/null | wc -l) +MONTHLY_COUNT=$(find "$MONTHLY_DIR" -name "*.enc" -type f 2>/dev/null | wc -l) + +log "=== Rotation complete ===" +log "Daily backups: $AFTER_COUNT files" +log "Weekly backups: $WEEKLY_COUNT files" +log "Monthly backups: $MONTHLY_COUNT files" +log "Total size: $AFTER_SIZE (was $BEFORE_SIZE)" + +# 5. Verify at least one backup exists +TOTAL_BACKUPS=$((AFTER_COUNT + WEEKLY_COUNT + MONTHLY_COUNT)) +if [ "$TOTAL_BACKUPS" -eq 0 ]; then + log "ERROR: No backups remaining after rotation!" + + # Try to send alert if mail is available + if command -v mail >/dev/null 2>&1; then + echo "CRITICAL: All backups deleted during rotation on $(hostname) at $(date)" | \ + mail -s "ALERT: Backup Rotation Error" admin@example.com 2>/dev/null || true + fi + + exit 1 +fi + +log "✅ Backup rotation successful (total: $TOTAL_BACKUPS backups)" diff --git a/packages/super-sync-server/tools/migrate-to-encrypted-volume.sh b/packages/super-sync-server/tools/migrate-to-encrypted-volume.sh new file mode 100755 index 0000000000..7b08de17f9 --- /dev/null +++ b/packages/super-sync-server/tools/migrate-to-encrypted-volume.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# SuperSync Migration to Encrypted Volume +# Migrates existing PostgreSQL data to LUKS-encrypted volume +# +# IMPORTANT: Run discovery script first to identify your actual container/volume names +# ./discover-docker-names.sh + +set -e # Exit on error +set -u # Exit on undefined variable + +# Configuration - UPDATE THESE TO MATCH YOUR DEPLOYMENT +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-postgres}" # From docker-compose.yml +POSTGRES_USER="${POSTGRES_USER:-supersync}" +POSTGRES_DB="${POSTGRES_DB:-supersync}" +SOURCE_VOLUME="${SOURCE_VOLUME:-/var/lib/docker/volumes/postgres-data/_data}" # Run: docker volume inspect postgres-data +TARGET_MOUNT="/mnt/pg-data-encrypted" +LOG_FILE="/var/log/supersync-migration-$(date +%Y%m%d-%H%M%S).log" + +log() { + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" | tee -a "$LOG_FILE" +} + +error() { + log "ERROR: $*" + exit 1 +} + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + error "This script must be run as root" +fi + +log "========================================" +log "SuperSync Migration to Encrypted Volume" +log "========================================" +log "" +log "Configuration:" +log " Container: $POSTGRES_CONTAINER" +log " Source: $SOURCE_VOLUME" +log " Target: $TARGET_MOUNT" +log " Log: $LOG_FILE" +log "" + +# Step 1: Pre-migration validation +log "=== Step 1/7: Pre-migration Validation ===" + +# Verify source exists +if [ ! -d "$SOURCE_VOLUME" ]; then + error "Source volume not found: $SOURCE_VOLUME" +fi +log "✅ Source volume exists" + +# Verify target is mounted and writable +if ! mountpoint -q "$TARGET_MOUNT"; then + error "Target not mounted: $TARGET_MOUNT (run unlock-encrypted-volume.sh first)" +fi +log "✅ Target is mounted" + +if [ ! -w "$TARGET_MOUNT" ]; then + error "Target not writable: $TARGET_MOUNT" +fi +log "✅ Target is writable" + +# Calculate disk space requirement +log "Calculating space requirements..." +SOURCE_SIZE=$(du -sb "$SOURCE_VOLUME" | cut -f1) +AVAILABLE_SPACE=$(df -B1 --output=avail "$TARGET_MOUNT" | tail -1) +REQUIRED_SPACE=$((SOURCE_SIZE + SOURCE_SIZE / 5)) # Source + 20% buffer + +log " Source size: $(numfmt --to=iec $SOURCE_SIZE)" +log " Available: $(numfmt --to=iec $AVAILABLE_SPACE)" +log " Required: $(numfmt --to=iec $REQUIRED_SPACE)" + +if [ "$AVAILABLE_SPACE" -lt "$REQUIRED_SPACE" ]; then + error "Insufficient disk space (need $(numfmt --to=iec $REQUIRED_SPACE), have $(numfmt --to=iec $AVAILABLE_SPACE))" +fi +log "✅ Sufficient disk space" + +# Verify container is running +if ! docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then + error "PostgreSQL container not running: $POSTGRES_CONTAINER" +fi +log "✅ PostgreSQL container is running" + +# Step 2: Create pre-migration backup +log "" +log "=== Step 2/7: Creating Pre-Migration Backup ===" +BACKUP_FILE="/var/backups/supersync/pre-migration-$(date +%Y%m%d-%H%M%S).sql.gz" +mkdir -p "$(dirname "$BACKUP_FILE")" + +log "Creating backup: $BACKUP_FILE" +docker exec "$POSTGRES_CONTAINER" pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" | gzip > "$BACKUP_FILE" + +if [ ! -f "$BACKUP_FILE" ] || [ ! -s "$BACKUP_FILE" ]; then + error "Backup creation failed" +fi + +BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1) +log "✅ Backup created: $BACKUP_SIZE" +log " Location: $BACKUP_FILE" + +# Step 3: Stop PostgreSQL cleanly +log "" +log "=== Step 3/7: Stopping PostgreSQL Gracefully ===" +log "Shutting down PostgreSQL (smart mode, 60s timeout)..." + +# Try graceful shutdown first +docker exec "$POSTGRES_CONTAINER" pg_ctl stop -D /var/lib/postgresql/data -m smart -t 60 || { + log "⚠️ Graceful shutdown failed, forcing stop..." + docker exec "$POSTGRES_CONTAINER" pg_ctl stop -D /var/lib/postgresql/data -m fast -t 30 || true +} + +# Stop container +log "Stopping container: $POSTGRES_CONTAINER" +docker compose stop "$POSTGRES_CONTAINER" + +# Verify stopped +if docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then + error "Container still running after stop command" +fi +log "✅ PostgreSQL stopped cleanly" + +# Step 4: Copy data with hard link preservation +log "" +log "=== Step 4/7: Copying Data to Encrypted Volume ===" +log "This may take several minutes depending on database size..." +log "" + +START_TIME=$(date +%s) + +rsync -aH --info=progress2 \ + --log-file="$LOG_FILE" \ + "$SOURCE_VOLUME/" \ + "$TARGET_MOUNT/" || error "rsync failed" + +END_TIME=$(date +%s) +DURATION=$((END_TIME - START_TIME)) + +log "" +log "✅ Data copy completed in $DURATION seconds" + +# Step 5: Verify data integrity +log "" +log "=== Step 5/7: Verifying Data Integrity ===" + +# File count verification +log "Comparing file counts..." +SOURCE_COUNT=$(find "$SOURCE_VOLUME" -type f 2>/dev/null | wc -l) +TARGET_COUNT=$(find "$TARGET_MOUNT" -type f 2>/dev/null | wc -l) + +log " Source files: $SOURCE_COUNT" +log " Target files: $TARGET_COUNT" + +if [ "$SOURCE_COUNT" -ne "$TARGET_COUNT" ]; then + error "File count mismatch (source: $SOURCE_COUNT, target: $TARGET_COUNT)" +fi +log "✅ File counts match" + +# Size verification +log "Comparing total sizes..." +SOURCE_SIZE_VERIFY=$(du -sb "$SOURCE_VOLUME" | cut -f1) +TARGET_SIZE=$(du -sb "$TARGET_MOUNT" | cut -f1) + +log " Source size: $(numfmt --to=iec $SOURCE_SIZE_VERIFY)" +log " Target size: $(numfmt --to=iec $TARGET_SIZE)" + +if [ "$SOURCE_SIZE_VERIFY" -ne "$TARGET_SIZE" ]; then + error "Size mismatch (source: $SOURCE_SIZE_VERIFY, target: $TARGET_SIZE)" +fi +log "✅ Sizes match" + +# PostgreSQL-specific checks +log "Verifying PostgreSQL data files..." +CRITICAL_FILES=("PG_VERSION" "base" "global") +for file in "${CRITICAL_FILES[@]}"; do + if [ ! -e "$TARGET_MOUNT/$file" ]; then + error "Critical file missing: $file" + fi +done +log "✅ All critical PostgreSQL files present" + +# Step 6: Set proper permissions +log "" +log "=== Step 6/7: Setting Permissions ===" +chown -R 999:999 "$TARGET_MOUNT" +chmod 700 "$TARGET_MOUNT" +log "✅ Permissions set (999:999, mode 700)" + +# Step 7: Summary +log "" +log "=========================================" +log "✅ Migration Completed Successfully!" +log "=========================================" +log "" +log "Summary:" +log " Files migrated: $TARGET_COUNT" +log " Total size: $(numfmt --to=iec $TARGET_SIZE)" +log " Duration: $DURATION seconds" +log " Backup: $BACKUP_FILE" +log " Log: $LOG_FILE" +log "" +log "NEXT STEPS:" +log " 1. Update docker-compose to use encrypted volume:" +log " docker compose -f docker-compose.yml -f docker-compose.encrypted.yaml config" +log "" +log " 2. Start PostgreSQL on encrypted volume:" +log " docker compose -f docker-compose.yml -f docker-compose.encrypted.yaml up -d postgres" +log "" +log " 3. Verify database integrity:" +log " ./verify-migration.sh" +log "" +log " 4. Test SuperSync operations" +log "" +log " 5. If issues occur, restore from backup:" +log " gunzip < $BACKUP_FILE | docker exec -i $POSTGRES_CONTAINER psql -U $POSTGRES_USER $POSTGRES_DB" +log "" diff --git a/packages/super-sync-server/tools/test-environment-setup.sh b/packages/super-sync-server/tools/test-environment-setup.sh new file mode 100755 index 0000000000..359c05ebcd --- /dev/null +++ b/packages/super-sync-server/tools/test-environment-setup.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Test Environment Setup for Encryption Testing +# Creates sample database with test data for migration dry-run + +set -e + +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-postgres}" +POSTGRES_USER="${POSTGRES_USER:-supersync}" +POSTGRES_DB="${POSTGRES_DB:-supersync}" + +echo "=== SuperSync Test Environment Setup ===" +echo "Container: $POSTGRES_CONTAINER" +echo "Database: $POSTGRES_DB" +echo "" + +# Check if PostgreSQL is running +if ! docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then + echo "Starting PostgreSQL..." + docker compose up -d postgres + + # Wait for PostgreSQL to be ready + echo "Waiting for PostgreSQL to be ready..." + until docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" >/dev/null 2>&1; do + sleep 1 + done +fi + +echo "✅ PostgreSQL is running" + +# Verify database exists +if ! docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -lqt | cut -d \| -f 1 | grep -qw "$POSTGRES_DB"; then + echo "Creating database: $POSTGRES_DB" + docker exec "$POSTGRES_CONTAINER" createdb -U "$POSTGRES_USER" "$POSTGRES_DB" +fi + +echo "✅ Database exists" + +# Create sample schema (simple test data) +echo "Creating test schema..." +docker exec -i "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" <<'EOF' +-- Drop existing test tables +DROP TABLE IF EXISTS test_operations CASCADE; +DROP TABLE IF EXISTS test_users CASCADE; +DROP TABLE IF EXISTS test_snapshots CASCADE; + +-- Users table +CREATE TABLE test_users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +); + +-- Operations table (simulating sync operations) +CREATE TABLE test_operations ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES test_users(id), + operation_type VARCHAR(50), + payload TEXT, + created_at TIMESTAMP DEFAULT NOW() +); + +-- Snapshots table +CREATE TABLE test_snapshots ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES test_users(id), + snapshot_data BYTEA, + created_at TIMESTAMP DEFAULT NOW() +); + +-- Insert test users +INSERT INTO test_users (email) +SELECT 'user' || generate_series(1, 10) || '@example.com'; + +-- Insert test operations (simulate sync activity) +INSERT INTO test_operations (user_id, operation_type, payload) +SELECT + (random() * 9 + 1)::INTEGER, + CASE (random() * 3)::INTEGER + WHEN 0 THEN 'CREATE' + WHEN 1 THEN 'UPDATE' + ELSE 'DELETE' + END, + 'Test payload data ' || generate_series(1, 1000); + +-- Insert test snapshots +INSERT INTO test_snapshots (user_id, snapshot_data) +SELECT + generate_series(1, 10), + decode(repeat('deadbeef', 1000), 'hex'); -- 4KB of data per snapshot + +-- Create indexes (similar to production) +CREATE INDEX idx_operations_user_id ON test_operations(user_id); +CREATE INDEX idx_operations_created_at ON test_operations(created_at); +CREATE INDEX idx_snapshots_user_id ON test_snapshots(user_id); + +EOF + +echo "✅ Test schema created" + +# Display statistics +echo "" +echo "=== Test Data Summary ===" +docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" -c " + SELECT + (SELECT COUNT(*) FROM test_users) AS users, + (SELECT COUNT(*) FROM test_operations) AS operations, + (SELECT COUNT(*) FROM test_snapshots) AS snapshots; +" + +# Display database size +echo "" +echo "=== Database Size ===" +docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" -c " + SELECT pg_size_pretty(pg_database_size('$POSTGRES_DB')) AS size; +" + +# Display table sizes +echo "" +echo "=== Table Sizes ===" +docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" -c " + SELECT + schemaname || '.' || tablename AS table, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size + FROM pg_tables + WHERE schemaname = 'public' + ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC; +" + +echo "" +echo "=========================================" +echo "✅ Test Environment Ready!" +echo "=========================================" +echo "" +echo "You can now test:" +echo " 1. Migration: sudo ./tools/migrate-to-encrypted-volume.sh" +echo " 2. Backup: sudo ./tools/backup-encrypted.sh" +echo " 3. Verification: sudo ./tools/verify-migration.sh" +echo "" +echo "To clean up test data:" +echo " docker exec $POSTGRES_CONTAINER psql -U $POSTGRES_USER $POSTGRES_DB -c \"DROP TABLE test_operations, test_users, test_snapshots CASCADE;\"" +echo "" diff --git a/packages/super-sync-server/tools/verify-migration.sh b/packages/super-sync-server/tools/verify-migration.sh new file mode 100755 index 0000000000..4d85aeee78 --- /dev/null +++ b/packages/super-sync-server/tools/verify-migration.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Post-Migration Verification Script +# Verifies data integrity after migration to encrypted volume +# ENHANCED: Includes checksum verification to detect silent corruption + +set -e + +# Configuration +SOURCE_DIR="${SOURCE_DIR:-/var/lib/docker/volumes/postgres-data/_data}" +TARGET_DIR="${TARGET_DIR:-/mnt/pg-data-encrypted}" +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-postgres}" +POSTGRES_USER="${POSTGRES_USER:-supersync}" +POSTGRES_DB="${POSTGRES_DB:-supersync}" +CHECKSUM_TEMP="/tmp/migration-checksums-$$" + +echo "=== SuperSync Migration Verification ===" +echo "Source: $SOURCE_DIR" +echo "Target: $TARGET_DIR" +echo "" + +# Create temp directory for checksums +mkdir -p "$CHECKSUM_TEMP" +trap "rm -rf $CHECKSUM_TEMP" EXIT + +# Step 1: File count verification +echo "Step 1/5: Verifying file counts..." +SOURCE_COUNT=$(find "$SOURCE_DIR" -type f 2>/dev/null | wc -l) +TARGET_COUNT=$(find "$TARGET_DIR" -type f 2>/dev/null | wc -l) + +echo " Source files: $SOURCE_COUNT" +echo " Target files: $TARGET_COUNT" + +if [ "$SOURCE_COUNT" -ne "$TARGET_COUNT" ]; then + echo "❌ FAIL: File count mismatch" + exit 1 +fi +echo " ✅ File counts match" + +# Step 2: Total size verification +echo "" +echo "Step 2/5: Verifying total sizes..." +SOURCE_SIZE=$(du -sb "$SOURCE_DIR" 2>/dev/null | cut -f1) +TARGET_SIZE=$(du -sb "$TARGET_DIR" 2>/dev/null | cut -f1) + +echo " Source size: $(numfmt --to=iec $SOURCE_SIZE)" +echo " Target size: $(numfmt --to=iec $TARGET_SIZE)" + +if [ "$SOURCE_SIZE" -ne "$TARGET_SIZE" ]; then + echo "❌ FAIL: Size mismatch" + exit 1 +fi +echo " ✅ Sizes match" + +# Step 3: Checksum verification (detects silent corruption) +echo "" +echo "Step 3/5: Computing and comparing checksums..." +echo " This may take a few minutes for large databases..." + +# Generate source checksums (PostgreSQL critical files only for speed) +echo " Computing source checksums..." +find "$SOURCE_DIR" -type f \( -name "*.conf" -o -name "pg_*" -o -path "*/base/*" \) 2>/dev/null | \ + sort | \ + xargs -r md5sum 2>/dev/null > "$CHECKSUM_TEMP/source.md5" || true + +# Generate target checksums +echo " Computing target checksums..." +find "$TARGET_DIR" -type f \( -name "*.conf" -o -name "pg_*" -o -path "*/base/*" \) 2>/dev/null | \ + sort | \ + xargs -r md5sum 2>/dev/null > "$CHECKSUM_TEMP/target.md5" || true + +# Normalize paths (remove directory prefixes) +sed "s|$SOURCE_DIR/||" "$CHECKSUM_TEMP/source.md5" > "$CHECKSUM_TEMP/source-normalized.md5" +sed "s|$TARGET_DIR/||" "$CHECKSUM_TEMP/target.md5" > "$CHECKSUM_TEMP/target-normalized.md5" + +# Compare checksums +if ! diff -q "$CHECKSUM_TEMP/source-normalized.md5" "$CHECKSUM_TEMP/target-normalized.md5" >/dev/null 2>&1; then + echo "❌ FAIL: Checksum mismatch detected - data corruption or incomplete copy" + echo "" + echo "First 10 differences:" + diff "$CHECKSUM_TEMP/source-normalized.md5" "$CHECKSUM_TEMP/target-normalized.md5" | head -20 + exit 1 +fi + +CHECKSUM_COUNT=$(wc -l < "$CHECKSUM_TEMP/source-normalized.md5") +echo " ✅ Checksums match ($CHECKSUM_COUNT files verified)" + +# Step 4: PostgreSQL-specific integrity checks +echo "" +echo "Step 4/5: Verifying PostgreSQL data integrity..." + +# Check for critical PostgreSQL files +CRITICAL_FILES=( + "PG_VERSION" + "postgresql.conf" + "base" + "global" + "pg_wal" +) + +for file in "${CRITICAL_FILES[@]}"; do + if [ ! -e "$TARGET_DIR/$file" ]; then + echo "❌ FAIL: Critical file missing: $file" + exit 1 + fi +done +echo " ✅ All critical PostgreSQL files present" + +# Verify PG_VERSION matches +if [ -f "$SOURCE_DIR/PG_VERSION" ] && [ -f "$TARGET_DIR/PG_VERSION" ]; then + SOURCE_VERSION=$(cat "$SOURCE_DIR/PG_VERSION") + TARGET_VERSION=$(cat "$TARGET_DIR/PG_VERSION") + + if [ "$SOURCE_VERSION" != "$TARGET_VERSION" ]; then + echo "❌ FAIL: PostgreSQL version mismatch (source: $SOURCE_VERSION, target: $TARGET_VERSION)" + exit 1 + fi + echo " ✅ PostgreSQL version matches: $TARGET_VERSION" +fi + +# Step 5: Database-level verification (if PostgreSQL is running) +echo "" +echo "Step 5/5: Database row count verification..." + +if docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then + echo " PostgreSQL is running - verifying row counts..." + + # Get table list and row counts + TABLES=$(docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" -t -c " + SELECT tablename FROM pg_tables + WHERE schemaname = 'public' + ORDER BY tablename; + " 2>/dev/null | tr -d ' ' | grep -v '^$' || echo "") + + if [ -n "$TABLES" ]; then + TOTAL_ROWS=0 + TABLE_COUNT=0 + + while IFS= read -r table; do + [ -z "$table" ] && continue + + ROW_COUNT=$(docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" "$POSTGRES_DB" -t -c " + SELECT COUNT(*) FROM \"$table\"; + " 2>/dev/null | tr -d ' ' || echo "0") + + TOTAL_ROWS=$((TOTAL_ROWS + ROW_COUNT)) + TABLE_COUNT=$((TABLE_COUNT + 1)) + + echo " $table: $ROW_COUNT rows" + done <<< "$TABLES" + + echo " ✅ Database accessible: $TABLE_COUNT tables, $TOTAL_ROWS total rows" + else + echo " ⚠️ No tables found (may be a fresh database)" + fi +else + echo " ⚠️ PostgreSQL not running - skipping database verification" + echo " Start PostgreSQL and re-run for full verification" +fi + +# Summary +echo "" +echo "=========================================" +echo "✅ Migration Verification PASSED" +echo "=========================================" +echo "" +echo "Summary:" +echo " Files: $TARGET_COUNT" +echo " Size: $(numfmt --to=iec $TARGET_SIZE)" +echo " Checksums verified: $CHECKSUM_COUNT files" +echo " No corruption detected" +echo "" +echo "Migration integrity confirmed!" +echo ""