diff --git a/angular.json b/angular.json index 0744920eae..40d817d320 100644 --- a/angular.json +++ b/angular.json @@ -254,7 +254,8 @@ "src/**/*.html", "e2e/**/*.ts", "electron/**/*.ts", - "packages/sync-core/**/*.ts" + "packages/sync-core/**/*.ts", + "packages/sync-providers/**/*.ts" ] } } diff --git a/docs/long-term-plans/sync-core-extraction-plan.md b/docs/long-term-plans/sync-core-extraction-plan.md index 2bf644f830..02c631fbc1 100644 --- a/docs/long-term-plans/sync-core-extraction-plan.md +++ b/docs/long-term-plans/sync-core-extraction-plan.md @@ -2,10 +2,13 @@ > **Status: In progress - PR 1, PR 2 guardrails/logger adapter work, PR 3a > vector-clock ownership, full-state op classification config, PR 3b pure -> helper slices, PR 4a port contracts, and the current PR 4b small -> orchestration/planning helper set are present on this branch. Remaining -> cleanup is targeted future `SyncLogger` routing for files as they move plus -> deciding whether PR 4c should extract any part of `OperationApplierService`.** +> helper slices, PR 4a port contracts, PR 4b small orchestration/planning +> helpers, and PR 4c's narrow operation replay coordinator are present. PR 5 +> has started with the `@sp/sync-providers` scaffold, provider boundary lint, +> provider-neutral contracts, a credential-store port, and app shims. Remaining +> provider work should move implementations behind provider-package ports while +> keeping app-owned IDs, OAuth routing, config UI, and platform bridges +> app-side.** **Goal:** Carve the sync engine out of `src/app/op-log/` into a reusable, framework-agnostic, **domain-agnostic** `@sp/sync-core` package, plus a sibling @@ -156,27 +159,59 @@ Current extraction state and remaining immediate debt: - PR 4a is present with `packages/sync-core/src/ports.ts`. The package now exports minimal contracts for operation application, action dispatch, remote-apply windows, deferred local action flushing, archive side effects, - and operation-store persistence. The existing Angular services satisfy those - contracts app-side. + operation-store persistence, conflict UI, and sync configuration. The existing + Angular services satisfy these contracts app-side. Conflict UI and sync config + adapters remain app-side and are not used by package orchestration yet. - PR 4b's current small helper set is present: remote-apply crash-safety ordering, upload last-server-sequence planning, full-state snapshot upload follow-up partitioning, download gap/full-state/encryption planning, and file-snapshot hydration skip planning. Provider calls, encryption/decryption, IndexedDB reads, UI, diagnostics, and result assembly remain app-side. +- PR 4c is present with `replayOperationBatch()` in `@sp/sync-core`. It owns + only the strict replay ordering around remote-apply windows, bulk dispatch, + the required event-loop yield, archive side-effect processing, post-sync + cooldown, and deferred local-action flushing. The Angular + `OperationApplierService` still owns NgRx action construction, + operation-to-action conversion, archive predicates, `remoteArchiveDataApplied`, + `Injector` usage, and diagnostics. +- Pre-P5 readiness cleanup is complete for this branch: movable core code no + longer depends on `OpLog`, generic prefix/error/compression helpers are + package-side with app-owned diagnostics, sync-core source comments were + rechecked for SP entity examples, and the core boundary grep was rerun with no + forbidden source imports. +- PR 5 has its initial package boundary: `packages/sync-providers/` exists with + tsup/Vitest scaffolding, root scripts, build-package wiring, the + `@sp/sync-providers` path alias, package-local generated-artifact ignores, + and ESLint restrictions that reject Angular, NgRx, app imports, + `@sp/shared-schema`, sync-core internals, and dynamic imports. +- Provider-neutral contracts now live in `@sp/sync-providers`: generic + string-ID provider contracts, operation-sync response types, file provider + response types, a credential-store port, and the local file-adapter port. The + app-side `provider.interface.ts` and local `file-adapter.interface.ts` remain + compatibility shims that specialize those contracts with `SyncProviderId` and + `PrivateCfgByProviderId`. +- File-based sync envelope contracts now live in `@sp/sync-providers` with + generic host-owned state, compact-operation, and archive payload parameters. + The app-side `file-based-sync.types.ts` shim binds those generics to + `CompactOperation` and `ArchiveModel`. +- Dropbox PKCE code generation now lives in `@sp/sync-providers`, including + the existing WebCrypto-first and `hash-wasm` fallback behavior. The app-side + Dropbox helper path remains a compatibility re-export. Suggested next order: -1. Finish PR 2 documentation and verification. -2. Continue targeted `SyncLogger` routing for files as they become movable. -3. Treat the PR 3b pure conflict-resolution and sync-import slices as complete +1. Treat PR 2 documentation/verification and the targeted `SyncLogger` routing + needed before provider extraction as complete for this branch. Continue + logger routing only when additional files actually move. +2. Treat the PR 3b pure conflict-resolution and sync-import slices as complete for this round. -4. Treat the current PR 4a/4b port and small-helper slices as complete for this - round; only revisit `OperationApplierService` under PR 4c after more - verification. -5. Continue logger/config cleanup before moving app error classes; prefix - parsing/formatting, generic error-message extraction, and generic - compression helpers now have package-side helpers with app-owned diagnostics. -6. Defer provider extraction until core boundaries and PR 4c are settled. +3. Treat the current PR 4a/4b/4c port, small-helper, and replay-coordinator + slices as complete for this round; keep the Angular `OperationApplierService` + shell app-side unless a later port proves another small extraction safe. +4. Finish PR 5 in three larger implementation slices behind the new provider + contracts: HTTP file providers first, SuperSync integration second, and + LocalFile last. Provider-specific `SyncLog`/`OpLog` routing should be + handled as provider files move behind provider-package ports. ## PR 1 - Thin First Slice (#7546) @@ -356,7 +391,9 @@ Remaining PR 2 follow-up: - `@sp/shared-schema` - `src/app/*` and relative app imports such as `../../src/app/*` - Keep package exceptions explicit for packages that cannot yet be linted. -- Add the same rule for `packages/sync-providers/**` once that package exists. +- `packages/sync-providers/**` now has the same boundary shape, with an + additional ban on sync-core internal import paths. It may import public + `@sp/sync-core` only. - The rule was proved with a temporary `@angular/core` import under `packages/sync-core/src/`; scoped lint failed as expected with `no-restricted-imports`, and the file was removed. @@ -461,6 +498,21 @@ Initial candidate-file audit: route through `SyncLogger` with sanitized operation metadata plus payload type/count summaries only; raw payload values and raw payload keys stay out of exportable logs. +- `op-log/validation/auto-fix-typia-errors.ts`: Typia repair attempts and + applied fixes now route through `SyncLogger` with path/type/count metadata + only; raw invalid values, defaults, and full Typia error objects stay out of + exportable logs. +- `op-log/validation/repair-menu-tree.ts`: menu-tree repair logs now use + `SyncLogger` metadata for removed references/invalid nodes; raw node objects + and folder names stay out of exportable logs. +- `op-log/validation/validation-fn.ts`: schema validation failures now route + through `SyncLogger` with counts, paths, expected types, and data shape + summaries only; raw validation result data and invalid values stay out of + exportable logs. +- `op-log/validation/is-related-model-data-valid.ts` and the invalid-date + repair branch in `data-repair.ts`: cross-model validation and date repair + diagnostics now keep raw app state, titles, and corrupted date strings out of + exportable logs. - `op-log/util/sync-file-prefix.ts`: now delegates to the package helper with app-supplied prefix and error construction. The app-facing shim should remain until consumers are deliberately switched to injected/configured helpers. @@ -669,8 +721,8 @@ Introduce orchestration ports without moving the orchestrators yet. This reduces the risk of the later service moves. Status: implemented for the current branch slice. `@sp/sync-core` exports the -first minimal port contracts, and these app services now explicitly satisfy -them: +first minimal replay/storage port contracts, and these app services now +explicitly satisfy them: - `OperationApplierService` implements `OperationApplyPort` and uses `ActionDispatchPort` for its NgRx dispatch seam. @@ -680,6 +732,18 @@ them: - `OperationLogStoreService` implements `OperationStorePort`. +`ConflictUiPort` and `SyncConfigPort` are also exported and satisfied by +app-side services: + +- `SyncImportConflictDialogService` implements + `ConflictUiPort` for the sync-import conflict + dialog while keeping its app-specific `SyncImportConflictData` API. +- `GlobalConfigService` implements `SyncConfigPort` by exposing the current + `selectSyncConfig` snapshot without leaking NgRx selectors into + `@sp/sync-core`. + +These adapters are intentionally not used by package orchestration yet. + This is contract-only: NgRx dispatch, hydration windows, archive IndexedDB handling, and deferred local action processing remain app-side. @@ -818,6 +882,28 @@ IndexedDB reads, and result assembly remain app-side. Only after 4a/4b are stable, decide whether any part of `OperationApplierService` belongs in `@sp/sync-core`. +Status: implemented for the current branch slice. The extracted part is the +narrow `replayOperationBatch()` coordinator in `packages/sync-core/src/replay-coordinator.ts`. +It is intentionally generic and calls host-supplied ports/callbacks in a strict +order: + +1. open the remote-apply window; +2. dispatch the host-created bulk replay action; +3. yield after dispatch so host reducers finish before side effects; +4. run remote archive side effects after dispatch when configured; +5. yield around archive side effects to preserve UI responsiveness; +6. start post-sync cooldown before ending the remote-apply window; +7. end the remote-apply window and flush deferred local actions. + +Package-level Vitest coverage now asserts dispatch-yield ordering, local +hydration behavior, archive failure reporting, archive notification timing, +cooldown failure handling, and empty-batch no-op behavior. + +The Angular `OperationApplierService` delegates to this coordinator but keeps +all app-specific work app-side: `bulkApplyOperations`, `convertOpToAction`, +`isArchiveAffectingAction`, `remoteArchiveDataApplied`, `Injector` access to +`OperationLogEffects`, and `OpLog` diagnostics. + Acceptable extraction: - a small generic replay coordinator that calls ports in a strict order; @@ -842,6 +928,27 @@ Hard requirements from `CLAUDE.md`: --- +## Pre-P5 Readiness Check + +Status: complete for this branch. + +- No remaining pre-P5 `SyncLogger` routing is needed in core: files already made + movable either live in `@sp/sync-core` without app logging, accept a + `SyncLogger` port, or stay app-side because their diagnostics/recovery + behavior is still SP-specific. +- `sync-file-prefix`, generic error-message extraction, and gzip/base64 + compression helpers are package-side behind host-owned configuration/error + factories. +- `OperationApplierService` logging remains app-side intentionally; the moved + replay coordinator has no logging dependency. +- Provider-specific logging and credential diagnostics remain in + `src/app/op-log/sync-providers/` and should be handled during PR 5 when those + files move to `@sp/sync-providers`. +- Boundary verification was rerun for `packages/sync-core/src` and found no + forbidden Angular, NgRx, `src/app`, or `@sp/shared-schema` imports. + +--- + ## PR 5 - Lift Providers Into `@sp/sync-providers` Pull bundled providers out of `src/app/op-log/sync-providers/` so engine, @@ -874,6 +981,125 @@ providers, and app wiring each live in their own package. - Provider package must not import `@sp/sync-core` internals beyond public ports/types. +### Current First Slice + +- `packages/sync-providers/` mirrors the `sync-core` package scaffolding: + `package.json`, tsup build, Vitest config, strict package `tsconfig`, and a + package-local `.gitignore` for generated artifacts. +- Root wiring is in place: `sync-providers:build`, + `sync-providers:test`, the `packages:test` aggregate used by root + `npm test`, `build-packages.js`, `prepare`, the `@sp/sync-providers` path + alias, package-lock workspace metadata, and Angular lint coverage. +- Boundary lint rejects Angular, NgRx, app source imports, `@sp/shared-schema`, + sync-core internals, and dynamic imports under `packages/sync-providers/**`. +- Provider-neutral type contracts moved first. App-owned `SyncProviderId`, + provider constants, OAuth routing, config UI, and the IndexedDB credential + store implementation remain app-side. +- `SyncCredentialStore` now implements the package + `SyncCredentialStorePort`, while `src/app/op-log/sync-providers/` keeps + shims so existing call sites keep their imports. + +### Current Second Slice + +- `FileBasedSyncData`, `SyncFileCompactOp`, and + `FILE_BASED_SYNC_CONSTANTS` moved into `@sp/sync-providers`. +- The package contracts stay host-agnostic by accepting generic state, + compact-operation, and archive payload types. +- `src/app/op-log/sync-providers/file-based/file-based-sync.types.ts` remains + the compatibility shim that binds the package envelope to app-owned + `CompactOperation` and `ArchiveModel`. + +### Current Third Slice + +- Provider-owned PKCE utilities moved into `@sp/sync-providers`: + `generateCodeVerifier`, `generateCodeChallenge`, and `generatePKCECodes`. +- The implementation keeps the existing browser WebCrypto behavior and the + `hash-wasm` fallback needed when `crypto.subtle` is unavailable. +- `src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts` + remains as a compatibility re-export for existing Dropbox call sites. + +### Current Fourth Slice + +- Provider-owned native HTTP retry helpers moved into `@sp/sync-providers`: + `executeNativeRequestWithRetry`, `isTransientNetworkError`, and the + `NativeHttpExecutor` / `NativeHttpRequestConfig` / `NativeHttpResponse` + contracts. +- The package version is platform-agnostic: callers inject a + `NativeHttpExecutor` (CapacitorHttp on Android, fetch on web/Electron, a + test double in unit tests) and an optional `SyncLogger` from + `@sp/sync-core`. Retry policy (2 attempts, 1s/2s backoff, transient + network errors only) is preserved. +- Retry log entries flow as safe `SyncLogMeta` primitives (url, attempt, + errorName, errorCode) rather than raw error objects, aligning with the + package's privacy-aware logger contract. +- `src/app/op-log/sync-providers/native-http-retry.ts` remains as the + app-side adapter that wires `CapacitorHttp` and `OP_LOG_SYNC_LOGGER` + through to the package helper so existing Dropbox and SuperSync callers + keep working unchanged. +- The full Dropbox and WebDAV provider moves were deferred from this + slice because their dependency surface (provider error classes, + per-platform fetch hacks, `tryCatchInlineAsync`, Capacitor plugin + registration, OAuth glue) needs additional package ports that should + be designed and reviewed in their own slice. Updated plan below. + +### Remaining Slice Plan + +Finish PR 5 in four slices rather than three: + +1. **Dropbox provider slice** (next) + - Move provider-specific error classes (`AuthFailSPError`, + `RemoteFileNotFoundAPIError`, `HttpNotOkAPIError`, + `MissingCredentialsSPError`, `NoRevAPIError`, `InvalidDataSPError`, + `EmptyRemoteBodySPError`, `MissingRefreshTokenAPIError`, + `TooManyRequestsAPIError`, `UploadRevToMatchMismatchAPIError`, + `PotentialCorsError`, `RemoteFileChangedUnexpectedly`) into the + package. Drop the `AdditionalLogErrorBase` constructor-time logging + side effect; rely on catch-site logging. The app's `sync-errors.ts` + re-exports the moved classes so consumer call sites stay unchanged. + - Introduce a `ProviderPlatformInfo` port for `isNativePlatform` / + `isIosNative` flags so provider code stops importing + `@capacitor/core` directly. Same applies to the iOS + `CapacitorWebFetch` lookup — wrap as a `WebFetchProvider` port. + - Move `tryCatchInlineAsync` into the package (small util, no + dependencies). Move the `DropboxFileMetadata` shape (already + duplicated under `imex/sync/dropbox/`). + - Move `dropbox.ts`, `dropbox-api.ts`, `dropbox-api.spec.ts`, + `dropbox-auth-helper.spec.ts`, `generate-pkce-codes.spec.ts` + into `packages/sync-providers/src/file-based/dropbox/`. Convert + Jasmine specs to Vitest. Replace `SyncProviderId.Dropbox` with a + `PROVIDER_ID_DROPBOX = 'Dropbox'` constant inside the package. + - Leave thin app-side shims that re-export the moved Dropbox class + wired with the app's logger + platform + credential-store + instances so `sync-providers.factory.ts` keeps working. +2. **WebDAV + Nextcloud slice** + - Reuse the same error-class, platform-info, and HTTP-port surface + introduced for Dropbox. + - Move `webdav-base-provider.ts`, `webdav-api.ts`, + `webdav-xml-parser.ts`, `webdav.const.ts`, `webdav.model.ts`, + `webdav.ts`, `nextcloud.ts`, `nextcloud.model.ts` and their + specs into the package. + - `webdav-http-adapter.ts` currently calls a Capacitor-registered + `WebDavHttp` plugin for native platforms. Keep the Capacitor + plugin registration app-side and inject a + `WebDavNativeHttpExecutor` port that resolves to the registered + plugin on Android/iOS or to `fetch` on web/Electron. + - Move `md5HashSync` (or replace with `hash-wasm` already in the + package) since WebDAV uses content hashing for revs. +3. **SuperSync integration slice** + - Move SuperSync provider implementation behind the same package boundary, + reusing the HTTP/native-fetch ports introduced by the file-provider slices. + - Move only provider implementation code; keep app state selectors, + provider lists, config UI, and Angular credential-store implementation in + `src/app`. + - Tighten provider factory/registry shims enough that app call sites keep + working while package providers no longer import app-owned IDs. +4. **LocalFile final slice** + - Move LocalFile provider implementation last. + - Put Electron/local-file APIs behind an app-provided file port and keep the + Electron bridge implementation app-side. + - Keep Android/browser LocalFile behavior covered by app shims while the + package owns only platform-neutral provider logic. + ### Verification - Per-provider unit specs. @@ -889,7 +1115,8 @@ This is now a final audit rather than the first boundary rule. ### Goals -- Extend the boundary rules to `packages/sync-providers/**`. +- Recheck the boundary rules for `packages/sync-core/**` and + `packages/sync-providers/**`. - Audit package manifests for accidental runtime deps. - Audit public exports for SP names and app-only concepts. - Add a small architecture note that explains the package boundaries and allowed @@ -904,19 +1131,59 @@ This is now a final audit rather than the first boundary rule. --- +## PR 7 - Optional Polish (Post-Provider Lift) + +Non-blocking cleanups surfaced during the PR 5 provider lift. None of these +change behaviour or boundaries — they remove duplication, tighten tests, and +retire deprecated aliases once consumers have migrated. + +### Candidates + +- **Consolidate PKCE helpers.** `packages/sync-providers/src/pkce.ts` currently + duplicates `src/app/util/pkce.util.ts`. The package needs to stand alone, so + the duplication is intentional during the scaffold, but the remaining + consumers (`src/app/plugins/oauth/plugin-oauth.service.ts`, + `src/app/plugins/oauth/pkce.util.spec.ts`, + `src/app/op-log/sync-providers/file-based/dropbox/dropbox-auth-helper.spec.ts`) + should migrate onto `@sp/sync-providers` so `src/app/util/pkce.util.ts` can be + deleted. Drift between the two implementations is the risk this resolves. +- **Drop the dead `_length` arg in `generatePKCECodes`.** Pre-existing from the + original helper; the parameter is unused. Either remove it (single call site) + or document why it is kept for API compatibility. +- **Tighten the PKCE verifier-length assertion.** `tests/pkce.spec.ts` bounds + the verifier with `toBeLessThanOrEqual(128)`, but a 32-byte random buffer + always encodes to exactly 43 base64url chars. Replace with an exact length + check so the test actually constrains the output. +- **Retire `SyncProviderServiceInterface` alias.** Marked `@deprecated` in + `src/app/op-log/sync-providers/provider.interface.ts`. Sweep callers to + `SyncProviderBase` / `FileSyncProvider` and remove the alias. +- **Trim duplicate ESLint pattern depths.** The `packages/sync-providers/**` + block in `eslint.config.js` lists `../sync-core/**`, `../../sync-core/**`, and + `**/sync-core/**` (plus shared-schema equivalents). The `**/...` form already + covers the relative variants; collapse for readability. + +### Verification + +- `npm run lint`, `npm test`, `npm run packages:test`. +- Grep for `pkce.util` and `SyncProviderServiceInterface` after the cleanup — + both should return zero hits outside of `packages/sync-providers`. + +--- + ## Summary Timeline -| PR | Scope | Risk | Notes | -| ------ | ---------------------------------------------------------- | ----------- | -------------------------------------- | -| **1** | Stand up `@sp/sync-core` with generic primitives and stubs | Low | Present on branch | -| **2** | Boundary lint, registry types, privacy-aware logger port | Medium | Groundwork present; finish follow-ups | -| **3a** | Vector-clock ownership and package test harness | Medium | Present on branch | -| **3b** | Pure algorithmic core | Medium | No Angular/NgRx/IndexedDB | -| **4a** | Port contracts only | Medium | Current slice present | -| **4b** | Move small orchestration units behind ports | High | Current slice present | -| **4c** | Revisit `OperationApplierService` extraction | High | Extract only if the boundary is proven | -| **5** | Lift providers into `@sp/sync-providers` | Medium-High | Provider deps stay out of core | -| **6** | Final boundary hardening and architecture note | Low | Audit and lock down | +| PR | Scope | Risk | Notes | +| ------ | ---------------------------------------------------------- | ----------- | ------------------------------------- | +| **1** | Stand up `@sp/sync-core` with generic primitives and stubs | Low | Present on branch | +| **2** | Boundary lint, registry types, privacy-aware logger port | Medium | Groundwork present; finish follow-ups | +| **3a** | Vector-clock ownership and package test harness | Medium | Present on branch | +| **3b** | Pure algorithmic core | Medium | No Angular/NgRx/IndexedDB | +| **4a** | Port contracts only | Medium | Current slice present | +| **4b** | Move small orchestration units behind ports | High | Current slice present | +| **4c** | Revisit `OperationApplierService` extraction | High | Narrow replay coordinator present | +| **5** | Lift providers into `@sp/sync-providers` | Medium-High | Provider deps stay out of core | +| **6** | Final boundary hardening and architecture note | Low | Audit and lock down | +| **7** | Optional polish: dedupe PKCE, retire deprecated aliases | Low | Non-blocking cleanup | After the final PR, `@sp/sync-core` should be the domain-agnostic sync engine and abstractions, `@sp/sync-providers` should contain bundled provider diff --git a/docs/wiki/3.01-API.md b/docs/wiki/3.01-API.md index 06939993a7..328233b50f 100755 --- a/docs/wiki/3.01-API.md +++ b/docs/wiki/3.01-API.md @@ -256,7 +256,7 @@ All responses are JSON with a consistent envelope: | ------------- | ------- | ----------------------------------------------------------- | | `query` | string | Filter by title (case-insensitive, contains) | | `projectId` | string | Filter by project ID | -| `tagId` | string | Filter by tag ID | +| `tagId` | string | Filter by tag ID. Use `TODAY` for tasks scheduled for today | | `includeDone` | boolean | Include completed tasks (default: false) | | `source` | string | `"active"` \| `"archived"` \| `"all"` (default: `"active"`) | @@ -272,6 +272,9 @@ curl "http://127.0.0.1:3876/tasks?source=archived" # Search tasks containing "meeting" curl "http://127.0.0.1:3876/tasks?query=meeting" +# List tasks scheduled for today +curl "http://127.0.0.1:3876/tasks?tagId=TODAY" + # Create a task curl -X POST http://127.0.0.1:3876/tasks \ -H "Content-Type: application/json" \ diff --git a/e2e/store-video/CLAUDE.md b/e2e/store-video/CLAUDE.md index 4eba99a888..e82ff0b08f 100644 --- a/e2e/store-video/CLAUDE.md +++ b/e2e/store-video/CLAUDE.md @@ -7,6 +7,14 @@ Playwright-driven generation of the marketing gif/video for the landing page and ```bash npm run video # tight default (~17s) → dist/video/reel*.{mp4,webm,gif} npm run video:full # full variant (~21s) → dist/video/reel-full*.{...} +npm run video:shorts # 9:16 portrait (~12s) → dist/video/reel-shorts*.{...} + # 1080×1920 for TikTok / YouTube Shorts / Instagram Reels. + # Skips the side-panel drag beat (no horizontal room). +npm run video:keyboard + # keyboard-first reel demonstrating SP shortcuts. +npm run video:mobile # 19.5:9 phone aspect (1080×2340) → reel-mobile*.{...} + # hasTouch context + isMobile UA; tap-ripple replaces + # the cursor ring; tap-driven choreography. npx cross-env MS_STORE_AUDIO_SOURCE=path/to/audio.ext npm run video:ms-store # 16:9 Store trailer → dist/video/reel-ms-store.mp4 + thumbnail @@ -24,14 +32,16 @@ Variant recordings are isolated under `.tmp/video/recordings//` (`defau ## Files -| File | Responsibility | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `playwright.store-video.config.ts` | Single chromium project, `video: 'off'` at project level (the fixture handles `recordVideo` itself because `browser.newContext()` doesn't inherit `use.video`). | -| `store-video/fixture.ts` | Custom context with `recordVideo` enabled at 1024×1024 / DPR 2, or 1920×1080 / DPR 1 for `REEL_VARIANT=ms-store`. Reuses the screenshot pipeline's seed builder. Init scripts handle: cursor highlight ring, dialog/snack/tooltip/mention suppression, app zoom. | -| `store-video/overlays.ts` | DOM-injected overlay primitives: `showOverlay`, `showCaption`, `showIntegrationsCard`, `showEndCard`, `cutToScene`, `fadeTransition`, `loopBoundary`, `attachDragGhost`, `smoothMouseMove`. Plus inline brand SVGs in the `LOGOS` constant. | -| `store-video/scenarios/reel.spec.ts` | Six-beat choreography. `REEL_VARIANT=full` triggers the optional "No account. No tracking." beat and relaxes hold timings. | -| `store-video/build-video.ts` | Picks the most recent `.webm` under `.tmp/video/recordings/`, applies the trim sidecar (cuts the seed-import lead-in), produces mp4/webm/gif via ffmpeg, optionally `gifsicle`-optimizes. For `ms-store`, produces a 1920×1080 H.264/AAC MP4 and PNG thumbnail. | -| `store-video/open-video.ts` | Opens an autoplay browser preview after `npm run video`. Prefers mp4, respects `REEL_VARIANT`, seeks slightly past the black first frame for preview only, and skips auto-open in CI. | +| File | Responsibility | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `playwright.store-video.config.ts` | Single chromium project, `video: 'off'` at project level (the fixture handles `recordVideo` itself because `browser.newContext()` doesn't inherit `use.video`). | +| `store-video/fixture.ts` | Custom context with `recordVideo` enabled at 1024×1024 / DPR 2, or 1920×1080 / DPR 1 for `REEL_VARIANT=ms-store`. Reuses the screenshot pipeline's seed builder. Init scripts handle: cursor highlight ring, dialog/snack/tooltip/mention suppression, app zoom. | +| `store-video/overlays.ts` | DOM-injected overlay primitives: `showOverlay`, `showCaption`, `showIntegrationsCard`, `showEndCard`, `cutToScene`, `fadeTransition`, `loopBoundary`, `attachDragGhost`, `smoothMouseMove`. Plus inline brand SVGs in the `LOGOS` constant. | +| `store-video/scenarios/reel.spec.ts` | Six-beat choreography. `REEL_VARIANT=full` triggers the optional "No account. No tracking." beat and relaxes hold timings. `REEL_VARIANT=shorts` skips the side-panel drag beat and tightens holds for portrait 9:16 capture. Skipped entirely for `REEL_VARIANT=keyboard`. | +| `store-video/scenarios/keyboard.spec.ts` | Keyboard-first reel (`REEL_VARIANT=keyboard` only). Five beats: tagline → `Shift+A` capture → `J`/`K` navigate → `F` focus mode → end card. Each chip is a real `page.keyboard.press()` so cause-and-effect is honest. | +| `store-video/scenarios/mobile.spec.ts` | Mobile-touch reel (`REEL_VARIANT=mobile` only). 1080×2340 phone aspect with `hasTouch`/`isMobile`. Four beats: tagline → tap-to-capture → tap-to-focus → end card. The fixture installs a tap-ripple on `touchstart`/`pointerdown` in lieu of the cursor highlight. | +| `store-video/build-video.ts` | Picks the most recent `.webm` under `.tmp/video/recordings/`, applies the trim sidecar (cuts the seed-import lead-in), produces mp4/webm/gif via ffmpeg, optionally `gifsicle`-optimizes. For `ms-store`, produces a 1920×1080 H.264/AAC MP4 and PNG thumbnail. | +| `store-video/open-video.ts` | Opens an autoplay browser preview after `npm run video`. Prefers mp4, respects `REEL_VARIANT`, seeks slightly past the black first frame for preview only, and skips auto-open in CI. | ## Beat structure (current) @@ -130,7 +140,7 @@ Optional Store env vars: ## Open polish ideas (not yet shipped) - **Theme + locale matrix.** Fixture supports both via `test.use({ theme, locale })`. Add Playwright projects per variant; matrix run produces `reel-en-dark.gif`, `reel-en-light.gif`, etc. Mirrors the screenshot pipeline pattern. -- **Aspect ratio variants.** 9:16 (1080×1920) for mobile social. Different `VIDEO_SIZE` per matrix entry. +- ~~**Aspect ratio variants.** 9:16 (1080×1920) for mobile social.~~ Shipped as `npm run video:shorts`. Different `VIDEO_SIZE` per variant lives in `fixture.ts:getVideoProfile`. - **5-second social cut.** `npm run video:short` produces beats {1, 3, 5}. Trivial extension of the variant flag. - **Drop-slot highlight.** Brief CSS pulse on the schedule slot the dragged task lands in — gives beat 2 a closing punctuation. - **Brand-color flash on logo entrance.** Currently logos are flat brand colors. Could flash bright then settle. diff --git a/e2e/store-video/fixture.ts b/e2e/store-video/fixture.ts index 01d55f1688..4da28a7dae 100644 --- a/e2e/store-video/fixture.ts +++ b/e2e/store-video/fixture.ts @@ -53,6 +53,27 @@ const getVideoProfile = (): VideoProfile => { }; } + if (process.env.REEL_VARIANT === 'shorts') { + return { + // 9:16 portrait at 1080x1920 — the canonical short-form video size for + // TikTok / YouTube Shorts / Instagram Reels / Mastodon. DPR 1 keeps the + // backing surface at 1080x1920 (DPR 2 would render 2160x3840 per frame + // and the recorder starts dropping frames). + size: { width: 1080, height: 1920 }, + deviceScaleFactor: 1, + }; + } + + if (process.env.REEL_VARIANT === 'mobile') { + return { + // 19.5:9 phone aspect (iPhone Pro Max / Pixel Pro). Renders as a + // recognizable "phone screen" frame for app-store mobile previews + // and Play Store "feature graphic" promo clips. + size: { width: 1080, height: 2340 }, + deviceScaleFactor: 1, + }; + } + return { // Square 1024x1024 plays well on social embeds and matches the rhythm of // the GitHub README. DPR 2 renders the page at 2x physical pixels, then @@ -127,6 +148,7 @@ export const test = base.extend({ // before recording really starts means we under-estimate by that amount, // i.e. trim slightly less — safer than over-trimming into beat 1's fade-in. recordingState.startMs = Date.now(); + const isMobile = VARIANT === 'mobile'; const context = await browser.newContext({ baseURL: baseURL ?? 'http://localhost:4242', userAgent: `PLAYWRIGHT-VIDEO-${testInfo.workerIndex}`, @@ -136,6 +158,11 @@ export const test = base.extend({ locale: 'en-US', viewport: VIDEO_SIZE, deviceScaleFactor: DEVICE_SCALE_FACTOR, + // Mobile variant: enable real touch dispatch so `page.touchscreen.tap` + // fires pointer/touch events the app's drag/click handlers recognize + // as a finger, and so any responsive `isMobile` branches activate. + hasTouch: isMobile, + isMobile, recordVideo: { dir: RECORDING_DIR, size: VIDEO_SIZE, @@ -146,6 +173,15 @@ export const test = base.extend({ await page.clock.install({ time: SCREENSHOT_BASE_DATE }); await page.addInitScript(ONBOARDING_INIT); + await page.addInitScript((variant) => { + // Stash variant on body so overlays.ts / scenarios can branch via + // attribute selectors without re-reading process.env in the page. + const apply = (): void => { + document.body.dataset.spVideoVariant = variant || 'default'; + }; + if (document.body) apply(); + else document.addEventListener('DOMContentLoaded', apply, { once: true }); + }, VARIANT); await page.addInitScript((darkMode) => { try { localStorage.setItem('DARK_MODE', darkMode); @@ -157,6 +193,69 @@ export const test = base.extend({ (window as unknown as { __spCurrentLocale?: string }).__spCurrentLocale = initialLocale; }, locale); + // Mobile variant: tap-ripple instead of cursor ring. The recorder + // doesn't draw a touch indicator on its own, so taps would otherwise + // read as instant state changes with no on-frame cause. Each + // touchstart spawns a short-lived expanding ring at the touch point. + if (isMobile) { + await page.addInitScript(() => { + const attach = (): void => { + if (document.getElementById('__sp-video-tap-ripple-style')) return; + const style = document.createElement('style'); + style.id = '__sp-video-tap-ripple-style'; + style.textContent = ` + .__sp-video-tap-ripple { + position: fixed; + top: 0; + left: 0; + width: 0; + height: 0; + border-radius: 50%; + background: radial-gradient(rgba(255,255,255,0.85) 0%,rgba(255,255,255,0.4) 40%,rgba(255,255,255,0) 75%); + z-index: 2147483640; + pointer-events: none; + opacity: 0.9; + transform: translate3d(-9999px,-9999px,0); + transition: width 520ms ease-out, height 520ms ease-out, opacity 620ms ease-out, transform 520ms ease-out; + } + `; + document.head.appendChild(style); + const spawn = (clientX: number, clientY: number): void => { + const r = document.createElement('div'); + r.className = '__sp-video-tap-ripple'; + r.style.transform = `translate3d(${clientX}px,${clientY}px,0)`; + document.body.appendChild(r); + void r.offsetWidth; + const finalSize = 220; + const half = finalSize / 2; + r.style.width = `${finalSize}px`; + r.style.height = `${finalSize}px`; + r.style.marginLeft = `${-half}px`; + r.style.marginTop = `${-half}px`; + r.style.opacity = '0'; + window.setTimeout(() => r.remove(), 700); + }; + document.addEventListener( + 'touchstart', + (e) => { + for (const t of Array.from(e.touches)) spawn(t.clientX, t.clientY); + }, + { passive: true, capture: true }, + ); + // Synthetic taps from Playwright sometimes route through pointer + // events without touch events — listen to both for coverage. + document.addEventListener( + 'pointerdown', + (e) => { + if (e.pointerType !== 'mouse') spawn(e.clientX, e.clientY); + }, + { passive: true, capture: true }, + ); + }; + if (document.body) attach(); + else document.addEventListener('DOMContentLoaded', attach, { once: true }); + }); + } // Inject a soft ring that follows the cursor — at 1024×1024 the OS // pointer is small and easy to miss; this makes drag motions read on // the gif. The ring is at z-index 2147483640 (under the full-screen @@ -164,44 +263,47 @@ export const test = base.extend({ // Spec code can also toggle visibility per-beat by adding/removing the // `__sp-hide-cursor-highlight` class on body — used during the capture // beat where the cursor sits in the middle of the focused input and - // would otherwise read as a stray white dot. - await page.addInitScript(() => { - const attach = (): void => { - const id = '__sp-video-cursor-highlight'; - if (document.getElementById(id)) return; - const dot = document.createElement('div'); - dot.id = id; - dot.style.cssText = [ - 'position:fixed', - 'top:0', - 'left:0', - 'width:36px', - 'height:36px', - 'margin:-18px 0 0 -18px', - 'border-radius:50%', - 'background:radial-gradient(rgba(255,255,255,0.55) 0%,rgba(255,255,255,0.18) 45%,rgba(255,255,255,0) 70%)', - 'z-index:2147483640', - 'pointer-events:none', - 'transform:translate3d(-9999px,-9999px,0)', - 'will-change:transform', - 'transition:opacity 150ms ease-out', - ].join(';'); - document.body.appendChild(dot); - const visibilityStyle = document.createElement('style'); - visibilityStyle.textContent = - 'body.__sp-hide-cursor-highlight #__sp-video-cursor-highlight{opacity:0!important}'; - document.head.appendChild(visibilityStyle); - document.addEventListener( - 'mousemove', - (e) => { - dot.style.transform = `translate3d(${e.clientX}px,${e.clientY}px,0)`; - }, - { passive: true }, - ); - }; - if (document.body) attach(); - else document.addEventListener('DOMContentLoaded', attach, { once: true }); - }); + // would otherwise read as a stray white dot. Skipped for the mobile + // variant: no cursor on touch, the tap ripple handles it. + if (!isMobile) { + await page.addInitScript(() => { + const attach = (): void => { + const id = '__sp-video-cursor-highlight'; + if (document.getElementById(id)) return; + const dot = document.createElement('div'); + dot.id = id; + dot.style.cssText = [ + 'position:fixed', + 'top:0', + 'left:0', + 'width:36px', + 'height:36px', + 'margin:-18px 0 0 -18px', + 'border-radius:50%', + 'background:radial-gradient(rgba(255,255,255,0.55) 0%,rgba(255,255,255,0.18) 45%,rgba(255,255,255,0) 70%)', + 'z-index:2147483640', + 'pointer-events:none', + 'transform:translate3d(-9999px,-9999px,0)', + 'will-change:transform', + 'transition:opacity 150ms ease-out', + ].join(';'); + document.body.appendChild(dot); + const visibilityStyle = document.createElement('style'); + visibilityStyle.textContent = + 'body.__sp-hide-cursor-highlight #__sp-video-cursor-highlight{opacity:0!important}'; + document.head.appendChild(visibilityStyle); + document.addEventListener( + 'mousemove', + (e) => { + dot.style.transform = `translate3d(${e.clientX}px,${e.clientY}px,0)`; + }, + { passive: true }, + ); + }; + if (document.body) attach(); + else document.addEventListener('DOMContentLoaded', attach, { once: true }); + }); + } // Suppress UI noise that fights with the choreographed reel: // - Material/CDK tooltips (cursor lingering would otherwise pop one) @@ -250,6 +352,21 @@ export const test = base.extend({ app-root { zoom: 1.4; } + /* Shorts (9:16, 1080x1920): zoom up further so the work-view fills + the portrait canvas. The inner viewport is 1080/1.6 = 675 wide, + 1920/1.6 = 1200 tall — wide enough for the task list at the + sidenav-collapsed width, tall enough that the seeded task rows + dominate the frame instead of leaving a half-empty top half. */ + body[data-sp-video-variant="shorts"] app-root { + zoom: 1.6; + } + /* Mobile (1080x2340 phone aspect): SP's responsive layout already + collapses the sidenav and switches to a touch-friendly toolbar + when isMobile=true, so a much smaller zoom is enough. 1.0 keeps + the layout in its native mobile breakpoint. */ + body[data-sp-video-variant="mobile"] app-root { + zoom: 1; + } /* The right-panel sizes itself to 250px (MIN_WIDTH) via SUP_RIGHT_PANEL_WIDTH localStorage seeded in ONBOARDING_INIT. No width override needed here — the panel's own resize logic diff --git a/e2e/store-video/overlays.ts b/e2e/store-video/overlays.ts index 1504b483e8..337295fee0 100644 --- a/e2e/store-video/overlays.ts +++ b/e2e/store-video/overlays.ts @@ -118,10 +118,12 @@ export type IntegrationsCardContent = { const STYLE_ID = '__sp-video-overlay-style'; const OVERLAY_ID_PREFIX = '__sp-video-overlay-'; +const KEY_CHIP_ID_PREFIX = '__sp-video-keychip-'; const END_CARD_ID = '__sp-video-end-card'; const CAPTION_ID = '__sp-video-caption'; let overlayCounter = 0; +let keyChipCounter = 0; const ensureStyleInjected = async (page: Page): Promise => { await page.evaluate((id) => { @@ -315,6 +317,12 @@ const ensureStyleInjected = async (page: Page): Promise => { gap: 64px 80px; justify-items: center; } + /* Portrait shorts (1080x1920) get a 2x3 grid so the logos read big + instead of cramped at the top of a tall frame. */ + body[data-sp-video-variant="shorts"] .__sp-video-int-card-logos { + grid-template-columns: repeat(2, auto); + gap: 88px 120px; + } .__sp-video-int-card-logo { display: flex; flex-direction: column; @@ -358,6 +366,45 @@ const ensureStyleInjected = async (page: Page): Promise => { color: #b6bbcd; font-weight: 500; } + /* Keycap chip — physically-modeled key. Used by the keyboard reel + to make each shortcut "land" visually before / during its action. + Anchored top-right by default so it doesn't fight the cursor or + the lower-third overlay text. */ + .__sp-video-keychip { + position: fixed; + top: 56px; + right: 56px; + z-index: 2147483641; + display: flex; + align-items: center; + gap: 12px; + padding: 18px 28px; + border-radius: 14px; + background: linear-gradient(180deg, #2a2f44 0%, #161a2c 100%); + box-shadow: + inset 0 -4px 0 rgba(0, 0, 0, 0.55), + inset 0 1px 0 rgba(255, 255, 255, 0.15), + 0 10px 30px rgba(0, 0, 0, 0.45); + color: #f7f8fb; + font-family: 'JetBrains Mono', 'SF Mono', Menlo, Consolas, monospace; + font-weight: 700; + font-size: clamp(26px, 2.6vw, 44px); + letter-spacing: 0.04em; + opacity: 0; + transform: translateY(-12px) scale(0.92); + transition: + opacity var(--__sp-fade-ms, 220ms) ease-out, + transform var(--__sp-fade-ms, 220ms) cubic-bezier(0.34, 1.56, 0.64, 1); + pointer-events: none; + } + .__sp-video-keychip.visible { + opacity: 1; + transform: translateY(0) scale(1); + } + .__sp-video-keychip-plus { + color: #8a90ad; + font-weight: 500; + } `; document.head.appendChild(style); }, STYLE_ID); @@ -869,6 +916,70 @@ export const showIntegrationsCard = async ( }; }; +/** + * Render a keyboard keycap chip in the top-right corner. The label is + * split on `+` and rendered with a muted "+" separator, so "Shift+A" + * looks like two keys joined by a thin plus. + * + * const chip = await showKeyChip(page, 'Shift+A'); + * await page.keyboard.press('Shift+A'); + * await chip.hide(); + * + * The chip uses a brief pop-in (transform scale 0.92 → 1) so each key in + * a fast sequence reads as a discrete event, not a static badge. + */ +export const showKeyChip = async ( + page: Page, + key: string, + options: { fadeMs?: number; noWait?: boolean } = {}, +): Promise => { + const fadeMs = options.fadeMs ?? 220; + await ensureStyleInjected(page); + keyChipCounter += 1; + const id = `${KEY_CHIP_ID_PREFIX}${keyChipCounter}`; + await page.evaluate( + (args) => { + const el = document.createElement('div'); + el.id = args.id; + el.className = '__sp-video-keychip'; + el.style.setProperty('--__sp-fade-ms', `${args.fadeMs}ms`); + const parts = args.key.split('+').map((s) => s.trim()); + parts.forEach((part, i) => { + if (i > 0) { + const plus = document.createElement('span'); + plus.className = '__sp-video-keychip-plus'; + plus.textContent = '+'; + el.appendChild(plus); + } + const span = document.createElement('span'); + span.textContent = part; + el.appendChild(span); + }); + document.body.appendChild(el); + void el.offsetWidth; + el.classList.add('visible'); + }, + { id, key, fadeMs }, + ); + if (!options.noWait) { + await page.waitForTimeout(fadeMs); + } + return { + hide: async (): Promise => { + await page.evaluate( + (args) => { + const el = document.getElementById(args.id); + if (!el) return; + el.classList.remove('visible'); + window.setTimeout(() => el.remove(), args.fadeMs + 50); + }, + { id, fadeMs }, + ); + await page.waitForTimeout(fadeMs); + }, + }; +}; + export const showEndCard = async ( page: Page, content: EndCardContent, diff --git a/e2e/store-video/scenarios/keyboard.spec.ts b/e2e/store-video/scenarios/keyboard.spec.ts new file mode 100644 index 0000000000..d7c7270e55 --- /dev/null +++ b/e2e/store-video/scenarios/keyboard.spec.ts @@ -0,0 +1,344 @@ +/** + * Keyboard reel — five-beat choreography demonstrating Super Productivity's + * keyboard-first design. Each beat anchors a visible keycap chip to a real + * `page.keyboard.press()` so the cause-and-effect reads honestly: the chip + * appears, the shortcut fires, the app reacts. + * + * Lead-in Black fades to SP task list. + * 1 "Keyboard-first." tagline overlay. + * 2 Shift+A → global add-task-bar opens, types "Read book 30m", Enter. + * 3 J / K → moves task focus down then back up, with both chips. + * 4 F → focus mode opens on the highlighted task. + * 5 End card "Made for keyboards." with platforms line. + * + * Activated only by `REEL_VARIANT=keyboard` so the default capture run + * still produces the canonical marketing reel. + */ +import { test } from '../fixture'; +import { loopBoundary, showEndCard, showKeyChip, showOverlay } from '../overlays'; + +const VARIANT = process.env.REEL_VARIANT ?? ''; +const NEW_TASK_TITLE = 'Read book 30m'; + +const parkCursor = async (page: import('@playwright/test').Page): Promise => { + try { + await page.mouse.move(0, 0); + } catch { + /* noop */ + } +}; + +/** + * Aggressively clear CDK overlay blockers that swallow SP's keyboard shortcuts. + * + * SP's `ShortcutService.handleKeyDown` bails when `_hasOpenCdkOverlay` finds + * ANY `.cdk-overlay-pane` in the overlay container with `childElementCount > 0` + * (excluding tooltip panes). The check is purely DOM-structural — `display: + * none` does NOT exempt a pane. The fixture hides snack-bar / dialog / + * mention-list / add-task-bar panes via CSS only, so their hosting panes + * linger in the DOM with children intact and silently block every J/K/F press. + * + * Also blurs editable focus targets (input/textarea/contenteditable) so the + * shortcut handler's `isInputElement` check doesn't bail. Crucially: does + * NOT blur a focused , because beat 3 relies on that focus staying + * put between keypresses. + */ +const clearShortcutBlockers = async ( + page: import('@playwright/test').Page, +): Promise => { + await page.evaluate(() => { + document.querySelectorAll('.cdk-overlay-pane').forEach((pane) => { + if (pane.classList.contains('mat-mdc-tooltip-panel')) return; + pane.remove(); + }); + const active = document.activeElement as HTMLElement | null; + if (!active) return; + const tag = active.tagName; + if ( + tag === 'INPUT' || + tag === 'TEXTAREA' || + (active as HTMLElement).isContentEditable + ) { + active.blur(); + } + }); +}; + +/** + * Drive SP's task focus state from the test side, without using the store. + * + * The shortcut handler needs TWO things to route `focusNext()`: + * 1. `_taskFocusService.focusedTaskId()` set, OR an active `` element + * whose `data-task-id` the recovery path can read. + * 2. `_taskFocusService.lastFocusedTaskComponent()` set — this only happens + * from the task component's `focusin` HostListener, gated by + * `_isInnermostTaskFor(ev.target)` which requires + * `ev.target.closest('task') === host`. + * + * Both `taskEl.focus()` (real focus) and a follow-up `dispatchEvent(new + * FocusEvent('focusin', { bubbles: true }))` are issued. The browser already + * fires focusin on `.focus()` in normal pages, but we re-dispatch to harden + * against any test-runner edge case where the bubbling focusin doesn't run + * the Angular HostListener in time. + * + * Returns the `data-task-id` of the newly focused task, or `null` if no + * `` exists. + */ +const ensureTaskFocused = async ( + page: import('@playwright/test').Page, +): Promise => { + return await page.evaluate(() => { + const active = document.activeElement as HTMLElement | null; + const currentTaskEl = active?.closest('task') as HTMLElement | null; + const taskEl = + currentTaskEl ?? (document.querySelector('task') as HTMLElement | null); + if (!taskEl) return null; + taskEl.focus(); + taskEl.dispatchEvent(new FocusEvent('focusin', { bubbles: true })); + return taskEl.getAttribute('data-task-id'); + }); +}; + +test.describe('@video keyboard reel', () => { + test.skip(VARIANT !== 'keyboard', 'keyboard reel only runs when REEL_VARIANT=keyboard'); + test.use({ locale: 'en', theme: 'dark' }); + + test('keyboard reel', async ({ seededPage, markBeatsStart }) => { + const page = seededPage; + + // Forward in-page diagnostics + SP's own Log.warn output so we can see + // whether the shortcut handler bailed at `lastFocusedTaskComponent === + // null` or the id-mismatch guard. + page.on('console', (msg) => { + const text = msg.text(); + if ( + text.startsWith('[keyboard-reel]') || + text.includes('No focused task component') || + text.includes('does not match shortcut target') || + text.includes('Method ') || + msg.type() === 'warning' + ) { + process.stdout.write(`[page:${msg.type()}] ${text}\n`); + } + }); + + // ── Pre-roll (trimmed off the reel) ────────────────────────────────── + await page.goto('/#/tag/TODAY/tasks'); + await page.locator('task').first().waitFor({ state: 'visible', timeout: 15_000 }); + await parkCursor(page); + await page.waitForTimeout(300); + markBeatsStart(); + + // ── Lead-in ────────────────────────────────────────────────────────── + await loopBoundary(page, 'in', 460); + + // ── Beat 1 — "Keyboard-first." ─────────────────────────────────────── + const b1 = await showOverlay(page, 'Keyboard-first.'); + await page.waitForTimeout(900); + void b1.hide(); + await page.waitForTimeout(200); + + // ── Beat 2 — Shift+A → quick capture ───────────────────────────────── + const chipAdd = await showKeyChip(page, 'Shift+A'); + await clearShortcutBlockers(page); + await page.keyboard.press('Shift+A'); + const globalInput = page.locator('add-task-bar.global input').first(); + if (!(await globalInput.isVisible().catch(() => false))) { + await page.evaluate(() => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + helper?.store?.dispatch({ type: '[Layout] Show AddTaskBar' }); + }); + } + await globalInput.waitFor({ state: 'visible', timeout: 5_000 }); + await page.waitForTimeout(220); + await page.evaluate(() => document.body.classList.add('__sp-hide-cursor-highlight')); + await globalInput.pressSequentially(NEW_TASK_TITLE, { delay: 55 }); + await page.waitForTimeout(360); + await page.keyboard.press('Enter'); + await page.waitForTimeout(450); + await page.evaluate(() => + document.body.classList.remove('__sp-hide-cursor-highlight'), + ); + const backdrop = page.locator('.backdrop').first(); + if (await backdrop.isVisible().catch(() => false)) { + await backdrop.click({ force: true }); + await backdrop.waitFor({ state: 'hidden', timeout: 2_000 }).catch(() => undefined); + } + await page + .locator('add-task-bar.global') + .first() + .waitFor({ state: 'hidden', timeout: 3_000 }) + .catch(() => undefined); + await chipAdd.hide(); + + // Focus the first task before pressing J/K. SP's focusin handler only + // routes `setSelectedId` (which opens the detail panel) when + // `selectedTaskId` is already set; if we start with no selection, + // focusin just registers the task with TaskFocusService and the + // shortcut handler's `focusNext()` walks the list via `:focus` + // styling — no panel side effects. + await clearShortcutBlockers(page); + const firstTask = page.locator('task').first(); + await firstTask.scrollIntoViewIfNeeded().catch(() => undefined); + const initialTaskId = await ensureTaskFocused(page); + await page.evaluate((id) => { + const tasks = Array.from(document.querySelectorAll('task')); + const msg = + `[keyboard-reel] initial focus task=${id ?? 'null'} ` + + `taskCount=${tasks.length}`; + console.log(msg); + }, initialTaskId); + await page.waitForTimeout(200); + + /** + * One step in the J/K navigation beat. Chip already up; press the real + * key. SP's task-shortcut service calls `focusNext()` / `focusPrevious()` + * on the focused task component, which moves DOM focus to the next / + * previous `` host element — the `:focus` border (from + * `_task-base.scss`) is the visible cue. + * + * Logs activeElement + task index + overlay-pane count before each + * press, then the activeElement + task index after, so trace inspection + * can confirm focus actually moved. If `taskIndex` is unchanged, the + * shortcut handler bailed (overlay pane present, component reference + * null/mismatched) — check the forwarded `[page:warning]` lines. + */ + const step = async (direction: 'next' | 'prev'): Promise => { + await clearShortcutBlockers(page); + await ensureTaskFocused(page); + const snap = async (label: string): Promise => { + const info = await page.evaluate(() => { + const a = document.activeElement as HTMLElement | null; + const taskEl = a?.closest('task') as HTMLElement | null; + const all = Array.from(document.querySelectorAll('task')); + const idx = taskEl ? all.indexOf(taskEl) : -1; + const paneCount = Array.from( + document.querySelectorAll('.cdk-overlay-pane'), + ).filter( + (p) => + !p.classList.contains('mat-mdc-tooltip-panel') && p.childElementCount > 0, + ).length; + return { + tag: a?.tagName ?? null, + taskId: taskEl?.getAttribute('data-task-id') ?? null, + taskIndex: idx, + taskCount: all.length, + paneCount, + }; + }); + await page.evaluate( + (payload) => { + const msg = + `[keyboard-reel] ${payload.label} tag=${payload.tag} ` + + `task=${payload.taskId} ` + + `idx=${payload.taskIndex}/${payload.taskCount} ` + + `panes=${payload.paneCount}`; + console.log(msg); + }, + { ...info, label }, + ); + }; + await snap(`before-${direction}`); + await page.keyboard.press(direction === 'next' ? 'j' : 'k'); + await snap(`after-${direction}`); + }; + + // ── Beat 3 — J / K → navigate task list ────────────────────────────── + const chipJ = await showKeyChip(page, 'J'); + await step('next'); + await page.waitForTimeout(420); + await step('next'); + await page.waitForTimeout(420); + await chipJ.hide(); + await page.waitForTimeout(80); + const chipK = await showKeyChip(page, 'K'); + await step('prev'); + await page.waitForTimeout(420); + await chipK.hide(); + await page.waitForTimeout(120); + + // ── Beat 4 — F → focus mode ────────────────────────────────────────── + const chipF = await showKeyChip(page, 'F'); + await clearShortcutBlockers(page); + await page.keyboard.press('f'); + const focusVisible = await page + .locator('focus-mode-main') + .first() + .waitFor({ state: 'visible', timeout: 2_000 }) + .then(() => true) + .catch(() => false); + if (!focusVisible) { + const focusedTaskId = await page.evaluate(() => { + const focused = document.querySelector( + 'task:focus, task.isCurrent, task[class*="isSelected"]', + ); + return focused?.getAttribute('data-task-id') ?? null; + }); + await page.evaluate((id) => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + if (!helper?.store) return; + if (id) helper.store.dispatch({ type: '[Task] SetCurrentTask', id }); + helper.store.dispatch({ type: '[FocusMode] Show Overlay' }); + helper.store.dispatch({ + type: '[FocusMode] Start Session', + duration: 1500000, + }); + }, focusedTaskId); + await page + .locator('focus-mode-main') + .first() + .waitFor({ state: 'visible', timeout: 8_000 }) + .catch(() => undefined); + } + await page.clock.runFor(5500).catch(() => undefined); + await page + .locator('focus-mode-main .bottom-controls') + .first() + .waitFor({ state: 'visible', timeout: 5_000 }) + .catch(() => undefined); + await page.clock.resume().catch(() => undefined); + await page.waitForTimeout(1500); + await chipF.hide(); + await page.waitForTimeout(200); + + // ── Beat 4 → 5 — dismiss focus mode behind the end card ────────────── + await showEndCard( + page, + { + logo: { + src: '/assets/icons/sp.svg', + alt: 'Super Productivity', + monochrome: true, + }, + title: 'Made for keyboards.', + subtitle: 'superproductivity.com', + stats: [ + { template: '{n}+ shortcuts', to: 40 }, + 'Web · iOS · Android · macOS · Linux · Windows', + ], + }, + { fadeMs: 560 }, + ); + await page.evaluate(() => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + helper?.store?.dispatch({ type: '[FocusMode] Hide Overlay' }); + helper?.store?.dispatch({ type: '[FocusMode] Cancel Session' }); + }); + await page.waitForTimeout(2200); + + // ── Loop boundary ──────────────────────────────────────────────────── + await loopBoundary(page, 'out', 460); + }); +}); diff --git a/e2e/store-video/scenarios/mobile.spec.ts b/e2e/store-video/scenarios/mobile.spec.ts new file mode 100644 index 0000000000..bc5f345bf1 --- /dev/null +++ b/e2e/store-video/scenarios/mobile.spec.ts @@ -0,0 +1,175 @@ +/** + * Mobile-touch reel — four-beat choreography demonstrating Super Productivity + * on a phone. The fixture enables `hasTouch` and `isMobile` for this variant + * so the context dispatches real touch events; each beat uses + * `page.touchscreen.tap()` and the fixture's tap-ripple init script spawns a + * visible ring at each touch point. + * + * Lead-in Black fades to SP work-view (mobile layout). + * 1 "On the go." tagline overlay. + * 2 Tap "+" → quick add task, type, confirm. + * 3 Tap task → focus mode on tapped task. + * 4 End card "Mobile · iOS · Android" with stat counter. + * + * Activated only by `REEL_VARIANT=mobile`. Output lands as + * `dist/video/reel-mobile.{mp4,webm,gif}` at 1080×2340. + */ +import type { Locator, Page } from '@playwright/test'; +import { test } from '../fixture'; +import { loopBoundary, showEndCard, showOverlay } from '../overlays'; + +const VARIANT = process.env.REEL_VARIANT ?? ''; +const NEW_TASK_TITLE = 'Plan trip 30m'; +const NEW_TASK_DISPLAY = 'Plan trip'; + +const tapCenter = async (page: Page, locator: Locator): Promise => { + const box = await locator.boundingBox(); + if (!box) return; + const halfW = box.width / 2; + const halfH = box.height / 2; + const cx = box.x + halfW; + const cy = box.y + halfH; + await page.touchscreen.tap(cx, cy); +}; + +test.describe('@video mobile reel', () => { + test.skip(VARIANT !== 'mobile', 'mobile reel only runs when REEL_VARIANT=mobile'); + test.use({ locale: 'en', theme: 'dark' }); + + test('mobile reel', async ({ seededPage, markBeatsStart }) => { + const page = seededPage; + + // ── Pre-roll (trimmed off the reel) ────────────────────────────────── + await page.goto('/#/tag/TODAY/tasks'); + await page.locator('task').first().waitFor({ state: 'visible', timeout: 15_000 }); + await page.waitForTimeout(300); + markBeatsStart(); + + // ── Lead-in ────────────────────────────────────────────────────────── + await loopBoundary(page, 'in', 460); + + // ── Beat 1 — "On the go." ──────────────────────────────────────────── + const b1 = await showOverlay(page, 'On the go.'); + await page.waitForTimeout(900); + void b1.hide(); + await page.waitForTimeout(220); + + // ── Beat 2 — Tap + → quick capture ─────────────────────────────────── + const b2 = await showOverlay(page, 'Tap to capture.'); + // Open the global add-task bar via store dispatch — the mobile FAB + // selector varies across breakpoints, but the bar itself is the same + // surface and the tap-ripple before dispatch makes the cause visible. + const fab = page + .locator( + 'button.e2e-add-task-fab, .add-task-fab button, button[aria-label*="Add Task" i]', + ) + .first(); + if (await fab.isVisible().catch(() => false)) { + await tapCenter(page, fab); + } else { + await page.evaluate(() => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + helper?.store?.dispatch({ type: '[Layout] Show AddTaskBar' }); + }); + } + const globalInput = page.locator('add-task-bar.global input').first(); + await globalInput.waitFor({ state: 'visible', timeout: 5_000 }); + await page.waitForTimeout(220); + await globalInput.pressSequentially(NEW_TASK_TITLE, { delay: 55 }); + await page.waitForTimeout(360); + await page.keyboard.press('Enter'); + await page.waitForTimeout(500); + const backdrop = page.locator('.backdrop').first(); + if (await backdrop.isVisible().catch(() => false)) { + await backdrop.click({ force: true }); + await backdrop.waitFor({ state: 'hidden', timeout: 2_000 }).catch(() => undefined); + } + await page + .locator('add-task-bar.global') + .first() + .waitFor({ state: 'hidden', timeout: 3_000 }) + .catch(() => undefined); + void b2.hide(); + await page.waitForTimeout(220); + + // ── Beat 3 — Tap captured task → focus mode ────────────────────────── + const b3 = await showOverlay(page, 'Tap to focus.'); + const newTask = page.locator('task').filter({ hasText: NEW_TASK_DISPLAY }).first(); + await newTask.waitFor({ state: 'visible', timeout: 5_000 }); + const newTaskId = await newTask.getAttribute('data-task-id').catch(() => null); + await tapCenter(page, newTask); + await page.waitForTimeout(160); + // The tap visibly lands on the task; the focus-mode entry happens via + // dispatch so it doesn't depend on whichever overflow menu the tap + // routes through on this breakpoint. + if (newTaskId) { + await page.evaluate((id) => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + if (!helper?.store) return; + helper.store.dispatch({ type: '[Task] SetCurrentTask', id }); + helper.store.dispatch({ type: '[FocusMode] Show Overlay' }); + helper.store.dispatch({ + type: '[FocusMode] Start Session', + duration: 1500000, + }); + }, newTaskId); + await page + .locator('focus-mode-main') + .first() + .waitFor({ state: 'visible', timeout: 8_000 }) + .catch(() => undefined); + await page.clock.runFor(5500).catch(() => undefined); + await page + .locator('focus-mode-main .bottom-controls') + .first() + .waitFor({ state: 'visible', timeout: 5_000 }) + .catch(() => undefined); + await page.clock.resume().catch(() => undefined); + } + await page.waitForTimeout(1600); + void b3.hide(); + await page.waitForTimeout(220); + + // ── Beat 4 — End card "Mobile · iOS · Android" ────────────────────── + await showEndCard( + page, + { + logo: { + src: '/assets/icons/sp.svg', + alt: 'Super Productivity', + monochrome: true, + }, + title: 'Take it anywhere.', + subtitle: 'superproductivity.com', + stats: [ + { template: '{n} ★ on Google Play', to: 4.8, decimals: 1 }, + 'iOS · Android · Web · Desktop', + ], + }, + { fadeMs: 560 }, + ); + // Tear focus mode down behind the card so the loop-out doesn't flash + // it between end card and black. + await page.evaluate(() => { + const helper = ( + window as unknown as { + __e2eTestHelpers?: { store?: { dispatch: (a: unknown) => void } }; + } + ).__e2eTestHelpers; + helper?.store?.dispatch({ type: '[FocusMode] Hide Overlay' }); + helper?.store?.dispatch({ type: '[FocusMode] Cancel Session' }); + }); + await page.waitForTimeout(2300); + + // ── Loop boundary ──────────────────────────────────────────────────── + await loopBoundary(page, 'out', 460); + }); +}); diff --git a/e2e/store-video/scenarios/reel.spec.ts b/e2e/store-video/scenarios/reel.spec.ts index f3261deb41..962909833b 100644 --- a/e2e/store-video/scenarios/reel.spec.ts +++ b/e2e/store-video/scenarios/reel.spec.ts @@ -38,6 +38,11 @@ import { const VARIANT = process.env.REEL_VARIANT ?? ''; const isFull = VARIANT === 'full'; +// 9:16 portrait variant for TikTok / YouTube Shorts / Instagram Reels. Skips +// beat 2 (the side-panel drag doesn't translate to portrait — the schedule +// panel makes no sense without horizontal room) and tightens hold timings +// so the reel lands in the ~12-14s sweet spot for short-form algorithms. +const isShorts = VARIANT === 'shorts'; const parkCursor = async (page: import('@playwright/test').Page): Promise => { // Park the cursor offscreen so any matTooltip dismisses and the cursor @@ -58,6 +63,11 @@ const CAPTURED_TASK_TITLE = 'A task 1h'; const CAPTURED_TASK_DISPLAY_TITLE = 'A task'; test.describe('@video reel', () => { + // The keyboard and mobile variants have their own choreography (see + // keyboard.spec.ts / mobile.spec.ts); skip this spec when either is + // active so a single capture run doesn't record two unrelated webms. + test.skip(VARIANT === 'keyboard', 'keyboard variant runs keyboard.spec.ts'); + test.skip(VARIANT === 'mobile', 'mobile variant runs mobile.spec.ts'); test.use({ locale: 'en', theme: 'dark' }); test('marketing reel', async ({ seededPage, markBeatsStart }) => { @@ -67,7 +77,7 @@ test.describe('@video reel', () => { await page.goto('/#/tag/TODAY/tasks'); await page.locator('task').first().waitFor({ state: 'visible', timeout: 15_000 }); const scheduleBtn = page.locator('.e2e-toggle-schedule-day-panel').first(); - if (await scheduleBtn.isVisible().catch(() => false)) { + if (!isShorts && (await scheduleBtn.isVisible().catch(() => false))) { await scheduleBtn.click(); await page.locator('schedule-day-panel').first().waitFor({ state: 'visible', @@ -164,6 +174,9 @@ test.describe('@video reel', () => { bExtra = await showOverlay(page, 'No account. No tracking.', { noWait: true, }); + } else if (isShorts) { + // Shorts skip beat 2's drag entirely — go straight to Focus. + b2 = await showOverlay(page, 'Focus on what matters.', { noWait: true }); } else { b2 = await showOverlay(page, 'Plan your day.', { noWait: true }); } @@ -191,14 +204,18 @@ test.describe('@video reel', () => { } // ── Beat 2 — Plan your day. (native app drag) ──────────────────────── + // Shorts variant skips this entirely: the side schedule panel doesn't + // open on portrait, and there's nothing meaningful to drag onto. const schedulePanel = page.locator('schedule-day-panel').first(); const dragSource = page .locator('task') .filter({ hasText: CAPTURED_TASK_DISPLAY_TITLE }) .first(); - await dragSource.waitFor({ state: 'visible', timeout: 5_000 }); - const taskBox = await dragSource.boundingBox(); - const panelBox = await schedulePanel.boundingBox(); + if (!isShorts) { + await dragSource.waitFor({ state: 'visible', timeout: 5_000 }); + } + const taskBox = !isShorts ? await dragSource.boundingBox() : null; + const panelBox = !isShorts ? await schedulePanel.boundingBox() : null; if (taskBox && panelBox) { const taskHalfW = taskBox.width * 0.5; const taskHalfH = taskBox.height * 0.5; @@ -218,15 +235,17 @@ test.describe('@video reel', () => { await page.waitForTimeout(isFull ? 250 : 150); await parkCursor(page); } - await page.waitForTimeout(isFull ? 900 : 600); + await page.waitForTimeout(isFull ? 900 : isShorts ? 0 : 600); // ── Beat 2 → 3 transition: cut to black, dispatch focus mode ───────── + // Shorts: b2 already says "Focus on what matters." so reuse it rather + // than hide-and-respawn (which would re-fade the same words). let b3: OverlayHandle | undefined; await cutToScene( page, async () => { - void b2!.hide(); - if (await scheduleBtn.isVisible().catch(() => false)) { + if (!isShorts) void b2!.hide(); + if (!isShorts && (await scheduleBtn.isVisible().catch(() => false))) { await scheduleBtn.click(); } if (capturedTaskId) { @@ -258,14 +277,16 @@ test.describe('@video reel', () => { await page.clock.resume().catch(() => undefined); } await parkCursor(page); - b3 = await showOverlay(page, 'Focus on what matters.', { noWait: true }); + b3 = isShorts + ? b2 + : await showOverlay(page, 'Focus on what matters.', { noWait: true }); }, { fadeMs: 260, label: 'beat 2 to 3', }, ); - await page.waitForTimeout(isFull ? 1800 : 1200); + await page.waitForTimeout(isFull ? 1800 : isShorts ? 900 : 1200); // ── Beat 3 → 4 transition: cut to black, swap to integrations card ── let b4: OverlayHandle | undefined; @@ -308,7 +329,7 @@ test.describe('@video reel', () => { label: 'beat 3 to 4', }, ); - await page.waitForTimeout(isFull ? 2500 : 2000); + await page.waitForTimeout(isFull ? 2500 : isShorts ? 1700 : 2000); // ── Beat 4 → 5 transition: crossfade between controlled cards ─────── await showEndCard( @@ -331,7 +352,7 @@ test.describe('@video reel', () => { ); await page.waitForTimeout(260); if (b4) void b4.hide(); - await page.waitForTimeout(isFull ? 2740 : 2240); + await page.waitForTimeout(isFull ? 2740 : isShorts ? 1940 : 2240); // ── Loop boundary ──────────────────────────────────────────────────── // Don't hide the end card — let it stay live. The loop boundary diff --git a/e2e/tests/sync/supersync-keep-local-archive-preservation.spec.ts b/e2e/tests/sync/supersync-keep-local-archive-preservation.spec.ts index 6e1767abbb..e61eaea628 100644 --- a/e2e/tests/sync/supersync-keep-local-archive-preservation.spec.ts +++ b/e2e/tests/sync/supersync-keep-local-archive-preservation.spec.ts @@ -132,6 +132,11 @@ test.describe('@supersync @archive Keep Local Archive Preservation', () => { // ============ PHASE 3: Client B syncs and chooses "Keep Local" ============ console.log('[KeepLocal] Phase 3: Client B syncs and chooses Keep Local'); + // The imported fixture may be auto-repaired during import because it intentionally + // exercises backup compatibility. This test is about validation errors caused by + // the keep-local conflict path, so only collect errors from that point onward. + consoleErrors.length = 0; + await clientB.sync.setupSuperSync({ ...syncConfig, waitForInitialSync: false }); const dialog = clientB.page.locator('dialog-sync-import-conflict'); diff --git a/eslint.config.js b/eslint.config.js index 242f2914ab..1f9dfc0de7 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -167,6 +167,58 @@ module.exports = tseslint.config( ], }, }, + { + files: ['packages/sync-providers/**/*.ts'], + rules: { + 'no-restricted-imports': [ + 'error', + { + paths: [ + { + name: '@sp/shared-schema', + message: '@sp/sync-providers must not import SP-specific schema packages.', + }, + ], + patterns: [ + { + group: [ + '@angular/*', + '@ngrx/*', + '@sp/shared-schema/*', + '@sp/sync-core/*', + '../sync-core/*', + '../sync-core/**', + '../shared-schema/*', + '../shared-schema/**', + '../../shared-schema/*', + '../../shared-schema/**', + '../../sync-core/*', + '../../sync-core/**', + '**/shared-schema/*', + '**/shared-schema/**', + '**/sync-core/*', + '**/sync-core/**', + 'src/app/*', + 'src/app/**', + '**/src/app/*', + '**/src/app/**', + ], + message: + '@sp/sync-providers must use only public @sp/sync-core exports and must not import Angular, NgRx, app code, or SP-specific schema packages.', + }, + ], + }, + ], + 'no-restricted-syntax': [ + 'error', + { + selector: 'ImportExpression', + message: + '@sp/sync-providers must not use dynamic imports; they bypass package-boundary checks.', + }, + ], + }, + }, // NgRx effects files - require hydration guards on selector-based effects { files: ['**/*.effects.ts'], diff --git a/package-lock.json b/package-lock.json index 8c118e0d5c..8a83b10a98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9542,6 +9542,10 @@ "resolved": "packages/sync-core", "link": true }, + "node_modules/@sp/sync-providers": { + "resolved": "packages/sync-providers", + "link": true + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -29972,6 +29976,248 @@ } } }, + "packages/sync-providers": { + "name": "@sp/sync-providers", + "version": "1.0.0", + "dependencies": { + "@sp/sync-core": "*", + "hash-wasm": "^4.12.0" + }, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.0.0", + "vitest": "^3.2.4" + } + }, + "packages/sync-providers/node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/sync-providers/node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "packages/sync-providers/node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "packages/sync-providers/node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "packages/sync-providers/node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "packages/sync-providers/node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "packages/sync-providers/node_modules/vitest/node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, "packages/vite-plugin": { "name": "@super-productivity/vite-plugin", "version": "1.0.0", diff --git a/package.json b/package.json index 487ad100d1..4a7415f747 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "assemble:android:prod": "cd android && ./gradlew assembleRelease && cd ..", "assemble:android:stage": "cd android && ./gradlew assembleDebug && cd ..", "prebuild": "npm run env && node ./tools/git-version.js && npm run build:packages", - "prepare": "husky && ts-patch install && npm run env && npm run sync-core:build && npm run shared-schema:build && npm run plugin-api:build", + "prepare": "husky && ts-patch install && npm run env && npm run sync-core:build && npm run sync-providers:build && npm run shared-schema:build && npm run plugin-api:build", "build": "npm run buildAllElectron:noTests:prod", "build:packages": "node ./packages/build-packages.js", "buildAllElectron:noTests:prod": "npm run lint && npm run buildFrontend:prod:es6 && npm run electron:build", @@ -86,6 +86,9 @@ "video:open": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\",\"moduleResolution\":\"node\"}' npx ts-node --transpile-only e2e/store-video/open-video.ts", "video:full": "cross-env REEL_VARIANT=full npm run video", "video:ms-store": "cross-env REEL_VARIANT=ms-store npm run video", + "video:shorts": "cross-env REEL_VARIANT=shorts npm run video", + "video:keyboard": "cross-env REEL_VARIANT=keyboard npm run video", + "video:mobile": "cross-env REEL_VARIANT=mobile npm run video", "electron": "NODE_ENV=PROD electron .", "electron:build": "node ./tools/build-wayland-idle-helper.js && tsc -p electron/tsconfig.electron.json && node electron/scripts/bundle-preload.js", "electron:verify-asar": "node tools/verify-electron-requires.js .tmp/app-builds/linux-unpacked/resources/app.asar", @@ -125,7 +128,7 @@ "sync:ios": "npx cap sync ios", "dist:ios:prod": "npm run buildFrontend:prodWeb && npm run sync:ios", "stats": "ng build --configuration production --source-map --stats-json && npx esbuild-visualizer --metadata .tmp/angular-dist/stats.json && xdg-open stats.html", - "test": "npm run release-notes:test && cross-env TZ='Europe/Berlin' ng test --watch=false && npm run test:tz:ci", + "test": "npm run packages:test && npm run release-notes:test && cross-env TZ='Europe/Berlin' ng test --watch=false && npm run test:tz:ci", "test:once": "cross-env TZ='Europe/Berlin' ng test --watch=false", "test:watch": "cross-env TZ='Europe/Berlin' ng test --browsers ChromeHeadless", "test:tz:ci": "npm run test:tz:la", @@ -158,8 +161,11 @@ "release-notes:test": "node --test tools/release-notes.test.js", "clean:translations": "node ./tools/clean-translations.js", "plugin-api:build": "cd packages/plugin-api && npm run build", + "packages:test": "npm run sync-core:test && npm run sync-providers:test", "sync-core:build": "cd packages/sync-core && npm run build", "sync-core:test": "cd packages/sync-core && npm test", + "sync-providers:build": "cd packages/sync-providers && npm run build", + "sync-providers:test": "cd packages/sync-providers && npm test", "shared-schema:build": "cd packages/shared-schema && npm run build", "plugin-api:build:watch": "cd packages/plugin-api && npm run build:watch", "vite-plugin:build": "cd packages/vite-plugin && npm run build", diff --git a/packages/build-packages.js b/packages/build-packages.js index 7511d4d6c2..b13f7c0f5e 100755 --- a/packages/build-packages.js +++ b/packages/build-packages.js @@ -48,6 +48,15 @@ async function getPlugins() { skipCopy: true, }); + // Provider contracts and implementations depend on public sync-core exports + // but must stay outside the core package boundary. + plugins.push({ + name: 'sync-providers', + path: 'packages/sync-providers', + buildCommand: 'npm run build', + skipCopy: true, + }); + // Add shared-schema after sync-core as it's needed for app/server type resolution. plugins.push({ name: 'shared-schema', diff --git a/packages/super-sync-server/docker-compose.yml b/packages/super-sync-server/docker-compose.yml index b1cebca6a9..c3c39e6e3d 100644 --- a/packages/super-sync-server/docker-compose.yml +++ b/packages/super-sync-server/docker-compose.yml @@ -23,7 +23,7 @@ services: - NODE_ENV=${NODE_ENV:-production} - PORT=1900 - DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-supersync}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-supersync}} - - RUN_MIGRATIONS_ON_STARTUP=${RUN_MIGRATIONS_ON_STARTUP:-false} + - RUN_MIGRATIONS_ON_STARTUP=${RUN_MIGRATIONS_ON_STARTUP:-true} - JWT_SECRET=${JWT_SECRET} - PUBLIC_URL=${PUBLIC_URL:-http://localhost:1900} - CORS_ORIGINS=${CORS_ORIGINS:-https://app.super-productivity.com} diff --git a/packages/super-sync-server/prisma/migrations/20260511000000_add_entity_sequence_index/migration.sql b/packages/super-sync-server/prisma/migrations/20260511000000_add_entity_sequence_index/migration.sql index 3bb96e1744..7752187c8d 100644 --- a/packages/super-sync-server/prisma/migrations/20260511000000_add_entity_sequence_index/migration.sql +++ b/packages/super-sync-server/prisma/migrations/20260511000000_add_entity_sequence_index/migration.sql @@ -1,10 +1,7 @@ --- This migration is intentionally non-blocking. --- --- The original migration created --- "operations_user_id_entity_type_entity_id_server_seq_idx" with --- CREATE INDEX CONCURRENTLY. On production-sized operation logs this can run --- longer than a normal app deploy and interact badly with Prisma's advisory --- migration lock. --- --- Build the physical index as a separate off-hours deploy step instead: --- RUN_POST_MIGRATION_INDEXES=true ./scripts/deploy.sh +-- Add an index that satisfies conflict detection's latest-entity-op query. +-- Keep the existing entity lookup index so this migration is additive. +-- Avoid name-only idempotency here: interrupted concurrent builds can leave an +-- invalid index with the same name, which should fail loudly instead of being +-- marked as an applied migration. +CREATE INDEX CONCURRENTLY "operations_user_id_entity_type_entity_id_server_seq_idx" + ON "operations"("user_id", "entity_type", "entity_id", "server_seq"); diff --git a/packages/super-sync-server/prisma/migrations/20260512000000_add_full_state_sequence_index_drop_redundant_indexes/migration.sql b/packages/super-sync-server/prisma/migrations/20260512000000_add_full_state_sequence_index_drop_redundant_indexes/migration.sql new file mode 100644 index 0000000000..d46a0fb751 --- /dev/null +++ b/packages/super-sync-server/prisma/migrations/20260512000000_add_full_state_sequence_index_drop_redundant_indexes/migration.sql @@ -0,0 +1,15 @@ +-- Speed up latest/oldest full-state operation lookups without adding write +-- cost for regular operations. +-- Drop first so a rerun can recover from a failed CONCURRENTLY build that left +-- an INVALID index behind. +DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_full_state_server_seq_idx"; +CREATE INDEX CONCURRENTLY "operations_user_id_full_state_server_seq_idx" + ON "operations"("user_id", "server_seq") + WHERE "op_type" IN ('SYNC_IMPORT', 'BACKUP_IMPORT', 'REPAIR'); + +-- Remove redundant indexes while reducing write amplification on the hot table. +-- Restore-point op_type lookups are covered by the partial index above; add a +-- targeted index if future code filters by other op_type values. +DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_op_type_idx"; +DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_entity_type_entity_id_idx"; +DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_server_seq_idx"; diff --git a/packages/super-sync-server/prisma/schema.prisma b/packages/super-sync-server/prisma/schema.prisma index 625f86f84c..b23723b425 100644 --- a/packages/super-sync-server/prisma/schema.prisma +++ b/packages/super-sync-server/prisma/schema.prisma @@ -80,13 +80,10 @@ model Operation { user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@unique([userId, serverSeq]) - @@index([userId, serverSeq]) - @@index([userId, entityType, entityId]) - // Optional conflict-detection index including server_seq is built out-of-band - // by scripts/deploy.sh with RUN_POST_MIGRATION_INDEXES=true. + @@index([userId, entityType, entityId, serverSeq]) @@index([userId, clientId]) @@index([userId, receivedAt]) - @@index([userId, opType]) // Speeds up restore point listing (queries for SYNC_IMPORT/BACKUP_IMPORT ops) + // Restore-point opType lookups use a raw partial index in the 20260512000000 migration. @@map("operations") } diff --git a/packages/super-sync-server/scripts/deploy.sh b/packages/super-sync-server/scripts/deploy.sh index 6db39e7508..cf247ee736 100755 --- a/packages/super-sync-server/scripts/deploy.sh +++ b/packages/super-sync-server/scripts/deploy.sh @@ -54,75 +54,24 @@ echo "" cd "$SERVER_DIR" -trim_deploy_env_value() { - local value="$1" - - value="${value#"${value%%[![:space:]]*}"}" - value="${value%"${value##*[![:space:]]}"}" - printf '%s' "$value" -} - -load_deploy_env() { - local line - local key - local value - - [ -f ".env" ] || return 0 - - while IFS= read -r line || [ -n "$line" ]; do - line="${line%$'\r'}" - [[ "$line" =~ ^[[:space:]]*# ]] && continue - [[ "$line" != *=* ]] && continue - - key="${line%%=*}" - value="${line#*=}" - key="${key#"${key%%[![:space:]]*}"}" - key="${key%"${key##*[![:space:]]}"}" - - case "$key" in - GHCR_USER|GHCR_TOKEN|RUN_POST_MIGRATION_INDEXES|MIGRATION_WAIT_TIMEOUT|MIGRATION_RETRY_INTERVAL|DEPLOY_WAIT_TIMEOUT|POSTGRES_WAIT_TIMEOUT|POSTGRES_SERVICE) ;; - *) continue ;; - esac - - if [ -z "${!key+x}" ]; then - value="$(trim_deploy_env_value "$value")" - if [[ "$value" == \"*\" && "$value" == *\" ]]; then - value="${value:1:${#value}-2}" - elif [[ "$value" == \'*\' && "$value" == *\' ]]; then - value="${value:1:${#value}-2}" - else - value="${value%%[[:space:]]#*}" - value="$(trim_deploy_env_value "$value")" - fi - export "$key=$value" - fi - done < ".env" -} - -has_placeholder_ghcr_credentials() { - [ "${GHCR_USER:-}" = "your-github-username" ] || [ "${GHCR_TOKEN:-}" = "your-github-token" ] -} - # Pull latest code (scripts, docker-compose.yml, etc.) echo "==> Pulling latest code..." git pull --ff-only || { echo "WARNING: git pull failed — continuing with current files"; } echo "" -# Load deploy flags and GHCR credentials from .env. Docker Compose reads the -# service configuration from .env itself; this only covers values used directly -# by this script. -load_deploy_env +# Load GHCR credentials from .env (for private images) +if [ -f ".env" ]; then + GHCR_VARS=$(grep -E '^(GHCR_USER|GHCR_TOKEN)=' ".env" 2>/dev/null | xargs || true) + if [ -n "$GHCR_VARS" ]; then + export $GHCR_VARS + fi +fi # Login to GHCR if credentials provided if [ -n "${GHCR_TOKEN:-}" ] && [ -n "${GHCR_USER:-}" ]; then - if has_placeholder_ghcr_credentials; then - echo "==> Skipping GHCR login (placeholder credentials in .env)" - echo "" - else - echo "==> Logging in to GHCR..." - echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin - echo "" - fi + echo "==> Logging in to GHCR..." + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin + echo "" fi # Check if monitoring compose exists and include it @@ -131,14 +80,6 @@ if [ -f "docker-compose.monitoring.yml" ]; then COMPOSE_FILES="$COMPOSE_FILES -f docker-compose.monitoring.yml" fi -EXTERNAL_DB_START_SERVICES="supersync caddy" -if [ -f "docker-compose.monitoring.yml" ]; then - EXTERNAL_DB_START_SERVICES="$EXTERNAL_DB_START_SERVICES dozzle uptime-kuma" -fi - -ENTITY_SEQUENCE_INDEX_MIGRATION="20260511000000_add_entity_sequence_index" -ENTITY_SEQUENCE_INDEX_NAME="operations_user_id_entity_type_entity_id_server_seq_idx" - # Validate Caddyfile syntax before deploying CADDY_IMAGE=$(grep 'image:.*caddy:' docker-compose.yml | head -1 | awk '{print $2}' | tr -d '"'"'" || true) if [ -z "$CADDY_IMAGE" ]; then @@ -166,350 +107,11 @@ else docker compose $COMPOSE_FILES pull supersync fi -run_prisma() { - docker compose $COMPOSE_FILES run --rm --no-deps supersync npx prisma "$@" -} - -run_database_scalar() { - docker compose $COMPOSE_FILES run --rm --no-deps -T \ - -v "$SERVER_DIR/scripts/deploy-db-scalar.mjs:/app/deploy-db-scalar.mjs:ro" \ - supersync node /app/deploy-db-scalar.mjs "$1" -} - -run_database_execute() { - local sql="$1" - - printf '%s\n' "$sql" | docker compose $COMPOSE_FILES run --rm --no-deps -T supersync npx prisma db execute --stdin --schema prisma/schema.prisma -} - -is_prisma_advisory_lock_error() { - grep -qiE 'advisory[[:space:]-]+lock' <<<"$1" -} - -quote_sql_identifier() { - local identifier="${1//\"/\"\"}" - - printf '"%s"' "$identifier" -} - -get_current_db_schema() { - run_database_scalar "SELECT current_schema();" -} - -entity_sequence_index_state_sql() { - cat <&1); then - printf '%s\n' "$output" - return 0 - else - exit_code=$? - fi - - printf '%s\n' "$output" - - if ! is_prisma_advisory_lock_error "$output"; then - return "$exit_code" - fi - - elapsed=$(($(date +%s) - started_at)) - if [ "$elapsed" -ge "$retry_timeout" ]; then - echo "" - echo "ERROR: Timed out waiting for Prisma advisory migration lock after ${retry_timeout}s" - echo " Another supersync container may still be running migrations." - return "$exit_code" - fi - - echo "" - echo " Another Prisma migration is still holding the advisory lock while running: $description" - echo " Waiting ${retry_interval}s before retrying (elapsed ${elapsed}/${retry_timeout}s)..." - sleep "$retry_interval" - done -} - -run_migrations_with_retry() { - local exit_code - - if run_prisma_with_advisory_lock_retry "migrate deploy" migrate deploy; then - return 0 - else - exit_code=$? - fi - - echo "" - echo "==> migrate deploy failed; checking known index migration recovery..." - recover_failed_entity_sequence_index_migration - if [ "$KNOWN_INDEX_MIGRATION_RECOVERED" = "true" ]; then - run_prisma_with_advisory_lock_retry "migrate deploy after known index migration recovery" migrate deploy - return - fi - - return "$exit_code" -} - -drop_invalid_entity_sequence_index() { - local index_state - local drop_index_sql - - if ! index_state="$(get_entity_sequence_index_state)"; then - echo "ERROR: Failed to query invalid $ENTITY_SEQUENCE_INDEX_NAME index state" - return 1 - fi - - if [ "$index_state" = "invalid" ]; then - echo " Dropping invalid optional index $ENTITY_SEQUENCE_INDEX_NAME" - drop_index_sql="$(build_drop_entity_sequence_index_sql)" - run_database_execute "$drop_index_sql" - fi -} - -recover_failed_entity_sequence_index_migration() { - local failed_migration - local failed_migration_sql - local migration_table_exists - local advisory_lock_available - local advisory_lock_sql - - KNOWN_INDEX_MIGRATION_RECOVERED=false - - if ! migration_table_exists="$(run_database_scalar "SELECT (to_regclass(format('%I._prisma_migrations', current_schema())) IS NOT NULL)::text;")"; then - echo "ERROR: Failed to check Prisma migration table state" - return 1 - fi - if [ "$migration_table_exists" != "true" ]; then - return 0 - fi - - failed_migration_sql="SELECT 1 FROM _prisma_migrations WHERE migration_name = '$ENTITY_SEQUENCE_INDEX_MIGRATION' AND finished_at IS NULL AND rolled_back_at IS NULL LIMIT 1;" - if ! failed_migration="$(run_database_scalar "$failed_migration_sql")"; then - echo "ERROR: Failed to check $ENTITY_SEQUENCE_INDEX_MIGRATION migration state" - return 1 - fi - if [ "$failed_migration" != "1" ]; then - return 0 - fi - - # Prisma 5.x migration-engine uses this session-level advisory lock. This is - # only a snapshot probe; migrate resolve still uses its own advisory-lock retry. - advisory_lock_sql='SELECT CASE WHEN pg_try_advisory_lock(72707369) THEN pg_advisory_unlock(72707369)::text ELSE false::text END;' - if ! advisory_lock_available="$(run_database_scalar "$advisory_lock_sql")"; then - echo "ERROR: Failed to check Prisma advisory migration lock state" - return 1 - fi - if [ "$advisory_lock_available" != "true" ]; then - echo "" - echo "==> $ENTITY_SEQUENCE_INDEX_MIGRATION still appears to be active" - echo " Skipping automatic recovery and letting migrate deploy wait for Prisma's advisory lock." - return 0 - fi - - echo "" - echo "==> Recovering failed $ENTITY_SEQUENCE_INDEX_MIGRATION migration..." - drop_invalid_entity_sequence_index - run_prisma_with_advisory_lock_retry \ - "migrate resolve --rolled-back $ENTITY_SEQUENCE_INDEX_MIGRATION" \ - migrate resolve --rolled-back "$ENTITY_SEQUENCE_INDEX_MIGRATION" - KNOWN_INDEX_MIGRATION_RECOVERED=true -} - -warn_if_entity_sequence_index_missing() { - local index_state - - if ! index_state="$(get_entity_sequence_index_state)"; then - echo "ERROR: Failed to query optional $ENTITY_SEQUENCE_INDEX_NAME index state" - return 1 - fi - - case "$index_state" in - valid) - return 0 - ;; - missing) - echo "" - echo "WARNING: Optional index $ENTITY_SEQUENCE_INDEX_NAME is not present." - echo " Conflict-detection queries may be slower until it is built." - echo " Run RUN_POST_MIGRATION_INDEXES=true ./scripts/deploy.sh off-hours." - ;; - invalid) - echo "" - echo "WARNING: Optional index $ENTITY_SEQUENCE_INDEX_NAME exists but is invalid." - echo " Run RUN_POST_MIGRATION_INDEXES=true ./scripts/deploy.sh off-hours to rebuild it." - ;; - wrong) - echo "" - echo "WARNING: Optional index $ENTITY_SEQUENCE_INDEX_NAME exists with an unexpected definition." - echo " Drop or rename it manually before rebuilding optional indexes." - ;; - *) - echo "ERROR: Unexpected $ENTITY_SEQUENCE_INDEX_NAME index state: $index_state" - return 1 - ;; - esac -} - -run_post_migration_indexes() { - local create_index_sql - local index_state - - if [ "${RUN_POST_MIGRATION_INDEXES:-false}" != "true" ]; then - echo "" - echo "==> Skipping optional post-migration index builds" - echo " Set RUN_POST_MIGRATION_INDEXES=true during an off-hours deploy to build $ENTITY_SEQUENCE_INDEX_NAME" - warn_if_entity_sequence_index_missing || echo "WARNING: Could not verify optional $ENTITY_SEQUENCE_INDEX_NAME index state; continuing." - return 0 - fi - - echo "" - echo "==> Building post-migration indexes..." - - drop_invalid_entity_sequence_index - - if ! index_state="$(get_entity_sequence_index_state)"; then - echo "ERROR: Failed to query optional $ENTITY_SEQUENCE_INDEX_NAME index state" - return 1 - fi - if [ "$index_state" = "wrong" ]; then - require_valid_entity_sequence_index_definition - fi - - create_index_sql="$(build_create_entity_sequence_index_sql)" - run_database_execute "$create_index_sql" - require_valid_entity_sequence_index_definition -} - # Run migrations before replacing the app container. This keeps the currently # running app available while online index builds run, and it fails the deploy # before the app is restarted if Prisma cannot apply a migration. POSTGRES_WAIT_TIMEOUT="${POSTGRES_WAIT_TIMEOUT:-60}" -POSTGRES_SERVICE="${POSTGRES_SERVICE-postgres}" +POSTGRES_SERVICE="${POSTGRES_SERVICE:-postgres}" echo "" if [ -n "$POSTGRES_SERVICE" ]; then echo "==> Ensuring $POSTGRES_SERVICE is running (wait timeout: ${POSTGRES_WAIT_TIMEOUT}s)..." @@ -520,13 +122,13 @@ fi echo "" echo "==> Applying database migrations before app restart..." -run_migrations_with_retry -run_post_migration_indexes +docker compose $COMPOSE_FILES run --rm --no-deps supersync npx prisma migrate deploy -# The migration above already ran while the old app was still serving. Force -# startup migrations off for this compose update so the replacement app cannot -# race the deploy migrator or retry long-running index work in a restart loop. -export RUN_MIGRATIONS_ON_STARTUP=false +# The migration above already ran while the old app was still serving. Disable +# startup migrations for this compose update so the replacement app starts +# immediately after image creation. Direct docker-compose users keep the image +# default unless they also set RUN_MIGRATIONS_ON_STARTUP=false. +export RUN_MIGRATIONS_ON_STARTUP="${RUN_MIGRATIONS_ON_STARTUP:-false}" # Start containers and wait for all health checks. Online index migrations should # already be applied, but the longer timeout still covers slow image starts and @@ -534,13 +136,7 @@ export RUN_MIGRATIONS_ON_STARTUP=false WAIT_TIMEOUT="${DEPLOY_WAIT_TIMEOUT:-900}" echo "" echo "==> Starting containers (wait timeout: ${WAIT_TIMEOUT}s)..." -if [ -n "$POSTGRES_SERVICE" ]; then - START_COMMAND=(docker compose $COMPOSE_FILES up -d --wait --wait-timeout "$WAIT_TIMEOUT") -else - START_COMMAND=(docker compose $COMPOSE_FILES up -d --wait --wait-timeout "$WAIT_TIMEOUT" --no-deps $EXTERNAL_DB_START_SERVICES) -fi - -if ! "${START_COMMAND[@]}" 2>&1; then +if ! docker compose $COMPOSE_FILES up -d --wait --wait-timeout "$WAIT_TIMEOUT" 2>&1; then echo "" echo "==> Container startup failed!" diff --git a/packages/super-sync-server/src/sync/gzip.ts b/packages/super-sync-server/src/sync/gzip.ts index eb28093dea..9f799526b1 100644 --- a/packages/super-sync-server/src/sync/gzip.ts +++ b/packages/super-sync-server/src/sync/gzip.ts @@ -36,12 +36,18 @@ export const gzipAsync = (buffer: Buffer, options?: zlib.ZlibOptions): Promise { - if (!(cause instanceof Error)) return false; + const code = + cause !== null && typeof cause === 'object' && 'code' in cause + ? String((cause as { code?: unknown }).code) + : undefined; - const code = (cause as { code?: unknown }).code; + if (code === 'ERR_BUFFER_TOO_LARGE' || code === 'ERR_OUT_OF_RANGE') { + return true; + } + + const message = cause instanceof Error ? cause.message : ''; return ( - code === 'ERR_BUFFER_TOO_LARGE' || - code === 'ERR_OUT_OF_RANGE' || - cause.message.includes('maxOutputLength') + message.includes('maxOutputLength') || + message.includes('Cannot create a Buffer larger than') ); }; diff --git a/packages/super-sync-server/src/sync/services/operation-download.service.ts b/packages/super-sync-server/src/sync/services/operation-download.service.ts index 37abdeb784..f21ded36aa 100644 --- a/packages/super-sync-server/src/sync/services/operation-download.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-download.service.ts @@ -13,6 +13,59 @@ import { } from '../sync.types'; import { Logger } from '../../logger'; +const OPERATION_DOWNLOAD_SELECT = { + id: true, + serverSeq: true, + clientId: true, + actionType: true, + opType: true, + entityType: true, + entityId: true, + payload: true, + vectorClock: true, + schemaVersion: true, + clientTimestamp: true, + receivedAt: true, + isPayloadEncrypted: true, + syncImportReason: true, +} as const; + +type OperationDownloadRow = { + id: string; + serverSeq: number; + clientId: string; + actionType: string; + opType: string; + entityType: string; + entityId: string | null; + payload: unknown; + vectorClock: unknown; + schemaVersion: number; + clientTimestamp: bigint; + receivedAt: bigint; + isPayloadEncrypted: boolean; + syncImportReason: string | null; +}; + +const mapOperationRow = (row: OperationDownloadRow): ServerOperation => ({ + serverSeq: row.serverSeq, + op: { + id: row.id, + clientId: row.clientId, + actionType: row.actionType, + opType: row.opType as Operation['opType'], + entityType: row.entityType, + entityId: row.entityId ?? undefined, + payload: row.payload, + vectorClock: row.vectorClock as VectorClock, + schemaVersion: row.schemaVersion, + timestamp: Number(row.clientTimestamp), + isPayloadEncrypted: row.isPayloadEncrypted, + syncImportReason: row.syncImportReason ?? undefined, + }, + receivedAt: Number(row.receivedAt), +}); + export class OperationDownloadService { /** * Get operations since a given sequence number. @@ -34,26 +87,10 @@ export class OperationDownloadService { serverSeq: 'asc', }, take: limit, + select: OPERATION_DOWNLOAD_SELECT, }); - return ops.map((row) => ({ - serverSeq: row.serverSeq, - op: { - id: row.id, - clientId: row.clientId, - actionType: row.actionType, - opType: row.opType as Operation['opType'], - entityType: row.entityType, - entityId: row.entityId ?? undefined, - payload: row.payload, - vectorClock: row.vectorClock as unknown as VectorClock, - schemaVersion: row.schemaVersion, - timestamp: Number(row.clientTimestamp), - isPayloadEncrypted: row.isPayloadEncrypted, - syncImportReason: row.syncImportReason ?? undefined, - }, - receivedAt: Number(row.receivedAt), - })); + return ops.map(mapOperationRow); } /** @@ -63,12 +100,17 @@ export class OperationDownloadService { * BACKUP_IMPORT, REPAIR), we skip to that operation's sequence instead. This prevents * sending operations that will be filtered out by the client anyway, saving bandwidth * and processing time. + * + * includeSnapshotMetadata controls only the expensive vector-clock aggregate; the + * full-state fast-forward still applies so piggyback downloads can return the + * replacing operation without scanning superseded history. */ async getOpsSinceWithSeq( userId: number, sinceSeq: number, excludeClient?: string, limit: number = 500, + includeSnapshotMetadata: boolean = true, ): Promise<{ ops: ServerOperation[]; latestSeq: number; @@ -84,6 +126,23 @@ export class OperationDownloadService { }); const latestSeq = seqRow?.lastSeq ?? 0; + if (latestSeq === 0) { + const gapDetected = sinceSeq > 0; + if (gapDetected) { + Logger.warn( + `[user:${userId}] Gap detected: client at sinceSeq=${sinceSeq} but server is empty (latestSeq=0)`, + ); + } + + return { + ops: [], + latestSeq, + gapDetected, + latestSnapshotSeq: undefined, + snapshotVectorClock: undefined, + }; + } + // Find the latest full-state operation (SYNC_IMPORT, BACKUP_IMPORT, REPAIR) // These operations supersede all previous operations const latestFullStateOp = await tx.operation.findFirst({ @@ -112,38 +171,42 @@ export class OperationDownloadService { `(latest snapshot at seq ${latestSnapshotSeq})`, ); - // Compute aggregated vector clock from all ops up to and including the snapshot. - // This ensures clients know about all clock entries from skipped ops. - // Uses a SQL aggregate to avoid loading all individual ops' clocks into memory — - // the aggregation happens in PostgreSQL, returning only the final per-client max. - const clockRows = await tx.$queryRaw< - Array<{ client_id: string; max_counter: bigint }> - >` - SELECT kv.key AS client_id, MAX(kv.value::bigint) AS max_counter - FROM operations, LATERAL jsonb_each_text(vector_clock) AS kv(key, value) - WHERE user_id = ${userId} - AND server_seq <= ${latestSnapshotSeq} - AND jsonb_typeof(vector_clock) = 'object' - AND kv.value ~ '^[0-9]+$' - GROUP BY kv.key - `; + if (includeSnapshotMetadata) { + // Compute aggregated vector clock from all ops up to and including the snapshot. + // This ensures clients know about all clock entries from skipped ops. + // Uses a SQL aggregate to avoid loading all individual ops' clocks into memory — + // the aggregation happens in PostgreSQL, returning only the final per-client max. + const clockRows = await tx.$queryRaw< + Array<{ client_id: string; max_counter: bigint }> + >` + SELECT kv.key AS client_id, MAX(kv.value::bigint) AS max_counter + FROM operations, LATERAL jsonb_each_text(vector_clock) AS kv(key, value) + WHERE user_id = ${userId} + AND server_seq <= ${latestSnapshotSeq} + AND jsonb_typeof(vector_clock) = 'object' + AND kv.value ~ '^[0-9]+$' + GROUP BY kv.key + `; - snapshotVectorClock = {}; - for (const row of clockRows) { - snapshotVectorClock[row.client_id] = Number(row.max_counter); + snapshotVectorClock = {}; + for (const row of clockRows) { + snapshotVectorClock[row.client_id] = Number(row.max_counter); + } + + // Limit snapshot clock to MAX_VECTOR_CLOCK_SIZE to prevent oversized + // clocks from being sent to clients. Preserve the requesting client's ID + // and the snapshot author's ID to avoid false EQUAL in comparison. + const preserveIds: string[] = []; + if (excludeClient) preserveIds.push(excludeClient); + if (latestFullStateOp?.clientId) { + preserveIds.push(latestFullStateOp.clientId); + } + snapshotVectorClock = limitVectorClockSize(snapshotVectorClock, preserveIds); + + Logger.info( + `[user:${userId}] Computed snapshotVectorClock with ${Object.keys(snapshotVectorClock).length} entries: ${JSON.stringify(snapshotVectorClock)}`, + ); } - - // Limit snapshot clock to MAX_VECTOR_CLOCK_SIZE to prevent oversized - // clocks from being sent to clients. Preserve the requesting client's ID - // and the snapshot author's ID to avoid false EQUAL in comparison. - const preserveIds: string[] = []; - if (excludeClient) preserveIds.push(excludeClient); - if (latestFullStateOp?.clientId) preserveIds.push(latestFullStateOp.clientId); - snapshotVectorClock = limitVectorClockSize(snapshotVectorClock, preserveIds); - - Logger.info( - `[user:${userId}] Computed snapshotVectorClock with ${Object.keys(snapshotVectorClock).length} entries: ${JSON.stringify(snapshotVectorClock)}`, - ); } const ops = await tx.operation.findMany({ @@ -156,28 +219,24 @@ export class OperationDownloadService { serverSeq: 'asc', }, take: limit, + select: OPERATION_DOWNLOAD_SELECT, }); - // Get min sequence efficiently - const minSeqAgg = await tx.operation.aggregate({ - where: { userId, serverSeq: { lte: latestSeq } }, - _min: { serverSeq: true }, - }); + let minSeq: number | null = null; - const minSeq = minSeqAgg._min.serverSeq ?? null; + if (sinceSeq > 0 && latestSeq > 0) { + // Get min sequence efficiently, but only when gap detection can use it. + const minSeqAgg = await tx.operation.aggregate({ + where: { userId, serverSeq: { lte: latestSeq } }, + _min: { serverSeq: true }, + }); + minSeq = minSeqAgg._min.serverSeq ?? null; + } // Gap detection logic let gapDetected = false; - // Case 1: Client has history but server is empty - if (sinceSeq > 0 && latestSeq === 0) { - gapDetected = true; - Logger.warn( - `[user:${userId}] Gap detected: client at sinceSeq=${sinceSeq} but server is empty (latestSeq=0)`, - ); - } - - // Case 2: Client is ahead of server + // Case 1: Client is ahead of server if (sinceSeq > latestSeq && latestSeq > 0) { gapDetected = true; Logger.warn( @@ -186,7 +245,7 @@ export class OperationDownloadService { } if (sinceSeq > 0 && latestSeq > 0) { - // Case 3: Requested seq is purged + // Case 2: Requested seq is purged if (minSeq !== null && sinceSeq < minSeq - 1) { gapDetected = true; Logger.warn( @@ -194,7 +253,7 @@ export class OperationDownloadService { ); } - // Case 4: Gap in returned operations (use effectiveSinceSeq which accounts for snapshot skip) + // Case 3: Gap in returned operations (use effectiveSinceSeq which accounts for snapshot skip) if ( !excludeClient && ops.length > 0 && @@ -207,24 +266,7 @@ export class OperationDownloadService { } } - const mappedOps = ops.map((row) => ({ - serverSeq: row.serverSeq, - op: { - id: row.id, - clientId: row.clientId, - actionType: row.actionType, - opType: row.opType as Operation['opType'], - entityType: row.entityType, - entityId: row.entityId ?? undefined, - payload: row.payload, - vectorClock: row.vectorClock as unknown as VectorClock, - schemaVersion: row.schemaVersion, - timestamp: Number(row.clientTimestamp), - isPayloadEncrypted: row.isPayloadEncrypted, - syncImportReason: row.syncImportReason ?? undefined, - }, - receivedAt: Number(row.receivedAt), - })); + const mappedOps = ops.map(mapOperationRow); return { ops: mappedOps, diff --git a/packages/super-sync-server/src/sync/services/snapshot.service.ts b/packages/super-sync-server/src/sync/services/snapshot.service.ts index 4489e0e777..eab79c60bc 100644 --- a/packages/super-sync-server/src/sync/services/snapshot.service.ts +++ b/packages/super-sync-server/src/sync/services/snapshot.service.ts @@ -8,7 +8,7 @@ * generation for the same user. This prevents duplicate expensive computation. */ import { Prisma } from '@prisma/client'; -import { prisma, Operation as PrismaOperation } from '../../db'; +import { prisma } from '../../db'; import { Logger } from '../../logger'; import { CURRENT_SCHEMA_VERSION, @@ -70,6 +70,50 @@ export class EncryptedOpsNotSupportedError extends Error { } } +const REPLAY_OPERATION_SELECT = { + id: true, + serverSeq: true, + opType: true, + entityType: true, + entityId: true, + payload: true, + schemaVersion: true, + isPayloadEncrypted: true, +} as const; + +type ReplayOperationRow = { + id: string; + serverSeq: number; + opType: string; + entityType: string; + entityId: string | null; + payload: unknown; + schemaVersion: number; + isPayloadEncrypted: boolean; +}; + +const assertContiguousReplayBatch = ( + ops: ReplayOperationRow[], + expectedFirstSeq: number, + targetSeq: number, +): void => { + if (ops.length === 0) { + throw new Error( + `SNAPSHOT_REPLAY_INCOMPLETE: Missing operations from seq ${expectedFirstSeq} to ${targetSeq}.`, + ); + } + + let expectedSeq = expectedFirstSeq; + for (const op of ops) { + if (op.serverSeq !== expectedSeq) { + throw new Error( + `SNAPSHOT_REPLAY_INCOMPLETE: Expected seq ${expectedSeq} but got ${op.serverSeq} while replaying to seq ${targetSeq}.`, + ); + } + expectedSeq++; + } +}; + export interface SnapshotResult { state: unknown; serverSeq: number; @@ -351,19 +395,24 @@ export class SnapshotService { const batchOps = await tx.operation.findMany({ where: { userId, + // Keep the replay aligned with the advertised snapshot sequence. serverSeq: { gt: currentSeq, lte: latestSeq }, }, orderBy: { serverSeq: 'asc' }, take: BATCH_SIZE, + select: REPLAY_OPERATION_SELECT, }); + // _resolveExpectedFirstSeq permits a leading gap when the first + // surviving op is a full-state op (e.g. SYNC_IMPORT after a + // clean-slate upload), because full-state ops reset state anyway. const expectedFirstSeq = this._resolveExpectedFirstSeq( batchOps, currentSeq, startSeq, latestSeq, ); - this._assertContiguousReplayBatch(batchOps, expectedFirstSeq, latestSeq); + assertContiguousReplayBatch(batchOps, expectedFirstSeq, latestSeq); // Replay ops state = this.replayOpsToState(batchOps, state); @@ -572,10 +621,12 @@ export class SnapshotService { const batchOps = await tx.operation.findMany({ where: { userId, + // Historical restore points replay only up to the requested sequence. serverSeq: { gt: currentSeq, lte: targetSeq }, }, orderBy: { serverSeq: 'asc' }, take: BATCH_SIZE, + select: REPLAY_OPERATION_SELECT, }); const expectedFirstSeq = this._resolveExpectedFirstSeq( @@ -584,7 +635,7 @@ export class SnapshotService { startSeq, targetSeq, ); - this._assertContiguousReplayBatch(batchOps, expectedFirstSeq, targetSeq); + assertContiguousReplayBatch(batchOps, expectedFirstSeq, targetSeq); state = this.replayOpsToState(batchOps, state); currentSeq = batchOps[batchOps.length - 1].serverSeq; @@ -608,7 +659,7 @@ export class SnapshotService { * Used internally by snapshot generation methods. */ replayOpsToState( - ops: PrismaOperation[], + ops: ReplayOperationRow[], initialState: Record = {}, ): Record { const state = { ...(initialState as Record>) }; @@ -853,7 +904,7 @@ export class SnapshotService { * mid-stream gaps still indicate corruption and must throw. */ private _resolveExpectedFirstSeq( - batchOps: PrismaOperation[], + batchOps: ReplayOperationRow[], currentSeq: number, startSeq: number, targetSeq: number, @@ -876,26 +927,4 @@ export class SnapshotService { } return firstOp.serverSeq; } - - private _assertContiguousReplayBatch( - batchOps: PrismaOperation[], - expectedFirstSeq: number, - targetSeq: number, - ): void { - if (batchOps.length === 0) { - throw new Error( - `SNAPSHOT_REPLAY_INCOMPLETE: Missing operation at serverSeq ${expectedFirstSeq} while replaying to ${targetSeq}`, - ); - } - - for (let i = 0; i < batchOps.length; i++) { - const expectedSeq = expectedFirstSeq + i; - const actualSeq = batchOps[i].serverSeq; - if (actualSeq !== expectedSeq) { - throw new Error( - `SNAPSHOT_REPLAY_INCOMPLETE: Expected operation serverSeq ${expectedSeq} but got ${actualSeq} while replaying to ${targetSeq}`, - ); - } - } - } } diff --git a/packages/super-sync-server/src/sync/sync.routes.ts b/packages/super-sync-server/src/sync/sync.routes.ts index 760770e9f4..ec5ac501f2 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ts @@ -332,6 +332,7 @@ export const syncRoutes = async (fastify: FastifyInstance): Promise => { lastKnownServerSeq, clientId, PIGGYBACK_LIMIT, + false, ); newOps = opsResult.ops; latestSeq = opsResult.latestSeq; @@ -411,6 +412,7 @@ export const syncRoutes = async (fastify: FastifyInstance): Promise => { lastKnownServerSeq, clientId, PIGGYBACK_LIMIT, + false, ); newOps = opsResult.ops; latestSeq = opsResult.latestSeq; diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index 0992d55940..a31da0f869 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -674,6 +674,7 @@ export class SyncService { sinceSeq: number, excludeClient?: string, limit: number = 500, + includeSnapshotMetadata: boolean = true, ): Promise<{ ops: ServerOperation[]; latestSeq: number; @@ -686,6 +687,7 @@ export class SyncService { sinceSeq, excludeClient, limit, + includeSnapshotMetadata, ); } @@ -875,6 +877,7 @@ export class SyncService { }, orderBy: { serverSeq: 'asc' }, select: { serverSeq: true, opType: true }, + take: 2, }); if (restorePoints.length === 0) { @@ -997,7 +1000,7 @@ export class SyncService { }; } - // Count restore points remaining + // Only need to know whether at least two restore points remain. const restorePoints = await prisma.operation.findMany({ where: { userId, @@ -1005,6 +1008,7 @@ export class SyncService { }, orderBy: { serverSeq: 'asc' }, select: { serverSeq: true }, + take: 2, }); // Minimum: 1 restore point + all ops after it @@ -1038,7 +1042,7 @@ export class SyncService { Logger.info( `[user:${userId}] Auto-cleanup iteration: freed ${Math.round(result.freedBytes / 1024)}KB, ` + - `${restorePoints.length - 1} restore points remaining`, + `${restorePoints.length - 1} restore point(s) remaining in current cleanup window`, ); } diff --git a/packages/super-sync-server/tests/decompress-body.spec.ts b/packages/super-sync-server/tests/decompress-body.spec.ts index c5b02b25a8..0ea67515ca 100644 --- a/packages/super-sync-server/tests/decompress-body.spec.ts +++ b/packages/super-sync-server/tests/decompress-body.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import * as zlib from 'zlib'; import { promisify } from 'util'; import { @@ -30,12 +30,47 @@ describe('decompressBody helper', () => { expect(JSON.parse(decompressed.toString('utf-8'))).toEqual(testPayload); }); - it('should reject when decompressed output exceeds maxOutputLength', async () => { + it('should reject gzip output beyond maxOutputLength', async () => { const jsonPayload = JSON.stringify(testPayload); const compressed = await gzipAsync(Buffer.from(jsonPayload, 'utf-8')); await expect(decompressBody(compressed, undefined, 1)).rejects.toThrow(); }); + + it('should pass maxOutputLength to zlib.gunzip', async () => { + const gunzipMock = vi.fn( + ( + _body: Buffer, + _options: zlib.ZlibOptions | ((err: Error | null, result: Buffer) => void), + callback?: (err: Error | null, result: Buffer) => void, + ) => { + if (callback) { + callback(null, Buffer.from(JSON.stringify(testPayload), 'utf-8')); + } + }, + ); + + try { + vi.resetModules(); + vi.doMock('zlib', () => ({ gunzip: gunzipMock })); + + const { decompressBody: decompressBodyWithMockedZlib } = + await import('../src/sync/compressed-body-parser'); + await decompressBodyWithMockedZlib( + Buffer.from('mock-gzip-body'), + undefined, + 1024, + ); + + expect(gunzipMock).toHaveBeenCalled(); + expect(gunzipMock.mock.calls[0][1]).toMatchObject({ + maxOutputLength: 1024, + }); + } finally { + vi.doUnmock('zlib'); + vi.resetModules(); + } + }); }); describe('base64-encoded gzip (Android)', () => { @@ -165,6 +200,45 @@ describe('parseCompressedJsonBody helper', () => { } }); + it('should pass maxDecompressedSize to zlib.gunzip as maxOutputLength', async () => { + const gunzipMock = vi.fn( + ( + _body: Buffer, + _options: zlib.ZlibOptions | ((err: Error | null, result: Buffer) => void), + callback?: (err: Error | null, result: Buffer) => void, + ) => { + if (callback) { + callback(null, Buffer.from(JSON.stringify(testPayload), 'utf-8')); + } + }, + ); + + try { + vi.resetModules(); + vi.doMock('zlib', () => ({ gunzip: gunzipMock })); + + const { parseCompressedJsonBody: parseCompressedJsonBodyWithMockedZlib } = + await import('../src/sync/compressed-body-parser'); + const result = await parseCompressedJsonBodyWithMockedZlib( + Buffer.from('mock-gzip-body'), + undefined, + { + maxCompressedSize: 1024, + maxDecompressedSize: 456, + }, + ); + + expect(result.ok).toBe(true); + expect(gunzipMock).toHaveBeenCalled(); + expect(gunzipMock.mock.calls[0][1]).toMatchObject({ + maxOutputLength: 456, + }); + } finally { + vi.doUnmock('zlib'); + vi.resetModules(); + } + }); + it('should return 400 for non-buffer gzip bodies', async () => { const result = await parseCompressedJsonBody('not-a-buffer', undefined, { maxCompressedSize: 1024, diff --git a/packages/super-sync-server/tests/integration/snapshot-skip-optimization.integration.spec.ts b/packages/super-sync-server/tests/integration/snapshot-skip-optimization.integration.spec.ts index b35c9b099c..9b9576f64b 100644 --- a/packages/super-sync-server/tests/integration/snapshot-skip-optimization.integration.spec.ts +++ b/packages/super-sync-server/tests/integration/snapshot-skip-optimization.integration.spec.ts @@ -32,7 +32,9 @@ vi.mock('../../src/db', () => ({ // Mock the auth module to bypass authentication vi.mock('../../src/auth', () => ({ - verifyToken: vi.fn().mockResolvedValue({ userId: 1, email: 'test@test.com' }), + verifyToken: vi + .fn() + .mockResolvedValue({ valid: true, userId: 1, email: 'test@test.com' }), })); // Import after mocking @@ -65,7 +67,9 @@ describe('Snapshot Skip Optimization Integration', () => { vi.mocked(prisma.$transaction).mockImplementation(async (callback: any) => { const tx = { operation: { - findFirst: vi.fn().mockResolvedValue({ serverSeq: syncImportSeq }), + findFirst: vi + .fn() + .mockResolvedValue({ serverSeq: syncImportSeq, clientId: 'client-a' }), findMany: vi.fn().mockResolvedValue([ { id: uuidv7(), @@ -105,6 +109,7 @@ describe('Snapshot Skip Optimization Integration', () => { userSyncState: { findUnique: vi.fn().mockResolvedValue({ lastSeq: 11 }), }, + $queryRaw: vi.fn().mockResolvedValue([]), }; return callback(tx); }); @@ -128,7 +133,9 @@ describe('Snapshot Skip Optimization Integration', () => { vi.mocked(prisma.$transaction).mockImplementation(async (callback: any) => { const tx = { operation: { - findFirst: vi.fn().mockResolvedValue({ serverSeq: syncImportSeq }), + findFirst: vi + .fn() + .mockResolvedValue({ serverSeq: syncImportSeq, clientId: 'client-a' }), findMany: vi.fn().mockImplementation(async (args: any) => { // The query should start from seq 99 (syncImportSeq - 1), not 0 const startSeq = args.where.serverSeq.gt; @@ -174,6 +181,7 @@ describe('Snapshot Skip Optimization Integration', () => { userSyncState: { findUnique: vi.fn().mockResolvedValue({ lastSeq: 101 }), }, + $queryRaw: vi.fn().mockResolvedValue([]), }; return callback(tx); }); @@ -200,7 +208,9 @@ describe('Snapshot Skip Optimization Integration', () => { vi.mocked(prisma.$transaction).mockImplementation(async (callback: any) => { const tx = { operation: { - findFirst: vi.fn().mockResolvedValue({ serverSeq: syncImportSeq }), + findFirst: vi + .fn() + .mockResolvedValue({ serverSeq: syncImportSeq, clientId: 'client-a' }), findMany: vi.fn().mockImplementation(async (args: any) => { // Client already has ops up to seq 50, sinceSeq should NOT be modified expect(args.where.serverSeq.gt).toBe(50); @@ -303,7 +313,9 @@ describe('Snapshot Skip Optimization Integration', () => { const tx = { operation: { // findFirst with orderBy: { serverSeq: 'desc' } returns the LATEST - findFirst: vi.fn().mockResolvedValue({ serverSeq: latestFullStateSeq }), + findFirst: vi + .fn() + .mockResolvedValue({ serverSeq: latestFullStateSeq, clientId: 'client-a' }), findMany: vi.fn().mockImplementation(async (args: any) => { // Should skip to seq 49, not seq 9 expect(args.where.serverSeq.gt).toBe(latestFullStateSeq - 1); @@ -332,6 +344,7 @@ describe('Snapshot Skip Optimization Integration', () => { userSyncState: { findUnique: vi.fn().mockResolvedValue({ lastSeq: 50 }), }, + $queryRaw: vi.fn().mockResolvedValue([]), }; return callback(tx); }); diff --git a/packages/super-sync-server/tests/migration-sql.spec.ts b/packages/super-sync-server/tests/migration-sql.spec.ts index 5af1500e19..e622d96f53 100644 --- a/packages/super-sync-server/tests/migration-sql.spec.ts +++ b/packages/super-sync-server/tests/migration-sql.spec.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from 'vitest'; const currentDir = dirname(fileURLToPath(import.meta.url)); describe('performance migrations', () => { - it('keeps the entity sequence index out of blocking Prisma migrations', () => { + it('adds the entity sequence index without a blocking or destructive migration', () => { const migrationSql = readFileSync( join( currentDir, @@ -15,152 +15,65 @@ describe('performance migrations', () => { 'utf8', ); - expect(migrationSql).toContain('intentionally non-blocking'); - expect(migrationSql).toContain('RUN_POST_MIGRATION_INDEXES=true ./scripts/deploy.sh'); + expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY'); + expect(migrationSql).not.toMatch(/\bIF\s+NOT\s+EXISTS\b/i); expect(migrationSql).toContain( '"operations_user_id_entity_type_entity_id_server_seq_idx"', ); - expect(migrationSql).not.toMatch(/^\s*CREATE\s+INDEX\b/im); - expect(migrationSql).not.toMatch(/^\s*DROP\s+INDEX\b/im); + expect(migrationSql).toContain( + 'ON "operations"("user_id", "entity_type", "entity_id", "server_seq")', + ); + expect(migrationSql).not.toMatch(/\bDROP\s+INDEX\b/i); expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i); expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i); }); + it('adds partial full-state sequence index and drops redundant indexes', () => { + const migrationSql = readFileSync( + join( + currentDir, + '../prisma/migrations/20260512000000_add_full_state_sequence_index_drop_redundant_indexes/migration.sql', + ), + 'utf8', + ); + + expect(migrationSql).toContain('CREATE INDEX CONCURRENTLY'); + expect(migrationSql).toContain('"operations_user_id_full_state_server_seq_idx"'); + expect(migrationSql).toContain('ON "operations"("user_id", "server_seq")'); + expect(migrationSql).toContain( + `WHERE "op_type" IN ('SYNC_IMPORT', 'BACKUP_IMPORT', 'REPAIR')`, + ); + expect(migrationSql).toContain( + 'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_op_type_idx"', + ); + expect(migrationSql).toContain( + 'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_entity_type_entity_id_idx"', + ); + expect(migrationSql).toContain( + 'DROP INDEX CONCURRENTLY IF EXISTS "operations_user_id_server_seq_idx"', + ); + expect(migrationSql).not.toMatch(/\bDROP\s+TABLE\b/i); + expect(migrationSql).not.toMatch(/\bALTER\s+TABLE\b/i); + expect(migrationSql).not.toMatch(/\bBEGIN\b|\bCOMMIT\b/i); + }); + it('runs migrations before replacing the app during compose deploys', () => { const deployScript = readFileSync(join(currentDir, '../scripts/deploy.sh'), 'utf8'); const dockerfile = readFileSync(join(currentDir, '../Dockerfile'), 'utf8'); const composeFile = readFileSync(join(currentDir, '../docker-compose.yml'), 'utf8'); - const envExample = readFileSync(join(currentDir, '../env.example'), 'utf8'); - const readmeFile = readFileSync(join(currentDir, '../README.md'), 'utf8'); - const deployDbScalarScript = readFileSync( - join(currentDir, '../scripts/deploy-db-scalar.mjs'), - 'utf8', - ); - const buildComposeFile = readFileSync( - join(currentDir, '../docker-compose.build.yml'), - 'utf8', - ); - const schemaFile = readFileSync(join(currentDir, '../prisma/schema.prisma'), 'utf8'); - const migrationCommand = - 'docker compose $COMPOSE_FILES run --rm --no-deps supersync npx prisma "$@"'; - const indexSql = - 'CREATE INDEX CONCURRENTLY IF NOT EXISTS %s ON %s("user_id", "entity_type", "entity_id", "server_seq");'; - const failedMigrationSql = - "SELECT 1 FROM _prisma_migrations WHERE migration_name = '$ENTITY_SEQUENCE_INDEX_MIGRATION' AND finished_at IS NULL AND rolled_back_at IS NULL LIMIT 1;"; - const recoveryCall = '\n recover_failed_entity_sequence_index_migration\n'; - const migrationCall = '\nrun_migrations_with_retry\n'; - const indexCall = '\nrun_post_migration_indexes\n\n# The migration above'; + const migrationCommand = 'run --rm --no-deps supersync npx prisma migrate deploy'; const startCommand = 'up -d --wait --wait-timeout "$WAIT_TIMEOUT"'; - const recoveryCallIndex = deployScript.indexOf(recoveryCall); - const migrationCallIndex = deployScript.indexOf(migrationCall); - const indexCallIndex = deployScript.indexOf(indexCall); - const startCommandIndex = deployScript.indexOf(startCommand); - const recoveryFunction = deployScript.slice( - deployScript.indexOf('recover_failed_entity_sequence_index_migration()'), - deployScript.indexOf('run_post_migration_indexes()'), - ); - expect(deployScript).toContain('load_deploy_env'); - expect(deployScript).toContain('trim_deploy_env_value'); - expect(deployScript).toContain('inherit_errexit'); - expect(deployScript).toContain('value="${value%%[[:space:]]#*}"'); - expect(deployScript).toContain('EXTERNAL_DB_START_SERVICES="supersync caddy"'); - expect(deployScript).toContain('--no-deps $EXTERNAL_DB_START_SERVICES'); - expect(deployScript).toContain('run_database_scalar'); - expect(deployScript).toContain('has_placeholder_ghcr_credentials'); - expect(deployScript).toContain( - 'Skipping GHCR login (placeholder credentials in .env)', - ); - expect(deployScript).toContain( - '-v "$SERVER_DIR/scripts/deploy-db-scalar.mjs:/app/deploy-db-scalar.mjs:ro"', - ); - expect(deployScript).toContain('supersync node /app/deploy-db-scalar.mjs "$1"'); - expect(deployScript).toContain('prisma db execute --stdin'); - expect(deployScript).not.toContain('requires POSTGRES_SERVICE'); - expect(deployScript).toContain('GHCR_USER|GHCR_TOKEN|RUN_POST_MIGRATION_INDEXES'); expect(deployScript).toContain('POSTGRES_WAIT_TIMEOUT'); - expect(deployScript).toContain('POSTGRES_SERVICE="${POSTGRES_SERVICE-postgres}"'); - expect(deployScript).toContain('MIGRATION_WAIT_TIMEOUT'); - expect(deployScript).toContain('MIGRATION_RETRY_INTERVAL'); - expect(deployScript).toContain('MIGRATION_WAIT_TIMEOUT must be a positive integer'); - expect(deployScript).toContain('MIGRATION_RETRY_INTERVAL must be a positive integer'); - expect(deployScript).toContain('20260511000000_add_entity_sequence_index'); - expect(deployScript).toContain(failedMigrationSql); - expect(recoveryFunction).toContain('pg_try_advisory_lock(72707369)'); - expect(recoveryFunction).toContain('pg_advisory_unlock(72707369)'); - expect(recoveryFunction).toContain('if ! migration_table_exists='); - expect(recoveryFunction).toContain('if ! failed_migration='); - expect(recoveryFunction).toContain('if ! advisory_lock_available='); - expect(recoveryFunction).toContain('drop_invalid_entity_sequence_index'); - expect(deployScript).toContain('migrate resolve --rolled-back'); - expect(deployScript).toContain('migrate deploy after known index migration recovery'); - expect(deployScript).toContain('Prisma advisory migration lock'); - expect(deployScript).toContain("grep -qiE 'advisory[[:space:]-]+lock'"); - expect(deployScript).not.toContain('|P1002'); - expect(deployScript).toContain('quote_sql_identifier'); - expect(deployScript).toContain('get_current_db_schema'); - expect(deployScript).toContain('entity_sequence_index_state_sql'); - expect(deployScript).toContain('idx_ns.nspname = current_schema()'); - expect(deployScript).toContain('i.indnkeyatts AS key_column_count'); - expect(deployScript).toContain('WHERE k.ord <= i.indnkeyatts'); - expect(deployScript).toContain('key_column_count = 4'); - expect(deployScript).toContain( - "key_columns = ARRAY['user_id', 'entity_type', 'entity_id', 'server_seq']::name[]", - ); - expect(deployScript).toContain('require_valid_entity_sequence_index_definition'); - expect(deployScript).toContain('unexpected definition'); - expect(deployScript).toContain('warn_if_entity_sequence_index_missing'); - expect(deployScript).toContain( - 'warn_if_entity_sequence_index_missing || echo "WARNING: Could not verify optional $ENTITY_SEQUENCE_INDEX_NAME index state; continuing."', - ); - expect(deployScript).toContain('RUN_POST_MIGRATION_INDEXES'); - expect(deployScript).toContain(indexSql); - expect(deployScript).toContain('DROP INDEX CONCURRENTLY IF EXISTS'); - expect(deployScript).toContain('if ! index_state='); - expect(deployScript).not.toContain('migrate resolve --applied'); + expect(deployScript).toContain('POSTGRES_SERVICE="${POSTGRES_SERVICE:-postgres}"'); expect(deployScript).toContain(migrationCommand); - expect(deployScript).toMatch(/else\s+exit_code=\$\?/); - expect(deployScript).not.toMatch(/fi\s+exit_code=\$\?/); expect(deployScript).toContain('RUN_MIGRATIONS_ON_STARTUP'); - expect(deployScript).toContain('export RUN_MIGRATIONS_ON_STARTUP=false'); - expect(recoveryCallIndex).toBeGreaterThan(-1); - expect(migrationCallIndex).toBeGreaterThan(-1); - expect(indexCallIndex).toBeGreaterThan(-1); - expect(startCommandIndex).toBeGreaterThan(-1); - expect(recoveryCallIndex).toBeLessThan(migrationCallIndex); - expect(migrationCallIndex).toBeLessThan(startCommandIndex); - expect(migrationCallIndex).toBeLessThan(indexCallIndex); - expect(indexCallIndex).toBeLessThan(startCommandIndex); - expect(dockerfile).toContain('RUN_MIGRATIONS_ON_STARTUP'); - expect(dockerfile).toContain('ENV RUN_MIGRATIONS_ON_STARTUP=false'); - expect(dockerfile).toContain('npx prisma migrate deploy || exit 1'); - expect(dockerfile).toContain('exec node dist/src/index.js'); - expect(composeFile).toContain( - 'RUN_MIGRATIONS_ON_STARTUP=${RUN_MIGRATIONS_ON_STARTUP:-false}', + expect(deployScript.indexOf(migrationCommand)).toBeLessThan( + deployScript.indexOf(startCommand), ); - expect(composeFile).toContain('DATABASE_URL=${DATABASE_URL:-postgresql://'); - expect(buildComposeFile).toContain('./scripts/deploy.sh --build'); - expect(buildComposeFile).not.toContain('up -d --build'); - expect(envExample).not.toMatch(/^GHCR_USER=/m); - expect(envExample).not.toMatch(/^GHCR_TOKEN=/m); - expect(envExample).toMatch(/^# POSTGRES_SERVICE=$/m); - expect(envExample).not.toMatch(/^POSTGRES_SERVICE=/m); - expect(readmeFile).toContain('Upgrade note'); - expect(readmeFile).toMatch(/set\s+`POSTGRES_SERVICE=` to the empty value/); - expect(readmeFile).toContain('migrate resolve --rolled-back'); - expect(readmeFile).toContain('same schema/search path used by'); - expect(readmeFile).toContain('SET search_path TO public'); - expect(readmeFile).toContain('DROP INDEX CONCURRENTLY IF EXISTS'); - expect(readmeFile).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS'); - expect(readmeFile).toContain('out-of-band index'); - expect(readmeFile).toContain('dependencies disabled'); - expect(deployDbScalarScript).toContain('Scalar deploy probe'); - expect(deployDbScalarScript).toContain('$queryRawUnsafe'); - expect(deployDbScalarScript).toContain('$disconnect'); - expect(schemaFile).toContain('@@index([userId, entityType, entityId])'); - expect(schemaFile).toContain('built out-of-band'); - expect(schemaFile).not.toContain( - '@@index([userId, entityType, entityId, serverSeq])', + expect(dockerfile).toContain('RUN_MIGRATIONS_ON_STARTUP'); + expect(composeFile).toContain( + 'RUN_MIGRATIONS_ON_STARTUP=${RUN_MIGRATIONS_ON_STARTUP:-true}', ); }); }); diff --git a/packages/super-sync-server/tests/operation-download.service.spec.ts b/packages/super-sync-server/tests/operation-download.service.spec.ts index e2bfba8506..500a9b4943 100644 --- a/packages/super-sync-server/tests/operation-download.service.spec.ts +++ b/packages/super-sync-server/tests/operation-download.service.spec.ts @@ -28,6 +28,23 @@ vi.mock('../src/logger', () => ({ import { prisma } from '../src/db'; +const EXPECTED_OPERATION_DOWNLOAD_SELECT = { + id: true, + serverSeq: true, + clientId: true, + actionType: true, + opType: true, + entityType: true, + entityId: true, + payload: true, + vectorClock: true, + schemaVersion: true, + clientTimestamp: true, + receivedAt: true, + isPayloadEncrypted: true, + syncImportReason: true, +}; + // Helper to create a mock operation row (as returned by Prisma) const createMockOpRow = ( serverSeq: number, @@ -44,6 +61,7 @@ const createMockOpRow = ( clientTimestamp: bigint; receivedAt: bigint; isPayloadEncrypted: boolean; + syncImportReason: string | null; }> = {}, ) => ({ id: overrides.id ?? `op-${serverSeq}`, @@ -60,6 +78,7 @@ const createMockOpRow = ( clientTimestamp: overrides.clientTimestamp ?? BigInt(Date.now()), receivedAt: overrides.receivedAt ?? BigInt(Date.now()), isPayloadEncrypted: overrides.isPayloadEncrypted ?? false, + syncImportReason: overrides.syncImportReason ?? null, }); describe('OperationDownloadService', () => { @@ -88,6 +107,7 @@ describe('OperationDownloadService', () => { }, orderBy: { serverSeq: 'asc' }, take: 500, + select: EXPECTED_OPERATION_DOWNLOAD_SELECT, }); }); @@ -117,6 +137,7 @@ describe('OperationDownloadService', () => { }, orderBy: { serverSeq: 'asc' }, take: 500, + select: EXPECTED_OPERATION_DOWNLOAD_SELECT, }); }); @@ -160,6 +181,17 @@ describe('OperationDownloadService', () => { expect(result[0].op.isPayloadEncrypted).toBe(true); }); + + it('should map sync import reason when present', async () => { + const importOp = createMockOpRow(1, 'client-1', { + syncImportReason: 'recovery', + }); + vi.mocked(prisma.operation.findMany).mockResolvedValue([importOp as any]); + + const result = await service.getOpsSince(1, 0); + + expect(result[0].op.syncImportReason).toBe('recovery'); + }); }); describe('getOpsSinceWithSeq', () => { @@ -203,6 +235,173 @@ describe('OperationDownloadService', () => { expect(result.gapDetected).toBe(false); }); + it('should skip min sequence aggregate when gap detection cannot use it', async () => { + let capturedTx: any; + + vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { + capturedTx = { + operation: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + aggregate: vi.fn(), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 5 }), + }, + }; + return fn(capturedTx); + }); + + const result = await service.getOpsSinceWithSeq(1, 0); + + expect(result.gapDetected).toBe(false); + expect(capturedTx.operation.aggregate).not.toHaveBeenCalled(); + }); + + it('should return immediately for an empty server without operation queries', async () => { + let capturedTx: any; + + vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { + capturedTx = { + operation: { + findFirst: vi.fn(), + findMany: vi.fn(), + aggregate: vi.fn(), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 0 }), + }, + }; + return fn(capturedTx); + }); + + const result = await service.getOpsSinceWithSeq(1, 0); + + expect(result).toEqual({ + ops: [], + latestSeq: 0, + gapDetected: false, + latestSnapshotSeq: undefined, + snapshotVectorClock: undefined, + }); + expect(capturedTx.operation.findFirst).not.toHaveBeenCalled(); + expect(capturedTx.operation.findMany).not.toHaveBeenCalled(); + expect(capturedTx.operation.aggregate).not.toHaveBeenCalled(); + }); + + it('should select only download response fields inside atomic reads', async () => { + let capturedTx: any; + + vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { + capturedTx = { + operation: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + aggregate: vi.fn(), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 5 }), + }, + }; + return fn(capturedTx); + }); + + await service.getOpsSinceWithSeq(1, 0); + + expect(capturedTx.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + select: EXPECTED_OPERATION_DOWNLOAD_SELECT, + }), + ); + }); + + it('should use latestSeq as a stable upper bound for operation reads', async () => { + let capturedTx: any; + + vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { + capturedTx = { + operation: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([]), + aggregate: vi.fn().mockResolvedValue({ _min: { serverSeq: 1 } }), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 20 }), + }, + }; + return fn(capturedTx); + }); + + await service.getOpsSinceWithSeq(1, 10); + + expect(capturedTx.operation.findFirst).toHaveBeenCalledWith({ + where: { + userId: 1, + serverSeq: { lte: 20 }, + opType: { in: ['SYNC_IMPORT', 'BACKUP_IMPORT', 'REPAIR'] }, + }, + orderBy: { serverSeq: 'desc' }, + select: { serverSeq: true, clientId: true }, + }); + expect(capturedTx.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + userId: 1, + serverSeq: { gt: 10, lte: 20 }, + }), + }), + ); + expect(capturedTx.operation.aggregate).toHaveBeenCalledWith({ + where: { userId: 1, serverSeq: { lte: 20 } }, + _min: { serverSeq: true }, + }); + }); + + it('should skip snapshot vector-clock aggregation but still return the full-state op when metadata is not requested', async () => { + let capturedTx: any; + + vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { + capturedTx = { + operation: { + findFirst: vi.fn().mockResolvedValue({ + serverSeq: 50, + clientId: 'snapshot-author', + }), + findMany: vi.fn().mockResolvedValue([ + createMockOpRow(50, 'snapshot-author', { + opType: 'SYNC_IMPORT', + entityType: 'ALL', + entityId: null, + payload: { state: { task: {} } }, + }), + ]), + aggregate: vi.fn().mockResolvedValue({ _min: { serverSeq: 1 } }), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 60 }), + }, + $queryRaw: vi.fn(), + }; + return fn(capturedTx); + }); + + const result = await service.getOpsSinceWithSeq(1, 10, undefined, 500, false); + + expect(result.latestSnapshotSeq).toBe(50); + expect(result.snapshotVectorClock).toBeUndefined(); + expect(result.ops).toHaveLength(1); + expect(result.ops[0].serverSeq).toBe(50); + expect(result.ops[0].op.opType).toBe('SYNC_IMPORT'); + expect(capturedTx.$queryRaw).not.toHaveBeenCalled(); + expect(capturedTx.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + serverSeq: { gt: 49, lte: 60 }, + }), + }), + ); + }); + it('should detect gap when client is ahead of server', async () => { vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { const mockTx = { @@ -224,23 +423,27 @@ describe('OperationDownloadService', () => { }); it('should detect gap when client has history but server is empty', async () => { + let capturedTx: any; vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { - const mockTx = { + capturedTx = { operation: { - findFirst: vi.fn().mockResolvedValue(null), - findMany: vi.fn().mockResolvedValue([]), - aggregate: vi.fn().mockResolvedValue({ _min: { serverSeq: null } }), + findFirst: vi.fn(), + findMany: vi.fn(), + aggregate: vi.fn(), }, userSyncState: { findUnique: vi.fn().mockResolvedValue({ lastSeq: 0 }), }, }; - return fn(mockTx); + return fn(capturedTx); }); const result = await service.getOpsSinceWithSeq(1, 5); // Client at seq 5, server empty expect(result.gapDetected).toBe(true); + expect(capturedTx.operation.findFirst).not.toHaveBeenCalled(); + expect(capturedTx.operation.findMany).not.toHaveBeenCalled(); + expect(capturedTx.operation.aggregate).not.toHaveBeenCalled(); }); it('should detect gap when requested seq is purged', async () => { @@ -316,14 +519,16 @@ describe('OperationDownloadService', () => { orderBy: { serverSeq: 'desc' }, select: { serverSeq: true, clientId: true }, }); - expect(capturedTx.operation.findMany).toHaveBeenCalledWith({ - where: { - userId: 1, - serverSeq: { gt: 10, lte: 42 }, - }, - orderBy: { serverSeq: 'asc' }, - take: 500, - }); + expect(capturedTx.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + userId: 1, + serverSeq: { gt: 10, lte: 42 }, + }, + orderBy: { serverSeq: 'asc' }, + take: 500, + }), + ); expect(capturedTx.operation.aggregate).toHaveBeenCalledWith({ where: { userId: 1, serverSeq: { lte: 42 } }, _min: { serverSeq: true }, diff --git a/packages/super-sync-server/tests/setup.ts b/packages/super-sync-server/tests/setup.ts index a4d74f82e9..11b8d87a79 100644 --- a/packages/super-sync-server/tests/setup.ts +++ b/packages/super-sync-server/tests/setup.ts @@ -141,6 +141,12 @@ vi.mock('../src/db', () => { if (typeof where.opType === 'string' && op.opType !== where.opType) { return false; } + if ( + where.isPayloadEncrypted !== undefined && + op.isPayloadEncrypted !== where.isPayloadEncrypted + ) { + return false; + } return true; }; @@ -224,6 +230,11 @@ vi.mock('../src/db', () => { applySelect(op, args.select), ); }), + count: vi.fn().mockImplementation(async (args: any) => { + return Array.from(testData.operations.values()).filter((op) => + matchesWhere(op, args.where), + ).length; + }), aggregate: vi.fn().mockResolvedValue({ _min: { serverSeq: 1 } }), deleteMany: vi.fn().mockResolvedValue({ count: 0 }), }, @@ -289,6 +300,7 @@ vi.mock('../src/db', () => { createMany: vi.fn(), findFirst: vi.fn(), findMany: vi.fn(), + count: vi.fn().mockResolvedValue(0), aggregate: vi.fn().mockResolvedValue({ _min: { serverSeq: 1 } }), deleteMany: vi.fn().mockResolvedValue({ count: 0 }), }, diff --git a/packages/super-sync-server/tests/snapshot.service.spec.ts b/packages/super-sync-server/tests/snapshot.service.spec.ts index 49f34fa0b2..911be0a2af 100644 --- a/packages/super-sync-server/tests/snapshot.service.spec.ts +++ b/packages/super-sync-server/tests/snapshot.service.spec.ts @@ -26,6 +26,17 @@ vi.mock('../src/db', () => ({ import { prisma } from '../src/db'; +const EXPECTED_REPLAY_OPERATION_SELECT = { + id: true, + serverSeq: true, + opType: true, + entityType: true, + entityId: true, + payload: true, + schemaVersion: true, + isPayloadEncrypted: true, +}; + describe('SnapshotService', () => { let service: SnapshotService; @@ -254,6 +265,131 @@ describe('SnapshotService', () => { }); describe('generateSnapshot - FIX 1.7 concurrent lock', () => { + it('should only select replay fields needed to generate snapshots', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + id: 'op-1', + serverSeq: 1, + opType: 'CRT', + entityType: 'TASK', + entityId: 'task-1', + payload: { title: 'Test Task' }, + schemaVersion: 1, + isPayloadEncrypted: false, + }, + ]); + const mockTransaction = vi.mocked(prisma.$transaction); + + mockTransaction.mockImplementation(async (fn) => { + const mockTx = { + userSyncState: { + findUnique: vi + .fn() + .mockResolvedValueOnce({ lastSeq: 1 }) + .mockResolvedValueOnce({ + snapshotData: null, + lastSnapshotSeq: null, + snapshotAt: null, + snapshotSchemaVersion: null, + }), + update: vi.fn().mockResolvedValue({}), + }, + operation: { count: vi.fn().mockResolvedValue(0), findMany }, + }; + return fn(mockTx as any); + }); + + await service.generateSnapshot(1); + + expect(findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + userId: 1, + serverSeq: { gt: 0, lte: 1 }, + }, + select: EXPECTED_REPLAY_OPERATION_SELECT, + }), + ); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('clientId'); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('vectorClock'); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('receivedAt'); + }); + + it('should reject latest snapshot generation when encrypted ops are in the replay range', async () => { + const update = vi.fn(); + const findMany = vi.fn(); + + vi.mocked(prisma.$transaction).mockImplementation(async (fn) => { + const mockTx = { + userSyncState: { + findUnique: vi + .fn() + .mockResolvedValueOnce({ lastSeq: 1 }) + .mockResolvedValueOnce({ + snapshotData: null, + lastSnapshotSeq: null, + snapshotAt: null, + snapshotSchemaVersion: null, + }), + update, + }, + operation: { + count: vi.fn().mockResolvedValue(1), + findMany, + }, + }; + return fn(mockTx as any); + }); + + await expect(service.generateSnapshot(1)).rejects.toThrow( + 'ENCRYPTED_OPS_NOT_SUPPORTED', + ); + expect(findMany).not.toHaveBeenCalled(); + expect(update).not.toHaveBeenCalled(); + }); + + it('should reject latest snapshot generation when replay cannot reach latestSeq', async () => { + const update = vi.fn(); + + vi.mocked(prisma.$transaction).mockImplementation(async (fn) => { + const mockTx = { + userSyncState: { + findUnique: vi + .fn() + .mockResolvedValueOnce({ lastSeq: 2 }) + .mockResolvedValueOnce({ + snapshotData: null, + lastSnapshotSeq: null, + snapshotAt: null, + snapshotSchemaVersion: null, + }), + update, + }, + operation: { + count: vi.fn().mockResolvedValue(0), + findMany: vi.fn().mockResolvedValue([ + { + id: 'op-2', + serverSeq: 2, + opType: 'CRT', + entityType: 'TASK', + entityId: 'task-2', + payload: { title: 'Task 2' }, + schemaVersion: 1, + isPayloadEncrypted: false, + }, + ]), + }, + }; + return fn(mockTx as any); + }); + + await expect(service.generateSnapshot(1)).rejects.toThrow( + 'SNAPSHOT_REPLAY_INCOMPLETE', + ); + expect(update).not.toHaveBeenCalled(); + }); + it('should prevent concurrent snapshot generation for same user', async () => { // Create a delayed response to simulate long snapshot generation let resolveFirst: (value: any) => void; @@ -791,6 +927,32 @@ describe('SnapshotService', () => { ); }); + it('should reject historical snapshot generation when replay cannot reach targetSeq', async () => { + vi.mocked(prisma.$transaction).mockImplementation(async (fn) => { + const mockTx = { + userSyncState: { + findUnique: vi + .fn() + .mockResolvedValueOnce({ lastSeq: 5 }) + .mockResolvedValueOnce({ + snapshotData: null, + lastSnapshotSeq: null, + snapshotSchemaVersion: null, + }), + }, + operation: { + count: vi.fn().mockResolvedValue(0), + findMany: vi.fn().mockResolvedValue([]), + }, + }; + return fn(mockTx as any); + }); + + await expect(service.generateSnapshotAtSeq(1, 5)).rejects.toThrow( + 'SNAPSHOT_REPLAY_INCOMPLETE', + ); + }); + it('should throw error when replay range is not contiguous', async () => { vi.mocked(prisma.$transaction).mockImplementation(async (fn) => { const mockTx = { @@ -823,6 +985,52 @@ describe('SnapshotService', () => { 'SNAPSHOT_REPLAY_INCOMPLETE', ); }); + + it('should only select replay fields needed to generate snapshots at a sequence', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + id: 'op-1', + serverSeq: 1, + opType: 'CRT', + entityType: 'TASK', + entityId: 'task-1', + payload: { title: 'Test Task' }, + schemaVersion: 1, + isPayloadEncrypted: false, + }, + ]); + + vi.mocked(prisma.$transaction).mockImplementation(async (fn) => { + const mockTx = { + userSyncState: { + findUnique: vi + .fn() + .mockResolvedValueOnce({ lastSeq: 1 }) + .mockResolvedValueOnce({ + snapshotData: null, + lastSnapshotSeq: null, + snapshotSchemaVersion: null, + }), + }, + operation: { + count: vi.fn().mockResolvedValue(0), + findMany, + }, + }; + return fn(mockTx as any); + }); + + await service.generateSnapshotAtSeq(1, 1); + + expect(findMany).toHaveBeenCalledWith( + expect.objectContaining({ + select: EXPECTED_REPLAY_OPERATION_SELECT, + }), + ); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('clientId'); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('vectorClock'); + expect(findMany.mock.calls[0][0].select).not.toHaveProperty('receivedAt'); + }); }); describe('replayOpsToState', () => { diff --git a/packages/super-sync-server/tests/storage-quota-cleanup.spec.ts b/packages/super-sync-server/tests/storage-quota-cleanup.spec.ts index f3d368a721..cf0fbabfa4 100644 --- a/packages/super-sync-server/tests/storage-quota-cleanup.spec.ts +++ b/packages/super-sync-server/tests/storage-quota-cleanup.spec.ts @@ -375,6 +375,30 @@ describe('Storage Quota Cleanup', () => { expect(query).not.toContain('vector_clock::text'); }); + it('should only fetch enough restore points to decide cleanup behavior', async () => { + const { initSyncService, getSyncService } = + await import('../src/sync/sync.service'); + const { prisma } = await import('../src/db'); + initSyncService(); + const service = getSyncService(); + + createRestorePoint(clientId, userId); + createOp(clientId, userId); + createRestorePoint(clientId, userId); + + vi.mocked(prisma.operation.findMany).mockClear(); + + await service.deleteOldestRestorePointAndOps(userId); + + expect(prisma.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + orderBy: { serverSeq: 'asc' }, + select: { serverSeq: true, opType: true }, + take: 2, + }), + ); + }); + it('should keep single restore point but delete ops before it', async () => { const { initSyncService, getSyncService } = await import('../src/sync/sync.service'); @@ -544,6 +568,37 @@ describe('Storage Quota Cleanup', () => { expect(result.deletedOps).toBe(0); }); + it('should only fetch enough restore points before each cleanup iteration', async () => { + const { initSyncService, getSyncService } = + await import('../src/sync/sync.service'); + const { prisma } = await import('../src/db'); + initSyncService(); + const service = getSyncService(); + + testUsers.set(userId, { + id: userId, + email: 'test@test.com', + storageUsedBytes: BigInt(150 * 1024 * 1024), + storageQuotaBytes: BigInt(100 * 1024 * 1024), + }); + + createRestorePoint(clientId, userId); + createOp(clientId, userId); + createRestorePoint(clientId, userId); + + vi.mocked(prisma.operation.findMany).mockClear(); + + await service.freeStorageForUpload(userId, 1000); + + expect(prisma.operation.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + orderBy: { serverSeq: 'asc' }, + select: { serverSeq: true }, + take: 2, + }), + ); + }); + it('should return failure when only one restore point exists', async () => { const { initSyncService, getSyncService } = await import('../src/sync/sync.service'); diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index 1595663d87..d8de6cda15 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -10,7 +10,10 @@ const mocks = vi.hoisted(() => { checkStorageQuota: vi.fn(), uploadOps: vi.fn(), cacheRequestResults: vi.fn(), + cacheSnapshot: vi.fn(), + cacheSnapshotIfReplayable: vi.fn(), updateStorageUsage: vi.fn(), + freeStorageForUpload: vi.fn(), getLatestSeq: vi.fn(), getOpsSinceWithSeq: vi.fn(), }; @@ -70,7 +73,15 @@ describe('Sync compressed body routes', () => { quota: 100 * 1024 * 1024, }); mocks.syncService.uploadOps.mockResolvedValue([{ accepted: true, serverSeq: 1 }]); + mocks.syncService.cacheSnapshot.mockResolvedValue(undefined); + mocks.syncService.cacheSnapshotIfReplayable.mockResolvedValue(undefined); mocks.syncService.updateStorageUsage.mockResolvedValue(undefined); + mocks.syncService.freeStorageForUpload.mockResolvedValue({ + success: false, + freedBytes: 0, + deletedRestorePoints: 0, + deletedOps: 0, + }); mocks.syncService.getLatestSeq.mockResolvedValue(1); mocks.syncService.getOpsSinceWithSeq.mockResolvedValue({ ops: [], @@ -88,20 +99,63 @@ describe('Sync compressed body routes', () => { it('should accept plain JSON ops upload', async () => { const clientId = 'plain-json-client'; + const payload = { + ops: [createOp(clientId)], + clientId, + }; + const jsonPayload = JSON.stringify(payload); + const payloadSize = Buffer.byteLength(jsonPayload, 'utf-8'); const response = await app.inject({ method: 'POST', url: '/api/sync/ops', - headers: { authorization: `Bearer ${authToken}` }, - payload: { - ops: [createOp(clientId)], - clientId, + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + 'content-length': String(payloadSize), }, + payload: jsonPayload, }); expect(response.statusCode).toBe(200); expect(response.json().results[0].accepted).toBe(true); expect(mocks.syncService.uploadOps).toHaveBeenCalledOnce(); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, payloadSize); + }); + + it('should fall back to UTF-8 JSON byte size for plain JSON without content-length', async () => { + const clientId = 'plain-json-client'; + const payload = { + ops: [ + createOp(clientId), + { + ...createOp(clientId), + id: 'op-unicode', + entityId: 'task-unicode', + payload: { title: 'Übergrößenträger 🚀' }, + }, + ], + clientId, + }; + const jsonPayload = JSON.stringify(payload); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + 'content-length': 'not-a-number', + }, + payload: jsonPayload, + }); + + expect(response.statusCode).toBe(200); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith( + 1, + Buffer.byteLength(jsonPayload, 'utf-8'), + ); + expect(Buffer.byteLength(jsonPayload, 'utf-8')).toBeGreaterThan(jsonPayload.length); }); it('should accept base64 gzip ops upload', async () => { @@ -113,6 +167,7 @@ describe('Sync compressed body routes', () => { const compressedPayload = await gzipAsync( Buffer.from(JSON.stringify(payload), 'utf-8'), ); + const decompressedSize = Buffer.byteLength(JSON.stringify(payload), 'utf-8'); const response = await app.inject({ method: 'POST', @@ -129,6 +184,120 @@ describe('Sync compressed body routes', () => { expect(response.statusCode).toBe(200); expect(response.json().results[0].accepted).toBe(true); expect(mocks.syncService.uploadOps).toHaveBeenCalledOnce(); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, decompressedSize); + expect(decompressedSize).not.toBe( + Buffer.byteLength(compressedPayload.toString('base64'), 'utf-8'), + ); + }); + + it('should skip snapshot metadata for upload piggyback downloads', async () => { + const clientId = 'plain-json-client'; + const payload = { + ops: [createOp(clientId)], + clientId, + lastKnownServerSeq: 0, + }; + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + payload, + }); + + expect(response.statusCode).toBe(200); + expect(mocks.syncService.getOpsSinceWithSeq).toHaveBeenCalledWith( + 1, + 0, + clientId, + 500, + false, + ); + }); + + it('should skip snapshot metadata for deduplicated retry piggyback downloads', async () => { + const clientId = 'plain-json-client'; + const cachedResults = [{ accepted: true, serverSeq: 1 }]; + mocks.syncService.checkRequestDeduplication.mockReturnValue(cachedResults); + mocks.syncService.getOpsSinceWithSeq.mockResolvedValue({ + ops: [], + latestSeq: 4, + }); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + payload: { + ops: [createOp(clientId)], + clientId, + lastKnownServerSeq: 3, + requestId: 'retry-request', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ + results: cachedResults, + latestSeq: 4, + deduplicated: true, + }); + expect(mocks.syncService.checkStorageQuota).not.toHaveBeenCalled(); + expect(mocks.syncService.getOpsSinceWithSeq).toHaveBeenCalledWith( + 1, + 3, + clientId, + 500, + false, + ); + }); + + it('should charge compressed snapshot uploads by decompressed JSON size', async () => { + const clientId = 'base64-gzip-snapshot-client'; + const payload = { + state: { + tasks: { + 'task-1': { id: 'task-1', title: 'Snapshot Task' }, + }, + }, + clientId, + reason: 'recovery', + vectorClock: { [clientId]: 1 }, + schemaVersion: 1, + }; + const jsonPayload = JSON.stringify(payload); + const compressedPayload = await gzipAsync(Buffer.from(jsonPayload, 'utf-8')); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/snapshot', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + 'content-encoding': 'gzip', + 'content-transfer-encoding': 'base64', + }, + payload: compressedPayload.toString('base64'), + }); + + expect(response.statusCode).toBe(200); + expect(response.json().accepted).toBe(true); + expect(mocks.syncService.cacheSnapshotIfReplayable).toHaveBeenCalledWith( + 1, + payload.state, + 1, + false, + ); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith( + 1, + Buffer.byteLength(jsonPayload, 'utf-8'), + ); }); it('should preserve the invalid gzip route response', async () => { diff --git a/packages/super-sync-server/tests/sync-operations.spec.ts b/packages/super-sync-server/tests/sync-operations.spec.ts index eefec8e7b2..28837ff89f 100644 --- a/packages/super-sync-server/tests/sync-operations.spec.ts +++ b/packages/super-sync-server/tests/sync-operations.spec.ts @@ -145,6 +145,29 @@ vi.mock('../src/db', async () => { .sort((a: any, b: any) => a.serverSeq - b.serverSeq) .slice(0, args.take || 500); }), + count: vi.fn().mockImplementation(async (args: any) => { + const ops = Array.from(state.operations.values()); + return ops.filter((op: any) => { + if (args.where?.userId !== undefined && args.where.userId !== op.userId) + return false; + if ( + args.where?.serverSeq?.gt !== undefined && + op.serverSeq <= args.where.serverSeq.gt + ) + return false; + if ( + args.where?.serverSeq?.lte !== undefined && + op.serverSeq > args.where.serverSeq.lte + ) + return false; + if ( + args.where?.isPayloadEncrypted !== undefined && + op.isPayloadEncrypted !== args.where.isPayloadEncrypted + ) + return false; + return true; + }).length; + }), aggregate: vi.fn().mockImplementation(async (args: any) => { const ops = Array.from(state.operations.values()).filter( (op: any) => args.where?.userId === op.userId, @@ -290,6 +313,29 @@ vi.mock('../src/db', async () => { .sort((a: any, b: any) => a.serverSeq - b.serverSeq) .slice(0, args.take || 500); }), + count: vi.fn().mockImplementation(async (args: any) => { + const ops = Array.from(state.operations.values()); + return ops.filter((op: any) => { + if (args.where?.userId !== undefined && args.where.userId !== op.userId) + return false; + if ( + args.where?.serverSeq?.gt !== undefined && + op.serverSeq <= args.where.serverSeq.gt + ) + return false; + if ( + args.where?.serverSeq?.lte !== undefined && + op.serverSeq > args.where.serverSeq.lte + ) + return false; + if ( + args.where?.isPayloadEncrypted !== undefined && + op.isPayloadEncrypted !== args.where.isPayloadEncrypted + ) + return false; + return true; + }).length; + }), aggregate: vi.fn().mockImplementation(async (args: any) => { const ops = Array.from(state.operations.values()).filter( (op: any) => args.where?.userId === op.userId, diff --git a/packages/super-sync-server/tests/time-tracking-operations.spec.ts b/packages/super-sync-server/tests/time-tracking-operations.spec.ts index e80a480add..2dd8f341f3 100644 --- a/packages/super-sync-server/tests/time-tracking-operations.spec.ts +++ b/packages/super-sync-server/tests/time-tracking-operations.spec.ts @@ -161,6 +161,29 @@ vi.mock('../src/db', async () => { .sort((a: any, b: any) => a.serverSeq - b.serverSeq) .slice(0, args.take || 500); }), + count: vi.fn().mockImplementation(async (args: any) => { + const ops = Array.from(state.operations.values()); + return ops.filter((op: any) => { + if (args.where?.userId !== undefined && args.where.userId !== op.userId) + return false; + if ( + args.where?.serverSeq?.gt !== undefined && + op.serverSeq <= args.where.serverSeq.gt + ) + return false; + if ( + args.where?.serverSeq?.lte !== undefined && + op.serverSeq > args.where.serverSeq.lte + ) + return false; + if ( + args.where?.isPayloadEncrypted !== undefined && + op.isPayloadEncrypted !== args.where.isPayloadEncrypted + ) + return false; + return true; + }).length; + }), aggregate: vi.fn().mockImplementation(async (args: any) => { const ops = Array.from(state.operations.values()).filter( (op: any) => args.where?.userId === op.userId, @@ -277,6 +300,29 @@ vi.mock('../src/db', async () => { .sort((a: any, b: any) => a.serverSeq - b.serverSeq) .slice(0, args.take || 500); }), + count: vi.fn().mockImplementation(async (args: any) => { + const ops = Array.from(state.operations.values()); + return ops.filter((op: any) => { + if (args.where?.userId !== undefined && args.where.userId !== op.userId) + return false; + if ( + args.where?.serverSeq?.gt !== undefined && + op.serverSeq <= args.where.serverSeq.gt + ) + return false; + if ( + args.where?.serverSeq?.lte !== undefined && + op.serverSeq > args.where.serverSeq.lte + ) + return false; + if ( + args.where?.isPayloadEncrypted !== undefined && + op.isPayloadEncrypted !== args.where.isPayloadEncrypted + ) + return false; + return true; + }).length; + }), aggregate: vi.fn().mockImplementation(async (args: any) => { const ops = Array.from(state.operations.values()).filter( (op: any) => args.where?.userId === op.userId, diff --git a/packages/sync-core/src/entity-key.util.ts b/packages/sync-core/src/entity-key.util.ts index 52e44ee530..6b35134485 100644 --- a/packages/sync-core/src/entity-key.util.ts +++ b/packages/sync-core/src/entity-key.util.ts @@ -2,7 +2,7 @@ * Creates a unique key for an entity by combining its type and ID. * Used for indexing and looking up entities across the operation log system. * - * @param entityType The type of the entity (e.g., 'TASK', 'PROJECT') + * @param entityType The host-defined type of the entity * @param entityId The unique ID of the entity * @returns A composite key in the format "ENTITY_TYPE:entityId" */ diff --git a/packages/sync-core/src/index.ts b/packages/sync-core/src/index.ts index 3341aae189..e0bcc71357 100644 --- a/packages/sync-core/src/index.ts +++ b/packages/sync-core/src/index.ts @@ -74,6 +74,13 @@ export type { LwwUpdateActionTypeHelpers } from './lww-update-action-types'; // Apply-operation result and option types. export type { ApplyOperationsResult, ApplyOperationsOptions } from './apply.types'; +// Generic operation replay coordinator. +export { replayOperationBatch, yieldToEventLoop } from './replay-coordinator'; +export type { + OperationReplayArchiveFailureContext, + OperationReplayCoordinatorOptions, +} from './replay-coordinator'; + // Remote operation application coordinator. export { applyRemoteOperations } from './remote-apply'; export type { @@ -119,11 +126,18 @@ export type { export type { ActionDispatchPort, ArchiveSideEffectPort, + ConflictUiDialogRequest, + ConflictUiNotification, + ConflictUiNotificationSeverity, + ConflictUiPort, DeferredLocalActionsPort, OperationApplyPort, OperationStorePort, RemoteApplyWindowPort, SyncActionLike, + SyncConfigPort, + SyncConfigSnapshot, + SyncPortMeta, } from './ports'; // Conflict-resolution helpers. diff --git a/packages/sync-core/src/lww-update-action-types.ts b/packages/sync-core/src/lww-update-action-types.ts index 9810a094c2..1bc4197035 100644 --- a/packages/sync-core/src/lww-update-action-types.ts +++ b/packages/sync-core/src/lww-update-action-types.ts @@ -22,8 +22,8 @@ export interface LwwUpdateActionTypeHelpers * Build LWW helpers for the given list of entity types. * * @example - * const lww = createLwwUpdateActionTypeHelpers(['TASK', 'PROJECT'] as const); - * lww.toLwwUpdateActionType('TASK'); // '[TASK] LWW Update' + * const lww = createLwwUpdateActionTypeHelpers(['ITEM', 'COLLECTION'] as const); + * lww.toLwwUpdateActionType('ITEM'); // '[ITEM] LWW Update' */ export const createLwwUpdateActionTypeHelpers = ( entityTypes: readonly TEntityType[], diff --git a/packages/sync-core/src/ports.ts b/packages/sync-core/src/ports.ts index 47cc4ab45f..db6c92bf99 100644 --- a/packages/sync-core/src/ports.ts +++ b/packages/sync-core/src/ports.ts @@ -1,6 +1,8 @@ import type { ApplyOperationsOptions, ApplyOperationsResult } from './apply.types'; import type { Operation, OperationLogEntry } from './operation.types'; +export type SyncPortMeta = Record; + /** * Minimal action shape used at sync-core boundaries. * @@ -70,3 +72,53 @@ export interface OperationStorePort< markSynced(seqs: number[]): Promise; markRejected(opIds: string[]): Promise; } + +/** + * Domain-free sync configuration snapshot. + * + * Provider IDs stay plain strings at the package boundary. Host applications can + * narrow them in their adapter layer. + */ +export interface SyncConfigSnapshot { + isEnabled: boolean; + syncProvider: TProviderId | null; + isEncryptionEnabled?: boolean; + isCompressionEnabled?: boolean; + isManualSyncOnly?: boolean; + syncInterval?: number; +} + +/** + * Port for reading host sync configuration without importing framework store + * selectors or host provider enums. + */ +export interface SyncConfigPort { + getSyncConfig(): Promise>; +} + +export interface ConflictUiDialogRequest { + conflictType: string; + scenario?: string; + reason?: string; + counts?: Record; + timestamps?: Record; + meta?: SyncPortMeta; +} + +export type ConflictUiNotificationSeverity = 'info' | 'warning' | 'error'; + +export interface ConflictUiNotification { + severity: ConflictUiNotificationSeverity; + message: string; + reason?: string; + meta?: SyncPortMeta; +} + +/** + * Port for conflict dialogs/snacks. Resolutions are strings so the host owns + * user-facing choices such as USE_LOCAL, USE_REMOTE, or CANCEL. + */ +export interface ConflictUiPort { + showConflictDialog(request: ConflictUiDialogRequest): Promise; + notify?(notification: ConflictUiNotification): Promise | void; +} diff --git a/packages/sync-core/src/replay-coordinator.ts b/packages/sync-core/src/replay-coordinator.ts new file mode 100644 index 0000000000..6cc02b825a --- /dev/null +++ b/packages/sync-core/src/replay-coordinator.ts @@ -0,0 +1,216 @@ +import type { ApplyOperationsOptions, ApplyOperationsResult } from './apply.types'; +import type { Operation } from './operation.types'; +import type { + ActionDispatchPort, + ArchiveSideEffectPort, + DeferredLocalActionsPort, + RemoteApplyWindowPort, + SyncActionLike, +} from './ports'; + +interface ReplayGlobals { + setTimeout?: (handler: () => void, timeout?: number) => unknown; +} + +export interface OperationReplayArchiveFailureContext< + TOperation extends Operation = Operation, +> { + op: TOperation; + processedCount: number; + error: unknown; +} + +export interface OperationReplayCoordinatorOptions< + TOperation extends Operation = Operation, + TBulkAction extends SyncActionLike = SyncActionLike, + TReplayAction extends SyncActionLike = SyncActionLike, +> { + ops: TOperation[]; + applyOptions?: ApplyOperationsOptions; + dispatcher: ActionDispatchPort; + createBulkApplyAction: (ops: TOperation[]) => TBulkAction; + remoteApplyWindow: RemoteApplyWindowPort; + deferredLocalActions: DeferredLocalActionsPort; + archiveSideEffects?: ArchiveSideEffectPort; + operationToAction?: (op: TOperation) => TReplayAction; + isArchiveAffectingAction?: (action: TReplayAction) => boolean; + onRemoteArchiveDataApplied?: () => void; + onArchiveSideEffectError?: ( + context: OperationReplayArchiveFailureContext, + ) => void; + onPostSyncCooldownError?: (error: unknown) => void; + yieldToEventLoop?: () => Promise; +} + +interface ArchiveSideEffectResult = Operation> { + appliedOps: TOperation[]; + hadArchiveAffectingOp: boolean; + failedOp?: { + op: TOperation; + error: Error; + }; +} + +const globals = (): ReplayGlobals => globalThis as unknown as ReplayGlobals; + +export const yieldToEventLoop = (): Promise => + new Promise((resolve) => { + const setTimeoutFn = globals().setTimeout; + if (setTimeoutFn === undefined) { + void Promise.resolve().then(resolve); + return; + } + setTimeoutFn(resolve, 0); + }); + +/** + * Replays an operation batch through host ports in the order required by sync. + * + * Host applications keep framework-specific action construction, operation + * conversion, archive predicates, diagnostics, and UI notifications outside the + * package. This coordinator only owns the generic ordering: + * + * 1. open the remote-apply window; + * 2. dispatch the host's bulk replay action; + * 3. yield once so host state reducers finish before side effects; + * 4. run remote archive side effects after dispatch, when configured; + * 5. start post-sync cooldown before closing the remote-apply window; + * 6. close the window and flush deferred local actions. + */ +export const replayOperationBatch = async < + TOperation extends Operation = Operation, + TBulkAction extends SyncActionLike = SyncActionLike, + TReplayAction extends SyncActionLike = SyncActionLike, +>({ + ops, + applyOptions = {}, + dispatcher, + createBulkApplyAction, + remoteApplyWindow, + deferredLocalActions, + archiveSideEffects, + operationToAction, + isArchiveAffectingAction = () => false, + onRemoteArchiveDataApplied, + onArchiveSideEffectError, + onPostSyncCooldownError, + yieldToEventLoop: waitForEventLoop = yieldToEventLoop, +}: OperationReplayCoordinatorOptions): Promise< + ApplyOperationsResult +> => { + if (ops.length === 0) { + return { appliedOps: [] }; + } + + const isLocalHydration = applyOptions.isLocalHydration ?? false; + + if (!isLocalHydration && archiveSideEffects !== undefined && !operationToAction) { + throw new Error( + 'replayOperationBatch requires operationToAction when archiveSideEffects is provided.', + ); + } + + remoteApplyWindow.startApplyingRemoteOps(); + try { + dispatcher.dispatch(createBulkApplyAction(ops)); + await waitForEventLoop(); + + if (!isLocalHydration && archiveSideEffects !== undefined && operationToAction) { + const archiveResult = await processArchiveSideEffects({ + ops, + archiveSideEffects, + operationToAction, + isArchiveAffectingAction, + onArchiveSideEffectError, + yieldToEventLoop: waitForEventLoop, + }); + + if (archiveResult.failedOp) { + return { + appliedOps: archiveResult.appliedOps, + failedOp: archiveResult.failedOp, + }; + } + + if (archiveResult.hadArchiveAffectingOp) { + onRemoteArchiveDataApplied?.(); + } + } + } finally { + if (!isLocalHydration) { + try { + remoteApplyWindow.startPostSyncCooldown(); + } catch (error) { + onPostSyncCooldownError?.(error); + } + } + + remoteApplyWindow.endApplyingRemoteOps(); + await deferredLocalActions.processDeferredActions(); + } + + return { appliedOps: ops }; +}; + +const processArchiveSideEffects = async < + TOperation extends Operation, + TReplayAction extends SyncActionLike, +>({ + ops, + archiveSideEffects, + operationToAction, + isArchiveAffectingAction, + onArchiveSideEffectError, + yieldToEventLoop: waitForEventLoop, +}: { + ops: TOperation[]; + archiveSideEffects: ArchiveSideEffectPort; + operationToAction: (op: TOperation) => TReplayAction; + isArchiveAffectingAction: (action: TReplayAction) => boolean; + onArchiveSideEffectError?: ( + context: OperationReplayArchiveFailureContext, + ) => void; + yieldToEventLoop: () => Promise; +}): Promise> => { + const appliedOps: TOperation[] = []; + let hadArchiveAffectingOp = false; + + for (const op of ops) { + try { + const action = operationToAction(op); + await archiveSideEffects.handleOperation(action); + + if (isArchiveAffectingAction(action)) { + hadArchiveAffectingOp = true; + await waitForEventLoop(); + } + + appliedOps.push(op); + } catch (error) { + onArchiveSideEffectError?.({ + op, + processedCount: appliedOps.length, + error, + }); + + return { + appliedOps, + hadArchiveAffectingOp, + failedOp: { + op, + error: toError(error), + }, + }; + } + } + + await waitForEventLoop(); + + return { + appliedOps, + hadArchiveAffectingOp, + }; +}; + +const toError = (error: unknown): Error => + error instanceof Error ? error : new Error(String(error)); diff --git a/packages/sync-core/tests/ports.spec.ts b/packages/sync-core/tests/ports.spec.ts new file mode 100644 index 0000000000..01f07ebee9 --- /dev/null +++ b/packages/sync-core/tests/ports.spec.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { ConflictUiPort, SyncConfigPort, SyncConfigSnapshot } from '../src'; + +describe('sync-core ports', () => { + it('keeps sync config provider IDs as host-owned strings', async () => { + type ProviderId = 'super-sync' | 'file-sync'; + const config: SyncConfigSnapshot = { + isEnabled: true, + syncProvider: 'super-sync', + isEncryptionEnabled: true, + isCompressionEnabled: false, + isManualSyncOnly: false, + syncInterval: 5, + }; + const port: SyncConfigPort = { + getSyncConfig: vi.fn().mockResolvedValue(config), + }; + + await expect(port.getSyncConfig()).resolves.toBe(config); + expect(port.getSyncConfig).toHaveBeenCalledOnce(); + }); + + it('keeps conflict UI reasons and resolutions host-owned strings', async () => { + type Resolution = 'USE_LOCAL' | 'USE_REMOTE' | 'CANCEL'; + const notify = vi.fn(); + const port: ConflictUiPort = { + showConflictDialog: vi.fn().mockResolvedValue('USE_LOCAL'), + notify, + }; + + await expect( + port.showConflictDialog({ + conflictType: 'sync-import', + scenario: 'LOCAL_IMPORT_FILTERS_REMOTE', + reason: 'BACKUP_RESTORE', + counts: { filteredOpCount: 3 }, + timestamps: { localImportTimestamp: 123 }, + meta: { providerId: 'super-sync' }, + }), + ).resolves.toBe('USE_LOCAL'); + + port.notify?.({ + severity: 'warning', + message: 'sync-conflict', + reason: 'BACKUP_RESTORE', + meta: { filteredOps: 3 }, + }); + + expect(port.showConflictDialog).toHaveBeenCalledOnce(); + expect(notify).toHaveBeenCalledWith({ + severity: 'warning', + message: 'sync-conflict', + reason: 'BACKUP_RESTORE', + meta: { filteredOps: 3 }, + }); + }); +}); diff --git a/packages/sync-core/tests/replay-coordinator.spec.ts b/packages/sync-core/tests/replay-coordinator.spec.ts new file mode 100644 index 0000000000..faa8cd087e --- /dev/null +++ b/packages/sync-core/tests/replay-coordinator.spec.ts @@ -0,0 +1,392 @@ +import { describe, expect, it, vi } from 'vitest'; +import { replayOperationBatch } from '../src/replay-coordinator'; +import type { + ActionDispatchPort, + ArchiveSideEffectPort, + DeferredLocalActionsPort, + Operation, + RemoteApplyWindowPort, + SyncActionLike, +} from '../src'; + +interface BulkReplayAction extends SyncActionLike { + operations: Operation[]; +} + +interface ReplayAction extends SyncActionLike { + opId: string; + archiveAffecting?: boolean; +} + +const createOperation = (id: string): Operation => ({ + id, + actionType: '[Test] Action', + opType: 'UPD', + entityType: 'TASK', + entityId: id, + payload: {}, + clientId: 'client-1', + vectorClock: { client1: 1 }, + timestamp: 1, + schemaVersion: 1, +}); + +const createRemoteApplyWindow = (callOrder: string[]): RemoteApplyWindowPort => ({ + startApplyingRemoteOps: vi.fn(() => { + callOrder.push('startApplyingRemoteOps'); + }), + endApplyingRemoteOps: vi.fn(() => { + callOrder.push('endApplyingRemoteOps'); + }), + startPostSyncCooldown: vi.fn(() => { + callOrder.push('startPostSyncCooldown'); + }), +}); + +const createDeferredLocalActions = (callOrder: string[]): DeferredLocalActionsPort => ({ + processDeferredActions: vi.fn(async () => { + callOrder.push('processDeferredActions'); + }), +}); + +describe('replayOperationBatch', () => { + it('dispatches a bulk action, yields, runs archive effects, then closes the sync window', async () => { + const callOrder: string[] = []; + const ops = [createOperation('op-1'), createOperation('op-2')]; + const dispatcher: ActionDispatchPort = { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + }), + }; + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async (action) => { + callOrder.push(`archive:${action.opId}`); + }), + }; + const yieldToEventLoop = vi.fn(async () => { + callOrder.push('yield'); + }); + const onRemoteArchiveDataApplied = vi.fn(() => { + callOrder.push('remoteArchiveDataApplied'); + }); + + const result = await replayOperationBatch({ + ops, + dispatcher, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + archiveSideEffects, + operationToAction: (op) => ({ + type: '[Test] Action', + opId: op.id, + archiveAffecting: op.id === 'op-2', + }), + isArchiveAffectingAction: (action) => action.archiveAffecting === true, + onRemoteArchiveDataApplied, + yieldToEventLoop, + }); + + expect(result).toEqual({ appliedOps: ops }); + expect(dispatcher.dispatch).toHaveBeenCalledWith({ + type: '[Test] Bulk Apply', + operations: ops, + }); + expect(archiveSideEffects.handleOperation).toHaveBeenCalledTimes(2); + expect(onRemoteArchiveDataApplied).toHaveBeenCalledTimes(1); + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'dispatchBulk', + 'yield', + 'archive:op-1', + 'archive:op-2', + 'yield', + 'yield', + 'remoteArchiveDataApplied', + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('waits for the post-dispatch yield before archive handling starts', async () => { + const ops = [createOperation('op-1')]; + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async () => undefined), + }; + let releaseFirstYield: (() => void) | undefined; + const yieldToEventLoop = vi.fn( + () => + new Promise((resolve) => { + if (releaseFirstYield === undefined) { + releaseFirstYield = resolve; + return; + } + resolve(); + }), + ); + + const applyPromise = replayOperationBatch({ + ops, + dispatcher: { dispatch: vi.fn() }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow([]), + deferredLocalActions: createDeferredLocalActions([]), + archiveSideEffects, + operationToAction: (op) => ({ type: '[Test] Action', opId: op.id }), + yieldToEventLoop, + }); + + await Promise.resolve(); + + expect(archiveSideEffects.handleOperation).not.toHaveBeenCalled(); + expect(releaseFirstYield).toBeDefined(); + + releaseFirstYield?.(); + await applyPromise; + + expect(archiveSideEffects.handleOperation).toHaveBeenCalledTimes(1); + }); + + it('skips archive handling and post-sync cooldown for local hydration', async () => { + const callOrder: string[] = []; + const op = createOperation('op-1'); + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async () => undefined), + }; + const remoteApplyWindow = createRemoteApplyWindow(callOrder); + + const result = await replayOperationBatch({ + ops: [op], + applyOptions: { isLocalHydration: true }, + dispatcher: { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + }), + }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow, + deferredLocalActions: createDeferredLocalActions(callOrder), + archiveSideEffects, + operationToAction: (operation) => ({ type: '[Test] Action', opId: operation.id }), + yieldToEventLoop: vi.fn(async () => { + callOrder.push('yield'); + }), + }); + + expect(result).toEqual({ appliedOps: [op] }); + expect(archiveSideEffects.handleOperation).not.toHaveBeenCalled(); + expect(remoteApplyWindow.startPostSyncCooldown).not.toHaveBeenCalled(); + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'dispatchBulk', + 'yield', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('reports partial archive failures and still flushes deferred local actions', async () => { + const callOrder: string[] = []; + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const archiveError = new Error('archive failed'); + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async (action) => { + callOrder.push(`archive:${action.opId}`); + if (action.opId === 'op-2') { + throw archiveError; + } + }), + }; + const onArchiveSideEffectError = vi.fn(); + const onRemoteArchiveDataApplied = vi.fn(); + + const result = await replayOperationBatch({ + ops: [op1, op2], + dispatcher: { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + }), + }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + archiveSideEffects, + operationToAction: (op) => ({ + type: '[Test] Action', + opId: op.id, + archiveAffecting: true, + }), + isArchiveAffectingAction: (action) => action.archiveAffecting === true, + onArchiveSideEffectError, + onRemoteArchiveDataApplied, + yieldToEventLoop: vi.fn(async () => { + callOrder.push('yield'); + }), + }); + + expect(result).toEqual({ + appliedOps: [op1], + failedOp: { + op: op2, + error: archiveError, + }, + }); + expect(onArchiveSideEffectError).toHaveBeenCalledWith({ + op: op2, + processedCount: 1, + error: archiveError, + }); + expect(onRemoteArchiveDataApplied).not.toHaveBeenCalled(); + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'dispatchBulk', + 'yield', + 'archive:op-1', + 'yield', + 'archive:op-2', + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('fails fast when archive side effects are configured without operation conversion', async () => { + const callOrder: string[] = []; + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async () => undefined), + }; + const dispatcher: ActionDispatchPort = { + dispatch: vi.fn(), + }; + + await expect( + replayOperationBatch({ + ops: [createOperation('op-1')], + dispatcher, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + archiveSideEffects, + }), + ).rejects.toThrow( + 'replayOperationBatch requires operationToAction when archiveSideEffects is provided.', + ); + + expect(dispatcher.dispatch).not.toHaveBeenCalled(); + expect(archiveSideEffects.handleOperation).not.toHaveBeenCalled(); + expect(callOrder).toEqual([]); + }); + + it('closes the sync window and flushes deferred actions when dispatch throws', async () => { + const callOrder: string[] = []; + const dispatchError = new Error('dispatch failed'); + + await expect( + replayOperationBatch({ + ops: [createOperation('op-1')], + dispatcher: { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + throw dispatchError; + }), + }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + yieldToEventLoop: vi.fn(async () => { + callOrder.push('yield'); + }), + }), + ).rejects.toBe(dispatchError); + + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'dispatchBulk', + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('closes the sync window even when post-sync cooldown throws', async () => { + const callOrder: string[] = []; + const cooldownError = new Error('cooldown failed'); + const remoteApplyWindow = createRemoteApplyWindow(callOrder); + vi.mocked(remoteApplyWindow.startPostSyncCooldown).mockImplementation(() => { + callOrder.push('startPostSyncCooldown'); + throw cooldownError; + }); + const onPostSyncCooldownError = vi.fn(); + + await replayOperationBatch({ + ops: [createOperation('op-1')], + dispatcher: { dispatch: vi.fn() }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow, + deferredLocalActions: createDeferredLocalActions(callOrder), + yieldToEventLoop: vi.fn(async () => undefined), + }); + + expect(onPostSyncCooldownError).not.toHaveBeenCalled(); + + await replayOperationBatch({ + ops: [createOperation('op-2')], + dispatcher: { dispatch: vi.fn() }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow, + deferredLocalActions: createDeferredLocalActions(callOrder), + onPostSyncCooldownError, + yieldToEventLoop: vi.fn(async () => undefined), + }); + + expect(onPostSyncCooldownError).toHaveBeenCalledWith(cooldownError); + expect(callOrder.slice(-3)).toEqual([ + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('does not open a sync window for an empty operation batch', async () => { + const callOrder: string[] = []; + const result = await replayOperationBatch({ + ops: [], + dispatcher: { dispatch: vi.fn() }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + }); + + expect(result).toEqual({ appliedOps: [] }); + expect(callOrder).toEqual([]); + }); +}); diff --git a/packages/sync-providers/.gitignore b/packages/sync-providers/.gitignore new file mode 100644 index 0000000000..1eae0cf670 --- /dev/null +++ b/packages/sync-providers/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/packages/sync-providers/package.json b/packages/sync-providers/package.json new file mode 100644 index 0000000000..55d438399d --- /dev/null +++ b/packages/sync-providers/package.json @@ -0,0 +1,35 @@ +{ + "name": "@sp/sync-providers", + "version": "1.0.0", + "description": "Framework-agnostic sync provider contracts for Super Productivity sync", + "main": "dist/index.js", + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } + }, + "dependencies": { + "@sp/sync-core": "*", + "hash-wasm": "^4.12.0" + }, + "scripts": { + "build": "tsup", + "build:tsc": "tsc", + "test": "vitest run", + "test:watch": "vitest" + }, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.0.0", + "vitest": "^3.2.4" + } +} diff --git a/packages/sync-providers/src/credential-store-port.ts b/packages/sync-providers/src/credential-store-port.ts new file mode 100644 index 0000000000..a12569e961 --- /dev/null +++ b/packages/sync-providers/src/credential-store-port.ts @@ -0,0 +1,13 @@ +export type CredentialChangeHandler = (data: { + providerId: PID; + privateCfg: TPrivateCfg; +}) => void; + +export interface SyncCredentialStorePort { + load(): Promise; + setComplete(privateCfg: TPrivateCfg): Promise; + updatePartial(updates: Partial): Promise; + upsertPartial(updates: Partial): Promise; + clear(): Promise; + onConfigChange?(callback: CredentialChangeHandler): void; +} diff --git a/packages/sync-providers/src/file-adapter.ts b/packages/sync-providers/src/file-adapter.ts new file mode 100644 index 0000000000..70f174c22f --- /dev/null +++ b/packages/sync-providers/src/file-adapter.ts @@ -0,0 +1,7 @@ +export interface FileAdapter { + readFile(filePath: string): Promise; + writeFile(filePath: string, dataStr: string): Promise; + deleteFile(filePath: string): Promise; + checkDirExists?(dirPath: string): Promise; + listFiles?(dirPath: string): Promise; +} diff --git a/packages/sync-providers/src/file-based-sync-data.ts b/packages/sync-providers/src/file-based-sync-data.ts new file mode 100644 index 0000000000..296876ba9e --- /dev/null +++ b/packages/sync-providers/src/file-based-sync-data.ts @@ -0,0 +1,88 @@ +import type { VectorClock } from '@sp/sync-core'; + +/** + * Compact operations stored in file-based sync data carry the syncVersion at + * which they were uploaded. The host owns the compact operation shape. + */ +export type SyncFileCompactOp< + TCompactOperation extends object = Record, +> = TCompactOperation & { sv?: number }; + +/** + * Schema for the shared sync file used by file-based providers. + * + * The host supplies the snapshot state, compact operation, and archive payload + * shapes. The provider package owns only the transport-level envelope. + */ +export interface FileBasedSyncData< + TState = unknown, + TCompactOperation extends object = Record, + TArchive = unknown, +> { + /** + * Schema version for this sync file format. + * Increment when making breaking changes to FileBasedSyncData structure. + */ + version: 2; + + /** + * Content-based optimistic lock counter. + * Incremented on each successful upload. + */ + syncVersion: number; + + /** + * Schema version of the host application data. + */ + schemaVersion: number; + + /** + * Causal state after all operations represented by this file. + */ + vectorClock: VectorClock; + + /** + * Timestamp of last successful sync (epoch ms). + */ + lastModified: number; + + /** + * Client ID that last modified this file. + */ + clientId: string; + + /** + * Complete host application state snapshot. + */ + state: TState; + + /** + * Recent or frequently mutable archive partition, host-defined. + */ + archiveYoung?: TArchive; + + /** + * Older archive partition, host-defined. + */ + archiveOld?: TArchive; + + /** + * Recent operations retained for conflict detection. + */ + recentOps: SyncFileCompactOp[]; + + /** + * The syncVersion of the oldest operation in recentOps. + */ + oldestOpSyncVersion?: number; +} + +export const FILE_BASED_SYNC_CONSTANTS = { + SYNC_FILE: 'sync-data.json', + BACKUP_FILE: 'sync-data.json.bak', + MIGRATION_LOCK_FILE: 'migration.lock', + FILE_VERSION: 2 as const, + MAX_RECENT_OPS: 500, + SYNC_VERSION_STORAGE_KEY_PREFIX: 'FILE_SYNC_VERSION_', + LEGACY_META_FILE: '__meta_', +} as const; diff --git a/packages/sync-providers/src/http/native-http-retry.ts b/packages/sync-providers/src/http/native-http-retry.ts new file mode 100644 index 0000000000..827985a8cf --- /dev/null +++ b/packages/sync-providers/src/http/native-http-retry.ts @@ -0,0 +1,156 @@ +import { NOOP_SYNC_LOGGER, toSyncLogError, type SyncLogger } from '@sp/sync-core'; + +/** + * Max retries for transient network errors (e.g., iOS NSURLErrorNetworkConnectionLost). + * iOS NSURLSession does NOT auto-retry POST requests (non-idempotent per RFC 7231). + * Retries are safe for Dropbox uploads: conditional writes (mode: 'update' with revToMatch + * or mode: 'add') ensure duplicates fail with UploadRevToMatchMismatchAPIError. + * Retries are safe for SuperSync: the server uses idempotent operations. + * @see https://developer.apple.com/library/archive/qa/qa1941/_index.html + */ +const MAX_RETRIES = 2; + +const DEFAULT_CONNECT_TIMEOUT_MS = 30_000; +const DEFAULT_READ_TIMEOUT_MS = 120_000; + +export interface NativeHttpRequestConfig { + url: string; + method: string; + headers: Record; + data?: string; + responseType?: 'text' | 'json'; + connectTimeout?: number; + readTimeout?: number; +} + +export interface NativeHttpResponse { + status: number; + headers: Record; + data: unknown; + url?: string; +} + +export type NativeHttpExecutor = ( + config: NativeHttpRequestConfig, +) => Promise; + +export interface ExecuteNativeRequestOptions { + executor: NativeHttpExecutor; + logger?: SyncLogger; + label?: string; + delay?: (ms: number) => Promise; +} + +const defaultDelay = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Execute a native HTTP request with retry logic for transient network errors. + * Only network-level errors thrown by the executor are retried; + * HTTP response errors (401, 404, etc.) are handled by the caller. + * + * The executor is injected so this helper stays platform-agnostic — hosts + * provide CapacitorHttp on native platforms, fetch on web/Electron, or a + * test double in unit tests. + */ +export const executeNativeRequestWithRetry = async ( + config: NativeHttpRequestConfig, + options: ExecuteNativeRequestOptions, +): Promise => { + const { + executor, + logger = NOOP_SYNC_LOGGER, + label = 'NativeHttp', + delay = defaultDelay, + } = options; + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + return await executor({ + url: config.url, + method: config.method, + headers: config.headers, + data: config.data, + responseType: config.responseType ?? 'text', + connectTimeout: config.connectTimeout ?? DEFAULT_CONNECT_TIMEOUT_MS, + readTimeout: config.readTimeout ?? DEFAULT_READ_TIMEOUT_MS, + }); + } catch (retryErr) { + if (attempt < MAX_RETRIES && isTransientNetworkError(retryErr)) { + const delayMs = 1000 * (attempt + 1); + // toSyncLogError drops .code on Error instances, so we read it directly + // off the raw error to preserve platform-specific transient codes + // (NSURLErrorDomain, SocketTimeoutException, …) in the retry log. + const rawCode = (retryErr as { code?: unknown } | null)?.code; + logger.warn( + `${label} transient network error, retrying in ${delayMs}ms ` + + `(attempt ${attempt + 1}/${MAX_RETRIES})`, + { + url: config.url, + attempt: attempt + 1, + maxRetries: MAX_RETRIES, + delayMs, + errorName: toSyncLogError(retryErr).name, + errorCode: + typeof rawCode === 'string' || typeof rawCode === 'number' + ? rawCode + : undefined, + }, + ); + await delay(delayMs); + continue; + } + throw retryErr; + } + } + // Unreachable: loop always returns or throws, but TypeScript needs this + throw new Error('All retry attempts exhausted'); +}; + +/** + * Checks if an error is a transient network error that should be retried. + * + * Uses a hybrid detection strategy: + * + * **iOS (primary):** Checks `error.code === 'NSURLErrorDomain'`. Capacitor's + * HttpRequestHandler.swift calls `call.reject(error.localizedDescription, (error as NSError).domain, error, nil)`, + * so the domain string (always `"NSURLErrorDomain"` for network errors) lands on `error.code` + * in JavaScript. This is locale-proof — it works regardless of the device language. + * Note: SSL errors also have NSURLErrorDomain, but retrying them is harmless + * (they fail instantly and consistently — at most 2 extra immediate failures). + * + * **Android (primary):** Checks `error.code` against Java exception class names like + * `'SocketTimeoutException'`, `'UnknownHostException'`, and `'ConnectException'`. + * Capacitor's CapacitorHttp.java uses `call.reject(e.getLocalizedMessage(), e.getClass().getSimpleName(), e)`. + * + * **Fallback:** English string matching on the error message for web/Electron/unknown platforms. + */ +export const isTransientNetworkError = (e: unknown): boolean => { + const errorCode = (e as { code?: string } | null)?.code; + if (typeof errorCode === 'string') { + if (errorCode === 'NSURLErrorDomain') { + return true; + } + if ( + errorCode === 'SocketTimeoutException' || + errorCode === 'UnknownHostException' || + errorCode === 'ConnectException' + ) { + return true; + } + } + + const message = (e instanceof Error ? e.message : String(e)).toLowerCase(); + return ( + message.includes('network connection was lost') || + // Intentionally broad: matches "request timed out", "connection timed out", "operation timed out", etc. + // Narrowing to "request timed out" would miss legitimate transient network errors. + message.includes('timed out') || + message.includes('not connected to the internet') || + message.includes('internet connection appears to be offline') || + message.includes('cannot find host') || + message.includes('hostname could not be found') || + message.includes('cannot connect to host') || + message.includes('could not connect to the server') + ); +}; diff --git a/packages/sync-providers/src/index.ts b/packages/sync-providers/src/index.ts new file mode 100644 index 0000000000..a6464ae521 --- /dev/null +++ b/packages/sync-providers/src/index.ts @@ -0,0 +1,49 @@ +export type { + CredentialChangeHandler, + SyncCredentialStorePort, +} from './credential-store-port'; +export { FILE_BASED_SYNC_CONSTANTS } from './file-based-sync-data'; +export type { FileBasedSyncData, SyncFileCompactOp } from './file-based-sync-data'; +export type { FileAdapter } from './file-adapter'; +export { generateCodeChallenge, generateCodeVerifier, generatePKCECodes } from './pkce'; +export type { + GenerateCodeChallengeOptions, + GenerateCodeVerifierOptions, + GeneratePkceCodesOptions, + PkceCrypto, + PkceSha256, +} from './pkce'; +export { + executeNativeRequestWithRetry, + isTransientNetworkError, +} from './http/native-http-retry'; +export type { + NativeHttpExecutor, + NativeHttpRequestConfig, + NativeHttpResponse, +} from './http/native-http-retry'; +export { + isFileSyncProvider, + type FileDownloadResponse, + type FileRevResponse, + type FileSnapshotOpDownloadResponse, + type FileSyncProvider, + type OpDownloadResponse, + type OpDownloadResponseBase, + type OpDownloadResponseForMode, + type OperationSyncCapable, + type OperationSyncProviderMode, + type OpUploadResponse, + type OpUploadResult, + type ProviderId, + type RestoreCapable, + type RestorePoint, + type RestorePointsResponse, + type RestoreSnapshotResponse, + type ServerSyncOperation, + type SnapshotUploadResponse, + type SuperSyncOpDownloadResponse, + type SyncOperation, + type SyncProviderAuthHelper, + type SyncProviderBase, +} from './provider.types'; diff --git a/packages/sync-providers/src/pkce.ts b/packages/sync-providers/src/pkce.ts new file mode 100644 index 0000000000..b736b1be45 --- /dev/null +++ b/packages/sync-providers/src/pkce.ts @@ -0,0 +1,117 @@ +import { sha256 as hashWasmSha256 } from 'hash-wasm'; + +const DEFAULT_PKCE_RANDOM_BYTES = 32; + +type PkceSubtleCrypto = { + digest(algorithm: 'SHA-256', data: Uint8Array): Promise; +}; + +export interface PkceCrypto { + getRandomValues(array: T): T; + subtle?: PkceSubtleCrypto; +} + +export type PkceSha256 = (data: Uint8Array) => Promise; + +export interface GenerateCodeVerifierOptions { + crypto?: PkceCrypto; + randomBytesLength?: number; +} + +export interface GenerateCodeChallengeOptions { + crypto?: PkceCrypto; + sha256Fallback?: PkceSha256; +} + +export interface GeneratePkceCodesOptions + extends GenerateCodeVerifierOptions, GenerateCodeChallengeOptions {} + +type BtoaGlobal = { + btoa?: (data: string) => string; +}; + +type CryptoGlobal = { + crypto?: PkceCrypto; +}; + +type TextEncoderConstructor = new () => { + encode(input?: string): Uint8Array; +}; + +type TextEncoderGlobal = { + TextEncoder?: TextEncoderConstructor; +}; + +const getOptionalDefaultCrypto = (): PkceCrypto | undefined => + (globalThis as CryptoGlobal).crypto; + +const getDefaultCrypto = (): PkceCrypto => { + const cryptoLike = getOptionalDefaultCrypto(); + if (!cryptoLike) { + throw new Error('Crypto API is unavailable'); + } + return cryptoLike; +}; + +const getBtoa = (): ((data: string) => string) => { + const btoaLike = (globalThis as BtoaGlobal).btoa; + if (!btoaLike) { + throw new Error('btoa is unavailable'); + } + return btoaLike; +}; + +const getTextEncoder = (): TextEncoderConstructor => { + const TextEncoderLike = (globalThis as TextEncoderGlobal).TextEncoder; + if (!TextEncoderLike) { + throw new Error('TextEncoder is unavailable'); + } + return TextEncoderLike; +}; + +const base64UrlEncode = (buffer: Uint8Array): string => { + const base64 = getBtoa()(String.fromCharCode(...buffer)); + return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +}; + +const hashWasmSha256ArrayBuffer: PkceSha256 = async (data) => { + const hexHash = await hashWasmSha256(data); + const bytes = new Uint8Array(hexHash.length / 2); + for (let i = 0; i < hexHash.length; i += 2) { + bytes[i / 2] = parseInt(hexHash.slice(i, i + 2), 16); + } + return bytes.buffer; +}; + +export const generateCodeVerifier = ( + options: GenerateCodeVerifierOptions = {}, +): string => { + const cryptoLike = options.crypto ?? getDefaultCrypto(); + const array = new Uint8Array(options.randomBytesLength ?? DEFAULT_PKCE_RANDOM_BYTES); + cryptoLike.getRandomValues(array); + return base64UrlEncode(array); +}; + +export const generateCodeChallenge = async ( + verifier: string, + options: GenerateCodeChallengeOptions = {}, +): Promise => { + const TextEncoderLike = getTextEncoder(); + const data = new TextEncoderLike().encode(verifier); + const cryptoLike = options.crypto ?? getOptionalDefaultCrypto(); + const subtle = cryptoLike?.subtle; + const digest = + subtle != null + ? await subtle.digest('SHA-256', data) + : await (options.sha256Fallback ?? hashWasmSha256ArrayBuffer)(data); + return base64UrlEncode(new Uint8Array(digest)); +}; + +export const generatePKCECodes = async ( + _length: number, + options: GeneratePkceCodesOptions = {}, +): Promise<{ codeVerifier: string; codeChallenge: string }> => { + const codeVerifier = generateCodeVerifier(options); + const codeChallenge = await generateCodeChallenge(codeVerifier, options); + return { codeVerifier, codeChallenge }; +}; diff --git a/packages/sync-providers/src/provider.types.ts b/packages/sync-providers/src/provider.types.ts new file mode 100644 index 0000000000..50c2d631e2 --- /dev/null +++ b/packages/sync-providers/src/provider.types.ts @@ -0,0 +1,199 @@ +import type { VectorClock } from '@sp/sync-core'; +import type { SyncCredentialStorePort } from './credential-store-port'; + +export type ProviderId = string; + +export interface SyncProviderAuthHelper { + authUrl?: string; + codeVerifier?: string; + + verifyCodeChallenge?(codeChallenge: string): Promise<{ + accessToken: string; + refreshToken: string; + expiresAt: number; + }>; +} + +export interface SyncProviderBase< + PID extends ProviderId = ProviderId, + TPrivateCfg = unknown, +> { + id: PID; + isUploadForcePossible?: boolean; + maxConcurrentRequests: number; + privateCfg: SyncCredentialStorePort; + + isReady(): Promise; + getAuthHelper?(): Promise; + setPrivateCfg(privateCfg: TPrivateCfg): Promise; + clearAuthCredentials?(): Promise; +} + +export interface FileRevResponse { + rev: string; +} + +export interface FileDownloadResponse extends FileRevResponse { + dataStr: string; +} + +export interface FileSyncProvider< + PID extends ProviderId = ProviderId, + TPrivateCfg = unknown, +> extends SyncProviderBase { + isLimitedToSingleFileSync?: boolean; + + getFileRev(targetPath: string, localRev: string | null): Promise; + downloadFile(targetPath: string): Promise; + uploadFile( + targetPath: string, + dataStr: string, + revToMatch: string | null, + isForceOverwrite?: boolean, + ): Promise; + removeFile(targetPath: string): Promise; + listFiles?(targetPath: string): Promise; +} + +export const isFileSyncProvider = < + PID extends ProviderId = ProviderId, + TPrivateCfg = unknown, +>( + provider: SyncProviderBase, +): provider is FileSyncProvider => { + return ( + 'getFileRev' in provider && + typeof (provider as Record).getFileRev === 'function' + ); +}; + +export type OperationSyncProviderMode = 'superSyncOps' | 'fileSnapshotOps'; + +export interface SyncOperation { + id: string; + clientId: string; + actionType: string; + opType: string; + entityType: string; + entityId?: string; + entityIds?: string[]; + payload: unknown; + vectorClock: VectorClock; + timestamp: number; + schemaVersion: number; + isPayloadEncrypted?: boolean; + syncImportReason?: string; +} + +export interface ServerSyncOperation { + serverSeq: number; + op: SyncOperation; + receivedAt: number; +} + +export interface OpUploadResult { + opId: string; + accepted: boolean; + serverSeq?: number; + error?: string; + errorCode?: string; + existingClock?: VectorClock; +} + +export interface OpUploadResponse { + results: OpUploadResult[]; + newOps?: ServerSyncOperation[]; + latestSeq: number; + hasMorePiggyback?: boolean; +} + +export interface OpDownloadResponseBase { + ops: ServerSyncOperation[]; + hasMore: boolean; + latestSeq: number; + latestSnapshotSeq?: number; + gapDetected?: boolean; + snapshotVectorClock?: VectorClock; + serverTime?: number; +} + +export interface SuperSyncOpDownloadResponse extends OpDownloadResponseBase { + snapshotState?: never; +} + +export interface FileSnapshotOpDownloadResponse extends OpDownloadResponseBase { + snapshotState?: unknown; +} + +export type OpDownloadResponse = + | SuperSyncOpDownloadResponse + | FileSnapshotOpDownloadResponse; + +export type OpDownloadResponseForMode = + M extends 'fileSnapshotOps' + ? FileSnapshotOpDownloadResponse + : SuperSyncOpDownloadResponse; + +export interface SnapshotUploadResponse { + accepted: boolean; + serverSeq?: number; + error?: string; +} + +export interface OperationSyncCapable< + M extends OperationSyncProviderMode = OperationSyncProviderMode, + TRestorePointType extends string = string, +> { + supportsOperationSync: boolean; + providerMode: M; + + uploadOps( + ops: SyncOperation[], + clientId: string, + lastKnownServerSeq?: number, + ): Promise; + downloadOps( + sinceSeq: number, + excludeClient?: string, + limit?: number, + ): Promise>; + getLastServerSeq(): Promise; + setLastServerSeq(seq: number): Promise; + uploadSnapshot( + state: unknown, + clientId: string, + reason: 'initial' | 'recovery' | 'migration', + vectorClock: VectorClock, + schemaVersion: number, + isPayloadEncrypted: boolean | undefined, + opId: string, + isCleanSlate?: boolean, + snapshotOpType?: TRestorePointType, + syncImportReason?: string, + ): Promise; + deleteAllData(): Promise<{ success: boolean }>; + getEncryptKey?(): Promise; +} + +export interface RestorePoint { + serverSeq: number; + timestamp: number; + type: TRestorePointType; + clientId: string; + description?: string; +} + +export interface RestorePointsResponse { + restorePoints: RestorePoint[]; +} + +export interface RestoreSnapshotResponse { + state: unknown; + serverSeq: number; + generatedAt: number; +} + +export interface RestoreCapable { + getRestorePoints(limit?: number): Promise[]>; + getStateAtSeq(serverSeq: number): Promise; +} diff --git a/packages/sync-providers/tests/native-http-retry.spec.ts b/packages/sync-providers/tests/native-http-retry.spec.ts new file mode 100644 index 0000000000..6ac1f423f2 --- /dev/null +++ b/packages/sync-providers/tests/native-http-retry.spec.ts @@ -0,0 +1,363 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + executeNativeRequestWithRetry, + isTransientNetworkError, + type NativeHttpExecutor, + type NativeHttpRequestConfig, + type NativeHttpResponse, +} from '../src'; +import type { SyncLogger } from '@sp/sync-core'; + +const successResponse: NativeHttpResponse = { + status: 200, + headers: {}, + data: 'ok', + url: 'https://example.com/api', +}; + +const baseConfig: NativeHttpRequestConfig = { + url: 'https://example.com/api', + method: 'POST', + headers: { Authorization: 'Bearer test' }, + data: 'test-data', +}; + +const noopLogger = (): SyncLogger => ({ + log: vi.fn(), + error: vi.fn(), + err: vi.fn(), + normal: vi.fn(), + verbose: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + critical: vi.fn(), + debug: vi.fn(), +}); + +describe('isTransientNetworkError', () => { + describe('iOS error.code detection (NSURLErrorDomain)', () => { + it('returns true for NSURLErrorDomain code', () => { + const error = Object.assign(new Error('Some localized message'), { + code: 'NSURLErrorDomain', + }); + expect(isTransientNetworkError(error)).toBe(true); + }); + + it('returns true for NSURLErrorDomain even with non-English message', () => { + const error = Object.assign( + new Error('Die Netzwerkverbindung wurde unterbrochen.'), + { code: 'NSURLErrorDomain' }, + ); + expect(isTransientNetworkError(error)).toBe(true); + }); + + it('returns true for NSURLErrorDomain with empty message', () => { + const error = Object.assign(new Error(''), { code: 'NSURLErrorDomain' }); + expect(isTransientNetworkError(error)).toBe(true); + }); + }); + + describe('Android error.code detection', () => { + it('returns true for SocketTimeoutException', () => { + const error = Object.assign(new Error('timeout'), { + code: 'SocketTimeoutException', + }); + expect(isTransientNetworkError(error)).toBe(true); + }); + + it('returns true for UnknownHostException', () => { + const error = Object.assign(new Error('host not found'), { + code: 'UnknownHostException', + }); + expect(isTransientNetworkError(error)).toBe(true); + }); + + it('returns true for ConnectException', () => { + const error = Object.assign(new Error('connection refused'), { + code: 'ConnectException', + }); + expect(isTransientNetworkError(error)).toBe(true); + }); + }); + + describe('Fallback string matching', () => { + it('returns true for "network connection was lost"', () => { + expect(isTransientNetworkError(new Error('The network connection was lost.'))).toBe( + true, + ); + }); + + it('returns true for "timed out"', () => { + expect(isTransientNetworkError(new Error('The request timed out.'))).toBe(true); + }); + + it('returns true for "internet connection appears to be offline"', () => { + expect( + isTransientNetworkError( + new Error('The Internet connection appears to be offline.'), + ), + ).toBe(true); + }); + + it('returns true for "not connected to the internet"', () => { + expect(isTransientNetworkError(new Error('not connected to the internet'))).toBe( + true, + ); + }); + + it('returns true for "hostname could not be found"', () => { + expect( + isTransientNetworkError( + new Error('A server with the specified hostname could not be found.'), + ), + ).toBe(true); + }); + + it('returns true for "cannot find host"', () => { + expect(isTransientNetworkError(new Error('cannot find host'))).toBe(true); + }); + + it('returns true for "could not connect to the server"', () => { + expect(isTransientNetworkError(new Error('Could not connect to the server.'))).toBe( + true, + ); + }); + + it('returns true for "cannot connect to host"', () => { + expect(isTransientNetworkError(new Error('cannot connect to host'))).toBe(true); + }); + + it('is case-insensitive', () => { + expect(isTransientNetworkError(new Error('THE NETWORK CONNECTION WAS LOST'))).toBe( + true, + ); + }); + + it('returns true for non-Error string with transient message', () => { + expect(isTransientNetworkError('network connection was lost')).toBe(true); + }); + }); + + describe('Negative cases', () => { + it('returns false for auth errors', () => { + expect(isTransientNetworkError(new Error('Unauthorized'))).toBe(false); + }); + + it('returns false for arbitrary errors', () => { + expect(isTransientNetworkError(new Error('Something unexpected happened'))).toBe( + false, + ); + }); + + it('returns false for HTTP status errors', () => { + expect(isTransientNetworkError(new Error('HTTP 409 Conflict'))).toBe(false); + }); + + it('returns false for non-Error string without transient message', () => { + expect(isTransientNetworkError('some other string')).toBe(false); + }); + + it('returns false for null', () => { + expect(isTransientNetworkError(null)).toBe(false); + }); + + it('returns false for undefined', () => { + expect(isTransientNetworkError(undefined)).toBe(false); + }); + + it('returns false for empty string', () => { + expect(isTransientNetworkError('')).toBe(false); + }); + + it('returns false for unrecognized error.code', () => { + const error = Object.assign(new Error('some error'), { code: 'SomeOtherDomain' }); + expect(isTransientNetworkError(error)).toBe(false); + }); + }); +}); + +describe('executeNativeRequestWithRetry', () => { + const collectDelays = (): { + delays: number[]; + delay: (ms: number) => Promise; + } => { + const delays: number[] = []; + return { + delays, + delay: async (ms) => { + delays.push(ms); + }, + }; + }; + + it('returns response on first success', async () => { + const executor = vi.fn().mockResolvedValue(successResponse); + const { delays, delay } = collectDelays(); + + const result = await executeNativeRequestWithRetry(baseConfig, { + executor, + label: 'Test', + delay, + }); + + expect(result).toBe(successResponse); + expect(executor).toHaveBeenCalledTimes(1); + expect(delays).toEqual([]); + }); + + it('retries on transient error and succeeds', async () => { + const transientError = Object.assign(new Error('connection lost'), { + code: 'NSURLErrorDomain', + }); + const executor = vi + .fn() + .mockRejectedValueOnce(transientError) + .mockResolvedValueOnce(successResponse); + const { delays, delay } = collectDelays(); + + const result = await executeNativeRequestWithRetry(baseConfig, { + executor, + label: 'Test', + delay, + }); + + expect(result).toBe(successResponse); + expect(executor).toHaveBeenCalledTimes(2); + expect(delays).toEqual([1000]); + }); + + it('retries twice and succeeds on third attempt', async () => { + const transientError = Object.assign(new Error('timeout'), { + code: 'SocketTimeoutException', + }); + const executor = vi + .fn() + .mockRejectedValueOnce(transientError) + .mockRejectedValueOnce(transientError) + .mockResolvedValueOnce(successResponse); + const { delays, delay } = collectDelays(); + + const result = await executeNativeRequestWithRetry(baseConfig, { + executor, + label: 'Test', + delay, + }); + + expect(result).toBe(successResponse); + expect(executor).toHaveBeenCalledTimes(3); + expect(delays).toEqual([1000, 2000]); + }); + + it('throws after exhausting all retries for transient errors', async () => { + const transientError = Object.assign(new Error('network lost'), { + code: 'NSURLErrorDomain', + }); + const executor = vi.fn().mockRejectedValue(transientError); + const { delays, delay } = collectDelays(); + + await expect( + executeNativeRequestWithRetry(baseConfig, { executor, label: 'Test', delay }), + ).rejects.toBe(transientError); + expect(executor).toHaveBeenCalledTimes(3); + expect(delays).toEqual([1000, 2000]); + }); + + it('immediately throws non-transient errors without retrying', async () => { + const nonTransientError = new Error('Unauthorized'); + const executor = vi.fn().mockRejectedValue(nonTransientError); + const { delays, delay } = collectDelays(); + + await expect( + executeNativeRequestWithRetry(baseConfig, { executor, label: 'Test', delay }), + ).rejects.toBe(nonTransientError); + expect(executor).toHaveBeenCalledTimes(1); + expect(delays).toEqual([]); + }); + + it('passes correct config to the executor', async () => { + const executor = vi.fn().mockResolvedValue(successResponse); + const { delay } = collectDelays(); + const config: NativeHttpRequestConfig = { + url: 'https://api.example.com/test', + method: 'GET', + headers: { Authorization: 'Bearer abc' }, + data: 'payload', + responseType: 'json', + connectTimeout: 5000, + readTimeout: 60000, + }; + + await executeNativeRequestWithRetry(config, { executor, label: 'Test', delay }); + + expect(executor).toHaveBeenCalledWith({ + url: 'https://api.example.com/test', + method: 'GET', + headers: { Authorization: 'Bearer abc' }, + data: 'payload', + responseType: 'json', + connectTimeout: 5000, + readTimeout: 60000, + }); + }); + + it('uses default responseType, connectTimeout, and readTimeout', async () => { + const executor = vi.fn().mockResolvedValue(successResponse); + const { delay } = collectDelays(); + + await executeNativeRequestWithRetry(baseConfig, { executor, label: 'Test', delay }); + + expect(executor).toHaveBeenCalledWith( + expect.objectContaining({ + responseType: 'text', + connectTimeout: 30000, + readTimeout: 120000, + }), + ); + }); + + it('logs a warn entry with safe meta on each retry', async () => { + const transientError = Object.assign(new Error('connection lost'), { + code: 'NSURLErrorDomain', + }); + const executor = vi + .fn() + .mockRejectedValueOnce(transientError) + .mockResolvedValueOnce(successResponse); + const { delay } = collectDelays(); + const logger = noopLogger(); + + await executeNativeRequestWithRetry(baseConfig, { + executor, + label: 'Test', + logger, + delay, + }); + + expect(logger.warn).toHaveBeenCalledTimes(1); + const [message, meta] = (logger.warn as ReturnType).mock.calls[0]; + expect(message).toContain('Test transient network error'); + expect(meta).toMatchObject({ + url: baseConfig.url, + attempt: 1, + maxRetries: 2, + delayMs: 1000, + errorName: 'Error', + errorCode: 'NSURLErrorDomain', + }); + }); + + it('runs with a default noop logger when none is provided', async () => { + const transientError = Object.assign(new Error('timeout'), { + code: 'SocketTimeoutException', + }); + const executor = vi + .fn() + .mockRejectedValueOnce(transientError) + .mockResolvedValueOnce(successResponse); + const { delay } = collectDelays(); + + await expect( + executeNativeRequestWithRetry(baseConfig, { executor, delay }), + ).resolves.toBe(successResponse); + }); +}); diff --git a/packages/sync-providers/tests/pkce.spec.ts b/packages/sync-providers/tests/pkce.spec.ts new file mode 100644 index 0000000000..8bd03ecf60 --- /dev/null +++ b/packages/sync-providers/tests/pkce.spec.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + generateCodeChallenge, + generateCodeVerifier, + generatePKCECodes, + type PkceCrypto, +} from '../src'; + +const createDeterministicCrypto = (): PkceCrypto => ({ + getRandomValues: (array: T): T => { + for (let i = 0; i < array.length; i++) { + array[i] = i + 1; + } + return array; + }, +}); + +describe('PKCE utilities', () => { + it('generates URL-safe code verifiers', () => { + const verifier = generateCodeVerifier({ crypto: createDeterministicCrypto() }); + + expect(verifier).toMatch(/^[A-Za-z0-9\-_]+$/); + expect(verifier.length).toBeGreaterThanOrEqual(43); + expect(verifier.length).toBeLessThanOrEqual(128); + }); + + it('generates the RFC 7636 S256 challenge for a known verifier', async () => { + await expect( + generateCodeChallenge('dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk'), + ).resolves.toBe('E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM'); + }); + + it('uses the hash fallback when subtle crypto is unavailable', async () => { + const sha256Fallback = vi.fn().mockResolvedValue(new Uint8Array(32).buffer); + + const challenge = await generateCodeChallenge('verifier', { + crypto: createDeterministicCrypto(), + sha256Fallback, + }); + + expect(sha256Fallback).toHaveBeenCalledOnce(); + expect(challenge).toBe('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'); + }); + + it('uses the hash fallback when global crypto is unavailable', async () => { + const cryptoDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'crypto'); + const sha256Fallback = vi.fn().mockResolvedValue(new Uint8Array(32).buffer); + + Object.defineProperty(globalThis, 'crypto', { + value: undefined, + configurable: true, + }); + + try { + const challenge = await generateCodeChallenge('verifier', { sha256Fallback }); + + expect(sha256Fallback).toHaveBeenCalledOnce(); + expect(challenge).toBe('AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'); + } finally { + if (cryptoDescriptor) { + Object.defineProperty(globalThis, 'crypto', cryptoDescriptor); + } else { + Reflect.deleteProperty(globalThis, 'crypto'); + } + } + }); + + it('generates a verifier and matching challenge pair', async () => { + const result = await generatePKCECodes(128, { + crypto: createDeterministicCrypto(), + }); + + await expect(generateCodeChallenge(result.codeVerifier)).resolves.toBe( + result.codeChallenge, + ); + }); +}); diff --git a/packages/sync-providers/tests/provider.types.spec.ts b/packages/sync-providers/tests/provider.types.spec.ts new file mode 100644 index 0000000000..57fa060889 --- /dev/null +++ b/packages/sync-providers/tests/provider.types.spec.ts @@ -0,0 +1,127 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + FILE_BASED_SYNC_CONSTANTS, + type FileBasedSyncData, + isFileSyncProvider, + type FileSyncProvider, + type OperationSyncCapable, + type SyncFileCompactOp, + type SyncCredentialStorePort, + type SyncProviderBase, +} from '../src'; + +type ProviderId = 'file' | 'ops'; +interface ProviderPrivateCfg { + token?: string; +} +interface HostCompactOp { + id: string; + action: string; +} +interface HostArchive { + records: number; +} + +const createCredentialStore = (): SyncCredentialStorePort< + ProviderId, + ProviderPrivateCfg +> => ({ + load: vi.fn().mockResolvedValue({ token: 'test-token' }), + setComplete: vi.fn().mockResolvedValue(undefined), + updatePartial: vi.fn().mockResolvedValue(undefined), + upsertPartial: vi.fn().mockResolvedValue(undefined), + clear: vi.fn().mockResolvedValue(undefined), +}); + +describe('sync provider contracts', () => { + it('keeps provider IDs and private config host-owned', async () => { + const provider: SyncProviderBase = { + id: 'ops', + maxConcurrentRequests: 4, + privateCfg: createCredentialStore(), + isReady: vi.fn().mockResolvedValue(true), + setPrivateCfg: vi.fn().mockResolvedValue(undefined), + }; + + await expect(provider.privateCfg.load()).resolves.toEqual({ + token: 'test-token', + }); + expect(provider.id).toBe('ops'); + }); + + it('identifies file sync providers by file operation capability', () => { + const fileProvider: FileSyncProvider = { + id: 'file', + maxConcurrentRequests: 2, + privateCfg: createCredentialStore(), + isReady: vi.fn().mockResolvedValue(true), + setPrivateCfg: vi.fn().mockResolvedValue(undefined), + getFileRev: vi.fn().mockResolvedValue({ rev: '1' }), + downloadFile: vi.fn().mockResolvedValue({ rev: '1', dataStr: '{}' }), + uploadFile: vi.fn().mockResolvedValue({ rev: '2' }), + removeFile: vi.fn().mockResolvedValue(undefined), + }; + const opsProvider: SyncProviderBase = { + id: 'ops', + maxConcurrentRequests: 4, + privateCfg: createCredentialStore(), + isReady: vi.fn().mockResolvedValue(true), + setPrivateCfg: vi.fn().mockResolvedValue(undefined), + }; + + expect(isFileSyncProvider(fileProvider)).toBe(true); + expect(isFileSyncProvider(opsProvider)).toBe(false); + }); + + it('keeps restore point strings supplied by the host app', async () => { + type RestorePointType = 'SYNC_IMPORT' | 'CUSTOM_REPAIR'; + const provider: OperationSyncCapable<'superSyncOps', RestorePointType> = { + supportsOperationSync: true, + providerMode: 'superSyncOps', + uploadOps: vi.fn().mockResolvedValue({ results: [], latestSeq: 1 }), + downloadOps: vi.fn().mockResolvedValue({ ops: [], hasMore: false, latestSeq: 1 }), + getLastServerSeq: vi.fn().mockResolvedValue(1), + setLastServerSeq: vi.fn().mockResolvedValue(undefined), + uploadSnapshot: vi.fn().mockResolvedValue({ accepted: true, serverSeq: 2 }), + deleteAllData: vi.fn().mockResolvedValue({ success: true }), + }; + + await expect( + provider.uploadSnapshot( + {}, + 'client-a', + 'recovery', + { clientA: 1 }, + 1, + false, + 'op-a', + false, + 'CUSTOM_REPAIR', + ), + ).resolves.toEqual({ accepted: true, serverSeq: 2 }); + }); + + it('keeps file-based sync data host payloads generic', () => { + const recentOp: SyncFileCompactOp = { + id: 'op-a', + action: 'create', + sv: 7, + }; + const data: FileBasedSyncData<{ entities: string[] }, HostCompactOp, HostArchive> = { + version: FILE_BASED_SYNC_CONSTANTS.FILE_VERSION, + syncVersion: 8, + schemaVersion: 1, + vectorClock: { clientA: 3 }, + lastModified: 1701700000000, + clientId: 'client-a', + state: { entities: ['a'] }, + archiveYoung: { records: 1 }, + archiveOld: { records: 2 }, + recentOps: [recentOp], + oldestOpSyncVersion: 7, + }; + + expect(data.recentOps[0].sv).toBe(7); + expect(FILE_BASED_SYNC_CONSTANTS.SYNC_FILE).toBe('sync-data.json'); + }); +}); diff --git a/packages/sync-providers/tsconfig.json b/packages/sync-providers/tsconfig.json new file mode 100644 index 0000000000..f81d5c1add --- /dev/null +++ b/packages/sync-providers/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "preserve", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "tests"] +} diff --git a/packages/sync-providers/tsup.config.ts b/packages/sync-providers/tsup.config.ts new file mode 100644 index 0000000000..e2f0728676 --- /dev/null +++ b/packages/sync-providers/tsup.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts'], + format: ['esm', 'cjs'], + dts: true, + sourcemap: true, + clean: true, +}); diff --git a/packages/sync-providers/vitest.config.ts b/packages/sync-providers/vitest.config.ts new file mode 100644 index 0000000000..1bd50c8653 --- /dev/null +++ b/packages/sync-providers/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['tests/**/*.spec.ts'], + }, +}); diff --git a/src/app/core/electron/local-rest-api-handler.service.spec.ts b/src/app/core/electron/local-rest-api-handler.service.spec.ts index 97d8186f1f..126b3a48d4 100644 --- a/src/app/core/electron/local-rest-api-handler.service.spec.ts +++ b/src/app/core/electron/local-rest-api-handler.service.spec.ts @@ -86,6 +86,17 @@ describe('LocalRestApiHandlerService', () => { return responsePromise; }; + const expectTaskIds = ( + response: LocalRestApiResponsePayload, + expectedIds: string[], + ): void => { + expect(response.body.ok).toBe(true); + if (!response.body.ok) { + throw new Error(`Expected success response, got ${response.body.error.code}`); + } + expect((response.body.data as Task[]).map((task) => task.id)).toEqual(expectedIds); + }; + beforeEach(() => { requestHandler = null; responsePromiseResolve = null; @@ -140,9 +151,13 @@ describe('LocalRestApiHandlerService', () => { }, ); - dateServiceMock = jasmine.createSpyObj('DateService', ['todayStr', 'isToday']); + dateServiceMock = jasmine.createSpyObj( + 'DateService', + ['todayStr', 'getStartOfNextDayDiffMs'], + {}, + ); dateServiceMock.todayStr.and.returnValue('2026-05-12'); - dateServiceMock.isToday.and.returnValue(false); + dateServiceMock.getStartOfNextDayDiffMs.and.returnValue(0); TestBed.configureTestingModule({ providers: [ @@ -208,6 +223,7 @@ describe('LocalRestApiHandlerService', () => { expect(response.body.ok).toBe(true); expect(response.status).toBe(200); + expectTaskIds(response, ['task-1', 'task-2']); }); it('should filter tasks by query', async () => { @@ -221,7 +237,7 @@ describe('LocalRestApiHandlerService', () => { createRequest('GET', '/tasks', { query: { query: 'milk' } }), ); - expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1']); }); it('should filter tasks by projectId', async () => { @@ -235,7 +251,7 @@ describe('LocalRestApiHandlerService', () => { createRequest('GET', '/tasks', { query: { projectId: 'project-1' } }), ); - expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1']); }); it('should filter tasks by tagId', async () => { @@ -249,7 +265,133 @@ describe('LocalRestApiHandlerService', () => { createRequest('GET', '/tasks', { query: { tagId: 'tag-1' } }), ); - expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1']); + }); + + it('should filter tasks by the virtual TODAY tag using dueDay', async () => { + const tasks = [ + createMockTask('task-1', { dueDay: '2026-05-12' }), + createMockTask('task-2', { dueDay: '2026-05-13' }), + createMockTask('task-3', { dueDay: '2026-05-11' }), + createMockTask('task-4'), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { query: { tagId: TODAY_TAG.id } }), + ); + + expectTaskIds(response, ['task-1']); + }); + + it('should combine the virtual TODAY tag filter with projectId and query', async () => { + const tasks = [ + createMockTask('task-1', { + title: 'Buy milk', + projectId: 'project-1', + dueDay: '2026-05-12', + }), + createMockTask('task-2', { + title: 'Buy bread', + projectId: 'project-2', + dueDay: '2026-05-12', + }), + createMockTask('task-3', { + title: 'Walk dog', + projectId: 'project-1', + dueDay: '2026-05-12', + }), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { + query: { tagId: TODAY_TAG.id, projectId: 'project-1', query: 'milk' }, + }), + ); + + expectTaskIds(response, ['task-1']); + }); + + it('should include done virtual TODAY tasks when includeDone=true', async () => { + const tasks = [ + createMockTask('task-1', { dueDay: '2026-05-12', isDone: false }), + createMockTask('task-2', { dueDay: '2026-05-12', isDone: true }), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { + query: { tagId: TODAY_TAG.id, includeDone: 'true' }, + }), + ); + + expectTaskIds(response, ['task-1', 'task-2']); + }); + + it('should filter virtual TODAY tasks by dueWithTime and start-of-next-day offset', async () => { + dateServiceMock.todayStr.and.returnValue('2026-02-15'); + dateServiceMock.getStartOfNextDayDiffMs.and.returnValue(4 * 60 * 60 * 1000); + const tasks = [ + createMockTask('task-1', { + dueWithTime: new Date(2026, 1, 16, 2, 0).getTime(), + }), + createMockTask('task-2', { + dueWithTime: new Date(2026, 1, 16, 5, 0).getTime(), + }), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { query: { tagId: TODAY_TAG.id } }), + ); + + expectTaskIds(response, ['task-1']); + }); + + it('should let dueWithTime take priority over dueDay for the virtual TODAY tag', async () => { + const tasks = [ + createMockTask('task-1', { + dueDay: '2026-05-12', + dueWithTime: new Date(2026, 4, 13, 10, 0).getTime(), + }), + createMockTask('task-2', { + dueDay: '2026-05-12', + }), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { query: { tagId: TODAY_TAG.id } }), + ); + + expectTaskIds(response, ['task-2']); + }); + + it('should not fail the virtual TODAY filter for invalid dueWithTime values', async () => { + const tasks = [ + createMockTask('task-1', { + dueDay: '2026-05-12', + dueWithTime: -1, + }), + createMockTask('task-2', { + dueWithTime: -1, + }), + createMockTask('task-3', { + dueDay: '2026-05-12', + dueWithTime: 8_640_000_000_000_001, + }), + createMockTask('task-4', { + dueWithTime: 8_640_000_000_000_001, + }), + ]; + Object.defineProperty(taskServiceMock, 'allTasks$', { get: () => of(tasks) }); + + const response = await sendRequestAndWait( + createRequest('GET', '/tasks', { query: { tagId: TODAY_TAG.id } }), + ); + + expectTaskIds(response, ['task-1', 'task-3']); }); it('should filter TODAY virtual tag by due fields', async () => { @@ -281,7 +423,7 @@ describe('LocalRestApiHandlerService', () => { const response = await sendRequestAndWait(createRequest('GET', '/tasks')); - expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1']); }); it('should include done tasks when includeDone=true', async () => { @@ -295,11 +437,11 @@ describe('LocalRestApiHandlerService', () => { createRequest('GET', '/tasks', { query: { includeDone: 'true' } }), ); - expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1', 'task-2']); }); it('should return archived tasks when source=archived', async () => { - const archivedTask = createMockTask('archivedTask1', { isDone: true }); + const archivedTask = createMockTask('archivedTask1'); (taskArchiveServiceMock as any).load.and.returnValue( Promise.resolve({ ids: ['archivedTask1'], @@ -312,6 +454,7 @@ describe('LocalRestApiHandlerService', () => { ); expect(response.body.ok).toBe(true); + expectTaskIds(response, ['archivedTask1']); expect(taskArchiveServiceMock.load).toHaveBeenCalled(); }); @@ -328,6 +471,7 @@ describe('LocalRestApiHandlerService', () => { ); expect(response.body.ok).toBe(true); + expectTaskIds(response, ['task-1']); expect(taskServiceMock.getAllTasksEverywhere).toHaveBeenCalled(); }); }); diff --git a/src/app/core/electron/local-rest-api-handler.service.ts b/src/app/core/electron/local-rest-api-handler.service.ts index 020ffd545d..6b20d22f28 100644 --- a/src/app/core/electron/local-rest-api-handler.service.ts +++ b/src/app/core/electron/local-rest-api-handler.service.ts @@ -7,6 +7,7 @@ import { ProjectService } from '../../features/project/project.service'; import { TagService } from '../../features/tag/tag.service'; import { TODAY_TAG } from '../../features/tag/tag.const'; import { DateService } from '../date/date.service'; +import { isTodayWithOffset } from '../../util/is-today.util'; import { LocalRestApiRequestPayload, LocalRestApiResponsePayload, @@ -111,6 +112,20 @@ const createSuccessResponse = ( type TaskSource = 'active' | 'archived' | 'all'; +const isValidTimestamp = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value) && new Date(value).getTime() > 0; + +const isTaskInToday = ( + task: Task, + todayStr: string, + startOfNextDayDiffMs: number, +): boolean => { + if (isValidTimestamp(task.dueWithTime)) { + return isTodayWithOffset(task.dueWithTime, todayStr, startOfNextDayDiffMs); + } + return task.dueDay === todayStr; +}; + @Injectable({ providedIn: 'root', }) @@ -296,10 +311,12 @@ export class LocalRestApiHandlerService { filtered = filtered.filter((t) => t.projectId === projectId); } - if (tagId) { - filtered = filtered.filter((t) => - tagId === TODAY_TAG.id ? this._isTaskInToday(t) : t.tagIds.includes(tagId), - ); + if (tagId === TODAY_TAG.id) { + const todayStr = this._dateService.todayStr(); + const startOfNextDayDiffMs = this._dateService.getStartOfNextDayDiffMs(); + filtered = filtered.filter((t) => isTaskInToday(t, todayStr, startOfNextDayDiffMs)); + } else if (tagId) { + filtered = filtered.filter((t) => t.tagIds.includes(tagId)); } if (!includeDone) { @@ -309,14 +326,6 @@ export class LocalRestApiHandlerService { return createSuccessResponse(requestId, 200, filtered); } - private _isTaskInToday(task: Task): boolean { - if (task.dueWithTime) { - return this._dateService.isToday(task.dueWithTime); - } - - return task.dueDay === this._dateService.todayStr(); - } - private async _handleCreateTask( requestId: string, body: unknown, diff --git a/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.spec.ts b/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.spec.ts index d74e0e5991..f6243ede76 100644 --- a/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.spec.ts +++ b/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.spec.ts @@ -20,7 +20,11 @@ import { DateService } from '../../core/date/date.service'; // Helper to access private methods for testing type PrivateService = { _sortAll(tasks: TaskCopy[]): TaskCopy[]; - _movePlannedTasksToToday(tasks: TaskCopy[]): void; + _movePlannedTasksToToday( + tasks: TaskCopy[], + today: string, + startOfNextDayDiffMs: number, + ): void; }; describe('AddTasksForTomorrowService', () => { @@ -156,12 +160,14 @@ describe('AddTasksForTomorrowService', () => { 'getLogicalTomorrowMs', 'getLogicalTodayDate', 'todayStr', + 'getStartOfNextDayDiffMs', ]); dateServiceSpy.getLogicalTomorrowMs.and.returnValue( new Date('2026-04-18T00:00:00Z').getTime(), ); dateServiceSpy.getLogicalTodayDate.and.returnValue(new Date('2026-04-17T00:00:00Z')); dateServiceSpy.todayStr.and.returnValue(todayStr); + dateServiceSpy.getStartOfNextDayDiffMs.and.returnValue(0); TestBed.configureTestingModule({ providers: [ @@ -355,6 +361,8 @@ describe('AddTasksForTomorrowService', () => { expect(dispatchSpy).toHaveBeenCalledWith( TaskSharedActions.planTasksForToday({ taskIds: ['task2'], // Only task2 + today: todayStr, + startOfNextDayDiffMs: 0, isSkipRemoveReminder: true, }), ); @@ -638,7 +646,7 @@ describe('AddTasksForTomorrowService', () => { const dispatchSpy = spyOn(store, 'dispatch'); const tasks = [mockTaskWithDueTimeTomorrow, mockTaskWithDueDayTomorrow]; - (service as unknown as PrivateService)._movePlannedTasksToToday(tasks); + (service as unknown as PrivateService)._movePlannedTasksToToday(tasks, todayStr, 0); // The service may order tasks differently based on the sorting algorithm expect(dispatchSpy).toHaveBeenCalled(); @@ -657,7 +665,7 @@ describe('AddTasksForTomorrowService', () => { it('should not dispatch when empty array', () => { const dispatchSpy = spyOn(store, 'dispatch'); - (service as unknown as PrivateService)._movePlannedTasksToToday([]); + (service as unknown as PrivateService)._movePlannedTasksToToday([], todayStr, 0); expect(dispatchSpy).not.toHaveBeenCalled(); }); diff --git a/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.ts b/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.ts index 1f4ff000bc..455f9fac58 100644 --- a/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.ts +++ b/src/app/features/add-tasks-for-tomorrow/add-tasks-for-tomorrow.service.ts @@ -69,6 +69,8 @@ export class AddTasksForTomorrowService { // NOTE: this gets a lot of interference from tagEffect.preventParentAndSubTaskInTodayList$: async addAllDueTomorrow(): Promise<'ADDED' | void> { + const todayStr = this._dateService.todayStr(); + const startOfNextDayDiffMs = this._dateService.getStartOfNextDayDiffMs(); const dueRepeatCfgs = await this._repeatableForTomorrow$.pipe(first()).toPromise(); const tomorrow = this._dateService.getLogicalTomorrowMs(); @@ -126,7 +128,7 @@ export class AddTasksForTomorrowService { }); } - this._movePlannedTasksToToday(allDueSorted); + this._movePlannedTasksToToday(allDueSorted, todayStr, startOfNextDayDiffMs); if (allDueSorted.length) { return 'ADDED'; @@ -138,6 +140,7 @@ export class AddTasksForTomorrowService { const todayDate = this._dateService.getLogicalTodayDate(); const todayTS = todayDate.getTime(); const todayStr = this._dateService.todayStr(); + const startOfNextDayDiffMs = this._dateService.getStartOfNextDayDiffMs(); TaskLog.log('[AddTasksForTomorrow] Starting addAllDueToday', { todayStr }); @@ -215,18 +218,24 @@ export class AddTasksForTomorrowService { }); } - this._movePlannedTasksToToday(allDueSorted); + this._movePlannedTasksToToday(allDueSorted, todayStr, startOfNextDayDiffMs); if (allDueSorted.length) { return 'ADDED'; } } - private _movePlannedTasksToToday(plannedTasks: TaskCopy[]): void { + private _movePlannedTasksToToday( + plannedTasks: TaskCopy[], + today: string, + startOfNextDayDiffMs: number, + ): void { if (plannedTasks.length) { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: plannedTasks.map((t) => t.id), + today, + startOfNextDayDiffMs, isSkipRemoveReminder: true, }), ); diff --git a/src/app/features/calendar-integration/calendar-event-regex-filter.spec.ts b/src/app/features/calendar-integration/calendar-event-regex-filter.spec.ts index 4eb7ea1723..a995a036b8 100644 --- a/src/app/features/calendar-integration/calendar-event-regex-filter.spec.ts +++ b/src/app/features/calendar-integration/calendar-event-regex-filter.spec.ts @@ -86,7 +86,11 @@ describe('passesCalendarEventRegexFilter', () => { }); it('should still apply a regex exactly at the limit', () => { - const padded = 'Meeting'.padEnd(CALENDAR_REGEX_FILTER_MAX_LENGTH, '.'); + // 256-char pattern that still matches 'Team Meeting' via alternation. A + // naive `'Meeting'.padEnd(256, '.')` fails because `.` matches any char, + // requiring a 256+ char title. + const padded = + 'Meeting|' + 'a'.repeat(CALENDAR_REGEX_FILTER_MAX_LENGTH - 'Meeting|'.length); expect(padded.length).toBe(CALENDAR_REGEX_FILTER_MAX_LENGTH); expect(passesCalendarEventRegexFilter(baseEvent, padded, null)).toBe(true); }); diff --git a/src/app/features/config/global-config.service.spec.ts b/src/app/features/config/global-config.service.spec.ts new file mode 100644 index 0000000000..98fed8e1e3 --- /dev/null +++ b/src/app/features/config/global-config.service.spec.ts @@ -0,0 +1,43 @@ +import { TestBed } from '@angular/core/testing'; +import { MockStore, provideMockStore } from '@ngrx/store/testing'; +import { SyncProviderId } from '../../op-log/sync-providers/provider.const'; +import { DEFAULT_GLOBAL_CONFIG } from './default-global-config.const'; +import { GlobalConfigService } from './global-config.service'; +import { selectSyncConfig } from './store/global-config.reducer'; +import type { SyncConfig } from './global-config.model'; + +describe('GlobalConfigService', () => { + it('should expose a sync-core config snapshot without private provider fields', async () => { + const syncConfig = { + ...DEFAULT_GLOBAL_CONFIG.sync, + isEnabled: true, + syncProvider: SyncProviderId.SuperSync, + isEncryptionEnabled: true, + isCompressionEnabled: true, + isManualSyncOnly: true, + syncInterval: 15, + encryptKey: 'private-key', + } as SyncConfig; + + TestBed.configureTestingModule({ + providers: [provideMockStore()], + }); + + // overrideSelector bypasses feature-state lookup and any cross-spec + // pollution of the underlying store state. + const store = TestBed.inject(MockStore); + store.overrideSelector(selectSyncConfig, syncConfig); + + const snapshot = await TestBed.inject(GlobalConfigService).getSyncConfig(); + + expect(snapshot).toEqual({ + isEnabled: true, + syncProvider: SyncProviderId.SuperSync, + isEncryptionEnabled: true, + isCompressionEnabled: true, + isManualSyncOnly: true, + syncInterval: 15, + }); + expect('encryptKey' in snapshot).toBeFalse(); + }); +}); diff --git a/src/app/features/config/global-config.service.ts b/src/app/features/config/global-config.service.ts index 8f821a986d..bbcc6f005a 100644 --- a/src/app/features/config/global-config.service.ts +++ b/src/app/features/config/global-config.service.ts @@ -1,8 +1,9 @@ import { Injectable, inject, Signal } from '@angular/core'; import { select, Store } from '@ngrx/store'; +import type { SyncConfigPort, SyncConfigSnapshot } from '@sp/sync-core'; import { toSignal } from '@angular/core/rxjs-interop'; import { updateGlobalConfigSection } from './store/global-config.actions'; -import { Observable } from 'rxjs'; +import { firstValueFrom, Observable } from 'rxjs'; import { DEFAULT_GLOBAL_CONFIG } from './default-global-config.const'; import { AppFeaturesConfig, @@ -46,7 +47,9 @@ import { distinctUntilChangedObject } from '../../util/distinct-until-changed-ob @Injectable({ providedIn: 'root', }) -export class GlobalConfigService { +export class GlobalConfigService implements SyncConfigPort< + NonNullable +> { private readonly _store = inject>(Store); // Keep observables for backward compatibility @@ -192,4 +195,26 @@ export class GlobalConfigService { }), ); } + + async getSyncConfig(): Promise< + SyncConfigSnapshot> + > { + const { + isEnabled, + syncProvider, + isEncryptionEnabled, + isCompressionEnabled, + isManualSyncOnly, + syncInterval, + } = await firstValueFrom(this.sync$); + + return { + isEnabled, + syncProvider, + isEncryptionEnabled, + isCompressionEnabled, + isManualSyncOnly, + syncInterval, + }; + } } diff --git a/src/app/features/tag/store/tag.reducer.ts b/src/app/features/tag/store/tag.reducer.ts index ceeccb29fd..775079aed1 100644 --- a/src/app/features/tag/store/tag.reducer.ts +++ b/src/app/features/tag/store/tag.reducer.ts @@ -62,6 +62,7 @@ export const selectAllTagsWithoutMyDay = createSelector( * This selector returns raw stored taskIds which may be stale or incomplete. * `selectTodayTaskIds` computes membership from task.dueDay (virtual tag pattern) * and is self-healing. + * Exception: cleanup code that removes stale raw TODAY_TAG ids needs this stored order. * * See: docs/ai/today-tag-architecture.md */ diff --git a/src/app/features/tasks/dialog-view-task-reminders/dialog-view-task-reminders.component.ts b/src/app/features/tasks/dialog-view-task-reminders/dialog-view-task-reminders.component.ts index fdcfa89ec6..36105cea4f 100644 --- a/src/app/features/tasks/dialog-view-task-reminders/dialog-view-task-reminders.component.ts +++ b/src/app/features/tasks/dialog-view-task-reminders/dialog-view-task-reminders.component.ts @@ -37,6 +37,7 @@ import { TaskSharedActions } from '../../../root-store/meta/task-shared.actions' import { PlannerActions } from '../../planner/store/planner.actions'; import { getDbDateStr } from '../../../util/get-db-date-str'; import { selectTodayTaskIds } from '../../work-context/store/work-context.selectors'; +import { DateService } from '../../../core/date/date.service'; const MINUTES_TO_MILLISECONDS = 1000 * 60; @@ -73,6 +74,7 @@ export class DialogViewTaskRemindersComponent implements OnDestroy { private _matDialog = inject(MatDialog); private _store = inject(Store); private _reminderService = inject(ReminderService); + private _dateService = inject(DateService); data = inject<{ reminders: TaskWithReminderData[]; }>(MAT_DIALOG_DATA); @@ -175,6 +177,8 @@ export class DialogViewTaskRemindersComponent implements OnDestroy { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), parentTaskMap: { [task.id]: task.parentId, }, @@ -335,6 +339,8 @@ export class DialogViewTaskRemindersComponent implements OnDestroy { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: selectedTasks.map((t) => t.id), + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), parentTaskMap: selectedTasks.reduce((acc, next: Task) => { return { ...acc, [next.id as string]: next.parentId }; }, {}), diff --git a/src/app/features/tasks/store/task-due.effects.spec.ts b/src/app/features/tasks/store/task-due.effects.spec.ts index d480381ad9..8623cdaa7e 100644 --- a/src/app/features/tasks/store/task-due.effects.spec.ts +++ b/src/app/features/tasks/store/task-due.effects.spec.ts @@ -3,25 +3,52 @@ import { provideMockActions } from '@ngrx/effects/testing'; import { provideMockStore, MockStore } from '@ngrx/store/testing'; import { Observable, BehaviorSubject, of, Subject, take } from 'rxjs'; import { Action } from '@ngrx/store'; -import { TaskDueEffects } from './task-due.effects'; +import { getOverdueIdsInTodayOrder, TaskDueEffects } from './task-due.effects'; import { GlobalTrackingIntervalService } from '../../../core/global-tracking-interval/global-tracking-interval.service'; import { SyncWrapperService } from '../../../imex/sync/sync-wrapper.service'; import { AddTasksForTomorrowService } from '../../add-tasks-for-tomorrow/add-tasks-for-tomorrow.service'; import { SyncTriggerService } from '../../../imex/sync/sync-trigger.service'; -import { selectOverdueTasksOnToday, selectTasksDueForDay } from './task.selectors'; +import { + selectOverdueTasksOnToday, + selectTasksDueForDay, + selectTasksWithDueTimeForRange, +} from './task.selectors'; import { selectTodayTaskIds } from '../../work-context/store/work-context.selectors'; import { DEFAULT_TASK, Task, TaskWithDueDay } from '../task.model'; import { getDbDateStr } from '../../../util/get-db-date-str'; import { HydrationStateService } from '../../../op-log/apply/hydration-state.service'; import { TaskSharedActions } from '../../../root-store/meta/task-shared.actions'; +import { + initialTagState, + selectTodayTagTaskIds, + TAG_FEATURE_NAME, +} from '../../tag/store/tag.reducer'; +import { + selectStartOfNextDayDiffMs, + selectTodayStr, +} from '../../../root-store/app-state/app-state.selectors'; +import { initialTaskState, TASK_FEATURE_NAME } from './task.reducer'; +import { appStateFeatureKey } from '../../../root-store/app-state/app-state.reducer'; -// These tests are skipped because TaskDueEffects imports SyncTriggerService which -// imports PfapiService which imports pfapi-config.ts. That file eagerly instantiates -// PFAPI_SYNC_PROVIDERS (including Dropbox) at module load time, which fails in the -// test environment. This requires architectural changes to lazy-load sync providers. -// See also: src/app/imex/sync/sync.effects.spec.ts for similar pattern. -// TODO: Refactor pfapi-config.ts to lazy-load sync providers to enable these tests. -xdescribe('TaskDueEffects', () => { +describe('getOverdueIdsInTodayOrder', () => { + it('returns overdue ids in raw Today tag order', () => { + expect( + getOverdueIdsInTodayOrder( + [{ id: 'overdue-3' }, { id: 'overdue-1' }], + ['task-0', 'overdue-1', 'task-2', 'overdue-3'], + ), + ).toEqual(['overdue-1', 'overdue-3']); + }); + + it('returns an empty list when overdue ids are not in the raw Today tag order', () => { + expect( + getOverdueIdsInTodayOrder([{ id: 'overdue-1' }], ['task-2', 'task-3']), + ).toEqual([]); + }); +}); + +describe('TaskDueEffects', () => { + let previousTimeout: number; const actions$: Observable = of(); let effects: TaskDueEffects; let store: MockStore; @@ -34,6 +61,22 @@ xdescribe('TaskDueEffects', () => { let addTasksForTomorrowService: jasmine.SpyObj; const todayStr = getDbDateStr(); + const startOfNextDayDiffMs = 0; + + const initialState = { + [TASK_FEATURE_NAME]: initialTaskState, + [TAG_FEATURE_NAME]: initialTagState, + [appStateFeatureKey]: { todayStr, startOfNextDayDiffMs }, + }; + + beforeAll(() => { + previousTimeout = jasmine.DEFAULT_TIMEOUT_INTERVAL; + jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000; + }); + + afterAll(() => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = previousTimeout; + }); const createTask = (id: string, partial: Partial = {}): Task => ({ ...DEFAULT_TASK, @@ -74,17 +117,26 @@ xdescribe('TaskDueEffects', () => { const hydrationStateServiceSpy = jasmine.createSpyObj('HydrationStateService', [ 'isApplyingRemoteOps', + 'isInSyncWindow', ]); hydrationStateServiceSpy.isApplyingRemoteOps.and.returnValue(false); + hydrationStateServiceSpy.isInSyncWindow.and.returnValue(false); + hydrationStateServiceSpy.isInSyncWindow$ = of(false); TestBed.configureTestingModule({ providers: [ TaskDueEffects, provideMockActions(() => actions$), provideMockStore({ + initialState, selectors: [ { selector: selectOverdueTasksOnToday, value: [] }, { selector: selectTodayTaskIds, value: [] }, + { selector: selectTodayTagTaskIds, value: [] }, + { selector: selectTodayStr, value: todayStr }, + { selector: selectStartOfNextDayDiffMs, value: startOfNextDayDiffMs }, + { selector: selectTasksDueForDay, value: [] }, + { selector: selectTasksWithDueTimeForRange, value: [] }, ], }), { @@ -156,25 +208,22 @@ xdescribe('TaskDueEffects', () => { }); it('should not react to duplicate date strings (distinctUntilChanged)', (done) => { - let emitCount = 0; + const subscription = effects.createRepeatableTasksAndAddDueToday$.subscribe(); - const subscription = effects.createRepeatableTasksAndAddDueToday$.subscribe(() => { - emitCount++; - }); - - // Emit initial date - globalTrackingIntervalService.todayDateStr$.next(todayStr); - syncWrapperService.afterCurrentSyncDoneOrSyncDisabled$.next(true); - - // Wait for debounce to pass, then check only one emission for same date + // Wait for the initial BehaviorSubject emission to pass through the debounce, + // then emit the same date again. The sync wrapper is also a BehaviorSubject + // in this setup, so each inner switchMap subscription receives the current + // "sync done" value without a manual next(). + // distinctUntilChanged should suppress the duplicate date before that point. setTimeout(() => { - // Emit same date again - should be ignored + const callCountAfterInitialEmission = + addTasksForTomorrowService.addAllDueToday.calls.count(); globalTrackingIntervalService.todayDateStr$.next(todayStr); - syncWrapperService.afterCurrentSyncDoneOrSyncDisabled$.next(true); setTimeout(() => { - // Only first emission should have triggered - expect(emitCount).toBeLessThanOrEqual(1); + expect(addTasksForTomorrowService.addAllDueToday.calls.count()).toBe( + callCountAfterInitialEmission, + ); subscription.unsubscribe(); done(); }, 1500); @@ -189,7 +238,7 @@ xdescribe('TaskDueEffects', () => { }); store.overrideSelector(selectOverdueTasksOnToday, [overdueTask]); - store.overrideSelector(selectTodayTaskIds, ['overdue-1', 'task-2']); + store.overrideSelector(selectTodayTagTaskIds, ['overdue-1', 'task-2']); store.refreshState(); const subscription = effects.removeOverdueFormToday$.pipe(take(1)).subscribe({ @@ -211,13 +260,13 @@ xdescribe('TaskDueEffects', () => { syncWrapperService.afterCurrentSyncDoneOrSyncDisabled$.next(true); }); - it('should preserve task order from todayTaskIds when removing overdue', (done) => { + it('should preserve task order from todayTagTaskIds when removing overdue', (done) => { const overdueTask1 = createTask('overdue-1'); const overdueTask2 = createTask('overdue-3'); store.overrideSelector(selectOverdueTasksOnToday, [overdueTask1, overdueTask2]); - // Note the specific order in todayTaskIds - store.overrideSelector(selectTodayTaskIds, [ + // Note the specific order in raw today tag task ids + store.overrideSelector(selectTodayTagTaskIds, [ 'task-0', 'overdue-1', 'task-2', @@ -230,7 +279,7 @@ xdescribe('TaskDueEffects', () => { next: (action) => { expect(action).toEqual( jasmine.objectContaining({ - taskIds: ['overdue-1', 'overdue-3'], // Order from todayTaskIds + taskIds: ['overdue-1', 'overdue-3'], // Order from raw today tag task ids }), ); subscription.unsubscribe(); @@ -263,7 +312,7 @@ xdescribe('TaskDueEffects', () => { }, 1500); }); - it('should not emit when overdue tasks exist but none are in todayTaskIds', (done) => { + it('should not emit when overdue tasks exist but none are in todayTagTaskIds', (done) => { // This tests the fix for the bug where removeTasksFromTodayTag was dispatched // with empty taskIds, causing "missing entityId/entityIds" error during sync const overdueTask = createTask('overdue-1', { @@ -271,8 +320,8 @@ xdescribe('TaskDueEffects', () => { }); store.overrideSelector(selectOverdueTasksOnToday, [overdueTask]); - // todayTaskIds does NOT contain overdue-1, so overdueIds will be empty - store.overrideSelector(selectTodayTaskIds, ['task-2', 'task-3']); + // raw today tag task ids do NOT contain overdue-1, so overdueIds will be empty + store.overrideSelector(selectTodayTagTaskIds, ['task-2', 'task-3']); store.refreshState(); let emitted = false; @@ -307,6 +356,8 @@ xdescribe('TaskDueEffects', () => { expect(action).toEqual( jasmine.objectContaining({ taskIds: ['due-today-1'], + today: todayStr, + startOfNextDayDiffMs, isSkipRemoveReminder: true, }), ); @@ -385,6 +436,8 @@ xdescribe('TaskDueEffects', () => { expect(action).toEqual( jasmine.objectContaining({ taskIds: ['subtask-1'], + today: todayStr, + startOfNextDayDiffMs, isSkipRemoveReminder: true, }), ); @@ -427,8 +480,11 @@ xdescribe('TaskDueEffects', () => { const hydrationStateServiceSpy2 = jasmine.createSpyObj('HydrationStateService', [ 'isApplyingRemoteOps', + 'isInSyncWindow', ]); hydrationStateServiceSpy2.isApplyingRemoteOps.and.returnValue(false); + hydrationStateServiceSpy2.isInSyncWindow.and.returnValue(false); + hydrationStateServiceSpy2.isInSyncWindow$ = of(false); TestBed.resetTestingModule(); TestBed.configureTestingModule({ @@ -436,9 +492,11 @@ xdescribe('TaskDueEffects', () => { TaskDueEffects, provideMockActions(() => emptyActions$), provideMockStore({ + initialState, selectors: [ { selector: selectOverdueTasksOnToday, value: [] }, { selector: selectTodayTaskIds, value: [] }, + { selector: selectTodayTagTaskIds, value: [] }, ], }), { diff --git a/src/app/features/tasks/store/task-due.effects.ts b/src/app/features/tasks/store/task-due.effects.ts index 5cc2ae6e0c..5926f26d64 100644 --- a/src/app/features/tasks/store/task-due.effects.ts +++ b/src/app/features/tasks/store/task-due.effects.ts @@ -32,6 +32,15 @@ import { SyncTriggerService } from '../../../imex/sync/sync-trigger.service'; import { environment } from '../../../../environments/environment'; import { HydrationStateService } from '../../../op-log/apply/hydration-state.service'; import { waitForSyncWindow } from '../../../util/wait-for-sync-window.operator'; +import { selectTodayTagTaskIds } from '../../tag/store/tag.reducer'; + +export const getOverdueIdsInTodayOrder = ( + overdue: ReadonlyArray<{ id: string }>, + todayTagTaskIds: readonly string[], +): string[] => { + const overdueIds = new Set(overdue.map((task) => task.id)); + return todayTagTaskIds.filter((id) => overdueIds.has(id)); +}; @Injectable() export class TaskDueEffects { @@ -117,12 +126,12 @@ export class TaskDueEffects { switchMap(() => this._syncWrapperService.afterCurrentSyncDoneOrSyncDisabled$), switchMap(() => this._store$.select(selectOverdueTasksOnToday).pipe(first())), filter((overdue) => !!overdue.length), - withLatestFrom(this._store$.select(selectTodayTaskIds)), + // Intentional raw TODAY_TAG order read. The computed selectTodayTaskIds + // excludes overdue tasks by design, so it cannot remove stale raw IDs. + withLatestFrom(this._store$.select(selectTodayTagTaskIds)), // we do this to maintain the order of tasks - switchMap(([overdue, todayTaskIds]) => { - const overdueIds = todayTaskIds.filter( - (id) => !!overdue.find((oT) => oT.id === id), - ); + switchMap(([overdue, todayTagTaskIds]) => { + const overdueIds = getOverdueIdsInTodayOrder(overdue, todayTagTaskIds); if (overdueIds.length === 0) { return EMPTY; } @@ -226,6 +235,8 @@ export class TaskDueEffects { return missingTaskIds.length > 0 ? TaskSharedActions.planTasksForToday({ taskIds: missingTaskIds, + today: todayStr, + startOfNextDayDiffMs, isSkipRemoveReminder: true, }) : null; diff --git a/src/app/features/tasks/store/task-related-model.effects.spec.ts b/src/app/features/tasks/store/task-related-model.effects.spec.ts index 90f0c9a468..d1682d37db 100644 --- a/src/app/features/tasks/store/task-related-model.effects.spec.ts +++ b/src/app/features/tasks/store/task-related-model.effects.spec.ts @@ -12,6 +12,7 @@ import { TaskSharedActions } from '../../../root-store/meta/task-shared.actions' import { DEFAULT_TASK, Task } from '../task.model'; import { getDbDateStr } from '../../../util/get-db-date-str'; import { selectTodayTaskIds } from '../../work-context/store/work-context.selectors'; +import { DateService } from '../../../core/date/date.service'; describe('TaskRelatedModelEffects', () => { let effects: TaskRelatedModelEffects; @@ -19,6 +20,7 @@ describe('TaskRelatedModelEffects', () => { let store: MockStore; let taskService: jasmine.SpyObj; let hydrationStateService: jasmine.SpyObj; + let dateService: jasmine.SpyObj; const createTask = (id: string, partial: Partial = {}): Task => ({ ...DEFAULT_TASK, @@ -37,6 +39,12 @@ describe('TaskRelatedModelEffects', () => { 'isApplyingRemoteOps', ]); hydrationStateServiceSpy.isApplyingRemoteOps.and.returnValue(false); + const dateServiceSpy = jasmine.createSpyObj('DateService', [ + 'todayStr', + 'getStartOfNextDayDiffMs', + ]); + dateServiceSpy.todayStr.and.returnValue(getDbDateStr()); + dateServiceSpy.getStartOfNextDayDiffMs.and.returnValue(0); TestBed.configureTestingModule({ providers: [ @@ -46,6 +54,7 @@ describe('TaskRelatedModelEffects', () => { selectors: [{ selector: selectTodayTaskIds, value: [] }], }), { provide: TaskService, useValue: taskServiceSpy }, + { provide: DateService, useValue: dateServiceSpy }, { provide: GlobalConfigService, useValue: { @@ -63,6 +72,7 @@ describe('TaskRelatedModelEffects', () => { hydrationStateService = TestBed.inject( HydrationStateService, ) as jasmine.SpyObj; + dateService = TestBed.inject(DateService) as jasmine.SpyObj; }); afterEach(() => { @@ -81,8 +91,10 @@ describe('TaskRelatedModelEffects', () => { effects.autoAddTodayTagOnMarkAsDone.subscribe({ next: (action) => { expect(action).toEqual( - TaskSharedActions.planTasksForToday({ + jasmine.objectContaining({ taskIds: ['task-1'], + today: dateService.todayStr(), + startOfNextDayDiffMs: dateService.getStartOfNextDayDiffMs(), }), ); done(); diff --git a/src/app/features/tasks/store/task-related-model.effects.ts b/src/app/features/tasks/store/task-related-model.effects.ts index 69732861d5..29b5d7bc3a 100644 --- a/src/app/features/tasks/store/task-related-model.effects.ts +++ b/src/app/features/tasks/store/task-related-model.effects.ts @@ -47,6 +47,8 @@ export class TaskRelatedModelEffects { map(([{ task }]) => TaskSharedActions.planTasksForToday({ taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), }), ), ), @@ -72,6 +74,8 @@ export class TaskRelatedModelEffects { map((task) => TaskSharedActions.planTasksForToday({ taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), }), ), ), diff --git a/src/app/features/tasks/store/task-ui.effects.spec.ts b/src/app/features/tasks/store/task-ui.effects.spec.ts index e519efb225..4a92bf27bb 100644 --- a/src/app/features/tasks/store/task-ui.effects.spec.ts +++ b/src/app/features/tasks/store/task-ui.effects.spec.ts @@ -1,7 +1,7 @@ import { TestBed } from '@angular/core/testing'; -import { of, Subject } from 'rxjs'; +import { of, Subject, take } from 'rxjs'; import { TaskUiEffects } from './task-ui.effects'; -import { provideMockStore } from '@ngrx/store/testing'; +import { MockStore, provideMockStore } from '@ngrx/store/testing'; import { TaskService } from '../task.service'; import { SnackService } from '../../../core/snack/snack.service'; import { SnackParams } from '../../../core/snack/snack.model'; @@ -20,6 +20,10 @@ import { LOCAL_ACTIONS } from '../../../util/local-actions.token'; import { LS } from '../../../core/persistence/storage-keys.const'; import { Action } from '@ngrx/store'; import { LayoutService } from '../../../core-ui/layout/layout.service'; +import { DateService } from '../../../core/date/date.service'; +import { HydrationStateService } from '../../../op-log/apply/hydration-state.service'; +import { selectUnplannedDeadlineTasksForToday } from './task.selectors'; +import { Banner } from '../../../core/banner/banner.model'; describe('TaskUiEffects', () => { let effects: TaskUiEffects; @@ -28,6 +32,7 @@ describe('TaskUiEffects', () => { let taskServiceMock: jasmine.SpyObj; let navigateToTaskServiceMock: jasmine.SpyObj; let layoutServiceMock: jasmine.SpyObj; + let bannerServiceMock: jasmine.SpyObj; const createMockTask = (overrides: Partial = {}): Task => ({ @@ -242,4 +247,91 @@ describe('TaskUiEffects', () => { localStorage.removeItem(LS.ONBOARDING_HINTS_DONE); }); }); + + describe('deadlineTodayBanner$', () => { + let store: MockStore; + + beforeEach(() => { + actions$ = new Subject(); + snackServiceMock = jasmine.createSpyObj('SnackService', ['open']); + taskServiceMock = jasmine.createSpyObj('TaskService', ['setSelectedId', 'update']); + navigateToTaskServiceMock = jasmine.createSpyObj('NavigateToTaskService', [ + 'navigate', + ]); + layoutServiceMock = jasmine.createSpyObj('LayoutService', ['hideAddTaskBar']); + bannerServiceMock = jasmine.createSpyObj('BannerService', ['open', 'dismiss']); + const dateServiceMock = jasmine.createSpyObj('DateService', [ + 'todayStr', + 'getStartOfNextDayDiffMs', + ]); + dateServiceMock.todayStr.and.returnValue('2024-06-14'); + dateServiceMock.getStartOfNextDayDiffMs.and.returnValue(60 * 60 * 1000); + + TestBed.configureTestingModule({ + providers: [ + TaskUiEffects, + { provide: LOCAL_ACTIONS, useValue: actions$ }, + provideMockStore({ + selectors: [ + { selector: selectProjectById, value: null }, + { + selector: selectUnplannedDeadlineTasksForToday, + value: [createMockTask({ id: 'deadline-1' })], + }, + ], + }), + { provide: SnackService, useValue: snackServiceMock }, + { provide: TaskService, useValue: taskServiceMock }, + { provide: NavigateToTaskService, useValue: navigateToTaskServiceMock }, + { provide: LayoutService, useValue: layoutServiceMock }, + { + provide: WorkContextService, + useValue: { mainListTaskIds$: of([]) }, + }, + { + provide: NotifyService, + useValue: jasmine.createSpyObj('NotifyService', ['notify']), + }, + { provide: BannerService, useValue: bannerServiceMock }, + { + provide: GlobalConfigService, + useValue: { sound$: of({ doneSound: null }) }, + }, + { provide: Router, useValue: jasmine.createSpyObj('Router', ['navigate']) }, + { provide: DateService, useValue: dateServiceMock }, + { + provide: HydrationStateService, + useValue: jasmine.createSpyObj('HydrationStateService', { + isApplyingRemoteOps: false, + }), + }, + ], + }); + + effects = TestBed.inject(TaskUiEffects); + store = TestBed.inject(MockStore); + spyOn(store, 'dispatch'); + }); + + it('should dispatch planTasksForToday with replay date fields from banner action', (done) => { + effects.deadlineTodayBanner$.pipe(take(1)).subscribe({ + next: () => { + const bannerParams = bannerServiceMock.open.calls.mostRecent() + .args[0] as Banner; + + bannerParams.action!.fn(); + + expect(store.dispatch).toHaveBeenCalledWith( + TaskSharedActions.planTasksForToday({ + taskIds: ['deadline-1'], + today: '2024-06-14', + startOfNextDayDiffMs: 60 * 60 * 1000, + }), + ); + done(); + }, + error: done.fail, + }); + }); + }); }); diff --git a/src/app/features/tasks/store/task-ui.effects.ts b/src/app/features/tasks/store/task-ui.effects.ts index d3753134cd..d5808885e8 100644 --- a/src/app/features/tasks/store/task-ui.effects.ts +++ b/src/app/features/tasks/store/task-ui.effects.ts @@ -41,6 +41,7 @@ import { NavigateToTaskService } from '../../../core-ui/navigate-to-task/navigat import { LayoutService } from '../../../core-ui/layout/layout.service'; import { LS } from '../../../core/persistence/storage-keys.const'; import { skipWhileApplyingRemoteOps } from '../../../util/skip-during-sync.operator'; +import { DateService } from '../../../core/date/date.service'; @Injectable() export class TaskUiEffects { @@ -55,6 +56,7 @@ export class TaskUiEffects { private _workContextService = inject(WorkContextService); private _navigateToTaskService = inject(NavigateToTaskService); private _layoutService = inject(LayoutService); + private _dateService = inject(DateService); taskCreatedSnack$ = createEffect( () => @@ -302,6 +304,9 @@ export class TaskUiEffects { this._store$.dispatch( TaskSharedActions.planTasksForToday({ taskIds: currentTasks.map((t) => t.id), + today: this._dateService.todayStr(), + startOfNextDayDiffMs: + this._dateService.getStartOfNextDayDiffMs(), }), ); } diff --git a/src/app/features/tasks/task-context-menu/task-context-menu-inner/task-context-menu-inner.component.ts b/src/app/features/tasks/task-context-menu/task-context-menu-inner/task-context-menu-inner.component.ts index 7e18857472..098e4699fc 100644 --- a/src/app/features/tasks/task-context-menu/task-context-menu-inner/task-context-menu-inner.component.ts +++ b/src/app/features/tasks/task-context-menu/task-context-menu-inner/task-context-menu-inner.component.ts @@ -510,6 +510,8 @@ export class TaskContextMenuInnerComponent implements AfterViewInit, OnDestroy { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: [this.task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), parentTaskMap: { [this.task.id]: this.task.parentId }, isShowSnack: true, }), diff --git a/src/app/features/tasks/task-list/task-list.component.ts b/src/app/features/tasks/task-list/task-list.component.ts index 9232c9c8a3..89080d2e70 100644 --- a/src/app/features/tasks/task-list/task-list.component.ts +++ b/src/app/features/tasks/task-list/task-list.component.ts @@ -44,6 +44,7 @@ import { TaskLog } from '../../../core/log'; import { ScheduleExternalDragService } from '../../schedule/schedule-week/schedule-external-drag.service'; import { DEFAULT_OPTIONS } from '../../task-view-customizer/types'; import { dragDelayForTouch } from '../../../util/input-intent'; +import { DateService } from '../../../core/date/date.service'; export type TaskListId = 'PARENT' | 'SUB'; export type ListModelId = DropListModelSource | string; @@ -101,6 +102,7 @@ export class TaskListComponent implements OnDestroy, AfterViewInit { private _issueService = inject(IssueService); private _taskViewCustomizerService = inject(TaskViewCustomizerService); private _scheduleExternalDragService = inject(ScheduleExternalDragService); + private _dateService = inject(DateService); dropListService = inject(DropListService); private _layoutService = inject(LayoutService); protected readonly dragDelayForTouch = dragDelayForTouch; @@ -433,7 +435,13 @@ export class TaskListComponent implements OnDestroy, AfterViewInit { const workContextType = this._workContextService .activeWorkContextType as WorkContextType; const afterTaskId = getAnchorFromDragDrop(taskId, newOrderedIds); - this._store.dispatch(TaskSharedActions.planTasksForToday({ taskIds: [taskId] })); + this._store.dispatch( + TaskSharedActions.planTasksForToday({ + taskIds: [taskId], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), + }), + ); this._store.dispatch( moveTaskInTodayList({ taskId, diff --git a/src/app/features/tasks/task.service.spec.ts b/src/app/features/tasks/task.service.spec.ts index 995ee5a8b5..e37e093f56 100644 --- a/src/app/features/tasks/task.service.spec.ts +++ b/src/app/features/tasks/task.service.spec.ts @@ -80,8 +80,12 @@ describe('TaskService', () => { }, ); - const dateServiceSpy = jasmine.createSpyObj('DateService', ['todayStr']); + const dateServiceSpy = jasmine.createSpyObj('DateService', [ + 'todayStr', + 'getStartOfNextDayDiffMs', + ]); dateServiceSpy.todayStr.and.returnValue('2026-01-05'); + dateServiceSpy.getStartOfNextDayDiffMs.and.returnValue(0); const routerSpy = jasmine.createSpyObj('Router', ['navigate']); routerSpy.navigate.and.returnValue(Promise.resolve(true)); @@ -284,7 +288,11 @@ describe('TaskService', () => { service.addToToday(task); expect(store.dispatch).toHaveBeenCalledWith( - TaskSharedActions.planTasksForToday({ taskIds: ['task-1'] }), + TaskSharedActions.planTasksForToday({ + taskIds: ['task-1'], + today: '2026-01-05', + startOfNextDayDiffMs: 0, + }), ); }); }); diff --git a/src/app/features/tasks/task.service.ts b/src/app/features/tasks/task.service.ts index 18f95621f3..a43743a428 100644 --- a/src/app/features/tasks/task.service.ts +++ b/src/app/features/tasks/task.service.ts @@ -435,7 +435,13 @@ export class TaskService { } addToToday(task: TaskWithSubTasks): void { - this._store.dispatch(TaskSharedActions.planTasksForToday({ taskIds: [task.id] })); + this._store.dispatch( + TaskSharedActions.planTasksForToday({ + taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), + }), + ); } remove(task: TaskWithSubTasks): void { @@ -940,7 +946,13 @@ export class TaskService { moveToCurrentWorkContext(task: TaskWithSubTasks | Task): void { if (this._workContextService.activeWorkContextType === WorkContextType.TAG) { if (this._workContextService.activeWorkContextId === TODAY_TAG.id) { - this._store.dispatch(TaskSharedActions.planTasksForToday({ taskIds: [task.id] })); + this._store.dispatch( + TaskSharedActions.planTasksForToday({ + taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), + }), + ); } else { this.updateTags(task, [this._workContextService.activeWorkContextId as string]); } diff --git a/src/app/features/tasks/task/task.component.ts b/src/app/features/tasks/task/task.component.ts index 4dd617d6a2..9958e1b7b3 100644 --- a/src/app/features/tasks/task/task.component.ts +++ b/src/app/features/tasks/task/task.component.ts @@ -918,6 +918,8 @@ export class TaskComponent implements OnDestroy, AfterViewInit { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: [task.id], + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), parentTaskMap: { [task.id]: task.parentId }, }), ); diff --git a/src/app/features/work-view/work-view.component.ts b/src/app/features/work-view/work-view.component.ts index e0b21f7b95..b5b668e251 100644 --- a/src/app/features/work-view/work-view.component.ts +++ b/src/app/features/work-view/work-view.component.ts @@ -84,6 +84,7 @@ import { TaskRepeatCfg } from '../task-repeat-cfg/task-repeat-cfg.model'; import { RepeatCfgPreviewComponent } from '../task-repeat-cfg/repeat-cfg-preview/repeat-cfg-preview.component'; import { recordSearchNavDebug } from '../../util/search-nav-debug'; import { dragDelayForTouch } from '../../util/input-intent'; +import { DateService } from '../../core/date/date.service'; @Component({ selector: 'work-view', @@ -137,6 +138,7 @@ export class WorkViewComponent implements OnInit, OnDestroy { private _globalConfigService = inject(GlobalConfigService); private _matDialog = inject(MatDialog); private _destroyRef = inject(DestroyRef); + private _dateService = inject(DateService); protected readonly dragDelayForTouch = dragDelayForTouch; isFinishDayEnabled = computed( @@ -451,6 +453,8 @@ export class WorkViewComponent implements OnInit, OnDestroy { this._store.dispatch( TaskSharedActions.planTasksForToday({ taskIds: overdueTasks.map((t) => t.id), + today: this._dateService.todayStr(), + startOfNextDayDiffMs: this._dateService.getStartOfNextDayDiffMs(), }), ); } diff --git a/src/app/op-log/apply/operation-applier.service.ts b/src/app/op-log/apply/operation-applier.service.ts index 00c5b0a126..c39042bf66 100644 --- a/src/app/op-log/apply/operation-applier.service.ts +++ b/src/app/op-log/apply/operation-applier.service.ts @@ -1,5 +1,6 @@ import { inject, Injectable, Injector } from '@angular/core'; import { Store } from '@ngrx/store'; +import { replayOperationBatch } from '@sp/sync-core'; import type { ActionDispatchPort, OperationApplyPort, @@ -85,112 +86,41 @@ export class OperationApplierService implements OperationApplyPort { ); } - // Mark that we're applying remote operations to suppress selector-based effects - this.hydrationState.startApplyingRemoteOps(); - try { - // STEP 1: Bulk dispatch all operations in a single NgRx update - // The bulkOperationsMetaReducer iterates through ops and applies each action. - // Effects don't see individual actions - they only see bulkApplyOperations - // which no effect listens for. - this.store.dispatch(bulkApplyOperations({ operations: ops })); - - // Yield to event loop to ensure store update is processed - await new Promise((resolve) => setTimeout(resolve, 0)); - - // STEP 2: Handle archive operations (only for remote sync, not local hydration) - // Archive data lives in IndexedDB, not NgRx state, so we need to persist it separately. - if (!isLocalHydration) { - const archiveResult = await this._processArchiveOperations(ops); - if (archiveResult.failedOp) { - return archiveResult; - } - + const result = await replayOperationBatch({ + ops, + applyOptions: options, + dispatcher: this.store, + createBulkApplyAction: (operations) => bulkApplyOperations({ operations }), + remoteApplyWindow: this.hydrationState, + deferredLocalActions: { + processDeferredActions: () => + this.injector.get(OperationLogEffects).processDeferredActions(), + }, + archiveSideEffects: this.archiveOperationHandler, + operationToAction: convertOpToAction, + isArchiveAffectingAction, + onRemoteArchiveDataApplied: () => { // Dispatch action to signal archive data was applied (for potential future use) // Note: The refreshWorklogAfterRemoteArchiveOps effect that used to listen to // this action is now disabled to prevent UI freezes during bulk archive sync. - if (archiveResult.hadArchiveAffectingOp) { - this.store.dispatch(remoteArchiveDataApplied()); - } - } - } finally { - // Start cooldown BEFORE ending remote ops flag to eliminate the timing gap - // where isInSyncWindow() returns false and selector-based effects can fire. - // Only needed for remote ops - local hydration doesn't cause the timing gap issue. - // Wrapped in try-catch so endApplyingRemoteOps() always runs even if this fails. - if (!isLocalHydration) { - try { - this.hydrationState.startPostSyncCooldown(); - } catch (e) { - OpLog.err('OperationApplierService: startPostSyncCooldown failed', e); - } - } - - this.hydrationState.endApplyingRemoteOps(); - - // Process any user actions that were buffered during sync replay. - // These get fresh vector clocks that include the newly-applied remote ops. - await this.injector.get(OperationLogEffects).processDeferredActions(); - } - - OpLog.normal('OperationApplierService: Finished applying operations.'); - return { appliedOps: ops }; - } - - /** - * Process archive operations after bulk state dispatch. - * Archive data lives in IndexedDB and needs to be persisted separately. - * - * The archive handler is called for all operations - it internally decides - * which operations need archive storage updates. - */ - private async _processArchiveOperations(ops: Operation[]): Promise<{ - appliedOps: Operation[]; - hadArchiveAffectingOp: boolean; - failedOp?: { op: Operation; error: Error }; - }> { - const appliedOps: Operation[] = []; - let hadArchiveAffectingOp = false; - - for (let i = 0; i < ops.length; i++) { - const op = ops[i]; - try { - const action = convertOpToAction(op); - - // Call handler for all operations - it internally checks if action affects archive - await this.archiveOperationHandler.handleOperation(action); - - // Track if any archive-affecting operations were processed (for UI refresh) - if (isArchiveAffectingAction(action)) { - hadArchiveAffectingOp = true; - // Yield after EACH archive-affecting operation to prevent UI freeze. - // Archive operations involve slow IndexedDB writes that can block the event loop. - await new Promise((resolve) => setTimeout(resolve, 0)); - } - - appliedOps.push(op); - } catch (e) { + this.store.dispatch(remoteArchiveDataApplied()); + }, + onArchiveSideEffectError: ({ op, processedCount, error }) => { OpLog.err( `OperationApplierService: Failed archive handling for operation ${op.id}. ` + - `${appliedOps.length} ops were processed before this failure.`, - e, + `${processedCount} ops were processed before this failure.`, + error, ); + }, + onPostSyncCooldownError: (e) => { + OpLog.err('OperationApplierService: startPostSyncCooldown failed', e); + }, + }); - return { - appliedOps, - hadArchiveAffectingOp, - failedOp: { - op, - error: e instanceof Error ? e : new Error(String(e)), - }, - }; - } + if (!result.failedOp) { + OpLog.normal('OperationApplierService: Finished applying operations.'); } - // Final yield after processing all operations to ensure last operation completes - if (ops.length > 0) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } - - return { appliedOps, hadArchiveAffectingOp }; + return result; } } diff --git a/src/app/op-log/apply/operation-converter.util.spec.ts b/src/app/op-log/apply/operation-converter.util.spec.ts index 0380587cfd..9d943aa8c7 100644 --- a/src/app/op-log/apply/operation-converter.util.spec.ts +++ b/src/app/op-log/apply/operation-converter.util.spec.ts @@ -361,6 +361,105 @@ describe('operation-converter utility', () => { } }); + describe('legacy planTasksForToday date backfill', () => { + it('injects today from the originating operation timestamp when missing', () => { + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_PLAN_FOR_TODAY, + timestamp: new Date(2024, 5, 14, 12, 0, 0, 0).getTime(), + payload: { taskIds: ['task-1'] }, + }); + const action = convertOpToAction(op); + + expect((action as any).today).toBe('2024-06-14'); + }); + + it('injects today for MultiEntityPayload operations created before the field existed', () => { + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_PLAN_FOR_TODAY, + timestamp: new Date(2024, 5, 14, 12, 0, 0, 0).getTime(), + payload: { + actionPayload: { taskIds: ['task-1'] }, + entityChanges: [], + }, + }); + const action = convertOpToAction(op); + + expect((action as any).today).toBe('2024-06-14'); + }); + + it('preserves today when the operation payload already contains it', () => { + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_PLAN_FOR_TODAY, + timestamp: new Date(2024, 5, 15, 12, 0, 0, 0).getTime(), + payload: { taskIds: ['task-1'], today: '2024-06-14' }, + }); + const action = convertOpToAction(op); + + expect((action as any).today).toBe('2024-06-14'); + }); + }); + + describe('updateTask done replay date backfill', () => { + it('injects doneOn and dueDay from the originating operation timestamp when missing', () => { + const timestamp = new Date(2024, 5, 14, 12, 0, 0, 0).getTime(); + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_UPDATE, + timestamp, + payload: { + actionPayload: { + task: { id: 'task-1', changes: { isDone: true } }, + }, + entityChanges: [], + }, + }); + const action = convertOpToAction(op) as any; + + expect(action.task.changes.doneOn).toBe(timestamp); + expect(action.task.changes.dueDay).toBe('2024-06-14'); + }); + + it('preserves existing doneOn and dueDay when present', () => { + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_UPDATE, + timestamp: new Date(2024, 5, 15, 12, 0, 0, 0).getTime(), + payload: { + actionPayload: { + task: { + id: 'task-1', + changes: { + isDone: true, + doneOn: 1718352000000, + dueDay: '2024-06-14', + }, + }, + }, + entityChanges: [], + }, + }); + const action = convertOpToAction(op) as any; + + expect(action.task.changes.doneOn).toBe(1718352000000); + expect(action.task.changes.dueDay).toBe('2024-06-14'); + }); + + it('does not inject done fields for undone updates', () => { + const op = createMockOperation({ + actionType: ActionType.TASK_SHARED_UPDATE, + timestamp: new Date(2024, 5, 14, 12, 0, 0, 0).getTime(), + payload: { + actionPayload: { + task: { id: 'task-1', changes: { isDone: false } }, + }, + entityChanges: [], + }, + }); + const action = convertOpToAction(op) as any; + + expect(action.task.changes.doneOn).toBeUndefined(); + expect(action.task.changes.dueDay).toBeUndefined(); + }); + }); + describe('multi-entity payload handling', () => { it('should extract actionPayload from MultiEntityPayload', () => { const op = createMockOperation({ diff --git a/src/app/op-log/apply/operation-converter.util.ts b/src/app/op-log/apply/operation-converter.util.ts index 722368f797..ce0bec2a76 100644 --- a/src/app/op-log/apply/operation-converter.util.ts +++ b/src/app/op-log/apply/operation-converter.util.ts @@ -1,4 +1,5 @@ import { + ActionType, extractActionPayload, FULL_STATE_OP_TYPES, Operation, @@ -8,6 +9,7 @@ import { isLwwUpdateActionType } from '../core/lww-update-action-types'; import { isSingletonEntityId } from '../core/entity-registry'; import { PersistentAction } from '../core/persistent-action.interface'; import { SyncLog } from '../../core/log'; +import { getDbDateStr } from '../../util/get-db-date-str'; /** * Maps old/renamed action types to their current names. @@ -42,6 +44,74 @@ const extractFullStatePayload = (payload: unknown): Record => { return { appDataComplete: payload }; }; +const addLegacyPlanForTodayDate = ( + actionType: string, + actionPayload: Record, + op: Operation, +): Record => { + if ( + actionType === ActionType.TASK_SHARED_PLAN_FOR_TODAY && + typeof actionPayload['today'] !== 'string' + ) { + // Legacy operations did not store the logical day, timezone, or start-of-next-day + // offset. The timestamp is the best available fallback, but it is interpreted in + // the replaying device's local timezone and can still be off near midnight or for + // dueWithTime values around a different original day-start offset. + return { + ...actionPayload, + today: getDbDateStr(op.timestamp), + }; + } + return actionPayload; +}; + +const addReplaySafeDoneFields = ( + actionType: string, + actionPayload: Record, + op: Operation, +): Record => { + if (actionType !== ActionType.TASK_SHARED_UPDATE) { + return actionPayload; + } + + const task = actionPayload['task']; + if (typeof task !== 'object' || task === null) { + return actionPayload; + } + + const taskUpdate = task as Record; + const changes = taskUpdate['changes']; + if (typeof changes !== 'object' || changes === null) { + return actionPayload; + } + + const taskChanges = changes as Record; + if (taskChanges['isDone'] !== true) { + return actionPayload; + } + + const replaySafeChanges = { + ...taskChanges, + doneOn: + typeof taskChanges['doneOn'] === 'number' ? taskChanges['doneOn'] : op.timestamp, + // Older done ops did not store the logical day, timezone, or start-of-next-day + // offset. This timestamp fallback is replay-stable, but still uses the replaying + // device's local calendar and can be off near custom day-start boundaries. + dueDay: + typeof taskChanges['dueDay'] === 'string' + ? taskChanges['dueDay'] + : getDbDateStr(op.timestamp), + }; + + return { + ...actionPayload, + task: { + ...taskUpdate, + changes: replaySafeChanges, + }, + }; +}; + /** * Converts an Operation from the operation log back into a PersistentAction. * Used during sync replay and recovery to re-dispatch operations. @@ -64,6 +134,9 @@ export const convertOpToAction = (op: Operation): PersistentAction => { ? extractFullStatePayload(op.payload) : (extractActionPayload(op.payload) as Record); + actionPayload = addLegacyPlanForTodayDate(actionType, actionPayload, op); + actionPayload = addReplaySafeDoneFields(actionType, actionPayload, op); + // Force `payload.id = op.entityId` for non-singleton LWW Update ops. The // op's `entityId` is the canonical identifier — producers also enforce // this when creating ops, but a malformed/older remote op (or any path diff --git a/src/app/op-log/capture/operation-log.effects.spec.ts b/src/app/op-log/capture/operation-log.effects.spec.ts index dffd4b7f71..870843f844 100644 --- a/src/app/op-log/capture/operation-log.effects.spec.ts +++ b/src/app/op-log/capture/operation-log.effects.spec.ts @@ -19,6 +19,8 @@ import { } from './operation-capture.meta-reducer'; import { ClientIdService } from '../../core/util/client-id.service'; import { OperationCaptureService } from './operation-capture.service'; +import { TaskSharedActions } from '../../root-store/meta/task-shared.actions'; +import { DateService } from '../../core/date/date.service'; describe('OperationLogEffects', () => { let effects: OperationLogEffects; @@ -32,6 +34,7 @@ describe('OperationLogEffects', () => { let mockImmediateUploadService: jasmine.SpyObj; let mockClientIdService: jasmine.SpyObj; let mockOperationCaptureService: jasmine.SpyObj; + let mockDateService: jasmine.SpyObj; const createPersistentAction = ( type: string, @@ -73,6 +76,7 @@ describe('OperationLogEffects', () => { mockOperationCaptureService = jasmine.createSpyObj('OperationCaptureService', [ 'dequeue', ]); + mockDateService = jasmine.createSpyObj('DateService', ['todayStr']); // Default mock implementations mockLockService.request.and.callFake( @@ -91,6 +95,7 @@ describe('OperationLogEffects', () => { mockStore.select.and.returnValue(of({})); // Return empty state observable mockClientIdService.loadClientId.and.returnValue(Promise.resolve('testClient')); mockOperationCaptureService.dequeue.and.returnValue([]); + mockDateService.todayStr.and.returnValue('2024-06-14'); TestBed.configureTestingModule({ providers: [ @@ -105,6 +110,7 @@ describe('OperationLogEffects', () => { { provide: ImmediateUploadService, useValue: mockImmediateUploadService }, { provide: ClientIdService, useValue: mockClientIdService }, { provide: OperationCaptureService, useValue: mockOperationCaptureService }, + { provide: DateService, useValue: mockDateService }, ], }); @@ -255,6 +261,75 @@ describe('OperationLogEffects', () => { }); }); + it('should persist planTasksForToday replay date fields in actionPayload', (done) => { + const action = TaskSharedActions.planTasksForToday({ + taskIds: ['task-1'], + today: '2024-06-14', + startOfNextDayDiffMs: 4 * 60 * 60 * 1000, + }); + actions$ = of(action); + + effects.persistOperation$.subscribe({ + complete: () => { + const operation = + mockOpLogStore.appendWithVectorClockUpdate.calls.mostRecent().args[0]; + + expect(operation.actionType).toBe(ActionType.TASK_SHARED_PLAN_FOR_TODAY); + expect(operation.payload).toEqual({ + actionPayload: { + taskIds: ['task-1'], + today: '2024-06-14', + startOfNextDayDiffMs: 4 * 60 * 60 * 1000, + }, + entityChanges: [], + }); + done(); + }, + }); + }); + + it('should persist replay date fields for done task updates', (done) => { + const now = new Date(2024, 5, 15, 2, 0, 0, 0); + jasmine.clock().install(); + jasmine.clock().mockDate(now); + + const action = TaskSharedActions.updateTask({ + task: { id: 'task-1', changes: { isDone: true } }, + }); + actions$ = of(action); + + effects.persistOperation$.subscribe({ + complete: () => { + try { + const operation = + mockOpLogStore.appendWithVectorClockUpdate.calls.mostRecent().args[0]; + + expect(operation.timestamp).toBe(now.getTime()); + expect(operation.payload).toEqual({ + actionPayload: { + task: { + id: 'task-1', + changes: { + isDone: true, + doneOn: now.getTime(), + dueDay: '2024-06-14', + }, + }, + }, + entityChanges: [], + }); + done(); + } finally { + jasmine.clock().uninstall(); + } + }, + error: (err) => { + jasmine.clock().uninstall(); + done.fail(err); + }, + }); + }); + it('should notify user on persistence error', (done) => { mockOpLogStore.appendWithVectorClockUpdate.and.rejectWith( new Error('Write failed'), diff --git a/src/app/op-log/capture/operation-log.effects.ts b/src/app/op-log/capture/operation-log.effects.ts index 52d6e5067a..090de7f2dc 100644 --- a/src/app/op-log/capture/operation-log.effects.ts +++ b/src/app/op-log/capture/operation-log.effects.ts @@ -30,6 +30,7 @@ import { ImmediateUploadService } from '../sync/immediate-upload.service'; import { getDeferredActions, isDeferredAction } from './operation-capture.meta-reducer'; import { ClientIdService } from '../../core/util/client-id.service'; import { SuperSyncStatusService } from '../sync/super-sync-status.service'; +import { DateService } from '../../core/date/date.service'; /** * NgRx Effects for persisting application state changes as operations to the @@ -63,6 +64,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { private operationCaptureService = inject(OperationCaptureService); private immediateUploadService = inject(ImmediateUploadService); private superSyncStatusService = inject(SuperSyncStatusService); + private dateService = inject(DateService); /** * Effect that persists local user actions to the operation log. @@ -135,7 +137,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // Extract payload (everything except type and meta) // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { type, meta, ...actionPayload } = action; + const { type, meta, ...rawActionPayload } = action; // Use the action's declared opType from meta. We don't derive from entity changes because // some operations have different semantic meaning than their state changes suggest. @@ -174,6 +176,13 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // enqueueing — there's no matching queue entry to dequeue. const entityChanges = skipDequeue ? [] : this.operationCaptureService.dequeue(); + const operationTimestamp = Date.now(); + const actionPayload = this.addReplayDateFieldsToActionPayload( + action, + rawActionPayload, + operationTimestamp, + ); + // Create multi-entity payload with action payload and computed changes const multiEntityPayload: MultiEntityPayload = { actionPayload: actionPayload as Record, @@ -205,7 +214,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { payload: multiEntityPayload, clientId: clientId, vectorClock: newClock, - timestamp: Date.now(), + timestamp: operationTimestamp, schemaVersion: CURRENT_SCHEMA_VERSION, }; @@ -359,6 +368,50 @@ export class OperationLogEffects implements DeferredLocalActionsPort { return false; } + private addReplayDateFieldsToActionPayload( + action: PersistentAction, + actionPayload: Record, + operationTimestamp: number, + ): Record { + if (action.type !== ActionType.TASK_SHARED_UPDATE) { + return actionPayload; + } + + const task = actionPayload['task']; + if (typeof task !== 'object' || task === null) { + return actionPayload; + } + + const taskUpdate = task as Record; + const changes = taskUpdate['changes']; + if (typeof changes !== 'object' || changes === null) { + return actionPayload; + } + + const taskChanges = changes as Record; + if (taskChanges['isDone'] !== true) { + return actionPayload; + } + + return { + ...actionPayload, + task: { + ...taskUpdate, + changes: { + ...taskChanges, + doneOn: + typeof taskChanges['doneOn'] === 'number' + ? taskChanges['doneOn'] + : operationTimestamp, + dueDay: + typeof taskChanges['dueDay'] === 'string' + ? taskChanges['dueDay'] + : this.dateService.todayStr(), + }, + }, + }; + } + /** * Handles storage quota exceeded by triggering emergency compaction * and retrying the failed operation. diff --git a/src/app/op-log/sync-providers/credential-store.service.ts b/src/app/op-log/sync-providers/credential-store.service.ts index b51bd9e3de..1f3184413d 100644 --- a/src/app/op-log/sync-providers/credential-store.service.ts +++ b/src/app/op-log/sync-providers/credential-store.service.ts @@ -1,4 +1,8 @@ import { DBSchema, IDBPDatabase, openDB } from 'idb'; +import type { + CredentialChangeHandler, + SyncCredentialStorePort, +} from '@sp/sync-providers'; import { SyncProviderId, PRIVATE_CFG_PREFIX } from './provider.const'; import { PrivateCfgByProviderId } from '../core/types/sync.types'; import { SyncLog } from '../../core/log'; @@ -30,10 +34,8 @@ interface SyncCredentialsDb extends DBSchema { /** * Callback type for configuration change notifications. */ -export type CredentialChangeCallback = (data: { - providerId: PID; - privateCfg: PrivateCfgByProviderId; -}) => void; +export type CredentialChangeCallback = + CredentialChangeHandler>; /** * Store for managing sync provider credentials. @@ -52,7 +54,9 @@ export type CredentialChangeCallback = (data: { * Credentials are stored with keys: `PRIVATE_CFG_PREFIX + providerId` * (e.g., `__sp_cred_Dropbox`) */ -export class SyncCredentialStore { +export class SyncCredentialStore< + PID extends SyncProviderId, +> implements SyncCredentialStorePort> { private static readonly L = 'SyncCredentialStore'; private readonly _dbKey: string; diff --git a/src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts b/src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts index 6d74c54444..92c474a288 100644 --- a/src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts +++ b/src/app/op-log/sync-providers/file-based/dropbox/generate-pkce-codes.ts @@ -1,9 +1 @@ -import { generateCodeVerifier, generateCodeChallenge } from '../../../../util/pkce.util'; - -export const generatePKCECodes = async ( - _length: number, -): Promise<{ codeVerifier: string; codeChallenge: string }> => { - const codeVerifier = generateCodeVerifier(); - const codeChallenge = await generateCodeChallenge(codeVerifier); - return { codeVerifier, codeChallenge }; -}; +export { generatePKCECodes } from '@sp/sync-providers'; diff --git a/src/app/op-log/sync-providers/file-based/file-based-sync.types.ts b/src/app/op-log/sync-providers/file-based/file-based-sync.types.ts index bf2d583bed..6a85cd70f5 100644 --- a/src/app/op-log/sync-providers/file-based/file-based-sync.types.ts +++ b/src/app/op-log/sync-providers/file-based/file-based-sync.types.ts @@ -1,130 +1,23 @@ -import { VectorClock } from '../../../core/util/vector-clock'; -import { CompactOperation } from '../../../core/persistence/operation-log/compact/compact-operation.types'; -import { ArchiveModel } from '../../../features/time-tracking/time-tracking.model'; +import { + FILE_BASED_SYNC_CONSTANTS, + type FileBasedSyncData as GenericFileBasedSyncData, + type SyncFileCompactOp as GenericSyncFileCompactOp, +} from '@sp/sync-providers'; +import type { CompactOperation } from '../../../core/persistence/operation-log/compact/compact-operation.types'; +import type { ArchiveModel } from '../../../features/time-tracking/time-tracking.model'; + +export { FILE_BASED_SYNC_CONSTANTS }; /** - * Wrapper type for compact ops stored in the sync file. - * Extends CompactOperation with `sv` (syncVersion) — the syncVersion at which - * this op was uploaded. Used for partial-trimming gap detection. - * - * This is a file-based sync transport concern, NOT a general operation property. + * App-specific compact op wrapper stored in the file-based sync envelope. */ -export type SyncFileCompactOp = CompactOperation & { sv?: number }; +export type SyncFileCompactOp = GenericSyncFileCompactOp; /** - * File-based sync data structure. - * This is the schema for `sync-data.json` stored on WebDAV/Dropbox/LocalFile providers. - * - * Key design decisions: - * - Single file contains both state snapshot AND recent operations - * - `syncVersion` provides content-based optimistic locking (works without server ETags) - * - `recentOps` enables entity-level conflict resolution even with full state uploads + * App-specific binding for the generic file-based sync envelope. */ -export interface FileBasedSyncData { - /** - * Schema version for this sync file format. - * Increment when making breaking changes to FileBasedSyncData structure. - */ - version: 2; - - /** - * Content-based optimistic lock counter. - * Incremented on each successful upload. - * Used to detect concurrent modifications without relying on server ETags. - */ - syncVersion: number; - - /** - * Schema version of the application data. - * Used for migration when data format changes. - */ - schemaVersion: number; - - /** - * Vector clock representing the causal state after all operations. - * Used for conflict detection and determining operation ordering. - */ - vectorClock: VectorClock; - - /** - * Timestamp of last successful sync (epoch ms). - */ - lastModified: number; - - /** - * Client ID that last modified this file. - */ - clientId: string; - - /** - * Complete application state snapshot. - * This is the full AppDataComplete - tasks, projects, tags, config, etc. - * Compressed and optionally encrypted before storage. - */ - state: unknown; - - /** - * Archive data for tasks archived within last 21 days. - * Includes time-tracking data that may still be modified. - * Optional for backward compatibility with older sync files. - */ - archiveYoung?: ArchiveModel; - - /** - * Archive data for tasks archived more than 21 days ago. - * Contains older, inert data that rarely changes. - * Optional for backward compatibility with older sync files. - */ - archiveOld?: ArchiveModel; - - /** - * Recent operations for conflict detection (last N operations). - * Even though we upload full state, we need operations to: - * 1. Detect which entities were modified - * 2. Apply LWW at entity/field level instead of file level - * 3. Merge non-conflicting changes from concurrent edits - * - * Limit: MAX_RECENT_OPS operations - */ - recentOps: SyncFileCompactOp[]; - - /** - * The syncVersion (upload batch number) of the oldest operation in recentOps. - * Used for partial-trimming gap detection: when recentOps hits MAX_RECENT_OPS - * and oldest ops are trimmed, a slow-syncing client compares this against its - * sinceSeq to detect missed ops — no cross-machine clock comparison needed. - * - * Undefined when recentOps is empty or when old ops lack `sv` (backward compat). - */ - oldestOpSyncVersion?: number; -} - -/** - * Constants for file-based sync - */ -export const FILE_BASED_SYNC_CONSTANTS = { - /** Main sync data file name */ - SYNC_FILE: 'sync-data.json', - - /** Backup file name */ - BACKUP_FILE: 'sync-data.json.bak', - - /** Migration lock file name */ - MIGRATION_LOCK_FILE: 'migration.lock', - - /** Current file format version */ - FILE_VERSION: 2 as const, - - /** Maximum number of recent operations to keep */ - MAX_RECENT_OPS: 500, - - /** Storage key prefix for last known sync version */ - SYNC_VERSION_STORAGE_KEY_PREFIX: 'FILE_SYNC_VERSION_', - - /** - * Legacy PFAPI metadata file name written by v16.x clients. - * Its presence on a provider (without sync-data.json) signals a version mismatch - * where the old client is still writing and the new client must not silently diverge. - */ - LEGACY_META_FILE: '__meta_', -} as const; +export type FileBasedSyncData = GenericFileBasedSyncData< + unknown, + CompactOperation, + ArchiveModel +>; diff --git a/src/app/op-log/sync-providers/file-based/local-file/file-adapter.interface.ts b/src/app/op-log/sync-providers/file-based/local-file/file-adapter.interface.ts index 70f174c22f..2e9cf23c3c 100644 --- a/src/app/op-log/sync-providers/file-based/local-file/file-adapter.interface.ts +++ b/src/app/op-log/sync-providers/file-based/local-file/file-adapter.interface.ts @@ -1,7 +1 @@ -export interface FileAdapter { - readFile(filePath: string): Promise; - writeFile(filePath: string, dataStr: string): Promise; - deleteFile(filePath: string): Promise; - checkDirExists?(dirPath: string): Promise; - listFiles?(dirPath: string): Promise; -} +export type { FileAdapter } from '@sp/sync-providers'; diff --git a/src/app/op-log/sync-providers/native-http-retry.spec.ts b/src/app/op-log/sync-providers/native-http-retry.spec.ts deleted file mode 100644 index a16b22f685..0000000000 --- a/src/app/op-log/sync-providers/native-http-retry.spec.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { - isTransientNetworkError, - executeNativeRequestWithRetry, - NativeRequestConfig, -} from './native-http-retry'; -import { HttpResponse } from '@capacitor/core'; - -describe('isTransientNetworkError', () => { - describe('iOS error.code detection (NSURLErrorDomain)', () => { - it('should return true for NSURLErrorDomain code', () => { - const error = Object.assign(new Error('Some localized message'), { - code: 'NSURLErrorDomain', - }); - expect(isTransientNetworkError(error)).toBe(true); - }); - - it('should return true for NSURLErrorDomain even with non-English message', () => { - const error = Object.assign( - new Error('Die Netzwerkverbindung wurde unterbrochen.'), - { code: 'NSURLErrorDomain' }, - ); - expect(isTransientNetworkError(error)).toBe(true); - }); - - it('should return true for NSURLErrorDomain with empty message', () => { - const error = Object.assign(new Error(''), { code: 'NSURLErrorDomain' }); - expect(isTransientNetworkError(error)).toBe(true); - }); - }); - - describe('Android error.code detection', () => { - it('should return true for SocketTimeoutException', () => { - const error = Object.assign(new Error('timeout'), { - code: 'SocketTimeoutException', - }); - expect(isTransientNetworkError(error)).toBe(true); - }); - - it('should return true for UnknownHostException', () => { - const error = Object.assign(new Error('host not found'), { - code: 'UnknownHostException', - }); - expect(isTransientNetworkError(error)).toBe(true); - }); - - it('should return true for ConnectException', () => { - const error = Object.assign(new Error('connection refused'), { - code: 'ConnectException', - }); - expect(isTransientNetworkError(error)).toBe(true); - }); - }); - - describe('Fallback string matching', () => { - it('should return true for "network connection was lost"', () => { - expect(isTransientNetworkError(new Error('The network connection was lost.'))).toBe( - true, - ); - }); - - it('should return true for "timed out"', () => { - expect(isTransientNetworkError(new Error('The request timed out.'))).toBe(true); - }); - - it('should return true for "internet connection appears to be offline"', () => { - expect( - isTransientNetworkError( - new Error('The Internet connection appears to be offline.'), - ), - ).toBe(true); - }); - - it('should return true for "not connected to the internet"', () => { - expect(isTransientNetworkError(new Error('not connected to the internet'))).toBe( - true, - ); - }); - - it('should return true for "hostname could not be found"', () => { - expect( - isTransientNetworkError( - new Error('A server with the specified hostname could not be found.'), - ), - ).toBe(true); - }); - - it('should return true for "cannot find host"', () => { - expect(isTransientNetworkError(new Error('cannot find host'))).toBe(true); - }); - - it('should return true for "could not connect to the server"', () => { - expect(isTransientNetworkError(new Error('Could not connect to the server.'))).toBe( - true, - ); - }); - - it('should return true for "cannot connect to host"', () => { - expect(isTransientNetworkError(new Error('cannot connect to host'))).toBe(true); - }); - - it('should be case-insensitive', () => { - expect(isTransientNetworkError(new Error('THE NETWORK CONNECTION WAS LOST'))).toBe( - true, - ); - }); - - it('should return true for non-Error string with transient message', () => { - expect(isTransientNetworkError('network connection was lost')).toBe(true); - }); - }); - - describe('Negative cases', () => { - it('should return false for auth errors', () => { - expect(isTransientNetworkError(new Error('Unauthorized'))).toBe(false); - }); - - it('should return false for arbitrary errors', () => { - expect(isTransientNetworkError(new Error('Something unexpected happened'))).toBe( - false, - ); - }); - - it('should return false for HTTP status errors', () => { - expect(isTransientNetworkError(new Error('HTTP 409 Conflict'))).toBe(false); - }); - - it('should return false for non-Error string without transient message', () => { - expect(isTransientNetworkError('some other string')).toBe(false); - }); - - it('should return false for null', () => { - expect(isTransientNetworkError(null)).toBe(false); - }); - - it('should return false for undefined', () => { - expect(isTransientNetworkError(undefined)).toBe(false); - }); - - it('should return false for empty string', () => { - expect(isTransientNetworkError('')).toBe(false); - }); - - it('should return false for unrecognized error.code', () => { - const error = Object.assign(new Error('some error'), { - code: 'SomeOtherDomain', - }); - expect(isTransientNetworkError(error)).toBe(false); - }); - }); -}); - -describe('executeNativeRequestWithRetry', () => { - let mockRequestFn: jasmine.Spy<(opts: NativeRequestConfig) => Promise>; - let delayCallArgs: number[]; - const noopDelay = async (ms: number): Promise => { - delayCallArgs.push(ms); - }; - - const baseConfig: NativeRequestConfig = { - url: 'https://example.com/api', - method: 'POST', - headers: { Authorization: 'Bearer test' }, - data: 'test-data', - }; - - const successResponse: HttpResponse = { - status: 200, - headers: {}, - data: 'ok', - url: 'https://example.com/api', - }; - - beforeEach(() => { - mockRequestFn = jasmine.createSpy('requestFn'); - delayCallArgs = []; - }); - - it('should return response on first success', async () => { - mockRequestFn.and.returnValue(Promise.resolve(successResponse)); - - const result = await executeNativeRequestWithRetry( - baseConfig, - 'Test', - mockRequestFn, - noopDelay, - ); - - expect(result).toBe(successResponse); - expect(mockRequestFn).toHaveBeenCalledTimes(1); - expect(delayCallArgs).toEqual([]); - }); - - it('should retry on transient error and succeed', async () => { - const transientError = Object.assign(new Error('connection lost'), { - code: 'NSURLErrorDomain', - }); - - mockRequestFn.and.returnValues( - Promise.reject(transientError), - Promise.resolve(successResponse), - ); - - const result = await executeNativeRequestWithRetry( - baseConfig, - 'Test', - mockRequestFn, - noopDelay, - ); - - expect(result).toBe(successResponse); - expect(mockRequestFn).toHaveBeenCalledTimes(2); - expect(delayCallArgs).toEqual([1000]); - }); - - it('should retry twice and succeed on third attempt', async () => { - const transientError = Object.assign(new Error('timeout'), { - code: 'SocketTimeoutException', - }); - - mockRequestFn.and.returnValues( - Promise.reject(transientError), - Promise.reject(transientError), - Promise.resolve(successResponse), - ); - - const result = await executeNativeRequestWithRetry( - baseConfig, - 'Test', - mockRequestFn, - noopDelay, - ); - - expect(result).toBe(successResponse); - expect(mockRequestFn).toHaveBeenCalledTimes(3); - expect(delayCallArgs).toEqual([1000, 2000]); - }); - - it('should throw after exhausting all retries for transient errors', async () => { - const transientError = Object.assign(new Error('network lost'), { - code: 'NSURLErrorDomain', - }); - - mockRequestFn.and.returnValues( - Promise.reject(transientError), - Promise.reject(transientError), - Promise.reject(transientError), - ); - - await expectAsync( - executeNativeRequestWithRetry(baseConfig, 'Test', mockRequestFn, noopDelay), - ).toBeRejectedWith(transientError); - expect(mockRequestFn).toHaveBeenCalledTimes(3); - expect(delayCallArgs).toEqual([1000, 2000]); - }); - - it('should immediately throw non-transient errors without retrying', async () => { - const nonTransientError = new Error('Unauthorized'); - - mockRequestFn.and.returnValue(Promise.reject(nonTransientError)); - - await expectAsync( - executeNativeRequestWithRetry(baseConfig, 'Test', mockRequestFn, noopDelay), - ).toBeRejectedWith(nonTransientError); - - expect(mockRequestFn).toHaveBeenCalledTimes(1); - expect(delayCallArgs).toEqual([]); - }); - - it('should pass correct config to the request function', async () => { - mockRequestFn.and.returnValue(Promise.resolve(successResponse)); - - const config: NativeRequestConfig = { - url: 'https://api.example.com/test', - method: 'GET', - headers: { Authorization: 'Bearer abc' }, - data: 'payload', - responseType: 'json', - connectTimeout: 5000, - readTimeout: 60000, - }; - - await executeNativeRequestWithRetry(config, 'Test', mockRequestFn, noopDelay); - - expect(mockRequestFn).toHaveBeenCalledWith({ - url: 'https://api.example.com/test', - method: 'GET', - headers: { Authorization: 'Bearer abc' }, - data: 'payload', - responseType: 'json', - connectTimeout: 5000, - readTimeout: 60000, - }); - }); - - it('should use default responseType, connectTimeout, and readTimeout', async () => { - mockRequestFn.and.returnValue(Promise.resolve(successResponse)); - - await executeNativeRequestWithRetry(baseConfig, 'Test', mockRequestFn, noopDelay); - - expect(mockRequestFn).toHaveBeenCalledWith( - jasmine.objectContaining({ - responseType: 'text', - connectTimeout: 30000, - readTimeout: 120000, - }), - ); - }); -}); diff --git a/src/app/op-log/sync-providers/native-http-retry.ts b/src/app/op-log/sync-providers/native-http-retry.ts index d9d573512e..a2da08dbe7 100644 --- a/src/app/op-log/sync-providers/native-http-retry.ts +++ b/src/app/op-log/sync-providers/native-http-retry.ts @@ -1,123 +1,40 @@ import { CapacitorHttp, HttpResponse } from '@capacitor/core'; -import { SyncLog } from '../../core/log'; +import { + executeNativeRequestWithRetry as packageExecuteNativeRequestWithRetry, + isTransientNetworkError as packageIsTransientNetworkError, + type NativeHttpExecutor, + type NativeHttpRequestConfig, + type NativeHttpResponse, +} from '@sp/sync-providers'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; + +export type NativeRequestConfig = NativeHttpRequestConfig; /** - * Max retries for transient network errors (e.g., iOS NSURLErrorNetworkConnectionLost). - * iOS NSURLSession does NOT auto-retry POST requests (non-idempotent per RFC 7231). - * Retries are safe for Dropbox uploads: conditional writes (mode: 'update' with revToMatch - * or mode: 'add') ensure duplicates fail with UploadRevToMatchMismatchAPIError. - * Retries are safe for SuperSync: the server uses idempotent operations. - * @see https://developer.apple.com/library/archive/qa/qa1941/_index.html - */ -const MAX_RETRIES = 2; - -export interface NativeRequestConfig { - url: string; - method: string; - headers: Record; - data?: string; - responseType?: 'text' | 'json'; - connectTimeout?: number; - readTimeout?: number; -} - -const defaultDelay = (ms: number): Promise => - new Promise((resolve) => setTimeout(resolve, ms)); - -/** - * Execute a CapacitorHttp request with retry logic for transient network errors. - * Only network-level errors thrown by CapacitorHttp are retried; - * HTTP response errors (401, 404, etc.) are handled by the caller. + * App-side adapter: wires CapacitorHttp + OP_LOG_SYNC_LOGGER into the + * package-owned retry helper. Existing callers keep their positional API. * - * @param config - CapacitorHttp request configuration - * @param label - Logging label for the caller (e.g., 'DropboxApi', 'SuperSync') - * @param requestFn - Optional custom request function (for testing). Defaults to CapacitorHttp.request. - * @param delayFn - Optional delay function (for testing). Defaults to setTimeout-based delay. + * The package version (with `{ executor, logger, label, delay }` options) + * is the canonical interface — provider files that move into + * `@sp/sync-providers` should call it directly. */ export const executeNativeRequestWithRetry = async ( config: NativeRequestConfig, label: string = 'NativeHttp', requestFn: (opts: NativeRequestConfig) => Promise = (opts) => CapacitorHttp.request(opts), - delayFn: (ms: number) => Promise = defaultDelay, + delayFn?: (ms: number) => Promise, ): Promise => { - for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { - try { - return await requestFn({ - url: config.url, - method: config.method, - headers: config.headers, - data: config.data, - responseType: config.responseType ?? 'text', - connectTimeout: config.connectTimeout ?? 30000, - readTimeout: config.readTimeout ?? 120000, - }); - } catch (retryErr) { - if (attempt < MAX_RETRIES && isTransientNetworkError(retryErr)) { - const delayMs = 1000 * (attempt + 1); - SyncLog.warn( - `${label} transient network error on ${config.url}, ` + - `retrying in ${delayMs}ms (attempt ${attempt + 1}/${MAX_RETRIES})`, - retryErr, - ); - await delayFn(delayMs); - continue; - } - throw retryErr; - } - } - // Unreachable: loop always returns or throws, but TypeScript needs this - throw new Error('All retry attempts exhausted'); + const executor: NativeHttpExecutor = (cfg) => + requestFn(cfg) as unknown as Promise; + + const response = await packageExecuteNativeRequestWithRetry(config, { + executor, + logger: OP_LOG_SYNC_LOGGER, + label, + delay: delayFn, + }); + return response as unknown as HttpResponse; }; -/** - * Checks if an error is a transient network error that should be retried. - * - * Uses a hybrid detection strategy: - * - * **iOS (primary):** Checks `error.code === 'NSURLErrorDomain'`. Capacitor's - * HttpRequestHandler.swift calls `call.reject(error.localizedDescription, (error as NSError).domain, error, nil)`, - * so the domain string (always `"NSURLErrorDomain"` for network errors) lands on `error.code` - * in JavaScript. This is locale-proof — it works regardless of the device language. - * Note: SSL errors also have NSURLErrorDomain, but retrying them is harmless - * (they fail instantly and consistently — at most 2 extra immediate failures). - * - * **Android (primary):** Checks `error.code` against Java exception class names like - * `'SocketTimeoutException'`, `'UnknownHostException'`, and `'ConnectException'`. - * Capacitor's CapacitorHttp.java uses `call.reject(e.getLocalizedMessage(), e.getClass().getSimpleName(), e)`. - * - * **Fallback:** English string matching on the error message for web/Electron/unknown platforms. - */ -export const isTransientNetworkError = (e: unknown): boolean => { - // Check error.code first (locale-proof for iOS and Android) - const errorCode = (e as { code?: string } | null)?.code; - if (typeof errorCode === 'string') { - // iOS: NSURLErrorDomain covers all network errors (connection lost, timeout, DNS, etc.) - if (errorCode === 'NSURLErrorDomain') { - return true; - } - // Android: Java exception class names - if ( - errorCode === 'SocketTimeoutException' || - errorCode === 'UnknownHostException' || - errorCode === 'ConnectException' - ) { - return true; - } - } - - // Fallback: English string matching for web/Electron/unknown platforms - const message = (e instanceof Error ? e.message : String(e)).toLowerCase(); - return ( - message.includes('network connection was lost') || - // Intentionally broad: matches "request timed out", "connection timed out", "operation timed out", etc. - // Narrowing to "request timed out" would miss legitimate transient network errors. - message.includes('timed out') || - message.includes('not connected to the internet') || - message.includes('internet connection appears to be offline') || - message.includes('cannot find host') || - message.includes('hostname could not be found') || - message.includes('cannot connect to host') || - message.includes('could not connect to the server') - ); -}; +export const isTransientNetworkError = packageIsTransientNetworkError; diff --git a/src/app/op-log/sync-providers/provider.interface.ts b/src/app/op-log/sync-providers/provider.interface.ts index 10607c4c5e..471392fc8c 100644 --- a/src/app/op-log/sync-providers/provider.interface.ts +++ b/src/app/op-log/sync-providers/provider.interface.ts @@ -1,127 +1,41 @@ -import { SyncProviderId } from './provider.const'; -import { SyncCredentialStore } from './credential-store.service'; -import { PrivateCfgByProviderId } from '../core/types/sync.types'; +import { + isFileSyncProvider as isGenericFileSyncProvider, + type FileDownloadResponse as GenericFileDownloadResponse, + type FileRevResponse as GenericFileRevResponse, + type FileSnapshotOpDownloadResponse as GenericFileSnapshotOpDownloadResponse, + type FileSyncProvider as GenericFileSyncProvider, + type OpDownloadResponse as GenericOpDownloadResponse, + type OpDownloadResponseBase as GenericOpDownloadResponseBase, + type OpDownloadResponseForMode as GenericOpDownloadResponseForMode, + type OperationSyncCapable as GenericOperationSyncCapable, + type OperationSyncProviderMode as GenericOperationSyncProviderMode, + type OpUploadResponse as GenericOpUploadResponse, + type OpUploadResult as GenericOpUploadResult, + type RestoreCapable as GenericRestoreCapable, + type RestorePoint as GenericRestorePoint, + type RestorePointsResponse as GenericRestorePointsResponse, + type RestoreSnapshotResponse as GenericRestoreSnapshotResponse, + type ServerSyncOperation as GenericServerSyncOperation, + type SnapshotUploadResponse as GenericSnapshotUploadResponse, + type SuperSyncOpDownloadResponse as GenericSuperSyncOpDownloadResponse, + type SyncOperation as GenericSyncOperation, + type SyncProviderAuthHelper as GenericSyncProviderAuthHelper, + type SyncProviderBase as GenericSyncProviderBase, +} from '@sp/sync-providers'; +import type { SyncProviderId } from './provider.const'; +import type { PrivateCfgByProviderId } from '../core/types/sync.types'; -/** - * Authentication helper interface for OAuth flows - */ -export interface SyncProviderAuthHelper { - /** URL to redirect user for authentication */ - authUrl?: string; - /** Code verifier for PKCE authentication flow */ - codeVerifier?: string; +export type SyncProviderAuthHelper = GenericSyncProviderAuthHelper; - /** - * Verifies the code challenge and returns authentication tokens - * @param codeChallenge The code challenge from the OAuth redirect - * @returns Authentication tokens and expiration - */ - verifyCodeChallenge?(codeChallenge: string): Promise<{ - accessToken: string; - refreshToken: string; - expiresAt: number; - }>; -} +export type SyncProviderBase = GenericSyncProviderBase< + PID, + PrivateCfgByProviderId +>; -/** - * Base sync provider properties shared by all provider types. - * Both file-based providers and operation-based providers (SuperSync) implement this. - */ -export interface SyncProviderBase { - /** Unique identifier for this sync provider */ - id: PID; - - /** Whether this provider supports force upload (overwriting conflicts) */ - isUploadForcePossible?: boolean; - - /** Maximum number of concurrent requests allowed */ - maxConcurrentRequests: number; - - /** Store for provider-specific private configuration */ - privateCfg: SyncCredentialStore; - - /** - * Checks if the provider is ready to perform sync operations - * @returns True if the provider is authenticated and ready - */ - isReady(): Promise; - - /** - * Gets authentication helper for initiating auth flows - * @returns Auth helper object or undefined if not supported - */ - getAuthHelper?(): Promise; - - /** - * Updates the provider's private configuration - * @param privateCfg New configuration to store - */ - setPrivateCfg(privateCfg: PrivateCfgByProviderId): Promise; - - /** - * Clears authentication credentials while preserving non-auth config (e.g., encryptKey). - * Called when auth errors occur to ensure re-auth flow can proceed. - */ - clearAuthCredentials?(): Promise; -} - -/** - * File-based sync provider (Dropbox, WebDAV, LocalFile). - * Extends the base with file operations for reading/writing sync data. - */ -export interface FileSyncProvider< - PID extends SyncProviderId, -> extends SyncProviderBase { - /** Whether this provider is limited to syncing a single file */ - isLimitedToSingleFileSync?: boolean; - - /** - * Gets the current revision for a file - * @param targetPath Path to the file - * @param localRev Current local revision; can be used to check if download is necessary - * @returns The current revision information - * @throws Error if the operation fails - */ - getFileRev(targetPath: string, localRev: string | null): Promise; - - /** - * Downloads file data from the sync provider - * @param targetPath Path to the file - * @returns The file data and revision information - * @throws Error if the download fails - */ - downloadFile(targetPath: string): Promise; - - /** - * Uploads file data to the sync provider - * @param targetPath Path to the file - * @param dataStr Serialized file data - * @param revToMatch If revision does not match, the upload will fail - * @param isForceOverwrite Whether to force overwrite existing file - * @returns The new revision information after successful upload - * @throws Error if the upload fails or revision conflict occurs - */ - uploadFile( - targetPath: string, - dataStr: string, - revToMatch: string | null, - isForceOverwrite?: boolean, - ): Promise; - - /** - * Removes a file from the sync provider - * @param targetPath Path to the file to remove - * @throws Error if the removal fails - */ - removeFile(targetPath: string): Promise; - - /** - * Lists files in a directory - * @param targetPath Path to the directory to list - * @returns List of file names/paths - */ - listFiles?(targetPath: string): Promise; -} +export type FileSyncProvider = GenericFileSyncProvider< + PID, + PrivateCfgByProviderId +>; /** * @deprecated Use `SyncProviderBase` for generic provider references @@ -130,312 +44,34 @@ export interface FileSyncProvider< export type SyncProviderServiceInterface = FileSyncProvider; -/** - * Type guard to check if a provider supports file-based sync operations. - */ export const isFileSyncProvider = ( provider: SyncProviderBase, ): provider is FileSyncProvider => { - return ( - 'getFileRev' in provider && - typeof (provider as Record).getFileRev === 'function' - ); + return isGenericFileSyncProvider(provider); }; -/** - * Response for file revision operations - */ -export interface FileRevResponse { - /** The current revision identifier for the file */ - rev: string; -} - -/** - * Response for file download operations - */ -export interface FileDownloadResponse extends FileRevResponse { - /** The file content as a string */ - dataStr: string; -} - -// ===== Operation Sync Types ===== - -export type OperationSyncProviderMode = 'superSyncOps' | 'fileSnapshotOps'; - -/** - * Operation structure for server sync - */ -export interface SyncOperation { - id: string; - clientId: string; - actionType: string; - opType: string; - entityType: string; - entityId?: string; - entityIds?: string[]; // For batch operations - payload: unknown; - vectorClock: Record; - timestamp: number; - schemaVersion: number; - /** True if payload is an encrypted string (E2E encryption enabled) */ - isPayloadEncrypted?: boolean; - /** Reason for a SYNC_IMPORT operation (e.g. 'PASSWORD_CHANGED', 'FILE_IMPORT') */ - syncImportReason?: string; -} - -/** - * Server operation with server-assigned sequence - */ -export interface ServerSyncOperation { - serverSeq: number; - op: SyncOperation; - receivedAt: number; -} - -/** - * Result of uploading a single operation - */ -export interface OpUploadResult { - opId: string; - accepted: boolean; - serverSeq?: number; - error?: string; - /** Structured error code for programmatic handling */ - errorCode?: string; - /** - * The existing entity's vector clock when rejecting due to conflict. - * Allows clients to create LWW updates that dominate the server's state. - */ - existingClock?: Record; -} - -/** - * Response from uploading operations - */ -export interface OpUploadResponse { - results: OpUploadResult[]; - newOps?: ServerSyncOperation[]; - latestSeq: number; - /** - * True when piggybacked ops were limited (more ops exist on server). - * Client should trigger a download to get the remaining operations. - */ - hasMorePiggyback?: boolean; -} - -/** - * Response from downloading operations - */ -export interface OpDownloadResponseBase { - ops: ServerSyncOperation[]; - hasMore: boolean; - latestSeq: number; - /** - * Server sequence of the latest full-state operation (SYNC_IMPORT, BACKUP_IMPORT, REPAIR). - * Fresh clients can use this to understand where the effective state starts. - */ - latestSnapshotSeq?: number; - /** - * Whether a gap was detected in the operation sequence. - * This can happen when: - * - The server was reset and client has stale lastServerSeq - * - Operations were purged from the server - * - Client is ahead of server (shouldn't happen normally) - * - * When true, the client should reset lastServerSeq to 0 and re-download. - */ - gapDetected?: boolean; - /** - * Aggregated vector clock from all ops before and including the snapshot. - * Only set when snapshot optimization is used (sinceSeq < latestSnapshotSeq). - * Clients need this to create merged updates that dominate all known clocks. - */ - snapshotVectorClock?: Record; - /** - * Server's current time when the response was generated (Unix timestamp). - * Used to detect clock drift between client and server. - * - * Note: This differs from `receivedAt` on individual ops, which is the time - * when each operation was originally uploaded to the server (could be hours ago). - * For clock drift detection, we need the server's CURRENT time, not old timestamps. - */ - serverTime?: number; -} - -/** - * Response from pure SuperSync operation downloads. - */ -export interface SuperSyncOpDownloadResponse extends OpDownloadResponseBase { - snapshotState?: never; -} - -/** - * Response from file-based providers wrapped as operation sync. - */ -export interface FileSnapshotOpDownloadResponse extends OpDownloadResponseBase { - /** - * Full state snapshot from file-based sync providers. - * Only set when downloading from seq 0 (fresh download) from a file-based provider. - * Contains the complete application state for bootstrapping a new client. - */ - snapshotState?: unknown; -} - -export type OpDownloadResponse = - | SuperSyncOpDownloadResponse - | FileSnapshotOpDownloadResponse; - +export type FileRevResponse = GenericFileRevResponse; +export type FileDownloadResponse = GenericFileDownloadResponse; +export type OperationSyncProviderMode = GenericOperationSyncProviderMode; +export type SyncOperation = GenericSyncOperation; +export type ServerSyncOperation = GenericServerSyncOperation; +export type OpUploadResult = GenericOpUploadResult; +export type OpUploadResponse = GenericOpUploadResponse; +export type OpDownloadResponseBase = GenericOpDownloadResponseBase; +export type SuperSyncOpDownloadResponse = GenericSuperSyncOpDownloadResponse; +export type FileSnapshotOpDownloadResponse = GenericFileSnapshotOpDownloadResponse; +export type OpDownloadResponse = GenericOpDownloadResponse; export type OpDownloadResponseForMode = - M extends 'fileSnapshotOps' - ? FileSnapshotOpDownloadResponse - : SuperSyncOpDownloadResponse; + GenericOpDownloadResponseForMode; -/** - * Extended sync provider interface with operation sync support - */ -export interface OperationSyncCapable< - M extends OperationSyncProviderMode = OperationSyncProviderMode, -> { - /** - * Whether this provider supports direct operation sync (vs file-based) - */ - supportsOperationSync: boolean; - /** - * Distinguishes pure SuperSync operation transport from file-based providers - * that expose operation sync through a snapshot-backed adapter. - */ - providerMode: M; - - /** - * Upload operations to the server - * @param ops Operations to upload - * @param clientId Client identifier - * @param lastKnownServerSeq Last known server sequence - */ - uploadOps( - ops: SyncOperation[], - clientId: string, - lastKnownServerSeq?: number, - ): Promise; - - /** - * Download operations from the server - * @param sinceSeq Download ops with serverSeq > sinceSeq - * @param excludeClient Exclude ops from this client - * @param limit Maximum number of ops to return - */ - downloadOps( - sinceSeq: number, - excludeClient?: string, - limit?: number, - ): Promise>; - - /** - * Get the last known server sequence for this user - */ - getLastServerSeq(): Promise; - - /** - * Update the last known server sequence - */ - setLastServerSeq(seq: number): Promise; - - /** - * Upload a full state snapshot (for backup imports, repairs, initial sync) - * This is more efficient than sending large payloads through the ops endpoint. - * @param state The complete application state - * @param clientId Client identifier - * @param reason Why the snapshot is being uploaded - * @param vectorClock Current vector clock state - * @param schemaVersion Schema version of the state - * @param isPayloadEncrypted Whether the state payload is E2E encrypted - * @param opId Client's operation ID - server MUST use this ID to prevent ID mismatch bugs - * @param isCleanSlate If true, server deletes all user data before accepting the snapshot - */ - uploadSnapshot( - state: unknown, - clientId: string, - reason: 'initial' | 'recovery' | 'migration', - vectorClock: Record, - schemaVersion: number, - isPayloadEncrypted: boolean | undefined, - opId: string, - isCleanSlate?: boolean, - snapshotOpType?: RestorePointType, - syncImportReason?: string, - ): Promise; - - /** - * Delete all sync data for this user on the server. - * Used for encryption password changes - deletes all operations, - * tombstones, devices, and resets sync state. - */ - deleteAllData(): Promise<{ success: boolean }>; - - /** - * Get the encryption key for E2E encryption (optional). - * For API-based providers (SuperSync), returns the key from privateCfg. - * For file-based providers, encryption is handled internally so this returns undefined. - */ - getEncryptKey?(): Promise; -} - -/** - * Response from uploading a snapshot - */ -export interface SnapshotUploadResponse { - accepted: boolean; - serverSeq?: number; - error?: string; -} - -// ===== Restore Point Types ===== - -/** - * Type of restore point - */ export type RestorePointType = 'SYNC_IMPORT' | 'BACKUP_IMPORT' | 'REPAIR'; -/** - * A point in time that can be restored to - */ -export interface RestorePoint { - serverSeq: number; - timestamp: number; - type: RestorePointType; - clientId: string; - description?: string; -} +export type OperationSyncCapable< + M extends OperationSyncProviderMode = OperationSyncProviderMode, +> = GenericOperationSyncCapable; -/** - * Response from getting restore points - */ -export interface RestorePointsResponse { - restorePoints: RestorePoint[]; -} - -/** - * Response from getting a restore snapshot - */ -export interface RestoreSnapshotResponse { - state: unknown; - serverSeq: number; - generatedAt: number; -} - -/** - * Extended sync provider interface with restore point support - */ -export interface RestoreCapable { - /** - * Get available restore points - * @param limit Maximum number of restore points to return - */ - getRestorePoints(limit?: number): Promise; - - /** - * Get state snapshot at a specific server sequence - * @param serverSeq The server sequence to restore to - */ - getStateAtSeq(serverSeq: number): Promise; -} +export type SnapshotUploadResponse = GenericSnapshotUploadResponse; +export type RestorePoint = GenericRestorePoint; +export type RestorePointsResponse = GenericRestorePointsResponse; +export type RestoreSnapshotResponse = GenericRestoreSnapshotResponse; +export type RestoreCapable = GenericRestoreCapable; diff --git a/src/app/op-log/sync/sync-import-conflict-dialog.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-dialog.service.spec.ts new file mode 100644 index 0000000000..91ce3dbbb6 --- /dev/null +++ b/src/app/op-log/sync/sync-import-conflict-dialog.service.spec.ts @@ -0,0 +1,106 @@ +import { TestBed } from '@angular/core/testing'; +import { MatDialog, MatDialogRef } from '@angular/material/dialog'; +import type { ConflictUiPort } from '@sp/sync-core'; +import { of } from 'rxjs'; +import { UserInputWaitStateService } from '../../imex/sync/user-input-wait-state.service'; +import { + DialogSyncImportConflictComponent, + SyncImportConflictData, + SyncImportConflictResolution, +} from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; +import { SyncImportConflictDialogService } from './sync-import-conflict-dialog.service'; + +describe('SyncImportConflictDialogService', () => { + let service: SyncImportConflictDialogService; + let matDialogSpy: jasmine.SpyObj; + let userInputWaitStateSpy: jasmine.SpyObj; + let stopWaitingSpy: jasmine.Spy; + + beforeEach(() => { + matDialogSpy = jasmine.createSpyObj('MatDialog', ['open']); + matDialogSpy.open.and.returnValue(createDialogRef('USE_LOCAL')); + + stopWaitingSpy = jasmine.createSpy('stopWaiting'); + userInputWaitStateSpy = jasmine.createSpyObj('UserInputWaitStateService', [ + 'startWaiting', + ]); + userInputWaitStateSpy.startWaiting.and.returnValue(stopWaitingSpy); + + TestBed.configureTestingModule({ + providers: [ + SyncImportConflictDialogService, + { provide: MatDialog, useValue: matDialogSpy }, + { provide: UserInputWaitStateService, useValue: userInputWaitStateSpy }, + ], + }); + + service = TestBed.inject(SyncImportConflictDialogService); + }); + + it('keeps the existing sync-import dialog data API', async () => { + const data: SyncImportConflictData = { + filteredOpCount: 2, + localImportTimestamp: 123, + syncImportReason: 'BACKUP_RESTORE', + scenario: 'LOCAL_IMPORT_FILTERS_REMOTE', + }; + + await expectAsync(service.showConflictDialog(data)).toBeResolvedTo('USE_LOCAL'); + + expect(matDialogSpy.open).toHaveBeenCalledWith(DialogSyncImportConflictComponent, { + data, + disableClose: true, + restoreFocus: true, + }); + expect(userInputWaitStateSpy.startWaiting).toHaveBeenCalledWith( + 'sync-import-conflict', + ); + expect(stopWaitingSpy).toHaveBeenCalledOnceWith(); + }); + + it('adapts the generic ConflictUiPort request shape', async () => { + const port: ConflictUiPort = service; + + await expectAsync( + port.showConflictDialog({ + conflictType: 'sync-import', + scenario: 'INCOMING_IMPORT', + reason: 'SERVER_MIGRATION', + counts: { filteredOpCount: 3 }, + timestamps: { localImportTimestamp: 456 }, + }), + ).toBeResolvedTo('USE_LOCAL'); + + expect(matDialogSpy.open).toHaveBeenCalledWith(DialogSyncImportConflictComponent, { + data: { + filteredOpCount: 3, + localImportTimestamp: 456, + syncImportReason: 'SERVER_MIGRATION', + scenario: 'INCOMING_IMPORT', + }, + disableClose: true, + restoreFocus: true, + }); + }); + + it('stops waiting when the dialog closes without a selected resolution', async () => { + matDialogSpy.open.and.returnValue(createDialogRef(undefined)); + + await expectAsync( + service.showConflictDialog({ + filteredOpCount: 1, + localImportTimestamp: 789, + scenario: 'INCOMING_IMPORT', + }), + ).toBeResolvedTo('CANCEL'); + + expect(stopWaitingSpy).toHaveBeenCalledOnceWith(); + }); +}); + +const createDialogRef = ( + result: SyncImportConflictResolution | undefined, +): MatDialogRef => + ({ + afterClosed: () => of(result), + }) as unknown as MatDialogRef; diff --git a/src/app/op-log/sync/sync-import-conflict-dialog.service.ts b/src/app/op-log/sync/sync-import-conflict-dialog.service.ts index 413e12b9dc..2fe96a5249 100644 --- a/src/app/op-log/sync/sync-import-conflict-dialog.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-dialog.service.ts @@ -1,5 +1,6 @@ import { inject, Injectable } from '@angular/core'; import { MatDialog } from '@angular/material/dialog'; +import type { ConflictUiDialogRequest, ConflictUiPort } from '@sp/sync-core'; import { firstValueFrom } from 'rxjs'; import { UserInputWaitStateService } from '../../imex/sync/user-input-wait-state.service'; import { @@ -16,7 +17,7 @@ import { * have been creating changes without knowledge of that import. */ @Injectable({ providedIn: 'root' }) -export class SyncImportConflictDialogService { +export class SyncImportConflictDialogService implements ConflictUiPort { private _matDialog = inject(MatDialog); private _userInputWaitState = inject(UserInputWaitStateService); @@ -27,13 +28,16 @@ export class SyncImportConflictDialogService { * @returns The user's chosen resolution: USE_LOCAL, USE_REMOTE, or CANCEL */ async showConflictDialog( - data: SyncImportConflictData, + data: SyncImportConflictData | ConflictUiDialogRequest, ): Promise { + const dialogData = isSyncImportConflictData(data) + ? data + : toSyncImportConflictData(data); const stopWaiting = this._userInputWaitState.startWaiting('sync-import-conflict'); try { const dialogRef = this._matDialog.open(DialogSyncImportConflictComponent, { - data, + data: dialogData, disableClose: true, restoreFocus: true, }); @@ -45,3 +49,38 @@ export class SyncImportConflictDialogService { } } } + +const isSyncImportConflictData = ( + data: SyncImportConflictData | ConflictUiDialogRequest, +): data is SyncImportConflictData => + 'filteredOpCount' in data && 'localImportTimestamp' in data; + +const toSyncImportConflictData = ( + request: ConflictUiDialogRequest, +): SyncImportConflictData => { + if (request.conflictType !== 'sync-import') { + throw new Error(`Unsupported conflict dialog type: ${request.conflictType}`); + } + + const scenario = request.scenario; + if (scenario !== 'INCOMING_IMPORT' && scenario !== 'LOCAL_IMPORT_FILTERS_REMOTE') { + throw new Error(`Unsupported sync import conflict scenario: ${String(scenario)}`); + } + + const filteredOpCount = request.counts?.filteredOpCount; + if (typeof filteredOpCount !== 'number') { + throw new Error('Sync import conflict requires counts.filteredOpCount'); + } + + const localImportTimestamp = request.timestamps?.localImportTimestamp; + if (typeof localImportTimestamp !== 'number') { + throw new Error('Sync import conflict requires timestamps.localImportTimestamp'); + } + + return { + filteredOpCount, + localImportTimestamp, + syncImportReason: request.reason as SyncImportConflictData['syncImportReason'], + scenario, + }; +}; diff --git a/src/app/op-log/testing/integration/plan-today-replay.integration.spec.ts b/src/app/op-log/testing/integration/plan-today-replay.integration.spec.ts new file mode 100644 index 0000000000..6740926aba --- /dev/null +++ b/src/app/op-log/testing/integration/plan-today-replay.integration.spec.ts @@ -0,0 +1,121 @@ +import { Action, ActionReducer } from '@ngrx/store'; +import { bulkOperationsMetaReducer } from '../../apply/bulk-hydration.meta-reducer'; +import { bulkApplyOperations } from '../../apply/bulk-hydration.action'; +import { ActionType, Operation, OpType } from '../../core/operation.types'; +import { taskSharedSchedulingMetaReducer } from '../../../root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer'; +import { + createBaseState, + createMockTask, +} from '../../../root-store/meta/task-shared-meta-reducers/test-utils'; +import { RootState } from '../../../root-store/root-state'; +import { TASK_FEATURE_NAME } from '../../../features/tasks/store/task.reducer'; +import { TAG_FEATURE_NAME } from '../../../features/tag/store/tag.reducer'; +import { TODAY_TAG } from '../../../features/tag/tag.const'; +import { Task } from '../../../features/tasks/task.model'; +import { Tag } from '../../../features/tag/tag.model'; +import { appStateFeatureKey } from '../../../root-store/app-state/app-state.reducer'; + +describe('planTasksForToday operation replay', () => { + const TASK_ID = 'task-legacy-plan-today'; + const ACTION_TODAY = '2024-06-14'; + const REPLAY_TODAY = '2024-06-15'; + const START_OF_NEXT_DAY_DIFF_MS = 4 * 60 * 60 * 1000; + + const createState = ( + taskOverrides: Partial = {}, + replayOffsetMs = 0, + ): RootState => { + const base = createBaseState(); + const task = createMockTask({ + id: TASK_ID, + dueDay: undefined, + dueWithTime: undefined, + ...taskOverrides, + }); + + return { + ...base, + [TASK_FEATURE_NAME]: { + ...base[TASK_FEATURE_NAME], + ids: [TASK_ID], + entities: { [TASK_ID]: task }, + }, + [TAG_FEATURE_NAME]: { + ...base[TAG_FEATURE_NAME], + entities: { + ...base[TAG_FEATURE_NAME].entities, + [TODAY_TAG.id]: { + ...base[TAG_FEATURE_NAME].entities[TODAY_TAG.id], + taskIds: [], + } as Tag, + }, + }, + [appStateFeatureKey]: { + ...base[appStateFeatureKey], + todayStr: REPLAY_TODAY, + startOfNextDayDiffMs: replayOffsetMs, + }, + }; + }; + + const createPlanOp = ( + actionPayload: Record, + timestamp = new Date(2024, 5, 14, 12, 0, 0, 0).getTime(), + ): Operation => ({ + id: 'op-plan-today', + actionType: ActionType.TASK_SHARED_PLAN_FOR_TODAY, + opType: OpType.Update, + entityType: 'TASK', + entityId: TASK_ID, + entityIds: [TASK_ID], + payload: { + actionPayload, + entityChanges: [], + }, + clientId: 'clientA', + vectorClock: { clientA: 1 }, + timestamp, + schemaVersion: 1, + }); + + const applyOperation = (state: RootState, op: Operation): RootState => { + const passThroughReducer: ActionReducer = (s) => s as RootState; + const reducer = bulkOperationsMetaReducer( + taskSharedSchedulingMetaReducer(passThroughReducer), + ); + return reducer(state, bulkApplyOperations({ operations: [op] })); + }; + + it('replays legacy operations using the operation timestamp date instead of replay date', () => { + const state = createState(); + const op = createPlanOp({ + taskIds: [TASK_ID], + parentTaskMap: {}, + }); + + const result = applyOperation(state, op); + const task = result[TASK_FEATURE_NAME].entities[TASK_ID] as Task; + const todayTaskIds = (result[TAG_FEATURE_NAME].entities[TODAY_TAG.id] as Tag).taskIds; + + expect(task.dueDay).toBe(ACTION_TODAY); + expect(task.dueDay).not.toBe(REPLAY_TODAY); + expect(todayTaskIds).toEqual([TASK_ID]); + }); + + it('replays new operations with the captured start-of-next-day offset', () => { + const scheduledAt = new Date(2024, 5, 15, 2, 0, 0, 0).getTime(); + const state = createState({ dueWithTime: scheduledAt }, 0); + const op = createPlanOp({ + taskIds: [TASK_ID], + today: ACTION_TODAY, + startOfNextDayDiffMs: START_OF_NEXT_DAY_DIFF_MS, + parentTaskMap: {}, + }); + + const result = applyOperation(state, op); + const task = result[TASK_FEATURE_NAME].entities[TASK_ID] as Task; + + expect(task.dueDay).toBe(ACTION_TODAY); + expect(task.dueWithTime).toBe(scheduledAt); + }); +}); diff --git a/src/app/op-log/testing/integration/task-done-replay.integration.spec.ts b/src/app/op-log/testing/integration/task-done-replay.integration.spec.ts new file mode 100644 index 0000000000..9686fffc09 --- /dev/null +++ b/src/app/op-log/testing/integration/task-done-replay.integration.spec.ts @@ -0,0 +1,81 @@ +import { Action, ActionReducer } from '@ngrx/store'; +import { bulkOperationsMetaReducer } from '../../apply/bulk-hydration.meta-reducer'; +import { bulkApplyOperations } from '../../apply/bulk-hydration.action'; +import { ActionType, Operation, OpType } from '../../core/operation.types'; +import { taskSharedCrudMetaReducer } from '../../../root-store/meta/task-shared-meta-reducers/task-shared-crud.reducer'; +import { + createBaseState, + createMockTask, +} from '../../../root-store/meta/task-shared-meta-reducers/test-utils'; +import { RootState } from '../../../root-store/root-state'; +import { TASK_FEATURE_NAME } from '../../../features/tasks/store/task.reducer'; +import { Task } from '../../../features/tasks/task.model'; +import { appStateFeatureKey } from '../../../root-store/app-state/app-state.reducer'; + +describe('done task operation replay', () => { + const TASK_ID = 'task-done-yesterday'; + const ACTION_TODAY = '2024-06-14'; + const REPLAY_TODAY = '2024-06-15'; + const DONE_TIMESTAMP = new Date(2024, 5, 14, 12, 0, 0, 0).getTime(); + + const createState = (): RootState => { + const base = createBaseState(); + const task = createMockTask({ + id: TASK_ID, + isDone: false, + doneOn: undefined, + dueDay: undefined, + dueWithTime: undefined, + }); + + return { + ...base, + [TASK_FEATURE_NAME]: { + ...base[TASK_FEATURE_NAME], + ids: [TASK_ID], + entities: { [TASK_ID]: task }, + }, + [appStateFeatureKey]: { + ...base[appStateFeatureKey], + todayStr: REPLAY_TODAY, + startOfNextDayDiffMs: 0, + }, + }; + }; + + const applyOperation = (state: RootState, op: Operation): RootState => { + const passThroughReducer: ActionReducer = (s) => s as RootState; + const reducer = bulkOperationsMetaReducer( + taskSharedCrudMetaReducer(passThroughReducer), + ); + return reducer(state, bulkApplyOperations({ operations: [op] })); + }; + + it('replays completed tasks with the operation date instead of the replay date', () => { + const op: Operation = { + id: 'op-done-yesterday', + actionType: ActionType.TASK_SHARED_UPDATE, + opType: OpType.Update, + entityType: 'TASK', + entityId: TASK_ID, + payload: { + actionPayload: { + task: { id: TASK_ID, changes: { isDone: true } }, + }, + entityChanges: [], + }, + clientId: 'clientA', + vectorClock: { clientA: 1 }, + timestamp: DONE_TIMESTAMP, + schemaVersion: 1, + }; + + const result = applyOperation(createState(), op); + const task = result[TASK_FEATURE_NAME].entities[TASK_ID] as Task; + + expect(task.isDone).toBe(true); + expect(task.doneOn).toBe(DONE_TIMESTAMP); + expect(task.dueDay).toBe(ACTION_TODAY); + expect(task.dueDay).not.toBe(REPLAY_TODAY); + }); +}); diff --git a/src/app/op-log/validation/auto-fix-typia-errors.spec.ts b/src/app/op-log/validation/auto-fix-typia-errors.spec.ts index fa3e15f65a..b2a80c46e4 100644 --- a/src/app/op-log/validation/auto-fix-typia-errors.spec.ts +++ b/src/app/op-log/validation/auto-fix-typia-errors.spec.ts @@ -1,36 +1,30 @@ import { autoFixTypiaErrors } from './auto-fix-typia-errors'; import { createAppDataCompleteMock } from '../../util/app-data-mock'; -import { createValidate } from 'typia'; +import type { IValidation } from 'typia'; import { initialTaskState } from '../../features/tasks/store/task.reducer'; -import { DEFAULT_TASK, TaskState } from '../../features/tasks/task.model'; -import { OpLog } from '../../core/log'; +import { DEFAULT_TASK } from '../../features/tasks/task.model'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; -interface TestInterface { - globalConfig: { - misc: { - startOfNextDay: number; - }; - }; - optionalObj?: { - optionalProp?: string; - bool: boolean; - }; - task?: TaskState; -} +const createTypiaError = ( + path: string, + expected: string, + value?: unknown, +): IValidation.IError => ({ path, expected, value }) as IValidation.IError; describe('autoFixTypiaErrors', () => { - const validate = createValidate(); - let errSpy: jasmine.Spy; + let warnSpy: jasmine.Spy; beforeEach(() => { - // Spy on OpLog.err to prevent test output cluttering - errSpy = spyOn(OpLog, 'err').and.stub(); + // Spy on sync logger methods to prevent test output cluttering. + errSpy = spyOn(OP_LOG_SYNC_LOGGER, 'err').and.stub(); + warnSpy = spyOn(OP_LOG_SYNC_LOGGER, 'warn').and.stub(); }); afterEach(() => { // Reset spies errSpy.calls.reset(); + warnSpy.calls.reset(); }); it('should return data unchanged when no errors', () => { @@ -47,9 +41,9 @@ describe('autoFixTypiaErrors', () => { }, }, } as any; - const validateResult = validate(d); - expect(validateResult.success).toBe(false); - const result = autoFixTypiaErrors(d, (validateResult as any).errors); + const result = autoFixTypiaErrors(d, [ + createTypiaError('$input.globalConfig.misc.startOfNextDay', 'number', '111'), + ]); expect(result).toEqual({ globalConfig: { misc: { @@ -59,6 +53,28 @@ describe('autoFixTypiaErrors', () => { } as any); }); + it('should log auto-fixes without raw validation values', () => { + const d = { + globalConfig: { + misc: { + startOfNextDay: '4321', + }, + }, + } as any; + + autoFixTypiaErrors(d, [ + createTypiaError('$input.globalConfig.misc.startOfNextDay', 'number', '4321'), + ]); + + const serializedLogArgs = JSON.stringify([ + ...errSpy.calls.allArgs(), + ...warnSpy.calls.allArgs(), + ]); + expect(serializedLogArgs).toContain('valueStringLength'); + expect(serializedLogArgs).toContain('replacementType'); + expect(serializedLogArgs).not.toContain('4321'); + }); + it('should use defaults for globalConfig if no other value could be added', () => { const d = { globalConfig: { @@ -67,9 +83,9 @@ describe('autoFixTypiaErrors', () => { }, }, } as any; - const validateResult = validate(d); - expect(validateResult.success).toBe(false); - const result = autoFixTypiaErrors(d, (validateResult as any).errors); + const result = autoFixTypiaErrors(d, [ + createTypiaError('$input.globalConfig.misc.startOfNextDay', 'number'), + ]); expect(result.globalConfig.misc.startOfNextDay).not.toEqual(111); expect(result.globalConfig.misc.startOfNextDay).toEqual(0); }); @@ -85,9 +101,9 @@ describe('autoFixTypiaErrors', () => { optionalProp: null, }, } as any; - const validateResult = validate(d); - expect(validateResult.success).toBe(false); - const result = autoFixTypiaErrors(d, (validateResult as any).errors); + const result = autoFixTypiaErrors(d, [ + createTypiaError('$input.optionalObj.optionalProp', 'string | undefined', null), + ]); expect(result.globalConfig.misc.startOfNextDay).toEqual(111); expect((result as any).optionalObj.optionalProp).toEqual(undefined); }); @@ -103,9 +119,9 @@ describe('autoFixTypiaErrors', () => { bool: null, }, } as any; - const validateResult = validate(d); - expect(validateResult.success).toBe(false); - const result = autoFixTypiaErrors(d, (validateResult as any).errors); + const result = autoFixTypiaErrors(d, [ + createTypiaError('$input.optionalObj.bool', 'boolean', null), + ]); expect((result as any).optionalObj.bool).toEqual(false); }); @@ -130,10 +146,11 @@ describe('autoFixTypiaErrors', () => { ids: ['task-1'], }, } as any; - const validateResult = validate(d); - expect(validateResult.success).toBe(false); - const result = autoFixTypiaErrors(d, (validateResult as any).errors); + const result = autoFixTypiaErrors(d, [ + createTypiaError('$input.task.entities["task-1"].timeEstimate', 'number', '0'), + createTypiaError('$input.task.entities["task-1"].timeSpent', 'number', '0'), + ]); expect((result as any).task.entities['task-1'].timeEstimate).toEqual(0); expect((result as any).task.entities['task-1'].timeSpent).toEqual(0); @@ -175,7 +192,16 @@ describe('autoFixTypiaErrors', () => { ], ).toBe(0); expect(errSpy).toHaveBeenCalledWith( - "Fixed: simpleCounter.entities['BpYFLFtlIGGgTNfZB-t2-'].countOnDay['2025-06-16'] from null to 0 for simpleCounter", + '[auto-fix-typia-errors] Applied validation auto-fix', + undefined, + jasmine.objectContaining({ + path: "simpleCounter.entities['BpYFLFtlIGGgTNfZB-t2-'].countOnDay['2025-06-16']", + pathDepth: 5, + pathRoot: 'simpleCounter', + fix: 'simple-counter-countOnDay-null-to-zero', + valueType: 'null', + replacementType: 'number', + }), ); }); diff --git a/src/app/op-log/validation/auto-fix-typia-errors.ts b/src/app/op-log/validation/auto-fix-typia-errors.ts index a0e12b8c3f..35453e4949 100644 --- a/src/app/op-log/validation/auto-fix-typia-errors.ts +++ b/src/app/op-log/validation/auto-fix-typia-errors.ts @@ -1,9 +1,85 @@ import { AppDataComplete } from '../model/model-config'; import { IValidation } from 'typia'; +import type { SyncLogMeta } from '@sp/sync-core'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; import { INBOX_PROJECT } from '../../features/project/project.const'; import { RECREATE_FALLBACK } from '../core/recreate-fallback.const'; -import { OpLog } from '../../core/log'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; + +const LOG_PREFIX = '[auto-fix-typia-errors]'; + +const getValueType = (value: unknown): string => { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +}; + +const getValueLogMeta = (prefix: string, value: unknown): SyncLogMeta => ({ + [`${prefix}Type`]: getValueType(value), + [`${prefix}StringLength`]: typeof value === 'string' ? value.length : undefined, + [`${prefix}ArrayLength`]: Array.isArray(value) ? value.length : undefined, + [`${prefix}KeyCount`]: + value !== null && typeof value === 'object' && !Array.isArray(value) + ? Object.keys(value).length + : undefined, +}); + +const getPathRoot = (keys: (string | number)[]): string | undefined => + typeof keys[0] === 'string' ? keys[0] : undefined; + +const getRepairLogMeta = ( + path: string, + keys: (string | number)[], + fix: string, +): SyncLogMeta => ({ + path, + pathDepth: keys.length, + pathRoot: getPathRoot(keys), + fix, +}); + +const logAutoFixAttempt = ( + error: IValidation.IError, + path: string, + keys: (string | number)[], + value: unknown, +): void => { + OP_LOG_SYNC_LOGGER.err(`${LOG_PREFIX} Attempting validation auto-fix`, undefined, { + path, + pathDepth: keys.length, + pathRoot: getPathRoot(keys), + expected: error.expected, + ...getValueLogMeta('value', value), + }); +}; + +const logAutoFixApplied = ( + path: string, + keys: (string | number)[], + fix: string, + value: unknown, + replacement: unknown, +): void => { + OP_LOG_SYNC_LOGGER.err(`${LOG_PREFIX} Applied validation auto-fix`, undefined, { + ...getRepairLogMeta(path, keys, fix), + ...getValueLogMeta('value', value), + ...getValueLogMeta('replacement', replacement), + }); +}; + +const logAutoFixWarning = ( + path: string, + keys: (string | number)[], + fix: string, + value: unknown, + replacement: unknown, +): void => { + OP_LOG_SYNC_LOGGER.warn(`${LOG_PREFIX} Applied validation auto-fix`, { + ...getRepairLogMeta(path, keys, fix), + ...getValueLogMeta('value', value), + ...getValueLogMeta('replacement', replacement), + }); +}; export const autoFixTypiaErrors = ( data: AppDataComplete, @@ -18,7 +94,7 @@ export const autoFixTypiaErrors = ( const path = error.path.replace('$input.', ''); const keys = parsePath(path); const value = getValueByPath(data, keys); - OpLog.err('Auto-fixing error:', error, keys, value); + logAutoFixAttempt(error, path, keys, value); if ( error.expected.includes('number') && @@ -27,38 +103,35 @@ export const autoFixTypiaErrors = ( ) { const parsedValue = parseFloat(value); setValueByPath(data, keys, parsedValue); - OpLog.err(`Fixed: ${path} from string "${value}" to number ${parsedValue}`); + logAutoFixApplied(path, keys, 'string-to-number', value, parsedValue); } else if (keys[0] === 'globalConfig') { const defaultValue = getValueByPath(DEFAULT_GLOBAL_CONFIG, keys.slice(1)); setValueByPath(data, keys, defaultValue); - OpLog.warn( - `Warning: ${path} had an invalid value and was set to default: ${defaultValue}`, - ); + logAutoFixWarning(path, keys, 'global-config-default', value, defaultValue); } else if (error.expected.includes('undefined') && value === null) { setValueByPath(data, keys, undefined); - OpLog.err(`Fixed: ${path} from null to undefined`); + logAutoFixApplied(path, keys, 'null-to-undefined', value, undefined); } else if (error.expected.includes('null') && value === 'null') { setValueByPath(data, keys, null); - OpLog.err(`Fixed: ${path} from string null to null`); + logAutoFixApplied(path, keys, 'string-null-to-null', value, null); } else if (error.expected.includes('undefined') && value === 'null') { setValueByPath(data, keys, undefined); - OpLog.err(`Fixed: ${path} from string null to null`); + logAutoFixApplied(path, keys, 'string-null-to-undefined', value, undefined); } else if (error.expected.includes('null') && value === undefined) { setValueByPath(data, keys, null); - OpLog.err(`Fixed: ${path} from undefined to null`); + logAutoFixApplied(path, keys, 'undefined-to-null', value, null); } else if (error.expected.includes('boolean') && !value) { setValueByPath(data, keys, false); - OpLog.err(`Fixed: ${path} to false (was ${value})`); + logAutoFixApplied(path, keys, 'falsey-to-false', value, false); } else if (keys[0] === 'task' && error.expected.includes('number')) { // If the value is a string that can be parsed to a number, parse it if (typeof value === 'string' && !isNaN(parseFloat(value))) { - setValueByPath(data, keys, parseFloat(value)); - OpLog.err( - `Fixed: ${path} from string "${value}" to number ${parseFloat(value)}`, - ); + const parsedValue = parseFloat(value); + setValueByPath(data, keys, parsedValue); + logAutoFixApplied(path, keys, 'task-string-to-number', value, parsedValue); } else { setValueByPath(data, keys, 0); - OpLog.err(`Fixed: ${path} to 0 (was ${value})`); + logAutoFixApplied(path, keys, 'task-number-default-zero', value, 0); } } else if ( // Issue #7330: a TASK entity recreated from a partial LWW Update can @@ -83,13 +156,16 @@ export const autoFixTypiaErrors = ( ? INBOX_PROJECT.id : (Object.keys(projectEntities)[0] ?? INBOX_PROJECT.id); setValueByPath(data, keys, fallback); - OpLog.err(`Fixed: ${path} from undefined to "${fallback}" for task.projectId`); + logAutoFixApplied(path, keys, 'task-project-id-fallback', value, fallback); } else { const defaultValue = RECREATE_FALLBACK.TASK.defaults[field]; setValueByPath(data, keys, defaultValue); - OpLog.err( - `Fixed: ${path} from undefined to ${JSON.stringify(defaultValue)} ` + - `for task.${field}`, + logAutoFixApplied( + path, + keys, + 'task-required-field-default', + value, + defaultValue, ); } } else if ( @@ -102,7 +178,7 @@ export const autoFixTypiaErrors = ( ) { // Fix for issue #4593: simpleCounter countOnDay null value setValueByPath(data, keys, 0); - OpLog.err(`Fixed: ${path} from null to 0 for simpleCounter`); + logAutoFixApplied(path, keys, 'simple-counter-countOnDay-null-to-zero', value, 0); } else if ( keys[0] === 'taskRepeatCfg' && keys[1] === 'entities' && @@ -118,7 +194,13 @@ export const autoFixTypiaErrors = ( const orderIndex = ids.indexOf(entityId); const orderValue = orderIndex >= 0 ? orderIndex : 0; setValueByPath(data, keys, orderValue); - OpLog.err(`Fixed: ${path} from null to ${orderValue} for taskRepeatCfg order`); + logAutoFixApplied( + path, + keys, + 'task-repeat-cfg-order-null-to-index', + value, + orderValue, + ); } else if ( keys[0] === 'metric' && keys[1] === 'entities' && @@ -131,7 +213,7 @@ export const autoFixTypiaErrors = ( // Fix deprecated metric array fields (obstructions, improvements, improvementsTomorrow) // These fields are marked "TODO remove" and will be removed in future setValueByPath(data, keys, []); - OpLog.err(`Fixed: ${path} to empty array for deprecated metric field`); + logAutoFixApplied(path, keys, 'metric-deprecated-array-default', value, []); } else if ( keys[0] === 'improvement' && keys[1] === 'hiddenImprovementBannerItems' && @@ -139,7 +221,7 @@ export const autoFixTypiaErrors = ( ) { // Fix improvement.hiddenImprovementBannerItems (deprecated) setValueByPath(data, keys, []); - OpLog.err(`Fixed: ${path} to empty array for deprecated improvement field`); + logAutoFixApplied(path, keys, 'improvement-deprecated-array-default', value, []); } } }); @@ -193,7 +275,6 @@ const setValueByPath = ( value: unknown, ): void => { if (!Array.isArray(path) || path.length === 0) return; - OpLog.err('Auto-fixing error =>', path, value); let current: Record = obj; for (let i = 0; i < path.length - 1; i++) { diff --git a/src/app/op-log/validation/data-repair.spec.ts b/src/app/op-log/validation/data-repair.spec.ts index f7c4cdc002..d796392516 100644 --- a/src/app/op-log/validation/data-repair.spec.ts +++ b/src/app/op-log/validation/data-repair.spec.ts @@ -14,6 +14,7 @@ import { import { IssueProvider } from '../../features/issue/issue.model'; import { AppDataComplete } from '../model/model-config'; import { dirtyDeepCopy } from '../../util/dirtyDeepCopy'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; const FAKE_PROJECT_ID = 'FAKE_PROJECT_ID'; describe('dataRepair()', () => { @@ -2267,6 +2268,8 @@ describe('dataRepair()', () => { describe('invalid dueDay/deadlineDay sanitization (#6908)', () => { it('should clear invalid dueDay from task', () => { + const invalidDueDay = '-/-/2026'; + const warnSpy = spyOn(OP_LOG_SYNC_LOGGER, 'warn').and.stub(); const taskState = { ...mock.task, ...fakeEntityStateFromArray([ @@ -2275,7 +2278,7 @@ describe('dataRepair()', () => { id: 'INVALID_DUE', title: 'INVALID_DUE', projectId: FAKE_PROJECT_ID, - dueDay: '-/-/2026' as any, + dueDay: invalidDueDay as any, }, ]), } as any; @@ -2287,6 +2290,15 @@ describe('dataRepair()', () => { }); expect(result.data.task.entities['INVALID_DUE']!.dueDay).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith( + '[data-repair] Clearing invalid task date field', + jasmine.objectContaining({ + taskId: 'INVALID_DUE', + field: 'dueDay', + valueStringLength: invalidDueDay.length, + }), + ); + expect(JSON.stringify(warnSpy.calls.allArgs())).not.toContain(invalidDueDay); }); it('should clear invalid deadlineDay from task', () => { diff --git a/src/app/op-log/validation/data-repair.ts b/src/app/op-log/validation/data-repair.ts index 79891c2055..c3366bd1a0 100644 --- a/src/app/op-log/validation/data-repair.ts +++ b/src/app/op-log/validation/data-repair.ts @@ -16,6 +16,7 @@ import { INBOX_PROJECT } from '../../features/project/project.const'; import { autoFixTypiaErrors } from './auto-fix-typia-errors'; import { IValidation } from 'typia'; import { OpLog } from '../../core/log'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; import { repairMenuTree } from './repair-menu-tree'; import { initialTimeTrackingState } from '../../features/time-tracking/store/time-tracking.reducer'; import { RepairSummary } from '../core/operation.types'; @@ -193,14 +194,22 @@ const _fixInvalidDueDateStrings = ( const t = taskState.entities[id] as TaskCopy; if (!t) continue; if (typeof t.dueDay === 'string' && !isDBDateStr(t.dueDay)) { - OpLog.warn(`[data-repair] Clearing invalid dueDay "${t.dueDay}" on task ${id}`); + OP_LOG_SYNC_LOGGER.warn('[data-repair] Clearing invalid task date field', { + taskId: id, + field: 'dueDay', + valueType: 'string', + valueStringLength: t.dueDay.length, + }); t.dueDay = undefined; fixedCount++; } if (typeof t.deadlineDay === 'string' && !isDBDateStr(t.deadlineDay)) { - OpLog.warn( - `[data-repair] Clearing invalid deadlineDay "${t.deadlineDay}" on task ${id}`, - ); + OP_LOG_SYNC_LOGGER.warn('[data-repair] Clearing invalid task date field', { + taskId: id, + field: 'deadlineDay', + valueType: 'string', + valueStringLength: t.deadlineDay.length, + }); t.deadlineDay = undefined; fixedCount++; } diff --git a/src/app/op-log/validation/hibernate-repro.integration.spec.ts b/src/app/op-log/validation/hibernate-repro.integration.spec.ts index a1b0323d18..c9767026fc 100644 --- a/src/app/op-log/validation/hibernate-repro.integration.spec.ts +++ b/src/app/op-log/validation/hibernate-repro.integration.spec.ts @@ -166,7 +166,7 @@ describe('Hibernate repro — cross-model invariants (issue #7330)', () => { console.log('[S2]', { isValid, err }); expect(isValid).toBe(false); - expect(err).toContain('tagId "ghost-tag"'); + expect(err).toContain('tagId from task not existing'); }); }); @@ -197,7 +197,7 @@ describe('Hibernate repro — cross-model invariants (issue #7330)', () => { console.log('[S3]', { isValid, err }); expect(isValid).toBe(false); - expect(err).toContain('projectId ghost-project'); + expect(err).toContain('projectId from task not existing'); }); }); @@ -260,7 +260,7 @@ describe('Hibernate repro — cross-model invariants (issue #7330)', () => { console.log('[S6]', { isValid, err }); expect(isValid).toBe(false); - expect(err).toContain('Missing task id ghost-task'); + expect(err).toContain('Inconsistent Task State: Missing task id for tag'); }); }); diff --git a/src/app/op-log/validation/is-related-model-data-valid.spec.ts b/src/app/op-log/validation/is-related-model-data-valid.spec.ts index ce2b797a1d..450885a608 100644 --- a/src/app/op-log/validation/is-related-model-data-valid.spec.ts +++ b/src/app/op-log/validation/is-related-model-data-valid.spec.ts @@ -1,6 +1,7 @@ import { AppDataComplete } from '../model/model-config'; import { OpLog } from '../../core/log'; import { TODAY_TAG } from '../../features/tag/tag.const'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; describe('isRelatedModelDataValid', () => { let isRelatedModelDataValid: any; @@ -13,6 +14,7 @@ describe('isRelatedModelDataValid', () => { spyOn(OpLog, 'warn'); spyOn(OpLog, 'err'); spyOn(OpLog, 'critical'); + spyOn(OP_LOG_SYNC_LOGGER, 'log'); /* eslint-disable @typescript-eslint/no-require-imports */ // Reset modules to allow re-importing with mocks // @ts-ignore @@ -58,6 +60,83 @@ describe('isRelatedModelDataValid', () => { expect(result).toBe(false); }); + it('should log validity error metadata without raw app data', () => { + const privateProjectTitle = 'Private Project Title'; + const data: any = { + project: { + ids: ['p1'], + entities: { + p1: { + id: 'p1', + title: privateProjectTitle, + taskIds: ['missing-task'], + backlogTaskIds: [], + noteIds: [], + }, + }, + }, + tag: { ids: [], entities: {} }, + task: { ids: [], entities: {} }, + taskRepeatCfg: { ids: [], entities: {} }, + archiveYoung: { task: { ids: [], entities: {} } }, + archiveOld: { task: { ids: [], entities: {} } }, + note: { ids: [], entities: {}, todayOrder: [] }, + issueProvider: { ids: [], entities: {} }, + reminders: [], + menuTree: { projectTree: [], tagTree: [] }, + }; + + const result = isRelatedModelDataValid(data as AppDataComplete); + + expect(result).toBe(false); + expect(OP_LOG_SYNC_LOGGER.log).toHaveBeenCalledWith( + '[is-related-model-data-valid] Validity error info', + jasmine.objectContaining({ + error: 'Missing task data for project', + tid: 'missing-task', + projectId: 'p1', + }), + ); + expect( + JSON.stringify((OP_LOG_SYNC_LOGGER.log as jasmine.Spy).calls.allArgs()), + ).not.toContain(privateProjectTitle); + }); + + it('should not log raw invalid menu tree node kinds', () => { + const privateNodeKind = 'Private Invalid Node Kind'; + const data: any = { + project: { ids: [], entities: {} }, + tag: { ids: [], entities: {} }, + task: { ids: [], entities: {} }, + taskRepeatCfg: { ids: [], entities: {} }, + archiveYoung: { task: { ids: [], entities: {} } }, + archiveOld: { task: { ids: [], entities: {} } }, + note: { ids: [], entities: {}, todayOrder: [] }, + issueProvider: { ids: [], entities: {} }, + reminders: [], + menuTree: { + projectTree: [{ id: 'tree-node-id', k: privateNodeKind }], + tagTree: [], + }, + }; + + const result = isRelatedModelDataValid(data as AppDataComplete); + + expect(result).toBe(false); + expect(OP_LOG_SYNC_LOGGER.log).toHaveBeenCalledWith( + '[is-related-model-data-valid] Validity error info', + jasmine.objectContaining({ + error: 'Invalid node kind in projectTree', + treeType: 'projectTree', + id: 'tree-node-id', + nodeKind: 'unknown', + }), + ); + expect( + JSON.stringify((OP_LOG_SYNC_LOGGER.log as jasmine.Spy).calls.allArgs()), + ).not.toContain(privateNodeKind); + }); + it('should fail validation when task has non-existent repeatCfgId', () => { const dataWithOrphanedRepeatCfgId: any = { project: { diff --git a/src/app/op-log/validation/is-related-model-data-valid.ts b/src/app/op-log/validation/is-related-model-data-valid.ts index c06511ea0d..b26dee12ab 100644 --- a/src/app/op-log/validation/is-related-model-data-valid.ts +++ b/src/app/op-log/validation/is-related-model-data-valid.ts @@ -1,7 +1,8 @@ import { devError } from '../../util/dev-error'; -import { environment } from '../../../environments/environment'; import { AppDataComplete } from '../model/model-config'; import { OpLog } from '../../core/log'; +import type { SyncLogMeta } from '@sp/sync-core'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; import { MenuTreeKind, MenuTreeTreeNode, @@ -19,12 +20,80 @@ import { TaskArchive } from '../../features/tasks/task.model'; let errorCount = 0; let lastValidityError: string | undefined; +const SAFE_VALIDITY_INFO_KEYS = new Set([ + 'archiveLabel', + 'defaultProjectId', + 'id', + 'ipId', + 'nid', + 'nodeKind', + 'parentId', + 'pid', + 'projectId', + 'repeatCfgId', + 'subId', + 'tagId', + 'taskId', + 'tid', + 'treeType', +]); + +const KNOWN_MENU_TREE_KINDS = new Set(Object.values(MenuTreeKind)); + +const getMenuTreeKindForLog = (kind: unknown): string => + typeof kind === 'string' && KNOWN_MENU_TREE_KINDS.has(kind) ? kind : 'unknown'; + +const getValueType = (value: unknown): string => { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +}; + +const getValidityInfoMeta = (additionalInfo: unknown): SyncLogMeta => { + if ( + additionalInfo === null || + typeof additionalInfo !== 'object' || + Array.isArray(additionalInfo) + ) { + return { + additionalInfoType: getValueType(additionalInfo), + additionalInfoArrayLength: Array.isArray(additionalInfo) + ? additionalInfo.length + : undefined, + }; + } + + const info = additionalInfo as Record; + const meta: SyncLogMeta = { + additionalInfoType: 'object', + additionalInfoKeyCount: Object.keys(info).length, + }; + + for (const key of SAFE_VALIDITY_INFO_KEYS) { + const value = info[key]; + if (key === 'nodeKind') { + meta[key] = getMenuTreeKindForLog(value); + continue; + } + if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' || + value === null + ) { + meta[key] = value; + } + } + + return meta; +}; + export const isRelatedModelDataValid = (d: AppDataComplete): boolean => { errorCount = 0; lastValidityError = undefined; // Reset at start of each validation if (!d) { - _validityError('Data is null or undefined', { d }); + _validityError('Data is null or undefined'); return false; } @@ -40,7 +109,9 @@ export const isRelatedModelDataValid = (d: AppDataComplete): boolean => { !d.issueProvider || !d.reminders ) { - _validityError('Missing required model data in AppDataComplete', { d }); + _validityError('Missing required model data in AppDataComplete', { + additionalInfoType: getValueType(d), + }); return false; } @@ -95,14 +166,10 @@ export const getLastValidityError = (): string | undefined => lastValidityError; const _validityError = (errTxt: string, additionalInfo?: unknown): void => { if (additionalInfo) { - OpLog.log('Validity Error Info: ', additionalInfo); - if (environment.production) { - try { - OpLog.log('Validity Error Info string: ', JSON.stringify(additionalInfo)); - } catch (e) { - OpLog.warn('Failed to stringify validity error info:', e); - } - } + OP_LOG_SYNC_LOGGER.log('[is-related-model-data-valid] Validity error info', { + error: errTxt, + ...getValidityInfoMeta(additionalInfo), + }); } if (errorCount <= 3) { devError(errTxt); @@ -199,7 +266,7 @@ const validateTasksToProjectsAndTags = ( for (const pid of d.project.ids) { const project = d.project.entities[pid]; if (!project) { - _validityError('No project', { pid, d }); + _validityError('No project', { pid }); return false; } @@ -214,15 +281,18 @@ const validateTasksToProjectsAndTags = ( for (const tid of projectTaskSet) { const task = d.task.entities[tid]; if (!task) { - _validityError(`Missing task data (tid: ${tid}) for Project ${project.title}`, { - project, - d, + _validityError('Missing task data for project', { + tid, + projectId: project.id, }); return false; } if (task.projectId !== project.id) { - _validityError('Inconsistent task projectId', { task, project, d }); + _validityError('Inconsistent task projectId', { + taskId: task.id, + projectId: project.id, + }); return false; } } @@ -232,20 +302,26 @@ const validateTasksToProjectsAndTags = ( for (const tid of d.task.ids) { const task = d.task.entities[tid]; if (!task) { - _validityError('Orphaned task ID in task.ids (no matching entity)', { tid, d }); + _validityError('Orphaned task ID in task.ids (no matching entity)', { tid }); return false; } // Check project reference if (task.projectId && !projectIds.has(task.projectId)) { - _validityError(`projectId ${task.projectId} from task not existing`, { task, d }); + _validityError('projectId from task not existing', { + taskId: task.id, + projectId: task.projectId, + }); return false; } // Check tag references for (const tagId of task.tagIds) { if (!tagIds.has(tagId)) { - _validityError(`tagId "${tagId}" from task not existing`, { task, d }); + _validityError('tagId from task not existing', { + taskId: task.id, + tagId, + }); return false; } } @@ -255,8 +331,8 @@ const validateTasksToProjectsAndTags = ( _validityError( `repeatCfgId "${task.repeatCfgId}" from task "${task.id}" not existing`, { - task, - d, + taskId: task.id, + repeatCfgId: task.repeatCfgId, }, ); return false; @@ -264,7 +340,7 @@ const validateTasksToProjectsAndTags = ( // Check if task has project or tag if (!task.parentId && !task.projectId && task.tagIds.length === 0) { - _validityError(`Task without project or tag`, { task, d }); + _validityError(`Task without project or tag`, { taskId: task.id }); return false; } } @@ -299,10 +375,10 @@ const validateTasksToProjectsAndTags = ( for (const tid of tag.taskIds) { if (!taskIds.has(tid)) { - _validityError( - `Inconsistent Task State: Missing task id ${tid} for Tag ${tag.title}`, - { tag, d }, - ); + _validityError(`Inconsistent Task State: Missing task id for tag`, { + tagId: tag.id, + tid, + }); return false; } } @@ -324,15 +400,18 @@ const validateNotes = ( for (const nid of project.noteIds) { const note = d.note.entities[nid]; if (!note) { - _validityError(`Missing note data (tid: ${nid}) for Project ${project.title}`, { - project, - d, + _validityError('Missing note data for project', { + nid, + projectId: project.id, }); return false; } if (note.projectId !== project.id) { - _validityError('Inconsistent note projectId', { note, project, d }); + _validityError('Inconsistent note projectId', { + nid: note.id, + projectId: project.id, + }); return false; } } @@ -343,7 +422,7 @@ const validateNotes = ( if (!noteIds.has(nid)) { _validityError( `Inconsistent Note State: Missing note id ${nid} for note.todayOrder`, - { d }, + { nid }, ); return false; } @@ -366,8 +445,8 @@ const validateSubTasks = ( // Check if parent exists if (task.parentId && !d.task.entities[task.parentId]) { _validityError(`Inconsistent Task State: Lonely Sub Task in Today ${task.id}`, { - task, - d, + taskId: task.id, + parentId: task.parentId, }); return false; } @@ -377,7 +456,7 @@ const validateSubTasks = ( if (!d.task.entities[subId]) { _validityError( `Inconsistent Task State: Missing sub task data in today ${subId}`, - { task, d }, + { taskId: task.id, subId }, ); return false; } @@ -396,8 +475,8 @@ const validateSubTasks = ( _validityError( `Inconsistent Task State: Lonely Sub Task in Archive ${task.id}`, { - task, - d, + taskId: task.id, + parentId: task.parentId, }, ); return false; @@ -410,7 +489,7 @@ const validateSubTasks = ( if (!d.archiveOld.task?.entities?.[subId]) { _validityError( `Inconsistent Task State: Missing sub task data in archive ${subId}`, - { task, d }, + { taskId: task.id, subId }, ); return false; } @@ -431,8 +510,8 @@ const validateSubTasks = ( _validityError( `Inconsistent Task State: Lonely Sub Task in Old Archive ${task.id}`, { - task, - d, + taskId: task.id, + parentId: task.parentId, }, ); return false; @@ -445,7 +524,7 @@ const validateSubTasks = ( if (!d.archiveYoung.task?.entities?.[subId]) { _validityError( `Inconsistent Task State: Missing sub task data in old archive ${subId}`, - { task, d }, + { taskId: task.id, subId }, ); return false; } @@ -463,7 +542,7 @@ const validateIssueProviders = (d: AppDataComplete, projectIds: Set): bo if (ip && ip.defaultProjectId && !projectIds.has(ip.defaultProjectId)) { _validityError( `defaultProjectId ${ip.defaultProjectId} from issueProvider not existing`, - { ip, d }, + { ipId: ip.id, defaultProjectId: ip.defaultProjectId }, ); return false; } @@ -489,7 +568,7 @@ const validateMenuTree = ( ): boolean => { for (const node of nodes) { if (!node || typeof node !== 'object') { - _validityError(`Invalid menuTree node in ${treeType}`, { node, d }); + _validityError(`Invalid menuTree node in ${treeType}`, { treeType }); return false; } @@ -497,16 +576,18 @@ const validateMenuTree = ( // Validate folder structure if (!node.id || !node.name) { _validityError(`Invalid folder node in ${treeType} - missing id or name`, { - node, - d, + treeType, + id: node.id, + nodeKind: node.k, }); return false; } if (!Array.isArray(node.children)) { _validityError(`Invalid folder node in ${treeType} - children not array`, { - node, - d, + treeType, + id: node.id, + nodeKind: node.k, }); return false; } @@ -518,33 +599,43 @@ const validateMenuTree = ( } else if (treeType === 'projectTree' && node.k === MenuTreeKind.PROJECT) { // Validate project reference if (!node.id) { - _validityError(`Project node in menuTree missing id`, { node, d }); + _validityError(`Project node in menuTree missing id`, { + treeType, + nodeKind: node.k, + }); return false; } if (!projectIds.has(node.id)) { _validityError( `Orphaned project reference in menuTree - project ${node.id} doesn't exist`, - { node, treeType, d }, + { id: node.id, treeType, nodeKind: node.k }, ); return false; } } else if (treeType === 'tagTree' && node.k === MenuTreeKind.TAG) { // Validate tag reference if (!node.id) { - _validityError(`Tag node in menuTree missing id`, { node, d }); + _validityError(`Tag node in menuTree missing id`, { + treeType, + nodeKind: node.k, + }); return false; } if (!tagIds.has(node.id)) { _validityError( `Orphaned tag reference in menuTree - tag ${node.id} doesn't exist`, - { node, treeType, d }, + { id: node.id, treeType, nodeKind: node.k }, ); return false; } } else { - _validityError(`Invalid node kind in ${treeType}: ${node.k}`, { node, d }); + _validityError(`Invalid node kind in ${treeType}`, { + treeType, + id: node.id, + nodeKind: node.k, + }); return false; } } @@ -554,7 +645,7 @@ const validateMenuTree = ( // Validate projectTree if (d.menuTree?.projectTree) { if (!Array.isArray(d.menuTree.projectTree)) { - _validityError('menuTree.projectTree is not an array', { d }); + _validityError('menuTree.projectTree is not an array'); return false; } if (!validateTreeNodes(d.menuTree.projectTree, 'projectTree')) { @@ -565,7 +656,7 @@ const validateMenuTree = ( // Validate tagTree if (d.menuTree?.tagTree) { if (!Array.isArray(d.menuTree.tagTree)) { - _validityError('menuTree.tagTree is not an array', { d }); + _validityError('menuTree.tagTree is not an array'); return false; } if (!validateTreeNodes(d.menuTree.tagTree, 'tagTree')) { diff --git a/src/app/op-log/validation/repair-menu-tree.spec.ts b/src/app/op-log/validation/repair-menu-tree.spec.ts index 00ce7271fa..a9f95e3cfe 100644 --- a/src/app/op-log/validation/repair-menu-tree.spec.ts +++ b/src/app/op-log/validation/repair-menu-tree.spec.ts @@ -3,8 +3,17 @@ import { MenuTreeKind, MenuTreeState, } from '../../features/menu-tree/store/menu-tree.model'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; describe('repairMenuTree', () => { + let logSpy: jasmine.Spy; + let warnSpy: jasmine.Spy; + + beforeEach(() => { + logSpy = spyOn(OP_LOG_SYNC_LOGGER, 'log').and.stub(); + warnSpy = spyOn(OP_LOG_SYNC_LOGGER, 'warn').and.stub(); + }); + it('should remove orphaned project references from projectTree', () => { const validProjectIds = new Set(['project1', 'project2']); const validTagIds = new Set(); @@ -188,4 +197,40 @@ describe('repairMenuTree', () => { expect(result.projectTree).toEqual([{ k: MenuTreeKind.PROJECT, id: 'project1' }]); expect(result.tagTree).toEqual([{ k: MenuTreeKind.TAG, id: 'tag1' }]); }); + + it('should log invalid nodes without raw node values', () => { + const privateNodeName = 'Private Folder Name'; + const validProjectIds = new Set(); + const validTagIds = new Set(); + + const menuTree: MenuTreeState = { + projectTree: [ + { + k: 'unexpected', + id: 'invalid-node-id', + name: privateNodeName, + } as any, + ], + tagTree: [], + }; + + const result = repairMenuTree(menuTree, validProjectIds, validTagIds); + + expect(result.projectTree).toEqual([]); + expect(warnSpy).toHaveBeenCalledWith( + '[repair-menu-tree] Removed invalid node', + jasmine.objectContaining({ + treeType: 'projectTree', + nodeKind: 'unknown', + hasNodeId: true, + }), + ); + + const serializedLogArgs = JSON.stringify([ + ...logSpy.calls.allArgs(), + ...warnSpy.calls.allArgs(), + ]); + expect(serializedLogArgs).not.toContain(privateNodeName); + expect(serializedLogArgs).not.toContain('invalid-node-id'); + }); }); diff --git a/src/app/op-log/validation/repair-menu-tree.ts b/src/app/op-log/validation/repair-menu-tree.ts index e055fefa8c..dcb32c94e7 100644 --- a/src/app/op-log/validation/repair-menu-tree.ts +++ b/src/app/op-log/validation/repair-menu-tree.ts @@ -3,7 +3,39 @@ import { MenuTreeState, MenuTreeTreeNode, } from '../../features/menu-tree/store/menu-tree.model'; -import { OpLog } from '../../core/log'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; + +const KNOWN_MENU_TREE_KINDS = new Set(Object.values(MenuTreeKind)); + +const getNodeKindForLog = (kind: unknown): string => + typeof kind === 'string' && KNOWN_MENU_TREE_KINDS.has(kind) ? kind : 'unknown'; + +const logOrphanedReferenceRemoval = ( + treeType: 'projectTree' | 'tagTree', + referenceType: 'project' | 'tag', + referenceId: string, +): void => { + OP_LOG_SYNC_LOGGER.log('[repair-menu-tree] Removed orphaned reference', { + treeType, + referenceType, + referenceId, + }); +}; + +const logInvalidNodeRemoval = ( + treeType: 'projectTree' | 'tagTree', + node: MenuTreeTreeNode, +): void => { + OP_LOG_SYNC_LOGGER.warn('[repair-menu-tree] Removed invalid node', { + treeType, + nodeKind: getNodeKindForLog(node.k), + hasNodeId: typeof node.id === 'string', + childCount: + node.k === MenuTreeKind.FOLDER && Array.isArray(node.children) + ? node.children.length + : undefined, + }); +}; /** * Repairs menuTree by removing orphaned project/tag references @@ -17,7 +49,7 @@ export const repairMenuTree = ( validProjectIds: Set, validTagIds: Set, ): MenuTreeState => { - OpLog.log('Repairing menuTree - removing orphaned references'); + OP_LOG_SYNC_LOGGER.log('[repair-menu-tree] Repairing orphaned references'); /** * Recursively filters tree nodes, removing orphaned project/tag references @@ -41,18 +73,18 @@ export const repairMenuTree = ( if (validProjectIds.has(node.id)) { filtered.push(node); } else { - OpLog.log(`Removing orphaned project reference ${node.id} from ${treeType}`); + logOrphanedReferenceRemoval(treeType, 'project', node.id); } } else if (treeType === 'tagTree' && node.k === MenuTreeKind.TAG) { // Keep tag only if it exists if (validTagIds.has(node.id)) { filtered.push(node); } else { - OpLog.log(`Removing orphaned tag reference ${node.id} from ${treeType}`); + logOrphanedReferenceRemoval(treeType, 'tag', node.id); } } else { // kind mismatch or unknown - OpLog.warn(`Removing invalid node from ${treeType}:`, node); + logInvalidNodeRemoval(treeType, node); } } diff --git a/src/app/op-log/validation/validation-fn.ts b/src/app/op-log/validation/validation-fn.ts index ad3fc1f85e..155cc715c8 100644 --- a/src/app/op-log/validation/validation-fn.ts +++ b/src/app/op-log/validation/validation-fn.ts @@ -24,7 +24,8 @@ import { MetricState } from '../../features/metric/metric.model'; import { GlobalConfigState } from '../../features/config/global-config.model'; import { AppDataComplete } from '../model/model-config'; import { ValidationResult } from '../core/types/sync.types'; -import { OpLog } from '../../core/log'; +import { OP_LOG_SYNC_LOGGER } from '../core/sync-logger.adapter'; +import { getValidationFailureLogMeta } from './validation-log-meta'; import { PluginMetaDataState, PluginUserDataState, @@ -58,7 +59,7 @@ const _validateSection = createValidate(); export const validateAllData = ( d: AppDataComplete | R, ): ValidationResult => { - const r = _wrapValidate(_validateAllData(d)); + const r = _wrapValidate(_validateAllData(d), d, false, 'appData'); return r as ValidationResult; // unfortunately that is quite a bit slower @@ -81,35 +82,59 @@ export const appDataValidators: { data: AppDataComplete[K] | R, ) => ValidationResult; } = { - task: (d: R | TaskState) => _wrapValidate(_validateTask(d), d, true), + task: (d: R | TaskState) => _wrapValidate(_validateTask(d), d, true, 'task'), taskRepeatCfg: (d: R | TaskRepeatCfgState) => - _wrapValidate(_validateTaskRepeatCfg(d), d, true), - archiveYoung: (d: R | ArchiveModel) => validateArchiveModel(d), - archiveOld: (d: R | ArchiveModel) => validateArchiveModel(d), - project: (d: R | ProjectState) => _wrapValidate(_validateProject(d), d, true), - menuTree: (d: R | MenuTreeState) => _wrapValidate(_validateMenuTree(d), d, false), - tag: (d: R | TagState) => _wrapValidate(_validateTag(d), d, true), + _wrapValidate(_validateTaskRepeatCfg(d), d, true, 'taskRepeatCfg'), + archiveYoung: (d: R | ArchiveModel) => validateArchiveModel(d, 'archiveYoung'), + archiveOld: (d: R | ArchiveModel) => validateArchiveModel(d, 'archiveOld'), + project: (d: R | ProjectState) => + _wrapValidate(_validateProject(d), d, true, 'project'), + menuTree: (d: R | MenuTreeState) => + _wrapValidate(_validateMenuTree(d), d, false, 'menuTree'), + tag: (d: R | TagState) => _wrapValidate(_validateTag(d), d, true, 'tag'), simpleCounter: (d: R | SimpleCounterState) => - _wrapValidate(_validateSimpleCounter(d), d, true), - note: (d) => _wrapValidate(_validateNote(d), d, true), - reminders: (d: R | Reminder[]) => _wrapValidate(_validateReminders(d)), - planner: (d: R | PlannerState) => _wrapValidate(_validatePlanner(d)), - boards: (d: R | BoardsState) => _wrapValidate(_validateBoards(d)), - issueProvider: (d) => _wrapValidate(_validateIssueProvider(d), d, true), - metric: (d: R | MetricState) => _wrapValidate(_validateMetric(d), d, true), - globalConfig: (d: R | GlobalConfigState) => _wrapValidate(_validateGlobalConfig(d)), - timeTracking: (d: R | TimeTrackingState) => _wrapValidate(_validateTimeTracking(d)), + _wrapValidate(_validateSimpleCounter(d), d, true, 'simpleCounter'), + note: (d) => _wrapValidate(_validateNote(d), d, true, 'note'), + reminders: (d: R | Reminder[]) => + _wrapValidate(_validateReminders(d), d, false, 'reminders'), + planner: (d: R | PlannerState) => + _wrapValidate(_validatePlanner(d), d, false, 'planner'), + boards: (d: R | BoardsState) => + _wrapValidate(_validateBoards(d), d, false, 'boards'), + issueProvider: (d) => + _wrapValidate(_validateIssueProvider(d), d, true, 'issueProvider'), + metric: (d: R | MetricState) => _wrapValidate(_validateMetric(d), d, true, 'metric'), + globalConfig: (d: R | GlobalConfigState) => + _wrapValidate(_validateGlobalConfig(d), d, false, 'globalConfig'), + timeTracking: (d: R | TimeTrackingState) => + _wrapValidate(_validateTimeTracking(d), d, false, 'timeTracking'), pluginUserData: (d: R | PluginUserDataState) => - _wrapValidate(_validatePluginUserData(d)), + _wrapValidate(_validatePluginUserData(d), d, false, 'pluginUserData'), pluginMetadata: (d: R | PluginMetaDataState) => - _wrapValidate(_validatePluginMetadata(d)), - section: (d: R | SectionState) => _wrapValidate(_validateSection(d), d, true), + _wrapValidate(_validatePluginMetadata(d), d, false, 'pluginMetadata'), + section: (d: R | SectionState) => + _wrapValidate(_validateSection(d), d, true, 'section'), } as const; -const validateArchiveModel = (d: ArchiveModel | R): ValidationResult => { +const logValidationFailure = ( + context: string, + result: ValidationResult, + inputData?: unknown, + isEntityCheck = false, +): void => { + OP_LOG_SYNC_LOGGER.log( + '[validation-fn] Validation failed', + getValidationFailureLogMeta({ context, result, inputData, isEntityCheck }), + ); +}; + +const validateArchiveModel = ( + d: ArchiveModel | R, + context: 'archiveYoung' | 'archiveOld', +): ValidationResult => { const r = _validateArchive(d); if (!r.success) { - OpLog.log('Validation failed', (r as any)?.errors, r.data); + logValidationFailure(context, r, d); } if (!isEntityStateConsistent((d as ArchiveModel).task)) { return { @@ -130,16 +155,17 @@ export const validateAppDataProperty = ( const _wrapValidate = ( result: ValidationResult, - d?: R, + d?: unknown, isEntityCheck = false, + context = 'unknown', ): ValidationResult => { if (!result.success) { - OpLog.log('Validation failed', (result as any)?.errors, result, d); + logValidationFailure(context, result, d, isEntityCheck); } if (isEntityCheck && !isEntityStateConsistent(d as any)) { return { success: false, - data: d, + data: d as R, errors: [{ expected: 'Valid Entity State', path: '.', value: d }], }; } diff --git a/src/app/op-log/validation/validation-log-meta.spec.ts b/src/app/op-log/validation/validation-log-meta.spec.ts new file mode 100644 index 0000000000..d51c64df63 --- /dev/null +++ b/src/app/op-log/validation/validation-log-meta.spec.ts @@ -0,0 +1,46 @@ +import { getValidationFailureLogMeta } from './validation-log-meta'; +import type { ValidationResult } from '../core/types/sync.types'; + +describe('getValidationFailureLogMeta', () => { + it('should summarize validation failures without raw data values', () => { + const privateTaskTitle = 'Private task title'; + const result = { + success: false, + data: { + title: privateTaskTitle, + }, + errors: [ + { + path: '$input.task.entities["task-id"].title', + expected: 'string', + value: privateTaskTitle, + }, + ], + } as ValidationResult; + + const meta = getValidationFailureLogMeta({ + context: 'task', + result, + inputData: { + title: privateTaskTitle, + }, + isEntityCheck: true, + }); + + expect(meta).toEqual( + jasmine.objectContaining({ + context: 'task', + success: false, + isEntityCheck: true, + errorCount: 1, + firstErrorPath: '$input.task.entities["task-id"].title', + firstErrorExpected: 'string', + resultDataType: 'object', + resultDataKeyCount: 1, + inputDataType: 'object', + inputDataKeyCount: 1, + }), + ); + expect(JSON.stringify(meta)).not.toContain(privateTaskTitle); + }); +}); diff --git a/src/app/op-log/validation/validation-log-meta.ts b/src/app/op-log/validation/validation-log-meta.ts new file mode 100644 index 0000000000..2ad206a9a9 --- /dev/null +++ b/src/app/op-log/validation/validation-log-meta.ts @@ -0,0 +1,53 @@ +import type { SyncLogMeta } from '@sp/sync-core'; +import type { ValidationResult } from '../core/types/sync.types'; + +const getValueType = (value: unknown): string => { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +}; + +const getObjectKeyCount = (value: unknown): number | undefined => + value !== null && typeof value === 'object' && !Array.isArray(value) + ? Object.keys(value).length + : undefined; + +const getArrayLength = (value: unknown): number | undefined => + Array.isArray(value) ? value.length : undefined; + +export const getValidationFailureLogMeta = ({ + context, + result, + inputData, + isEntityCheck, +}: { + context: string; + result: ValidationResult; + inputData?: unknown; + isEntityCheck?: boolean; +}): SyncLogMeta => { + if (result.success) { + return { + context, + success: true, + isEntityCheck, + }; + } + + const firstError = result.errors[0]; + + return { + context, + success: false, + isEntityCheck, + errorCount: result.errors.length, + firstErrorPath: firstError?.path, + firstErrorExpected: firstError?.expected, + resultDataType: getValueType(result.data), + resultDataKeyCount: getObjectKeyCount(result.data), + resultDataArrayLength: getArrayLength(result.data), + inputDataType: inputData === undefined ? undefined : getValueType(inputData), + inputDataKeyCount: getObjectKeyCount(inputData), + inputDataArrayLength: getArrayLength(inputData), + }; +}; diff --git a/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.spec.ts b/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.spec.ts index b6d23da6e6..bc846c3aa7 100644 --- a/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.spec.ts +++ b/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.spec.ts @@ -426,6 +426,66 @@ describe('taskSharedSchedulingMetaReducer', () => { expect(updatedSubtask.dueDay).toBe(getDbDateStr()); }); + it('should use the action date when replayed after the day changed', () => { + const actionToday = '2024-06-14'; + const replayToday = '2024-06-15'; + const testState = createStateWithExistingTasks([], [], [], []); + testState[appStateFeatureKey] = { + ...testState[appStateFeatureKey], + todayStr: replayToday, + }; + testState[TASK_FEATURE_NAME].entities.task1 = createMockTask({ + id: 'task1', + dueDay: undefined, + }); + testState[TASK_FEATURE_NAME].ids.push('task1'); + + const action = TaskSharedActions.planTasksForToday({ + taskIds: ['task1'], + today: actionToday, + parentTaskMap: {}, + }); + + metaReducer(testState, action); + const updatedState = mockReducer.calls.mostRecent().args[0]; + const updatedTask = updatedState[TASK_FEATURE_NAME].entities.task1; + + expect(updatedTask.dueDay).toBe(actionToday); + expect(updatedTask.dueDay).not.toBe(replayToday); + }); + + it('should use the action start-of-next-day offset when replayed after settings changed', () => { + const actionToday = '2024-06-14'; + const actionOffset = 4 * 60 * 60 * 1000; + const scheduledAt = new Date(2024, 5, 15, 2, 0, 0, 0).getTime(); + const testState = createStateWithExistingTasks([], [], [], []); + testState[appStateFeatureKey] = { + ...testState[appStateFeatureKey], + todayStr: actionToday, + startOfNextDayDiffMs: 0, + }; + testState[TASK_FEATURE_NAME].entities.task1 = createMockTask({ + id: 'task1', + dueDay: undefined, + dueWithTime: scheduledAt, + }); + testState[TASK_FEATURE_NAME].ids.push('task1'); + + const action = TaskSharedActions.planTasksForToday({ + taskIds: ['task1'], + today: actionToday, + startOfNextDayDiffMs: actionOffset, + parentTaskMap: {}, + }); + + metaReducer(testState, action); + const updatedState = mockReducer.calls.mostRecent().args[0]; + const updatedTask = updatedState[TASK_FEATURE_NAME].entities.task1; + + expect(updatedTask.dueWithTime).toBe(scheduledAt); + expect(updatedTask.dueDay).toBe(actionToday); + }); + it('should remove tasks from planner days when adding to Today', () => { const testState = { ...createStateWithExistingTasks([], [], [], []), diff --git a/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.ts b/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.ts index f4425445a7..befa896077 100644 --- a/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.ts +++ b/src/app/root-store/meta/task-shared-meta-reducers/task-shared-scheduling.reducer.ts @@ -178,11 +178,16 @@ const handlePlanTasksForToday = ( state: RootState, taskIds: string[], parentTaskMap: Record, + todayFromAction?: string, + startOfNextDayDiffMsFromAction?: number, isClearScheduledTime?: boolean, ): RootState => { const todayTag = getTag(state, TODAY_TAG.id); - const today = state[appStateFeatureKey]?.todayStr ?? getDbDateStr(); - const offsetMs = state[appStateFeatureKey]?.startOfNextDayDiffMs ?? 0; + const today = todayFromAction ?? state[appStateFeatureKey]?.todayStr ?? getDbDateStr(); + const offsetMs = + startOfNextDayDiffMsFromAction ?? + state[appStateFeatureKey]?.startOfNextDayDiffMs ?? + 0; // Filter out tasks that don't exist, are already in today, or whose parent is in today const newTasksForToday = taskIds.filter((taskId) => { @@ -339,10 +344,19 @@ const createActionHandlers = (state: RootState, action: Action): ActionHandlerMa [TaskSharedActions.planTasksForToday.type]: () => { const { taskIds, + today, + startOfNextDayDiffMs, parentTaskMap = {}, isClearScheduledTime, } = action as ReturnType; - return handlePlanTasksForToday(state, taskIds, parentTaskMap, isClearScheduledTime); + return handlePlanTasksForToday( + state, + taskIds, + parentTaskMap, + today, + startOfNextDayDiffMs, + isClearScheduledTime, + ); }, [TaskSharedActions.removeTasksFromTodayTag.type]: () => { const { taskIds } = action as ReturnType< diff --git a/src/app/root-store/meta/task-shared.actions.ts b/src/app/root-store/meta/task-shared.actions.ts index ddc561c4db..03d74ea9b3 100644 --- a/src/app/root-store/meta/task-shared.actions.ts +++ b/src/app/root-store/meta/task-shared.actions.ts @@ -268,6 +268,10 @@ export const TaskSharedActions = createActionGroup({ // Today Tag Management planTasksForToday: (taskProps: { taskIds: string[]; + // The logical day "today" referred to when the action was created. + // Optional only for legacy operation replay and older tests. + today?: string; + startOfNextDayDiffMs?: number; parentTaskMap?: { [taskId: string]: string | undefined }; isShowSnack?: boolean; isSkipRemoveReminder?: boolean; diff --git a/src/tsconfig.spec.json b/src/tsconfig.spec.json index b200e1872a..a24095a9cf 100644 --- a/src/tsconfig.spec.json +++ b/src/tsconfig.spec.json @@ -7,7 +7,8 @@ "paths": { "@super-productivity/plugin-api": ["packages/plugin-api/src/index.ts"], "@sp/shared-schema": ["packages/shared-schema/src/index.ts"], - "@sp/sync-core": ["packages/sync-core/src/index.ts"] + "@sp/sync-core": ["packages/sync-core/src/index.ts"], + "@sp/sync-providers": ["packages/sync-providers/src/index.ts"] } }, "files": ["test.ts", "polyfills.ts"], diff --git a/tsconfig.base.json b/tsconfig.base.json index 8f498cc4a7..d6073548d1 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -26,7 +26,8 @@ "paths": { "@super-productivity/plugin-api": ["packages/plugin-api/src/index.ts"], "@sp/shared-schema": ["packages/shared-schema/src/index.ts"], - "@sp/sync-core": ["packages/sync-core/src/index.ts"] + "@sp/sync-core": ["packages/sync-core/src/index.ts"], + "@sp/sync-providers": ["packages/sync-providers/src/index.ts"] }, "plugins": [ {